diff --git a/cmd/topo/projects.go b/cmd/topo/projects.go index f2bab895..2cdfae9c 100644 --- a/cmd/topo/projects.go +++ b/cmd/topo/projects.go @@ -27,13 +27,11 @@ var projectsCmd = &cobra.Command{ var projects []catalog.Project var err error - source := getSource(cmd) - switch source { - case builtinProjects: - projects, err = catalog.ListBuiltinProjects() - default: - projects, err = catalog.ListProjectsFromURL(ctx, source) + source, err := cmd.Flags().GetString(sourceFlag) + if err != nil { + panic(fmt.Sprintf("internal error: %s flag not registered: %v", sourceFlag, err)) } + projects, err = catalog.ListProjectsFromURL(ctx, source) if err != nil { return err } @@ -56,23 +54,6 @@ var projectsCmd = &cobra.Command{ func init() { addTargetFlag(projectsCmd) addTimeoutFlag(projectsCmd, defaultTimeout) - if experimentalFeaturesEnabled() { - projectsCmd.Flags().StringP(sourceFlag, "s", "", "where to source projects' data from") - } + projectsCmd.Flags().StringP(sourceFlag, "s", catalog.DefaultCatalogURL, "where to source projects' data from") rootCmd.AddCommand(projectsCmd) } - -const builtinProjects = "builtin" - -func getSource(cmd *cobra.Command) string { - if experimentalFeaturesEnabled() { - flagValue, err := cmd.Flags().GetString(sourceFlag) - if err != nil { - panic(fmt.Sprintf("internal error: %s flag not registered: %v", sourceFlag, err)) - } - if flagValue != "" { - return flagValue - } - } - return builtinProjects -} diff --git a/docs/development/DEVELOPMENT.md b/docs/development/DEVELOPMENT.md index 7a7a83e5..0d37827b 100644 --- a/docs/development/DEVELOPMENT.md +++ b/docs/development/DEVELOPMENT.md @@ -87,3 +87,19 @@ docker compose up ``` The documentation preview is available at `http://localhost:3000` and automatically reloads when files change. + +## Updating the catalog schema + +The catalog schema version is declared by the `go:generate` directive in `internal/catalog/catalog.go` and copied into `internal/catalog/catalog_schema_generated.go`. Its major component also selects the catalog version used by Topo. + +Generating the Go types requires Node.js 20 or newer, `npx`, and access to the internet. From the repository root, run: + +```sh +go generate ./internal/catalog +``` + +The generator downloads the configured version's `catalog.schema.json`, generates the catalog Go types with the pinned Quicktype version, and rewrites `internal/catalog/catalog_schema_generated.go`. To use another schema release, update the directive in `internal/catalog/catalog.go` and rerun the command. + +Commit the newly generated catalog schema types file and raise a PR to update the main branch. + +Available catalog versions and schemas are published in the [Topo Project Catalog Artifactory repository](https://artifacts.tools.arm.com/devx-topo-project-catalog/). diff --git a/internal/catalog/catalog.go b/internal/catalog/catalog.go index e48eac81..5e21908e 100644 --- a/internal/catalog/catalog.go +++ b/internal/catalog/catalog.go @@ -1,41 +1,28 @@ package catalog +//go:generate go run ../../scripts/generate_catalog_types v2.0.0 + import ( - "bytes" "context" - _ "embed" "encoding/json" "fmt" - "io" - "net/http" - "net/url" "os" "strings" - "github.com/santhosh-tekuri/jsonschema/v6" + "github.com/arm/topo/internal/fetch" ) -//go:embed data/catalog.json -var catalogJSON []byte - -//go:embed data/catalog.schema.json -var catalogSchemaJSON []byte - -type catalogDocument struct { - Schema string `json:"$schema,omitempty"` - Projects []Project `json:"projects"` -} +type Project = ProjectElement -type Project struct { - Name string `json:"name"` - Description string `json:"description"` - Features []string `json:"features"` - URL string `json:"url"` - Ref string `json:"ref"` -} +var ( + majorCatalogVersion = majorVersion(CatalogSchemaVersion) + defaultURL = "https://artifacts.tools.arm.com/devx-topo-project-catalog/" + majorCatalogVersion + "/catalog/" + DefaultCatalogURL = defaultURL + "catalog.json" +) -func ListBuiltinProjects() ([]Project, error) { - return parseProjects(catalogJSON) +func majorVersion(version string) string { + major, _, _ := strings.Cut(version, ".") + return major } func ListProjectsFromURL(ctx context.Context, url string) ([]Project, error) { @@ -47,39 +34,31 @@ func ListProjectsFromURL(ctx context.Context, url string) ([]Project, error) { } func parseProjects(b []byte) ([]Project, error) { - if err := validateAgainstSchema(b); err != nil { - return nil, fmt.Errorf("failed schema validation: %w", err) + catalogVersion, versionErr := unmarshalCatalogVersion(b) + schemaVersionMajor := majorVersion(CatalogSchemaVersion) + if majorVersion(catalogVersion) != schemaVersionMajor { + return nil, fmt.Errorf( + "failed to parse catalog: requested catalog version %q is incompatible with supported schema version %q: %w", + catalogVersion, + CatalogSchemaVersion, + versionErr, + ) } - - var catalog catalogDocument - if err := json.Unmarshal(b, &catalog); err != nil { - return nil, fmt.Errorf("failed to unmarshal projects: %w", err) + catalog, err := UnmarshalCatalogDocument(b) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal catalog: %w", err) } - return catalog.Projects, nil } -func validateAgainstSchema(b []byte) error { - const projectsSchemaURL = "https://raw.githubusercontent.com/arm/topo/main/internal/catalog/data/catalog.schema.json" - - compiler := jsonschema.NewCompiler() - schemaDoc, err := jsonschema.UnmarshalJSON(bytes.NewReader(catalogSchemaJSON)) - if err != nil { - return fmt.Errorf("failed to unmarshal schema: %w", err) - } - if err := compiler.AddResource(projectsSchemaURL, schemaDoc); err != nil { - return fmt.Errorf("failed to add schema resource: %w", err) +func unmarshalCatalogVersion(b []byte) (string, error) { + var header struct { + Version string `json:"version"` } - schema, err := compiler.Compile(projectsSchemaURL) - if err != nil { - return fmt.Errorf("failed to compile schema: %w", err) - } - - jsonDoc, err := jsonschema.UnmarshalJSON(bytes.NewReader(b)) - if err != nil { - return fmt.Errorf("failed to unmarshal projects: %w", err) + if err := json.Unmarshal(b, &header); err != nil { + return "", err } - return schema.Validate(jsonDoc) + return header.Version, nil } func fetchProjectsJSON(ctx context.Context, url string) ([]byte, error) { @@ -92,42 +71,9 @@ func fetchProjectsJSON(ctx context.Context, url string) ([]byte, error) { return data, nil } - data, err := httpGet(ctx, url) + data, err := fetch.Get(ctx, url) if err != nil { return nil, fmt.Errorf("failed to fetch project: %w", err) } return data, nil } - -func httpGet(ctx context.Context, rawURL string) ([]byte, error) { - parsedURL, err := url.Parse(rawURL) - if err != nil { - return nil, err - } - - if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" { - return nil, fmt.Errorf("unsupported URL scheme: %s", parsedURL.Scheme) - } - - req, err := http.NewRequestWithContext( - ctx, - http.MethodGet, - parsedURL.String(), - nil, - ) - if err != nil { - return nil, err - } - - resp, err := http.DefaultClient.Do(req) // #nosec G704 -- URL is explicitly provided by the CLI user and scheme-validated above. - if err != nil { - return nil, err - } - defer resp.Body.Close() // nolint:errcheck - - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - return nil, fmt.Errorf("request failed: %s", resp.Status) - } - - return io.ReadAll(resp.Body) -} diff --git a/internal/catalog/catalog_schema_generated.go b/internal/catalog/catalog_schema_generated.go new file mode 100644 index 00000000..67e01f3a --- /dev/null +++ b/internal/catalog/catalog_schema_generated.go @@ -0,0 +1,70 @@ +// Code generated from JSON Schema using quicktype. DO NOT EDIT. +// To parse and unparse this JSON data, add this code to your project and do: +// +// catalogDocument, err := UnmarshalCatalogDocument(bytes) +// bytes, err = catalogDocument.Marshal() + +package catalog + +import "encoding/json" + +func UnmarshalCatalogDocument(data []byte) (CatalogDocument, error) { + var r CatalogDocument + err := json.Unmarshal(data, &r) + return r, err +} + +func (r *CatalogDocument) Marshal() ([]byte, error) { + return json.Marshal(r) +} + +// A catalog of Topo Project repositories. +type CatalogDocument struct { + // Schema URL for editor and tooling discovery. + Schema *Schema `json:"$schema,omitempty"` + Projects []ProjectElement `json:"projects"` + // Catalog version. + Version string `json:"version"` +} + +type ProjectElement struct { + Description string `json:"description"` + // Optional list of hardware, runtime, or platform features used by the project. + Features []string `json:"features"` + Name string `json:"name"` + // Optional dictionary of parameter definitions used for parameterized projects. + Parameters map[string]ParameterValue `json:"parameters,omitempty"` + // Git ref to use when fetching the project repository. + Ref string `json:"ref"` + // Repository URL containing the project. This URL is the stable source identifier used to + // match catalog entries to configured sources. + URL string `json:"url"` +} + +// Parameter definition. +type ParameterValue struct { + // Value used if user skips input (only valid when not required). + Default *string `json:"default,omitempty"` + // Context displayed in user prompts. + Description *string `json:"description,omitempty"` + // Hint text displayed in help and prompts. + Example *string `json:"example,omitempty"` + // Advisory metadata that implementations may use to discover, filter, or suggest suitable + // parameter values. Unknown hint keys should be ignored. + Hints *Hints `json:"hints,omitempty"` + // If true, implementations must enforce input or error. + Required *bool `json:"required,omitempty"` +} + +// Advisory metadata that implementations may use to discover, filter, or suggest suitable +// parameter values. Unknown hint keys should be ignored. +type Hints struct { +} + +type Schema string + +const ( + HTTPSRawGithubusercontentCOMArmTopoProjectCatalogMainDataCatalogSchemaJSON Schema = "https://raw.githubusercontent.com/arm/topo-project-catalog/main/data/catalog.schema.json" +) + +const CatalogSchemaVersion = "v2.0.0" diff --git a/internal/catalog/catalog_test.go b/internal/catalog/catalog_test.go index b52c23e6..8ddbce13 100644 --- a/internal/catalog/catalog_test.go +++ b/internal/catalog/catalog_test.go @@ -44,17 +44,6 @@ func TestListProjectsFromURL(t *testing.T) { assert.Equal(t, projects, got) }) - t.Run("errors when payload doesn't validate against schema", func(t *testing.T) { - path := filepath.Join(t.TempDir(), "file.json") - projects := []catalog.Project{{Name: "aloha"}} - testutil.RequireWriteFile(t, path, string(asJSON(projects))) - - url := fmt.Sprintf("file://%s", path) - _, err := catalog.ListProjectsFromURL(context.Background(), url) - - require.Error(t, err) - }) - t.Run("errors when request fails", func(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.NotFound(w, r) @@ -76,15 +65,51 @@ func TestListProjectsFromURL(t *testing.T) { _, err := catalog.ListProjectsFromURL(context.Background(), url) require.Error(t, err) - assert.ErrorContains(t, err, "failed to unmarshal projects") + assert.ErrorContains(t, err, "failed to parse catalog") + assert.ErrorContains(t, err, `requested catalog version "" is incompatible`) + }) + + t.Run("errors when catalog fails to unmarshal", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "file.json") + testutil.RequireWriteFile(t, path, fmt.Sprintf( + `{"projects":"invalid","version":%q}`, + catalog.CatalogSchemaVersion, + )) + + url := fmt.Sprintf("file://%s", path) + _, err := catalog.ListProjectsFromURL(context.Background(), url) + + require.Error(t, err) + assert.ErrorContains(t, err, "failed to unmarshal catalog") + }) + + t.Run("reports incompatible catalog", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "file.json") + catalogVersion := "v0.0.0" + if catalogVersion == catalog.CatalogSchemaVersion { + catalogVersion = "v999.0.0" + } + testutil.RequireWriteFile(t, path, fmt.Sprintf(`{"projects":"invalid","version":%q}`, catalogVersion)) + + url := fmt.Sprintf("file://%s", path) + _, err := catalog.ListProjectsFromURL(context.Background(), url) + + require.Error(t, err) + assert.ErrorContains(t, err, fmt.Sprintf( + `requested catalog version %q is incompatible with supported schema version %q`, + catalogVersion, + catalog.CatalogSchemaVersion, + )) }) } func asJSON(projects []catalog.Project) []byte { data, err := json.Marshal(struct { Projects []catalog.Project `json:"projects"` + Version string `json:"version"` }{ Projects: projects, + Version: catalog.CatalogSchemaVersion, }) if err != nil { panic(err) diff --git a/internal/fetch/fetch.go b/internal/fetch/fetch.go new file mode 100644 index 00000000..aecec002 --- /dev/null +++ b/internal/fetch/fetch.go @@ -0,0 +1,42 @@ +package fetch + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" +) + +func Get(ctx context.Context, rawURL string) ([]byte, error) { + parsedURL, err := url.Parse(rawURL) + if err != nil { + return nil, fmt.Errorf("parsing URL failed: %w", err) + } + if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" { + return nil, fmt.Errorf("unsupported URL scheme: %s", parsedURL.Scheme) + } + + request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsedURL.String(), nil) + if err != nil { + return nil, fmt.Errorf("creating request failed: %w", err) + } + + // #nosec G704 -- callers explicitly provide the URL and its scheme is validated above. + response, err := http.DefaultClient.Do(request) + if err != nil { + return nil, fmt.Errorf("sending request failed: %w", err) + } + + if response.StatusCode != http.StatusOK { + statusErr := fmt.Errorf("request failed: HTTP %d (%s)", response.StatusCode, response.Status) + return nil, errors.Join(statusErr, response.Body.Close()) + } + + data, readErr := io.ReadAll(response.Body) + if err := errors.Join(readErr, response.Body.Close()); err != nil { + return nil, fmt.Errorf("failed to read response: %w", err) + } + return data, nil +} diff --git a/internal/fetch/fetch_test.go b/internal/fetch/fetch_test.go new file mode 100644 index 00000000..9359c7a7 --- /dev/null +++ b/internal/fetch/fetch_test.go @@ -0,0 +1,67 @@ +package fetch_test + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/arm/topo/internal/fetch" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGet(t *testing.T) { + t.Run("returns response body for successful request", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) { + response.WriteHeader(http.StatusOK) + _, _ = response.Write([]byte("contents")) + })) + defer server.Close() + + got, err := fetch.Get(context.Background(), server.URL) + + require.NoError(t, err) + assert.Equal(t, []byte("contents"), got) + }) + + t.Run("returns error for unsuccessful request", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) { + response.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + + _, err := fetch.Get(context.Background(), server.URL) + + assert.ErrorContains(t, err, "HTTP 404") + }) + + t.Run("rejects unsupported URL scheme", func(t *testing.T) { + _, err := fetch.Get(context.Background(), "file:///tmp/catalog.json") + + assert.ErrorContains(t, err, "unsupported URL scheme") + }) + + t.Run("respects context cancellation", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err := fetch.Get(ctx, "https://example.com") + + assert.ErrorIs(t, err, context.Canceled) + }) + + t.Run("respects caller timeout", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, request *http.Request) { + <-request.Context().Done() + })) + defer server.Close() + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond) + defer cancel() + + _, err := fetch.Get(ctx, server.URL) + + assert.ErrorIs(t, err, context.DeadlineExceeded) + }) +} diff --git a/internal/install/install.go b/internal/install/install.go index 9762154f..0511e093 100644 --- a/internal/install/install.go +++ b/internal/install/install.go @@ -4,24 +4,18 @@ import ( "context" "errors" "fmt" - "io" - "net/http" "net/url" "slices" "strings" - "time" archiveutil "github.com/arm/topo/internal/archive" "github.com/arm/topo/internal/command" + "github.com/arm/topo/internal/fetch" "github.com/arm/topo/internal/runner" "github.com/arm/topo/internal/ssh" "github.com/arm/topo/internal/version" ) -const ( - downloadTimeout = 2 * time.Minute -) - var defaultCandidatePaths = []string{"/usr/local/bin", "/usr/bin", "~/bin"} type PathCandidate struct { @@ -99,33 +93,6 @@ func FindPathDirs(r runner.Runner) ([]PathCandidate, error) { return validPaths, nil } -func downloadFile(ctx context.Context, url *url.URL) ([]byte, error) { - ctx, cancel := context.WithTimeout(ctx, downloadTimeout) - defer cancel() - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url.String(), nil) - if err != nil { - return nil, err - } - - // #nosec G704 -- Request is previously validated - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - - if resp.StatusCode != http.StatusOK { - statusErr := fmt.Errorf("unexpected fetch status code: %d", resp.StatusCode) - return nil, errors.Join(statusErr, resp.Body.Close()) - } - - b, readErr := io.ReadAll(resp.Body) - if err := errors.Join(readErr, resp.Body.Close()); err != nil { - return nil, err - } - return b, nil -} - func install(installPath string, r runner.Runner, binaries map[string][]byte) error { mode := "0755" @@ -191,12 +158,7 @@ func downloadLatestArtifactoryBinaries(ctx context.Context, artifactoryURL strin return nil, fmt.Errorf("failed to construct Artifactory download URL: %w", err) } - parsedArchiveURL, err := url.Parse(archiveURL) - if err != nil { - return nil, fmt.Errorf("failed to parse Artifactory download URL: %w", err) - } - - tarball, err := downloadFile(ctx, parsedArchiveURL) + tarball, err := fetch.Get(ctx, archiveURL) if err != nil { return nil, fmt.Errorf("failed to download latest release: %w", err) } diff --git a/internal/upgrade/upgrade.go b/internal/upgrade/upgrade.go index 9407eb72..2256f35d 100644 --- a/internal/upgrade/upgrade.go +++ b/internal/upgrade/upgrade.go @@ -4,13 +4,12 @@ import ( "context" "errors" "fmt" - "io" - "net/http" "os" "path/filepath" "runtime" archiveutil "github.com/arm/topo/internal/archive" + "github.com/arm/topo/internal/fetch" "github.com/arm/topo/internal/output/logger" "github.com/arm/topo/internal/output/term" "github.com/arm/topo/internal/version" @@ -100,26 +99,10 @@ func BinaryName(name string) string { } func downloadArchive(ctx context.Context, url string) ([]byte, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return nil, fmt.Errorf("failed to create download request: %w", err) - } - // #nosec G704 -- request to a hardcoded, trusted URL - resp, err := http.DefaultClient.Do(req) + data, err := fetch.Get(ctx, url) if err != nil { return nil, fmt.Errorf("failed to download archive: %w", err) } - defer resp.Body.Close() //nolint:errcheck - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("failed to download archive from %s: HTTP %d", url, resp.StatusCode) - } - - data, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read archive: %w", err) - } - return data, nil } diff --git a/internal/version/latest_artifactory.go b/internal/version/latest_artifactory.go index 3ee00e94..0db11db2 100644 --- a/internal/version/latest_artifactory.go +++ b/internal/version/latest_artifactory.go @@ -3,14 +3,12 @@ package version import ( "context" "fmt" - "io" "maps" - "net/http" "regexp" "slices" "sort" - "github.com/arm/topo/internal/output/logger" + "github.com/arm/topo/internal/fetch" ) const ArtifactoryBaseURL = "https://artifacts.tools.arm.com/topo" @@ -18,30 +16,10 @@ const ArtifactoryBaseURL = "https://artifacts.tools.arm.com/topo" var artifactoryVersionRe = regexp.MustCompile(`href="v?(\d+)\.(\d+)\.(\d+)/"`) func FetchLatestArtifactory(ctx context.Context, url string) (string, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return "", fmt.Errorf("creating request: %w", err) - } - // #nosec G704 -- request to a hardcoded, trusted URL - resp, err := http.DefaultClient.Do(req) + body, err := fetch.Get(ctx, url) if err != nil { return "", fmt.Errorf("fetching version index: %w", err) } - defer func() { - err = resp.Body.Close() - if err != nil { - logger.Error(fmt.Sprintf("failed to close version check response body: %v", err)) - } - }() - - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("fetching version index: HTTP %d", resp.StatusCode) - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return "", fmt.Errorf("reading version index: %w", err) - } matches := artifactoryVersionRe.FindAllStringSubmatch(string(body), -1) if len(matches) == 0 { diff --git a/internal/version/latest_homebrew.go b/internal/version/latest_homebrew.go index d5072303..1f79fb09 100644 --- a/internal/version/latest_homebrew.go +++ b/internal/version/latest_homebrew.go @@ -3,11 +3,9 @@ package version import ( "context" "fmt" - "io" - "net/http" "regexp" - "github.com/arm/topo/internal/output/logger" + "github.com/arm/topo/internal/fetch" ) const HomebrewFormulaURL = "https://raw.githubusercontent.com/arm/homebrew-topo/main/Formula/topo.rb" @@ -15,31 +13,10 @@ const HomebrewFormulaURL = "https://raw.githubusercontent.com/arm/homebrew-topo/ var homebrewFormulaVersionRe = regexp.MustCompile(`(?m)^\s*version\s+"([^"]+)"\s*$`) func FetchLatestHomebrew(ctx context.Context, formulaURL string) (string, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, formulaURL, nil) - if err != nil { - return "", fmt.Errorf("creating Homebrew formula request: %w", err) - } - - // #nosec G704 -- request to a hardcoded, trusted URL - resp, err := http.DefaultClient.Do(req) + body, err := fetch.Get(ctx, formulaURL) if err != nil { return "", fmt.Errorf("fetching Homebrew formula: %w", err) } - defer func() { - err = resp.Body.Close() - if err != nil { - logger.Error(fmt.Sprintf("failed to close Homebrew formula response body: %v", err)) - } - }() - - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("fetching Homebrew formula: HTTP %d", resp.StatusCode) - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return "", fmt.Errorf("reading Homebrew formula: %w", err) - } return ParseHomebrewFormulaVersion(string(body)) } diff --git a/scripts/generate_catalog_types/main.go b/scripts/generate_catalog_types/main.go new file mode 100644 index 00000000..a6c9054c --- /dev/null +++ b/scripts/generate_catalog_types/main.go @@ -0,0 +1,87 @@ +package main + +import ( + "errors" + "fmt" + "go/format" + "net/url" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" +) + +var catalogSchemaVersionPattern = regexp.MustCompile(`^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$`) + +const ( + quicktypeVersion = "26.0.0" + defaultSchemaBaseURL = "https://artifacts.tools.arm.com/devx-topo-project-catalog/" +) + +func main() { + if err := run(); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "generating catalog types failed: %v\n", err) + os.Exit(1) + } +} + +func run() error { + if len(os.Args) != 2 || os.Args[1] == "" { + return fmt.Errorf("expected one catalog version; usage: go run ./scripts/generate_catalog_types VERSION") + } + + catalogSchemaVersion := os.Args[1] + if err := validateCatalogVersion(catalogSchemaVersion); err != nil { + return err + } + schemaURL := defaultSchemaBaseURL + url.PathEscape(catalogSchemaVersion) + "/catalog/catalog.schema.json" + + outputFile, err := generatedOutputPath() + if err != nil { + return err + } + return generateTypes(schemaURL, catalogSchemaVersion, outputFile) +} + +func validateCatalogVersion(version string) error { + if !catalogSchemaVersionPattern.MatchString(version) { + return fmt.Errorf("catalog version %q must use vMAJOR.MINOR.PATCH format, for example v1.1.2", version) + } + return nil +} + +func generatedOutputPath() (string, error) { + _, sourceFile, _, ok := runtime.Caller(0) + if !ok { + return "", fmt.Errorf("determining repository root: generator source location is unavailable") + } + repositoryRoot := filepath.Clean(filepath.Join(filepath.Dir(sourceFile), "..", "..")) + return filepath.Join(repositoryRoot, "internal", "catalog", "catalog_schema_generated.go"), nil +} + +func generateTypes(schemaURL string, catalogSchemaVersion string, outputFile string) error { + // #nosec G702 -- schemaURL is passed as an argument without invoking a shell. + command := exec.Command("npx", "--yes", "quicktype@"+quicktypeVersion, + "--src-lang", "schema", "--lang", "go", "--package", "catalog", + "--top-level", "CatalogDocument", schemaURL) + command.Stderr = os.Stderr + generated, err := command.Output() + if err != nil { + if errors.Is(err, exec.ErrNotFound) { + return fmt.Errorf("starting quicktype failed: npx was not found; install Node.js 20 or newer: %w", err) + } + return fmt.Errorf("quicktype %s failed for schema %q: %w", quicktypeVersion, schemaURL, err) + } + + generated = fmt.Appendf(generated, "\nconst CatalogSchemaVersion = %q\n", catalogSchemaVersion) + formatted, err := format.Source(generated) + if err != nil { + return fmt.Errorf("failed to format quicktype output from schema %q: %w", schemaURL, err) + } + // #nosec G703 -- outputFile is derived from the generator source location. + if err := os.WriteFile(outputFile, formatted, 0o644); err != nil { + return fmt.Errorf("failed to write generated types to %q: %w", outputFile, err) + } + return nil +} diff --git a/scripts/update_projects/catalog.go b/scripts/update_projects/catalog.go deleted file mode 100644 index 083c6133..00000000 --- a/scripts/update_projects/catalog.go +++ /dev/null @@ -1,58 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" -) - -const ( - relativeCatalogOutputPath = "internal/catalog/data/catalog.json" -) - -type Catalog struct { - Schema string `json:"$schema"` - Projects []Project `json:"projects"` -} - -func ReadProjects(path string) ([]Project, error) { - file, err := os.Open(path) - if err != nil { - return nil, err - } - defer file.Close() //nolint:errcheck // Closing a read-only file cannot affect catalog generation. - - var document Catalog - if err := json.NewDecoder(file).Decode(&document); err != nil { - return nil, err - } - return document.Projects, nil -} - -func WriteCatalog(path string, document Catalog) error { - outputFile, err := os.Create(path) - if err != nil { - return fmt.Errorf("failed to create catalog output: %w", err) - } - enc := json.NewEncoder(outputFile) - enc.SetIndent("", " ") - writeErr := enc.Encode(document) - closeErr := outputFile.Close() - if writeErr != nil { - return fmt.Errorf("failed to write projects: %w", writeErr) - } - if closeErr != nil { - return fmt.Errorf("failed to close catalog output: %w", closeErr) - } - return nil -} - -func CatalogFilePath() (string, error) { - repoRoot, err := findModuleRoot() - if err != nil { - return "", err - } - - return filepath.Join(repoRoot, filepath.FromSlash(relativeCatalogOutputPath)), nil -} diff --git a/scripts/update_projects/catalog_schema.go b/scripts/update_projects/catalog_schema.go deleted file mode 100644 index d4f4fda6..00000000 --- a/scripts/update_projects/catalog_schema.go +++ /dev/null @@ -1,89 +0,0 @@ -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "os" - "path/filepath" - - "github.com/santhosh-tekuri/jsonschema/v6" -) - -const ( - relativeCatalogSchemaPath = "internal/catalog/data/catalog.schema.json" - catalogSchemaURL = "https://raw.githubusercontent.com/arm/topo/main/internal/catalog/data/catalog.schema.json" -) - -type CatalogSchema struct { - schemaURL string - schema *jsonschema.Schema -} - -func NewCatalogSchema(path string) (CatalogSchema, error) { - schemaJSON, err := os.ReadFile(path) - if err != nil { - return CatalogSchema{}, fmt.Errorf("failed to read catalog schema: %w", err) - } - return NewCatalogSchemaFromBytes(schemaJSON) -} - -func NewCatalogSchemaFromBytes(schemaJSON []byte) (CatalogSchema, error) { - compiler := jsonschema.NewCompiler() - schemaDoc, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaJSON)) - if err != nil { - return CatalogSchema{}, fmt.Errorf("failed to unmarshal schema: %w", err) - } - if err := compiler.AddResource(catalogSchemaURL, schemaDoc); err != nil { - return CatalogSchema{}, fmt.Errorf("failed to add schema resource: %w", err) - } - schema, err := compiler.Compile(catalogSchemaURL) - if err != nil { - return CatalogSchema{}, fmt.Errorf("failed to compile schema: %w", err) - } - - return CatalogSchema{ - schemaURL: catalogSchemaURL, - schema: schema, - }, nil -} - -func (v CatalogSchema) SchemaURL() string { - return v.schemaURL -} - -func (v CatalogSchema) ValidateProject(project Project) error { - document := Catalog{ - Schema: v.SchemaURL(), - Projects: []Project{project}, - } - if err := v.ValidateCatalog(document); err != nil { - return fmt.Errorf("invalid project document: %w", err) - } - return nil -} - -func (v CatalogSchema) ValidateCatalog(document Catalog) error { - jsonBytes, err := json.Marshal(document) - if err != nil { - return fmt.Errorf("failed to marshal catalog: %w", err) - } - - jsonDoc, err := jsonschema.UnmarshalJSON(bytes.NewReader(jsonBytes)) - if err != nil { - return fmt.Errorf("failed to unmarshal catalog: %w", err) - } - if err := v.schema.Validate(jsonDoc); err != nil { - return fmt.Errorf("failed schema validation: %w", err) - } - return nil -} - -func CatalogSchemaFilePath() (string, error) { - repoRoot, err := findModuleRoot() - if err != nil { - return "", err - } - - return filepath.Join(repoRoot, filepath.FromSlash(relativeCatalogSchemaPath)), nil -} diff --git a/scripts/update_projects/catalog_schema_test.go b/scripts/update_projects/catalog_schema_test.go deleted file mode 100644 index 112c5d48..00000000 --- a/scripts/update_projects/catalog_schema_test.go +++ /dev/null @@ -1,113 +0,0 @@ -package main - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestCatalogSchema(t *testing.T) { - t.Run("SchemaURL", func(t *testing.T) { - t.Run("returns catalog schema URL", func(t *testing.T) { - schemaPath, err := CatalogSchemaFilePath() - require.NoError(t, err) - validator, err := NewCatalogSchema(schemaPath) - require.NoError(t, err) - - got := validator.SchemaURL() - - assert.Equal(t, catalogSchemaURL, got) - }) - }) - - t.Run("ValidateProject", func(t *testing.T) { - t.Run("accepts project that matches catalog schema", func(t *testing.T) { - schemaPath, err := CatalogSchemaFilePath() - require.NoError(t, err) - validator, err := NewCatalogSchema(schemaPath) - require.NoError(t, err) - project := Project{ - XTopo: XTopo{ - Name: "Hello World", - Description: "A friendly project", - Features: []string{"web"}, - }, - URL: "https://github.com/Arm-Examples/topo-welcome.git", - Ref: "main", - } - - err = validator.ValidateProject(project) - - assert.NoError(t, err) - }) - - t.Run("rejects project that does not match catalog schema", func(t *testing.T) { - schemaPath, err := CatalogSchemaFilePath() - require.NoError(t, err) - validator, err := NewCatalogSchema(schemaPath) - require.NoError(t, err) - project := Project{ - XTopo: XTopo{ - Description: "Missing a name", - }, - URL: "https://github.com/Arm-Examples/topo-welcome.git", - Ref: "main", - } - - err = validator.ValidateProject(project) - - assert.Error(t, err) - }) - }) - - t.Run("ValidateCatalog", func(t *testing.T) { - t.Run("accepts document that matches catalog schema", func(t *testing.T) { - schemaPath, err := CatalogSchemaFilePath() - require.NoError(t, err) - validator, err := NewCatalogSchema(schemaPath) - require.NoError(t, err) - document := Catalog{ - Schema: catalogSchemaURL, - Projects: []Project{ - { - XTopo: XTopo{ - Name: "Hello World", - Description: "A friendly project", - }, - URL: "https://github.com/Arm-Examples/topo-welcome.git", - Ref: "main", - }, - }, - } - - err = validator.ValidateCatalog(document) - - assert.NoError(t, err) - }) - - t.Run("rejects document that does not match catalog schema", func(t *testing.T) { - schemaPath, err := CatalogSchemaFilePath() - require.NoError(t, err) - validator, err := NewCatalogSchema(schemaPath) - require.NoError(t, err) - document := Catalog{ - Schema: "https://example.com/catalog.schema.json", - Projects: []Project{ - { - XTopo: XTopo{ - Name: "Hello World", - Description: "A friendly project", - }, - URL: "https://github.com/Arm-Examples/topo-welcome.git", - Ref: "main", - }, - }, - } - - err = validator.ValidateCatalog(document) - - assert.Error(t, err) - }) - }) -} diff --git a/scripts/update_projects/catalog_test.go b/scripts/update_projects/catalog_test.go deleted file mode 100644 index 96abff50..00000000 --- a/scripts/update_projects/catalog_test.go +++ /dev/null @@ -1,78 +0,0 @@ -package main - -import ( - "encoding/json" - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestReadProjects(t *testing.T) { - t.Run("reads projects from catalog file", func(t *testing.T) { - path := filepath.Join(t.TempDir(), "catalog.json") - err := os.WriteFile(path, []byte(` -{ - "$schema": "https://raw.githubusercontent.com/arm/topo/main/internal/catalog/data/catalog.schema.json", - "projects": [ - { - "name": "death-star-trench-run", - "description": "Use the Force to benchmark impossible shots", - "features": ["X-wing", "Astromech", "Proton torpedoes"], - "url": "ssh://death-star.example", - "ref": "rebellion" - } - ] -} -`), 0o600) - require.NoError(t, err) - - got, err := ReadProjects(path) - - require.NoError(t, err) - want := []Project{ - { - XTopo: XTopo{ - Name: "death-star-trench-run", - Description: "Use the Force to benchmark impossible shots", - Features: []string{"X-wing", "Astromech", "Proton torpedoes"}, - }, - URL: "ssh://death-star.example", - Ref: "rebellion", - }, - } - assert.Equal(t, want, got) - }) -} - -func TestWriteCatalog(t *testing.T) { - t.Run("writes catalog document to file", func(t *testing.T) { - path := filepath.Join(t.TempDir(), "catalog.json") - want := Catalog{ - Schema: "https://raw.githubusercontent.com/arm/topo/main/internal/catalog/data/catalog.schema.json", - Projects: []Project{ - { - XTopo: XTopo{ - Name: "death-star-trench-run", - Description: "Use the Force to benchmark impossible shots", - Features: []string{"X-wing", "Astromech", "Proton torpedoes"}, - }, - URL: "ssh://death-star.example", - Ref: "rebellion", - }, - }, - } - - err := WriteCatalog(path, want) - - require.NoError(t, err) - gotBytes, err := os.ReadFile(path) - require.NoError(t, err) - var got Catalog - err = json.Unmarshal(gotBytes, &got) - require.NoError(t, err) - assert.Equal(t, want, got) - }) -} diff --git a/scripts/update_projects/github_client.go b/scripts/update_projects/github_client.go deleted file mode 100644 index bf1d292b..00000000 --- a/scripts/update_projects/github_client.go +++ /dev/null @@ -1,67 +0,0 @@ -package main - -import ( - "fmt" - "io" - "net/http" - "net/url" - "path" -) - -type GitHubClient struct { - httpClient *http.Client - token string -} - -func NewGitHubClient(token string) GitHubClient { - return GitHubClient{ - httpClient: http.DefaultClient, - token: token, - } -} - -func (c GitHubClient) FetchFile(source GitHubSource, repoFilePath string) ([]byte, error) { - // #nosec G704 -- URL is constructed from static GitHub source metadata. - req, err := http.NewRequest(http.MethodGet, c.fileURL(source, repoFilePath), nil) - if err != nil { - return nil, err - } - - req.Header.Set("User-Agent", "topo-project-update") - if c.token != "" { - req.Header.Set("Authorization", "token "+c.token) - } - req.Header.Set("Accept", "application/vnd.github.v3.raw") - - // #nosec G704 -- URL is constructed from static GitHub source metadata. - resp, err := c.httpClient.Do(req) - if err != nil { - return nil, fmt.Errorf("request failed: %w", err) - } - defer resp.Body.Close() //nolint:errcheck - - if resp.StatusCode == http.StatusNotFound { - return nil, fmt.Errorf("%s not found (status %d)", repoFilePath, resp.StatusCode) - } - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("unexpected status %d", resp.StatusCode) - } - - content, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read response body: %w", err) - } - return content, nil -} - -func (c GitHubClient) fileURL(source GitHubSource, repoFilePath string) string { - u := url.URL{ - Scheme: "https", - Host: "api.github.com", - Path: path.Join("repos", source.Repo, "contents", repoFilePath), - } - q := u.Query() - q.Set("ref", source.SHA) - u.RawQuery = q.Encode() - return u.String() -} diff --git a/scripts/update_projects/github_source.go b/scripts/update_projects/github_source.go deleted file mode 100644 index 0e7c9966..00000000 --- a/scripts/update_projects/github_source.go +++ /dev/null @@ -1,66 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" -) - -const relativeSourcesPath = "scripts/update_projects/github_sources.json" - -type GitHubSource struct { - Repo string `json:"repo"` - SHA string `json:"sha"` -} - -func (s GitHubSource) String() string { - return fmt.Sprintf("%s@%s", s.Repo, s.SHA) -} - -func (s GitHubSource) ID() ProjectSourceID { - return ProjectSourceID(s.URL()) -} - -func (s GitHubSource) URL() string { - return fmt.Sprintf("https://github.com/%s.git", s.Repo) -} - -func ListGithubSources(path string) ([]GitHubSource, error) { - sourcesFile, err := os.Open(path) - if err != nil { - return nil, err - } - defer sourcesFile.Close() //nolint:errcheck // Closing a read-only file cannot affect catalog generation. - - var sources []GitHubSource - if err := json.NewDecoder(sourcesFile).Decode(&sources); err != nil { - return nil, fmt.Errorf("failed to decode sources: %w", err) - } - if err := validateUniqueGitHubSourceIDs(sources); err != nil { - return nil, err - } - return sources, nil -} - -func GithubSourcesFilePath() (string, error) { - repoRoot, err := findModuleRoot() - if err != nil { - return "", err - } - - return filepath.Join(repoRoot, filepath.FromSlash(relativeSourcesPath)), nil -} - -func validateUniqueGitHubSourceIDs(sources []GitHubSource) error { - seen := make(map[ProjectSourceID]GitHubSource, len(sources)) - for _, source := range sources { - id := source.ID() - previous, exists := seen[id] - if exists { - return fmt.Errorf("duplicate source ID %s for %s and %s", id, previous, source) - } - seen[id] = source - } - return nil -} diff --git a/scripts/update_projects/github_source_test.go b/scripts/update_projects/github_source_test.go deleted file mode 100644 index 7bf0860d..00000000 --- a/scripts/update_projects/github_source_test.go +++ /dev/null @@ -1,25 +0,0 @@ -package main - -import ( - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestListGithubSources(t *testing.T) { - t.Run("disallows sources with duplicate ids", func(t *testing.T) { - sourcesJSON := `[ - {"repo":"example/repo","sha":"first-sha"}, - {"repo":"example/repo","sha":"second-sha"} - ]` - sourcesFilePath := filepath.Join(t.TempDir(), "github_sources.json") - require.NoError(t, os.WriteFile(sourcesFilePath, []byte(sourcesJSON), 0o600)) - - _, err := ListGithubSources(sourcesFilePath) - - assert.EqualError(t, err, "duplicate source ID https://github.com/example/repo.git for example/repo@first-sha and example/repo@second-sha") - }) -} diff --git a/scripts/update_projects/github_sources.json b/scripts/update_projects/github_sources.json deleted file mode 100644 index e4cd5aff..00000000 --- a/scripts/update_projects/github_sources.json +++ /dev/null @@ -1,6 +0,0 @@ -[ - {"repo": "Arm-Examples/topo-welcome", "sha": "main"}, - {"repo": "Arm-Examples/topo-lightbulb-moment", "sha": "main"}, - {"repo": "Arm-Examples/topo-llama-web-ui", "sha": "main"}, - {"repo": "Arm-Examples/topo-simd-visual-benchmark", "sha": "main"} -] diff --git a/scripts/update_projects/main.go b/scripts/update_projects/main.go deleted file mode 100644 index b2379253..00000000 --- a/scripts/update_projects/main.go +++ /dev/null @@ -1,83 +0,0 @@ -package main - -import ( - "log" - "os" - "strings" -) - -func main() { - githubToken := os.Getenv("GITHUB_TOKEN") - if githubToken == "" { - log.Println("⚠️ GITHUB_TOKEN is not set: you might get rate limited") - } - - githubClient := NewGitHubClient(githubToken) - - sourcesFilePath, err := GithubSourcesFilePath() - if err != nil { - log.Fatalf("failed to find sources file: %v\n", err) - } - - sources, err := ListGithubSources(sourcesFilePath) - if err != nil { - log.Fatalf("failed to list sources: %v\n", err) - } - - catalogFilePath, err := CatalogFilePath() - if err != nil { - log.Fatalf("failed to find catalog file: %v\n", err) - } - - currentProjects, err := ReadProjects(catalogFilePath) - if err != nil { - log.Fatalf("failed to read catalog file: %v\n", err) - } - - plan := PlanUpdate(sources, currentProjects) - log.Printf("update plan:\n%s", indent(plan.String())) - if !plan.HasChanges() { - log.Println("catalog already up to date") - return - } - - catalogSchemaFilePath, err := CatalogSchemaFilePath() - if err != nil { - log.Fatalf("failed to find catalog schema file: %v\n", err) - } - - validator, err := NewCatalogSchema(catalogSchemaFilePath) - if err != nil { - log.Fatalf("failed to create schema validator: %v\n", err) - } - - projects := append([]Project{}, plan.Unchanged...) - for _, source := range append(plan.ToAdd, plan.ToUpdate...) { - project, err := FetchProject(githubClient, source) - if err != nil { - log.Fatalf("failed to fetch %s: %v\n", source, err) - } - if err := validator.ValidateProject(project); err != nil { - log.Fatalf("invalid project %s: %v\n", source, err) - } - log.Printf("fetched %s\n", source) - projects = append(projects, project) - } - projects = ProjectsInSourceOrder(sources, projects) - - document := Catalog{ - Schema: validator.SchemaURL(), - Projects: projects, - } - if err := validator.ValidateCatalog(document); err != nil { - log.Fatalf("invalid catalog file: %v\n", err) - } - if err := WriteCatalog(catalogFilePath, document); err != nil { - log.Fatalf("failed to write catalog file: %v\n", err) - } - log.Printf("written catalog to %s\n", catalogFilePath) -} - -func indent(text string) string { - return " " + strings.ReplaceAll(text, "\n", "\n ") -} diff --git a/scripts/update_projects/module_root.go b/scripts/update_projects/module_root.go deleted file mode 100644 index 2a1e3610..00000000 --- a/scripts/update_projects/module_root.go +++ /dev/null @@ -1,25 +0,0 @@ -package main - -import ( - "os" - "path/filepath" -) - -func findModuleRoot() (string, error) { - dir, err := os.Getwd() - if err != nil { - return "", err - } - - for { - if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { - return dir, nil - } - - parent := filepath.Dir(dir) - if parent == dir { - return "", os.ErrNotExist - } - dir = parent - } -} diff --git a/scripts/update_projects/project.go b/scripts/update_projects/project.go deleted file mode 100644 index 53fe3eb2..00000000 --- a/scripts/update_projects/project.go +++ /dev/null @@ -1,87 +0,0 @@ -package main - -import ( - "bytes" - "fmt" - "io" - - "gopkg.in/yaml.v3" -) - -type ProjectSourceID string - -type Project struct { - XTopo - URL string `json:"url"` - Ref string `json:"ref"` -} - -type XTopo struct { - Name string `json:"name"` - Description string `json:"description"` - Features []string `json:"features"` - Parameters map[string]Parameter `json:"parameters,omitempty"` -} - -type Parameter struct { - Description string `json:"description,omitempty"` - Required bool `json:"required,omitempty"` - Default string `json:"default,omitempty"` - Example string `json:"example,omitempty"` - Hints map[string]any `json:"hints,omitempty"` -} - -func (t *XTopo) UnmarshalYAML(node *yaml.Node) error { - type rawXTopo struct { - Name string `yaml:"name"` - Description string `yaml:"description"` - Features []string `yaml:"features"` - Parameters map[string]Parameter `yaml:"parameters,omitempty"` - Args map[string]Parameter `yaml:"args,omitempty"` - } - - var raw rawXTopo - if err := node.Decode(&raw); err != nil { - return err - } - - t.Name = raw.Name - t.Description = raw.Description - t.Features = raw.Features - t.Parameters = raw.Parameters - if len(t.Parameters) == 0 && len(raw.Args) > 0 { - t.Parameters = raw.Args - } - - return nil -} - -func NewProject(source GitHubSource, composeFile io.Reader) (Project, error) { - type composeDocument struct { - XTopo XTopo `yaml:"x-topo"` - } - - var parsed composeDocument - decoder := yaml.NewDecoder(composeFile) - if err := decoder.Decode(&parsed); err != nil { - return Project{}, fmt.Errorf("failed to decode compose file: %w", err) - } - - return Project{ - XTopo: parsed.XTopo, - URL: source.URL(), - Ref: source.SHA, - }, nil -} - -func FetchProject(client GitHubClient, source GitHubSource) (Project, error) { - yamlBytes, err := client.FetchFile(source, "compose.yaml") - if err != nil { - return Project{}, err - } - return NewProject(source, bytes.NewReader(yamlBytes)) -} - -func (t Project) SourceID() ProjectSourceID { - return ProjectSourceID(t.URL) -} diff --git a/scripts/update_projects/project_sort.go b/scripts/update_projects/project_sort.go deleted file mode 100644 index b0e83a82..00000000 --- a/scripts/update_projects/project_sort.go +++ /dev/null @@ -1,21 +0,0 @@ -package main - -import "fmt" - -func ProjectsInSourceOrder(sources []GitHubSource, projects []Project) []Project { - projectByID := make(map[ProjectSourceID]Project, len(projects)) - for _, project := range projects { - projectByID[project.SourceID()] = project - } - - ordered := make([]Project, 0, len(sources)) - for _, source := range sources { - project, exists := projectByID[source.ID()] - if !exists { - panic(fmt.Sprintf("missing project for source %s", source)) - } - ordered = append(ordered, project) - } - - return ordered -} diff --git a/scripts/update_projects/project_sort_test.go b/scripts/update_projects/project_sort_test.go deleted file mode 100644 index 17979396..00000000 --- a/scripts/update_projects/project_sort_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package main - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestProjectsInSourceOrder(t *testing.T) { - t.Run("returns projects ordered to match sources", func(t *testing.T) { - sources := []GitHubSource{ - {Repo: "example/first", SHA: "first-sha"}, - {Repo: "example/second", SHA: "second-sha"}, - {Repo: "example/third", SHA: "third-sha"}, - } - projects := []Project{ - {URL: "https://github.com/example/third.git", Ref: "third-sha"}, - {URL: "https://github.com/example/orphan.git", Ref: "orphan-sha"}, - {URL: "https://github.com/example/second.git", Ref: "second-sha"}, - {URL: "https://github.com/example/first.git", Ref: "first-sha"}, - } - - got := ProjectsInSourceOrder(sources, projects) - - want := []Project{ - {URL: "https://github.com/example/first.git", Ref: "first-sha"}, - {URL: "https://github.com/example/second.git", Ref: "second-sha"}, - {URL: "https://github.com/example/third.git", Ref: "third-sha"}, - } - assert.Equal(t, want, got) - }) - - t.Run("panics when source has no matching project", func(t *testing.T) { - sources := []GitHubSource{ - {Repo: "example/missing", SHA: "missing-sha"}, - } - projects := []Project{} - - assert.PanicsWithValue(t, "missing project for source example/missing@missing-sha", func() { - ProjectsInSourceOrder(sources, projects) - }) - }) -} diff --git a/scripts/update_projects/project_test.go b/scripts/update_projects/project_test.go deleted file mode 100644 index 3ac98857..00000000 --- a/scripts/update_projects/project_test.go +++ /dev/null @@ -1,119 +0,0 @@ -package main - -import ( - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestNewProject(t *testing.T) { - t.Run("reads parameters from x-topo parameters", func(t *testing.T) { - source := GitHubSource{Repo: "Arm-Examples/topo-example", SHA: "main"} - composeFile := strings.NewReader(` -x-topo: - name: Hello World - description: A friendly project - features: - - web - parameters: - username: - description: User name - required: true - example: alice -`) - - got, err := NewProject(source, composeFile) - - require.NoError(t, err) - want := map[string]Parameter{ - "username": { - Description: "User name", - Required: true, - Example: "alice", - }, - } - assert.Equal(t, want, got.Parameters) - }) - - t.Run("reads deprecated args as parameters", func(t *testing.T) { - source := GitHubSource{Repo: "Arm-Examples/topo-example", SHA: "main"} - composeFile := strings.NewReader(` -x-topo: - name: Hello World - description: A friendly project - args: - username: - description: User name - required: true - default: alice -`) - - got, err := NewProject(source, composeFile) - - require.NoError(t, err) - want := map[string]Parameter{ - "username": { - Description: "User name", - Required: true, - Default: "alice", - }, - } - assert.Equal(t, want, got.Parameters) - }) - - t.Run("prefers parameters over args", func(t *testing.T) { - source := GitHubSource{Repo: "Arm-Examples/topo-example", SHA: "main"} - composeFile := strings.NewReader(` -x-topo: - name: Hello World - description: A friendly project - parameters: - username: - description: New name - args: - username: - description: Old name - token: - description: Secret token -`) - - got, err := NewProject(source, composeFile) - - require.NoError(t, err) - want := map[string]Parameter{ - "username": { - Description: "New name", - }, - } - assert.Equal(t, want, got.Parameters) - }) - - t.Run("reads parameters aliased to args", func(t *testing.T) { - source := GitHubSource{Repo: "Arm-Examples/topo-example", SHA: "main"} - composeFile := strings.NewReader(` -x-topo: - name: Hello World - description: A friendly project - args: &args - username: - description: User name - required: true - example: alice - parameters: *args -`) - - got, err := NewProject(source, composeFile) - - require.NoError(t, err) - want := map[string]Parameter{ - "username": { - Description: "User name", - Required: true, - Example: "alice", - }, - } - assert.Equal(t, want, got.Parameters) - }) -} diff --git a/scripts/update_projects/update_plan.go b/scripts/update_projects/update_plan.go deleted file mode 100644 index d53210fc..00000000 --- a/scripts/update_projects/update_plan.go +++ /dev/null @@ -1,79 +0,0 @@ -package main - -import ( - "fmt" - "strings" -) - -type UpdatePlan struct { - ToAdd []GitHubSource - ToUpdate []GitHubSource - ToRemove []Project - Unchanged []Project -} - -func (p UpdatePlan) HasChanges() bool { - return len(p.ToAdd) > 0 || len(p.ToUpdate) > 0 || len(p.ToRemove) > 0 -} - -func (p UpdatePlan) String() string { - var lines []string - lines = append(lines, fmt.Sprintf("🆕 %d to add", len(p.ToAdd))) - lines = appendSourceURLs(lines, p.ToAdd) - lines = append(lines, fmt.Sprintf("🔄 %d to update", len(p.ToUpdate))) - lines = appendSourceURLs(lines, p.ToUpdate) - lines = append(lines, fmt.Sprintf("🗑️ %d to remove", len(p.ToRemove))) - lines = appendProjectURLs(lines, p.ToRemove) - lines = append(lines, fmt.Sprintf("☑️ %d unchanged", len(p.Unchanged))) - return strings.Join(lines, "\n") -} - -func appendSourceURLs(lines []string, sources []GitHubSource) []string { - for _, source := range sources { - lines = append(lines, fmt.Sprintf(" - %s", source.URL())) - } - return lines -} - -func appendProjectURLs(lines []string, projects []Project) []string { - for _, project := range projects { - lines = append(lines, fmt.Sprintf(" - %s", project.URL)) - } - return lines -} - -func PlanUpdate(sources []GitHubSource, current []Project) UpdatePlan { - sourceByID := make(map[ProjectSourceID]GitHubSource, len(sources)) - for _, source := range sources { - sourceByID[source.ID()] = source - } - - currentByID := make(map[ProjectSourceID]Project, len(current)) - for _, project := range current { - currentByID[project.SourceID()] = project - } - - var plan UpdatePlan - for _, source := range sources { - project, exists := currentByID[source.ID()] - if !exists { - plan.ToAdd = append(plan.ToAdd, source) - continue - } - - if project.Ref != source.SHA { - plan.ToUpdate = append(plan.ToUpdate, source) - continue - } - - plan.Unchanged = append(plan.Unchanged, project) - } - - for _, project := range current { - if _, exists := sourceByID[project.SourceID()]; !exists { - plan.ToRemove = append(plan.ToRemove, project) - } - } - - return plan -} diff --git a/scripts/update_projects/update_plan_test.go b/scripts/update_projects/update_plan_test.go deleted file mode 100644 index 628a1e56..00000000 --- a/scripts/update_projects/update_plan_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package main - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestPlanUpdate(t *testing.T) { - t.Run("classifies added, updated and removed projects", func(t *testing.T) { - sources := []GitHubSource{ - {Repo: "example/unchanged", SHA: "same-sha"}, - {Repo: "example/updated", SHA: "new-sha"}, - {Repo: "example/added", SHA: "added-sha"}, - } - current := []Project{ - {URL: "https://github.com/example/removed.git", Ref: "removed-sha"}, - {URL: "https://github.com/example/unchanged.git", Ref: "same-sha"}, - {URL: "https://github.com/example/updated.git", Ref: "old-sha"}, - } - - got := PlanUpdate(sources, current) - - want := UpdatePlan{ - ToAdd: []GitHubSource{{Repo: "example/added", SHA: "added-sha"}}, - ToUpdate: []GitHubSource{{Repo: "example/updated", SHA: "new-sha"}}, - ToRemove: []Project{{URL: "https://github.com/example/removed.git", Ref: "removed-sha"}}, - Unchanged: []Project{{URL: "https://github.com/example/unchanged.git", Ref: "same-sha"}}, - } - assert.Equal(t, want, got) - }) -} - -func TestUpdatePlan(t *testing.T) { - t.Run("HasChanges", func(t *testing.T) { - t.Run("returns false when only projects are unchanged", func(t *testing.T) { - plan := UpdatePlan{ - Unchanged: []Project{{URL: "https://github.com/example/unchanged.git", Ref: "same-sha"}}, - } - - got := plan.HasChanges() - - assert.False(t, got) - }) - - t.Run("returns true when projects will be added", func(t *testing.T) { - plan := UpdatePlan{ - ToAdd: []GitHubSource{{Repo: "example/added", SHA: "added-sha"}}, - } - - got := plan.HasChanges() - - assert.True(t, got) - }) - - t.Run("returns true when projects will be updated", func(t *testing.T) { - plan := UpdatePlan{ - ToUpdate: []GitHubSource{{Repo: "example/updated", SHA: "new-sha"}}, - } - - got := plan.HasChanges() - - assert.True(t, got) - }) - - t.Run("returns true when projects will be removed", func(t *testing.T) { - plan := UpdatePlan{ - ToRemove: []Project{{URL: "https://github.com/example/removed.git", Ref: "removed-sha"}}, - } - - got := plan.HasChanges() - - assert.True(t, got) - }) - }) -}