From f7e15b4106f00f1bc1ba02aea51494758ae49a51 Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Fri, 24 May 2024 22:41:26 -0400 Subject: [PATCH 01/42] adding boilerplate for gitlab plugin --- internal/plugin/files.go | 3 ++ internal/plugin/gitlab.go | 57 +++++++++++++++++++++++++++++++++++ internal/plugin/includable.go | 2 ++ nix/flake/flakeref.go | 1 + 4 files changed, 63 insertions(+) create mode 100644 internal/plugin/gitlab.go diff --git a/internal/plugin/files.go b/internal/plugin/files.go index 70696b057c2..864204e878d 100644 --- a/internal/plugin/files.go +++ b/internal/plugin/files.go @@ -4,6 +4,7 @@ package plugin import ( + "fmt" "io/fs" "os" @@ -24,6 +25,8 @@ func getConfigIfAny(inc Includable, projectDir string) (*Config, error) { return nil, errors.WithStack(err) } return buildConfig(includable, projectDir, string(content)) + case *gitlabPlugin: + fmt.Print("Here") case *LocalPlugin: content, err := os.ReadFile(includable.Path()) if err != nil && !os.IsNotExist(err) { diff --git a/internal/plugin/gitlab.go b/internal/plugin/gitlab.go new file mode 100644 index 00000000000..7d02ec431fa --- /dev/null +++ b/internal/plugin/gitlab.go @@ -0,0 +1,57 @@ +package plugin + +import ( + "errors" + "strings" + + "github.com/samber/lo" + "go.jetpack.io/devbox/internal/cachehash" + "go.jetpack.io/devbox/nix/flake" +) + +type gitlabPlugin struct { + ref flake.Ref + name string +} + +func newGitlabPlugin(ref flake.Ref) (*gitlabPlugin, error) { + plugin := &gitlabPlugin{ref: ref} + // For backward compatibility, we don't strictly require name to be present + // in github plugins. If it's missing, we just use the directory as the name. + name, err := getPluginNameFromContent(plugin) + if err != nil && !errors.Is(err, errNameMissing) { + return nil, err + } + if name == "" { + name = strings.ReplaceAll(ref.Dir, "/", "-") + } + plugin.name = githubNameRegexp.ReplaceAllString( + strings.Join(lo.Compact([]string{ref.Owner, ref.Repo, name}), "."), + " ", + ) + return plugin, nil +} + +func (p *gitlabPlugin) CanonicalName() string { + return p.name +} + +func (p *gitlabPlugin) FileContent(subpath string) ([]byte, error) { + return []byte(subpath), nil +} + +func (p *gitlabPlugin) Hash() string { + return cachehash.Bytes([]byte(p.ref.String())) +} + +func (p *gitlabPlugin) LockfileKey() string { + return p.ref.String() +} + +func (p *gitlabPlugin) Fetch() ([]byte, error) { + content, err := p.FileContent(pluginConfigName) + if err != nil { + return nil, err + } + return jsonPurifyPluginContent(content) +} diff --git a/internal/plugin/includable.go b/internal/plugin/includable.go index 77045da139d..1fbabc4667b 100644 --- a/internal/plugin/includable.go +++ b/internal/plugin/includable.go @@ -26,6 +26,8 @@ func parseIncludable(includableRef, workingDir string) (Includable, error) { return newLocalPlugin(ref, workingDir) case flake.TypeGitHub: return newGithubPlugin(ref) + case flake.TypeGitLab: + return newGitlabPlugin(ref) default: return nil, fmt.Errorf("unsupported ref type %q", ref.Type) } diff --git a/nix/flake/flakeref.go b/nix/flake/flakeref.go index 96067de6293..f7f7baaf943 100644 --- a/nix/flake/flakeref.go +++ b/nix/flake/flakeref.go @@ -17,6 +17,7 @@ const ( TypeFile = "file" TypeGit = "git" TypeGitHub = "github" + TypeGitLab = "gitlab" TypeTarball = "tarball" ) From 5306195c98a603da1b468774795c214cd5d89992 Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Sat, 25 May 2024 12:43:07 -0400 Subject: [PATCH 02/42] basic functionality for public facing gitlab plugins working --- internal/plugin/files.go | 7 ++- internal/plugin/gitlab.go | 117 +++++++++++++++++++++++++++++++++++++- nix/flake/flakeref.go | 57 +++++++++++++++++++ 3 files changed, 178 insertions(+), 3 deletions(-) diff --git a/internal/plugin/files.go b/internal/plugin/files.go index 864204e878d..be369ecc7df 100644 --- a/internal/plugin/files.go +++ b/internal/plugin/files.go @@ -4,7 +4,6 @@ package plugin import ( - "fmt" "io/fs" "os" @@ -26,7 +25,11 @@ func getConfigIfAny(inc Includable, projectDir string) (*Config, error) { } return buildConfig(includable, projectDir, string(content)) case *gitlabPlugin: - fmt.Print("Here") + content, err := includable.Fetch() + if err != nil { + return nil, errors.WithStack(err) + } + return buildConfig(includable, projectDir, string(content)) case *LocalPlugin: content, err := os.ReadFile(includable.Path()) if err != nil && !os.IsNotExist(err) { diff --git a/internal/plugin/gitlab.go b/internal/plugin/gitlab.go index 7d02ec431fa..aa0cc61de8e 100644 --- a/internal/plugin/gitlab.go +++ b/internal/plugin/gitlab.go @@ -1,12 +1,22 @@ package plugin import ( + "cmp" "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" "strings" + "time" "github.com/samber/lo" + "go.jetpack.io/devbox/internal/boxcli/usererr" "go.jetpack.io/devbox/internal/cachehash" + "go.jetpack.io/devbox/internal/debug" "go.jetpack.io/devbox/nix/flake" + "go.jetpack.io/pkg/filecache" ) type gitlabPlugin struct { @@ -14,6 +24,8 @@ type gitlabPlugin struct { name string } +var gitlabCache = filecache.New[[]byte]("devbox/plugin/gitlab") + func newGitlabPlugin(ref flake.Ref) (*gitlabPlugin, error) { plugin := &gitlabPlugin{ref: ref} // For backward compatibility, we don't strictly require name to be present @@ -36,8 +48,111 @@ func (p *gitlabPlugin) CanonicalName() string { return p.name } +func (p *gitlabPlugin) request(contentURL string) (*http.Request, error) { + req, err := http.NewRequest(http.MethodGet, contentURL, nil) + if err != nil { + return nil, err + } + + // Add github token to request if available + glToken := os.Getenv("GITLAB_TOKEN") // TODO: @GITLAB_PLUGIN Is this right? + + if glToken != "" { + authValue := fmt.Sprintf("token %s", glToken) + req.Header.Add("Authorization", authValue) + } + + return req, nil +} + func (p *gitlabPlugin) FileContent(subpath string) ([]byte, error) { - return []byte(subpath), nil + contentURL, err := p.url(subpath) + + if err != nil { + return nil, err + } + + return gitlabCache.GetOrSet( + contentURL, + func() ([]byte, time.Duration, error) { + req, err := p.request(contentURL) + + if err != nil { + return nil, 0, err + } + + client := &http.Client{} + res, err := client.Do(req) + + if err != nil { + debug.Log(err.Error()) + return nil, 0, err + } + + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + return nil, 0, usererr.New( + "failed to get plugin %s @ %s (Status code %d). \nPlease make "+ + "sure a plugin.json file exists in plugin directory.", + p.LockfileKey(), + req.URL.String(), + res.StatusCode, + ) + } + + body, err := io.ReadAll(res.Body) + + debug.Log(string(body)) + + if err != nil { + return nil, 0, err + } + // Cache for 24 hours. Once we store the plugin in the lockfile, we + // should cache this indefinitely and only invalidate if the plugin + // is updated. + return body, 24 * time.Hour, nil + }, + ) +} + +func (p *gitlabPlugin) url(subpath string) (string, error) { + project, err := url.JoinPath(p.ref.Owner, p.ref.Repo) + + if err != nil { + return "", err + } + + file, err := url.JoinPath(p.ref.Dir, subpath) + + if err != nil { + return "", err + } + + path, err := url.JoinPath( + "https://gitlab.com/api/v4/projects", + url.PathEscape(project), + "repository", + "files", + url.PathEscape(file), + "raw", + ) + + if err != nil { + return "", err + } + + parsed, err := url.Parse(path) + + if err != nil { + return "", err + } + + query := parsed.Query() + query.Add("ref", cmp.Or(p.ref.Rev, p.ref.Ref, "main")) + parsed.RawQuery = query.Encode() + + return parsed.String(), nil } func (p *gitlabPlugin) Hash() string { diff --git a/nix/flake/flakeref.go b/nix/flake/flakeref.go index f7f7baaf943..1818d00847f 100644 --- a/nix/flake/flakeref.go +++ b/nix/flake/flakeref.go @@ -197,9 +197,14 @@ func parseURLRef(ref string) (parsed Ref, fragment string, err error) { if err := parseGitHubRef(refURL, &parsed); err != nil { return Ref{}, "", err } + case "gitlab": + if err := parseGitLabRef(refURL, &parsed); err != nil { + return Ref{}, "", err + } default: return Ref{}, "", redact.Errorf("unsupported flake reference URL scheme: %s", redact.Safe(refURL.Scheme)) } + return parsed, fragment, nil } @@ -249,6 +254,58 @@ func parseGitHubRef(refURL *url.URL, parsed *Ref) error { return nil } +func parseGitLabRef(refURL *url.URL, parsed *Ref) error { + // github:/(/)?(\?)? + + parsed.Type = TypeGitLab + + // Only split up to 3 times (owner, repo, ref/rev) so that we handle + // refs that have slashes in them. For example, + // "github:jetify-com/devbox/gcurtis/flakeref" parses as "gcurtis/flakeref". + split, err := splitPathOrOpaque(refURL, 3) + + if err != nil { + return err + } + + parsed.Owner = split[0] + parsed.Repo = split[1] + + if len(split) > 2 { + if revOrRef := split[2]; isGitHash(revOrRef) { + parsed.Rev = revOrRef + } else { + parsed.Ref = revOrRef + } + } + + parsed.Host = refURL.Query().Get("host") + parsed.Dir = refURL.Query().Get("dir") + + if qRef := refURL.Query().Get("ref"); qRef != "" { + if parsed.Rev != "" { + return redact.Errorf("gitlab flake reference has a ref and a rev") + } + if parsed.Ref != "" && qRef != parsed.Ref { + return redact.Errorf("gitlab flake reference has a ref in the path (%q) and a ref query parameter (%q)", parsed.Ref, qRef) + } + parsed.Ref = qRef + } + + if qRev := refURL.Query().Get("rev"); qRev != "" { + if parsed.Ref != "" { + return redact.Errorf("gitlab flake reference has a ref and a rev") + } + if parsed.Rev != "" && qRev != parsed.Rev { + return redact.Errorf("gitlab flake reference has a rev in the path (%q) and a rev query parameter (%q)", parsed.Rev, qRev) + } + parsed.Rev = qRev + } + + parsed.Dir = refURL.Query().Get("dir") + return nil +} + // String encodes the flake reference as a URL-like string. It normalizes the // result such that if two Ref values are equal, then their strings will also be // equal. From 34359d881b1c9f40005a5c041db156480e932d6c Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Sat, 25 May 2024 13:30:10 -0400 Subject: [PATCH 03/42] consolidated gitlab and github logic --- internal/plugin/files.go | 8 +- internal/plugin/git.go | 207 ++++++++++++++++++ .../plugin/{github_test.go => git_test.go} | 30 +-- internal/plugin/github.go | 139 ------------ internal/plugin/gitlab.go | 172 --------------- internal/plugin/includable.go | 12 +- nix/flake/flakeref.go | 40 ++-- 7 files changed, 255 insertions(+), 353 deletions(-) create mode 100644 internal/plugin/git.go rename internal/plugin/{github_test.go => git_test.go} (84%) delete mode 100644 internal/plugin/github.go delete mode 100644 internal/plugin/gitlab.go diff --git a/internal/plugin/files.go b/internal/plugin/files.go index be369ecc7df..9d682bb2e43 100644 --- a/internal/plugin/files.go +++ b/internal/plugin/files.go @@ -18,13 +18,7 @@ func getConfigIfAny(inc Includable, projectDir string) (*Config, error) { switch includable := inc.(type) { case *devpkg.Package: return getBuiltinPluginConfigIfExists(includable, projectDir) - case *githubPlugin: - content, err := includable.Fetch() - if err != nil { - return nil, errors.WithStack(err) - } - return buildConfig(includable, projectDir, string(content)) - case *gitlabPlugin: + case *gitPlugin: content, err := includable.Fetch() if err != nil { return nil, errors.WithStack(err) diff --git a/internal/plugin/git.go b/internal/plugin/git.go new file mode 100644 index 00000000000..bfb61074507 --- /dev/null +++ b/internal/plugin/git.go @@ -0,0 +1,207 @@ +package plugin + +import ( + "cmp" + "fmt" + "io" + "net/http" + "net/url" + "os" + "regexp" + "strings" + "time" + + "github.com/pkg/errors" + "github.com/samber/lo" + "go.jetpack.io/devbox/internal/boxcli/usererr" + "go.jetpack.io/devbox/internal/cachehash" + "go.jetpack.io/devbox/internal/debug" + "go.jetpack.io/devbox/nix/flake" + "go.jetpack.io/pkg/filecache" +) + +var githubCache = filecache.New[[]byte]("devbox/plugin/github") +var gitlabCache = filecache.New[[]byte]("devbox/plugin/gitlab") + +type gitPlugin struct { + ref flake.Ref + name string +} + +// Github only allows alphanumeric, hyphen, underscore, and period in repo names. +// but we clean up just in case. +var githubNameRegexp = regexp.MustCompile("[^a-zA-Z0-9-_.]+") + +func newGitPlugin(ref flake.Ref) (*gitPlugin, error) { + plugin := &gitPlugin{ref: ref} + // For backward compatibility, we don't strictly require name to be present + // in github plugins. If it's missing, we just use the directory as the name. + name, err := getPluginNameFromContent(plugin) + if err != nil && !errors.Is(err, errNameMissing) { + return nil, err + } + if name == "" { + name = strings.ReplaceAll(ref.Dir, "/", "-") + } + plugin.name = githubNameRegexp.ReplaceAllString( + strings.Join(lo.Compact([]string{ref.Owner, ref.Repo, name}), "."), + " ", + ) + return plugin, nil +} + +func (p *gitPlugin) Fetch() ([]byte, error) { + content, err := p.FileContent(pluginConfigName) + if err != nil { + return nil, err + } + return jsonPurifyPluginContent(content) +} + +func (p *gitPlugin) CanonicalName() string { + return p.name +} + +func (p *gitPlugin) Hash() string { + return cachehash.Bytes([]byte(p.ref.String())) +} + +func (p *gitPlugin) FileContent(subpath string) ([]byte, error) { + contentURL, err := p.url(subpath) + debug.Log(contentURL) + + if err != nil { + return nil, err + } + + callable := func() ([]byte, time.Duration, error) { + req, err := p.request(contentURL) + if err != nil { + return nil, 0, err + } + + client := &http.Client{} + res, err := client.Do(req) + if err != nil { + return nil, 0, err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return nil, 0, usererr.New( + "failed to get plugin %s @ %s (Status code %d). \nPlease make "+ + "sure a plugin.json file exists in plugin directory.", + p.LockfileKey(), + req.URL.String(), + res.StatusCode, + ) + } + body, err := io.ReadAll(res.Body) + if err != nil { + return nil, 0, err + } + // Cache for 24 hours. Once we store the plugin in the lockfile, we + // should cache this indefinitely and only invalidate if the plugin + // is updated. + return body, 24 * time.Hour, nil + } + + switch p.ref.Type { + + case flake.TypeGitHub: + return githubCache.GetOrSet(contentURL, callable) + case flake.TypeGitLab: + return gitlabCache.GetOrSet(contentURL, callable) + case flake.TypeBitBucket: + fallthrough // TODO + default: + return nil, err + } + +} + +func (p *gitPlugin) url(subpath string) (string, error) { + debug.Log(p.ref.Type) + switch p.ref.Type { + case flake.TypeGitLab: + return p.gitlabUrl(subpath) + case flake.TypeGitHub: + return p.githubUrl(subpath) + case flake.TypeBitBucket: + fallthrough // TODO + default: + return "", nil + } +} + +func (p *gitPlugin) githubUrl(subpath string) (string, error) { + // Github redirects "master" to "main" in new repos. They don't do the reverse + // so setting master here is better. + return url.JoinPath( + "https://raw.githubusercontent.com/", + p.ref.Owner, + p.ref.Repo, + cmp.Or(p.ref.Rev, p.ref.Ref, "master"), + p.ref.Dir, + subpath, + ) +} + +func (p *gitPlugin) gitlabUrl(subpath string) (string, error) { + project, err := url.JoinPath(p.ref.Owner, p.ref.Repo) + + if err != nil { + return "", err + } + + file, err := url.JoinPath(p.ref.Dir, subpath) + + if err != nil { + return "", err + } + + path, err := url.JoinPath( + "https://gitlab.com/api/v4/projects", + url.PathEscape(project), + "repository", + "files", + url.PathEscape(file), + "raw", + ) + + if err != nil { + return "", err + } + + parsed, err := url.Parse(path) + + if err != nil { + return "", err + } + + query := parsed.Query() + query.Add("ref", cmp.Or(p.ref.Rev, p.ref.Ref, "main")) + parsed.RawQuery = query.Encode() + + return parsed.String(), nil +} + +func (p *gitPlugin) request(contentURL string) (*http.Request, error) { + req, err := http.NewRequest(http.MethodGet, contentURL, nil) + if err != nil { + return nil, err + } + + // Add github token to request if available + ghToken := os.Getenv("GITHUB_TOKEN") + + if ghToken != "" { + authValue := fmt.Sprintf("token %s", ghToken) + req.Header.Add("Authorization", authValue) + } + + return req, nil +} + +func (p *gitPlugin) LockfileKey() string { + return p.ref.String() +} diff --git a/internal/plugin/github_test.go b/internal/plugin/git_test.go similarity index 84% rename from internal/plugin/github_test.go rename to internal/plugin/git_test.go index 5eb2c912fcf..5e87a6e6aa1 100644 --- a/internal/plugin/github_test.go +++ b/internal/plugin/git_test.go @@ -9,17 +9,17 @@ import ( "go.jetpack.io/devbox/nix/flake" ) -func TestNewGithubPlugin(t *testing.T) { +func TestNewGitPlugin(t *testing.T) { testCases := []struct { name string Include string - expected githubPlugin + expected gitPlugin expectedURL string }{ { name: "parse basic github plugin", Include: "github:jetify-com/devbox-plugins", - expected: githubPlugin{ + expected: gitPlugin{ ref: flake.Ref{ Type: "github", Owner: "jetify-com", @@ -32,7 +32,7 @@ func TestNewGithubPlugin(t *testing.T) { { name: "parse github plugin with dir param", Include: "github:jetify-com/devbox-plugins?dir=mongodb", - expected: githubPlugin{ + expected: gitPlugin{ ref: flake.Ref{ Type: "github", Owner: "jetify-com", @@ -46,7 +46,7 @@ func TestNewGithubPlugin(t *testing.T) { { name: "parse github plugin with dir param and rev", Include: "github:jetify-com/devbox-plugins/my-branch?dir=mongodb", - expected: githubPlugin{ + expected: gitPlugin{ ref: flake.Ref{ Type: "github", Owner: "jetify-com", @@ -61,7 +61,7 @@ func TestNewGithubPlugin(t *testing.T) { { name: "parse github plugin with dir param and rev", Include: "github:jetify-com/devbox-plugins/initials/my-branch?dir=mongodb", - expected: githubPlugin{ + expected: gitPlugin{ ref: flake.Ref{ Type: "github", Owner: "jetify-com", @@ -77,7 +77,7 @@ func TestNewGithubPlugin(t *testing.T) { for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { - actual, err := newGithubPluginForTest(testCase.Include) + actual, err := newGitPluginForTest(testCase.Include) assert.NoError(t, err) assert.Equal(t, &testCase.expected, actual) u, err := testCase.expected.url("") @@ -88,13 +88,13 @@ func TestNewGithubPlugin(t *testing.T) { } // keep in sync with newGithubPlugin -func newGithubPluginForTest(include string) (*githubPlugin, error) { +func newGitPluginForTest(include string) (*gitPlugin, error) { ref, err := flake.ParseRef(include) if err != nil { return nil, err } - plugin := &githubPlugin{ref: ref} + plugin := &gitPlugin{ref: ref} name := strings.ReplaceAll(ref.Dir, "/", "-") plugin.name = githubNameRegexp.ReplaceAllString( strings.Join(lo.Compact([]string{ref.Owner, ref.Repo, name}), "."), @@ -103,8 +103,8 @@ func newGithubPluginForTest(include string) (*githubPlugin, error) { return plugin, nil } -func TestGithubPluginAuth(t *testing.T) { - githubPlugin := githubPlugin{ +func TestGitPluginAuth(t *testing.T) { + gitPlugin := gitPlugin{ ref: flake.Ref{ Type: "github", Owner: "jetpack-io", @@ -116,9 +116,9 @@ func TestGithubPluginAuth(t *testing.T) { expectedURL := "https://raw.githubusercontent.com/jetpack-io/devbox-plugins/master/test" t.Run("generate request for public Github repository", func(t *testing.T) { - url, err := githubPlugin.url("test") + url, err := gitPlugin.url("test") assert.NoError(t, err) - actual, err := githubPlugin.request(url) + actual, err := gitPlugin.request(url) assert.NoError(t, err) assert.Equal(t, expectedURL, actual.URL.String()) assert.Equal(t, "", actual.Header.Get("Authorization")) @@ -126,9 +126,9 @@ func TestGithubPluginAuth(t *testing.T) { t.Run("generate request for private Github repository", func(t *testing.T) { t.Setenv("GITHUB_TOKEN", "gh_abcd") - url, err := githubPlugin.url("test") + url, err := gitPlugin.url("test") assert.NoError(t, err) - actual, err := githubPlugin.request(url) + actual, err := gitPlugin.request(url) assert.NoError(t, err) assert.Equal(t, expectedURL, actual.URL.String()) assert.Equal(t, "token gh_abcd", actual.Header.Get("Authorization")) diff --git a/internal/plugin/github.go b/internal/plugin/github.go deleted file mode 100644 index 972b815f731..00000000000 --- a/internal/plugin/github.go +++ /dev/null @@ -1,139 +0,0 @@ -package plugin - -import ( - "cmp" - "fmt" - "io" - "net/http" - "net/url" - "os" - "regexp" - "strings" - "time" - - "github.com/pkg/errors" - "github.com/samber/lo" - "go.jetpack.io/devbox/internal/boxcli/usererr" - "go.jetpack.io/devbox/internal/cachehash" - "go.jetpack.io/devbox/nix/flake" - "go.jetpack.io/pkg/filecache" -) - -var githubCache = filecache.New[[]byte]("devbox/plugin/github") - -type githubPlugin struct { - ref flake.Ref - name string -} - -// Github only allows alphanumeric, hyphen, underscore, and period in repo names. -// but we clean up just in case. -var githubNameRegexp = regexp.MustCompile("[^a-zA-Z0-9-_.]+") - -func newGithubPlugin(ref flake.Ref) (*githubPlugin, error) { - plugin := &githubPlugin{ref: ref} - // For backward compatibility, we don't strictly require name to be present - // in github plugins. If it's missing, we just use the directory as the name. - name, err := getPluginNameFromContent(plugin) - if err != nil && !errors.Is(err, errNameMissing) { - return nil, err - } - if name == "" { - name = strings.ReplaceAll(ref.Dir, "/", "-") - } - plugin.name = githubNameRegexp.ReplaceAllString( - strings.Join(lo.Compact([]string{ref.Owner, ref.Repo, name}), "."), - " ", - ) - return plugin, nil -} - -func (p *githubPlugin) Fetch() ([]byte, error) { - content, err := p.FileContent(pluginConfigName) - if err != nil { - return nil, err - } - return jsonPurifyPluginContent(content) -} - -func (p *githubPlugin) CanonicalName() string { - return p.name -} - -func (p *githubPlugin) Hash() string { - return cachehash.Bytes([]byte(p.ref.String())) -} - -func (p *githubPlugin) FileContent(subpath string) ([]byte, error) { - contentURL, err := p.url(subpath) - if err != nil { - return nil, err - } - return githubCache.GetOrSet( - contentURL, - func() ([]byte, time.Duration, error) { - req, err := p.request(contentURL) - if err != nil { - return nil, 0, err - } - - client := &http.Client{} - res, err := client.Do(req) - if err != nil { - return nil, 0, err - } - defer res.Body.Close() - if res.StatusCode != http.StatusOK { - return nil, 0, usererr.New( - "failed to get plugin %s @ %s (Status code %d). \nPlease make "+ - "sure a plugin.json file exists in plugin directory.", - p.LockfileKey(), - req.URL.String(), - res.StatusCode, - ) - } - body, err := io.ReadAll(res.Body) - if err != nil { - return nil, 0, err - } - // Cache for 24 hours. Once we store the plugin in the lockfile, we - // should cache this indefinitely and only invalidate if the plugin - // is updated. - return body, 24 * time.Hour, nil - }, - ) -} - -func (p *githubPlugin) url(subpath string) (string, error) { - // Github redirects "master" to "main" in new repos. They don't do the reverse - // so setting master here is better. - return url.JoinPath( - "https://raw.githubusercontent.com/", - p.ref.Owner, - p.ref.Repo, - cmp.Or(p.ref.Rev, p.ref.Ref, "master"), - p.ref.Dir, - subpath, - ) -} - -func (p *githubPlugin) request(contentURL string) (*http.Request, error) { - req, err := http.NewRequest(http.MethodGet, contentURL, nil) - if err != nil { - return nil, err - } - - // Add github token to request if available - ghToken := os.Getenv("GITHUB_TOKEN") - - if ghToken != "" { - authValue := fmt.Sprintf("token %s", ghToken) - req.Header.Add("Authorization", authValue) - } - - return req, nil -} - -func (p *githubPlugin) LockfileKey() string { - return p.ref.String() -} diff --git a/internal/plugin/gitlab.go b/internal/plugin/gitlab.go deleted file mode 100644 index aa0cc61de8e..00000000000 --- a/internal/plugin/gitlab.go +++ /dev/null @@ -1,172 +0,0 @@ -package plugin - -import ( - "cmp" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "os" - "strings" - "time" - - "github.com/samber/lo" - "go.jetpack.io/devbox/internal/boxcli/usererr" - "go.jetpack.io/devbox/internal/cachehash" - "go.jetpack.io/devbox/internal/debug" - "go.jetpack.io/devbox/nix/flake" - "go.jetpack.io/pkg/filecache" -) - -type gitlabPlugin struct { - ref flake.Ref - name string -} - -var gitlabCache = filecache.New[[]byte]("devbox/plugin/gitlab") - -func newGitlabPlugin(ref flake.Ref) (*gitlabPlugin, error) { - plugin := &gitlabPlugin{ref: ref} - // For backward compatibility, we don't strictly require name to be present - // in github plugins. If it's missing, we just use the directory as the name. - name, err := getPluginNameFromContent(plugin) - if err != nil && !errors.Is(err, errNameMissing) { - return nil, err - } - if name == "" { - name = strings.ReplaceAll(ref.Dir, "/", "-") - } - plugin.name = githubNameRegexp.ReplaceAllString( - strings.Join(lo.Compact([]string{ref.Owner, ref.Repo, name}), "."), - " ", - ) - return plugin, nil -} - -func (p *gitlabPlugin) CanonicalName() string { - return p.name -} - -func (p *gitlabPlugin) request(contentURL string) (*http.Request, error) { - req, err := http.NewRequest(http.MethodGet, contentURL, nil) - if err != nil { - return nil, err - } - - // Add github token to request if available - glToken := os.Getenv("GITLAB_TOKEN") // TODO: @GITLAB_PLUGIN Is this right? - - if glToken != "" { - authValue := fmt.Sprintf("token %s", glToken) - req.Header.Add("Authorization", authValue) - } - - return req, nil -} - -func (p *gitlabPlugin) FileContent(subpath string) ([]byte, error) { - contentURL, err := p.url(subpath) - - if err != nil { - return nil, err - } - - return gitlabCache.GetOrSet( - contentURL, - func() ([]byte, time.Duration, error) { - req, err := p.request(contentURL) - - if err != nil { - return nil, 0, err - } - - client := &http.Client{} - res, err := client.Do(req) - - if err != nil { - debug.Log(err.Error()) - return nil, 0, err - } - - defer res.Body.Close() - - if res.StatusCode != http.StatusOK { - return nil, 0, usererr.New( - "failed to get plugin %s @ %s (Status code %d). \nPlease make "+ - "sure a plugin.json file exists in plugin directory.", - p.LockfileKey(), - req.URL.String(), - res.StatusCode, - ) - } - - body, err := io.ReadAll(res.Body) - - debug.Log(string(body)) - - if err != nil { - return nil, 0, err - } - // Cache for 24 hours. Once we store the plugin in the lockfile, we - // should cache this indefinitely and only invalidate if the plugin - // is updated. - return body, 24 * time.Hour, nil - }, - ) -} - -func (p *gitlabPlugin) url(subpath string) (string, error) { - project, err := url.JoinPath(p.ref.Owner, p.ref.Repo) - - if err != nil { - return "", err - } - - file, err := url.JoinPath(p.ref.Dir, subpath) - - if err != nil { - return "", err - } - - path, err := url.JoinPath( - "https://gitlab.com/api/v4/projects", - url.PathEscape(project), - "repository", - "files", - url.PathEscape(file), - "raw", - ) - - if err != nil { - return "", err - } - - parsed, err := url.Parse(path) - - if err != nil { - return "", err - } - - query := parsed.Query() - query.Add("ref", cmp.Or(p.ref.Rev, p.ref.Ref, "main")) - parsed.RawQuery = query.Encode() - - return parsed.String(), nil -} - -func (p *gitlabPlugin) Hash() string { - return cachehash.Bytes([]byte(p.ref.String())) -} - -func (p *gitlabPlugin) LockfileKey() string { - return p.ref.String() -} - -func (p *gitlabPlugin) Fetch() ([]byte, error) { - content, err := p.FileContent(pluginConfigName) - if err != nil { - return nil, err - } - return jsonPurifyPluginContent(content) -} diff --git a/internal/plugin/includable.go b/internal/plugin/includable.go index 1fbabc4667b..dc70dc7b2b7 100644 --- a/internal/plugin/includable.go +++ b/internal/plugin/includable.go @@ -6,6 +6,7 @@ import ( "regexp" "go.jetpack.io/devbox/internal/boxcli/usererr" + "go.jetpack.io/devbox/internal/debug" "go.jetpack.io/devbox/nix/flake" ) @@ -21,13 +22,18 @@ func parseIncludable(includableRef, workingDir string) (Includable, error) { if err != nil { return nil, err } + + debug.Log(">>>> REF TYPE: " + ref.Type) + switch ref.Type { case flake.TypePath: return newLocalPlugin(ref, workingDir) - case flake.TypeGitHub: - return newGithubPlugin(ref) + case flake.TypeBitBucket: + fallthrough case flake.TypeGitLab: - return newGitlabPlugin(ref) + return newGitPlugin(ref) + case flake.TypeGitHub: + return newGitPlugin(ref) default: return nil, fmt.Errorf("unsupported ref type %q", ref.Type) } diff --git a/nix/flake/flakeref.go b/nix/flake/flakeref.go index 1818d00847f..7b737f802ae 100644 --- a/nix/flake/flakeref.go +++ b/nix/flake/flakeref.go @@ -12,13 +12,14 @@ import ( // Flake reference types supported by this package. const ( - TypeIndirect = "indirect" - TypePath = "path" - TypeFile = "file" - TypeGit = "git" - TypeGitHub = "github" - TypeGitLab = "gitlab" - TypeTarball = "tarball" + TypeIndirect = "indirect" + TypePath = "path" + TypeFile = "file" + TypeGit = "git" + TypeGitHub = "github" + TypeGitLab = "gitlab" + TypeBitBucket = "bitbucket" + TypeTarball = "tarball" ) // Ref is a parsed Nix flake reference. A flake reference is a subset of the @@ -193,12 +194,19 @@ func parseURLRef(ref string) (parsed Ref, fragment string, err error) { refURL.Scheme = refURL.Scheme[4:] // remove git+ } parsed.URL = refURL.String() - case "github": - if err := parseGitHubRef(refURL, &parsed); err != nil { + case "bitbucket": + parsed.Type = TypeBitBucket + if err := parseGitRef(refURL, &parsed); err != nil { return Ref{}, "", err } case "gitlab": - if err := parseGitLabRef(refURL, &parsed); err != nil { + parsed.Type = TypeGitLab + if err := parseGitRef(refURL, &parsed); err != nil { + return Ref{}, "", err + } + case "github": + parsed.Type = TypeGitHub + if err := parseGitRef(refURL, &parsed); err != nil { return Ref{}, "", err } default: @@ -208,11 +216,9 @@ func parseURLRef(ref string) (parsed Ref, fragment string, err error) { return parsed, fragment, nil } -func parseGitHubRef(refURL *url.URL, parsed *Ref) error { +func parseGitRef(refURL *url.URL, parsed *Ref) error { // github:/(/)?(\?)? - parsed.Type = TypeGitHub - // Only split up to 3 times (owner, repo, ref/rev) so that we handle // refs that have slashes in them. For example, // "github:jetify-com/devbox/gcurtis/flakeref" parses as "gcurtis/flakeref". @@ -234,19 +240,19 @@ func parseGitHubRef(refURL *url.URL, parsed *Ref) error { parsed.Dir = refURL.Query().Get("dir") if qRef := refURL.Query().Get("ref"); qRef != "" { if parsed.Rev != "" { - return redact.Errorf("github flake reference has a ref and a rev") + return redact.Errorf("%s flake reference has a ref and a rev", parsed.Type) } if parsed.Ref != "" && qRef != parsed.Ref { - return redact.Errorf("github flake reference has a ref in the path (%q) and a ref query parameter (%q)", parsed.Ref, qRef) + return redact.Errorf("%s flake reference has a ref in the path (%q) and a ref query parameter (%q)", parsed.Type, parsed.Ref, qRef) } parsed.Ref = qRef } if qRev := refURL.Query().Get("rev"); qRev != "" { if parsed.Ref != "" { - return redact.Errorf("github flake reference has a ref and a rev") + return redact.Errorf("%s flake reference has a ref and a rev", parsed.Type) } if parsed.Rev != "" && qRev != parsed.Rev { - return redact.Errorf("github flake reference has a rev in the path (%q) and a rev query parameter (%q)", parsed.Rev, qRev) + return redact.Errorf("%s flake reference has a rev in the path (%q) and a rev query parameter (%q)", parsed.Type, parsed.Rev, qRev) } parsed.Rev = qRev } From 73dc9afa18e06fecd947ae9ad0f079578831c88a Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Sun, 26 May 2024 16:26:07 -0400 Subject: [PATCH 04/42] still work in progress, but bitbucket and gitlab public urls are handled --- internal/plugin/git.go | 72 +++++++++++++++++++++++++++++------ internal/plugin/includable.go | 7 ++-- nix/flake/flakeref.go | 6 +++ 3 files changed, 70 insertions(+), 15 deletions(-) diff --git a/internal/plugin/git.go b/internal/plugin/git.go index bfb61074507..043ac01a374 100644 --- a/internal/plugin/git.go +++ b/internal/plugin/git.go @@ -7,6 +7,7 @@ import ( "net/http" "net/url" "os" + "os/exec" "regexp" "strings" "time" @@ -15,13 +16,14 @@ import ( "github.com/samber/lo" "go.jetpack.io/devbox/internal/boxcli/usererr" "go.jetpack.io/devbox/internal/cachehash" - "go.jetpack.io/devbox/internal/debug" "go.jetpack.io/devbox/nix/flake" "go.jetpack.io/pkg/filecache" ) +var gitCache = filecache.New[[]byte]("devbox/plugin/git") var githubCache = filecache.New[[]byte]("devbox/plugin/github") var gitlabCache = filecache.New[[]byte]("devbox/plugin/gitlab") +var bitbucketCache = filecache.New[[]byte]("devbox/plugin/bitbucket") type gitPlugin struct { ref flake.Ref @@ -68,24 +70,27 @@ func (p *gitPlugin) Hash() string { func (p *gitPlugin) FileContent(subpath string) ([]byte, error) { contentURL, err := p.url(subpath) - debug.Log(contentURL) if err != nil { return nil, err } - callable := func() ([]byte, time.Duration, error) { - req, err := p.request(contentURL) + retrieve := func() ([]byte, time.Duration, error) { + req, err := p.request(contentURL) // TODO: adjust this function to handle private repos + if err != nil { return nil, 0, err } client := &http.Client{} res, err := client.Do(req) + if err != nil { return nil, 0, err } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { return nil, 0, usererr.New( "failed to get plugin %s @ %s (Status code %d). \nPlease make "+ @@ -95,10 +100,13 @@ func (p *gitPlugin) FileContent(subpath string) ([]byte, error) { res.StatusCode, ) } + body, err := io.ReadAll(res.Body) + if err != nil { return nil, 0, err } + // Cache for 24 hours. Once we store the plugin in the lockfile, we // should cache this indefinitely and only invalidate if the plugin // is updated. @@ -106,33 +114,54 @@ func (p *gitPlugin) FileContent(subpath string) ([]byte, error) { } switch p.ref.Type { - + case flake.TypeGit: + return gitCache.GetOrSet(contentURL, retrieve) case flake.TypeGitHub: - return githubCache.GetOrSet(contentURL, callable) + return githubCache.GetOrSet(contentURL, retrieve) case flake.TypeGitLab: - return gitlabCache.GetOrSet(contentURL, callable) + return gitlabCache.GetOrSet(contentURL, retrieve) case flake.TypeBitBucket: - fallthrough // TODO + return bitbucketCache.GetOrSet(contentURL, retrieve) default: return nil, err } - } func (p *gitPlugin) url(subpath string) (string, error) { - debug.Log(p.ref.Type) switch p.ref.Type { + case flake.TypeGit: + return p.sshGitUrl() case flake.TypeGitLab: return p.gitlabUrl(subpath) case flake.TypeGitHub: return p.githubUrl(subpath) case flake.TypeBitBucket: - fallthrough // TODO + return p.bitbucketUrl(subpath) default: return "", nil } } +func (p *gitPlugin) sshGitUrl() (string, error) { + address, err := url.Parse(p.ref.URL) + + if err != nil { + return "", err + } + + defaultBranch := "main" + + if address.Host == flake.TypeGitHub { + // using master for GitHub repos for the same reasoning established in `githubUrl` + defaultBranch = "master" + } + + branch := cmp.Or(p.ref.Rev, p.ref.Ref, defaultBranch) + baseCommand := "git archive --format=tar --remote=git@" + + return fmt.Sprintf("%s%s:%s %s %s -o %s.tar ", baseCommand, address.Host, address.Path[1:], branch, p.ref.Dir, p.ref.Dir), nil +} + func (p *gitPlugin) githubUrl(subpath string) (string, error) { // Github redirects "master" to "main" in new repos. They don't do the reverse // so setting master here is better. @@ -146,6 +175,18 @@ func (p *gitPlugin) githubUrl(subpath string) (string, error) { ) } +func (p *gitPlugin) bitbucketUrl(subpath string) (string, error) { + return url.JoinPath( + "https://api.bitbucket.org/2.0/repositories", + p.ref.Owner, + p.ref.Repo, + "src", + cmp.Or(p.ref.Rev, p.ref.Ref, "main"), + p.ref.Dir, + subpath, + ) +} + func (p *gitPlugin) gitlabUrl(subpath string) (string, error) { project, err := url.JoinPath(p.ref.Owner, p.ref.Repo) @@ -186,7 +227,16 @@ func (p *gitPlugin) gitlabUrl(subpath string) (string, error) { } func (p *gitPlugin) request(contentURL string) (*http.Request, error) { + // TODO: Determine if private repo. Maybe use `git archive`? + // git archive --format=tar --remote=git@gitlab.com:astro-tec/devbox-plugin-test HEAD plugin -o plugin.tar + + if p.ref.Type == flake.TypeGit { + command := exec.Command(contentURL) + command.Wait() + } + req, err := http.NewRequest(http.MethodGet, contentURL, nil) + if err != nil { return nil, err } diff --git a/internal/plugin/includable.go b/internal/plugin/includable.go index dc70dc7b2b7..52a79688a27 100644 --- a/internal/plugin/includable.go +++ b/internal/plugin/includable.go @@ -6,7 +6,6 @@ import ( "regexp" "go.jetpack.io/devbox/internal/boxcli/usererr" - "go.jetpack.io/devbox/internal/debug" "go.jetpack.io/devbox/nix/flake" ) @@ -23,15 +22,15 @@ func parseIncludable(includableRef, workingDir string) (Includable, error) { return nil, err } - debug.Log(">>>> REF TYPE: " + ref.Type) - switch ref.Type { case flake.TypePath: return newLocalPlugin(ref, workingDir) + case flake.TypeGit: + fallthrough case flake.TypeBitBucket: fallthrough case flake.TypeGitLab: - return newGitPlugin(ref) + fallthrough case flake.TypeGitHub: return newGitPlugin(ref) default: diff --git a/nix/flake/flakeref.go b/nix/flake/flakeref.go index 7b737f802ae..4b14943794a 100644 --- a/nix/flake/flakeref.go +++ b/nix/flake/flakeref.go @@ -190,10 +190,16 @@ func parseURLRef(ref string) (parsed Ref, fragment string, err error) { q.Del("ref") q.Del("rev") refURL.RawQuery = q.Encode() + if len(refURL.Scheme) > 3 { refURL.Scheme = refURL.Scheme[4:] // remove git+ } + parsed.URL = refURL.String() + + if err := parseGitRef(refURL, &parsed); err != nil { + return Ref{}, "", err + } case "bitbucket": parsed.Type = TypeBitBucket if err := parseGitRef(refURL, &parsed); err != nil { From 844926fc0de09c74b1c69237c1eb590c913d54ee Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Sun, 16 Jun 2024 22:04:59 -0400 Subject: [PATCH 05/42] slightly closer --- internal/plugin/git.go | 25 ++++++++++---- internal/plugin/includable.go | 2 +- nix/flake/flakeref.go | 63 +++++------------------------------ 3 files changed, 28 insertions(+), 62 deletions(-) diff --git a/internal/plugin/git.go b/internal/plugin/git.go index 043ac01a374..ee82eea0e14 100644 --- a/internal/plugin/git.go +++ b/internal/plugin/git.go @@ -20,7 +20,7 @@ import ( "go.jetpack.io/pkg/filecache" ) -var gitCache = filecache.New[[]byte]("devbox/plugin/git") +var sshCache = filecache.New[[]byte]("devbox/plugin/ssh") var githubCache = filecache.New[[]byte]("devbox/plugin/github") var gitlabCache = filecache.New[[]byte]("devbox/plugin/gitlab") var bitbucketCache = filecache.New[[]byte]("devbox/plugin/bitbucket") @@ -114,8 +114,8 @@ func (p *gitPlugin) FileContent(subpath string) ([]byte, error) { } switch p.ref.Type { - case flake.TypeGit: - return gitCache.GetOrSet(contentURL, retrieve) + case flake.TypeSSH: + return sshCache.GetOrSet(contentURL, retrieve) case flake.TypeGitHub: return githubCache.GetOrSet(contentURL, retrieve) case flake.TypeGitLab: @@ -129,7 +129,7 @@ func (p *gitPlugin) FileContent(subpath string) ([]byte, error) { func (p *gitPlugin) url(subpath string) (string, error) { switch p.ref.Type { - case flake.TypeGit: + case flake.TypeSSH: return p.sshGitUrl() case flake.TypeGitLab: return p.gitlabUrl(subpath) @@ -158,8 +158,17 @@ func (p *gitPlugin) sshGitUrl() (string, error) { branch := cmp.Or(p.ref.Rev, p.ref.Ref, defaultBranch) baseCommand := "git archive --format=tar --remote=git@" + path, err := url.JoinPath(p.ref.Owner, p.ref.Repo) - return fmt.Sprintf("%s%s:%s %s %s -o %s.tar ", baseCommand, address.Host, address.Path[1:], branch, p.ref.Dir, p.ref.Dir), nil + formattedCommand := fmt.Sprintf("%s%s:%s %s %s -o %s.tar ", baseCommand, address.Query().Get("host"), path, branch, p.ref.Dir, p.ref.Dir) + + if err == nil { + return "", err + } + + // TODO: need to store the git archive in a temporary file...or something. Basically need to figure out how to handle this lol + + return formattedCommand, nil } func (p *gitPlugin) githubUrl(subpath string) (string, error) { @@ -176,6 +185,8 @@ func (p *gitPlugin) githubUrl(subpath string) (string, error) { } func (p *gitPlugin) bitbucketUrl(subpath string) (string, error) { + // bitbucket doesn't redirect master -> main or main -> master, so using "main" + // as the default in this case return url.JoinPath( "https://api.bitbucket.org/2.0/repositories", p.ref.Owner, @@ -219,6 +230,8 @@ func (p *gitPlugin) gitlabUrl(subpath string) (string, error) { return "", err } + // gitlab doesn't redirect master -> main or main -> master, so using "main" + // as the default in this case query := parsed.Query() query.Add("ref", cmp.Or(p.ref.Rev, p.ref.Ref, "main")) parsed.RawQuery = query.Encode() @@ -230,7 +243,7 @@ func (p *gitPlugin) request(contentURL string) (*http.Request, error) { // TODO: Determine if private repo. Maybe use `git archive`? // git archive --format=tar --remote=git@gitlab.com:astro-tec/devbox-plugin-test HEAD plugin -o plugin.tar - if p.ref.Type == flake.TypeGit { + if p.ref.Type == flake.TypeSSH { command := exec.Command(contentURL) command.Wait() } diff --git a/internal/plugin/includable.go b/internal/plugin/includable.go index 52a79688a27..5074cae0ab3 100644 --- a/internal/plugin/includable.go +++ b/internal/plugin/includable.go @@ -25,7 +25,7 @@ func parseIncludable(includableRef, workingDir string) (Includable, error) { switch ref.Type { case flake.TypePath: return newLocalPlugin(ref, workingDir) - case flake.TypeGit: + case flake.TypeSSH: fallthrough case flake.TypeBitBucket: fallthrough diff --git a/nix/flake/flakeref.go b/nix/flake/flakeref.go index 4b14943794a..965eeb67d0e 100644 --- a/nix/flake/flakeref.go +++ b/nix/flake/flakeref.go @@ -15,7 +15,7 @@ const ( TypeIndirect = "indirect" TypePath = "path" TypeFile = "file" - TypeGit = "git" + TypeSSH = "ssh" TypeGitHub = "github" TypeGitLab = "gitlab" TypeBitBucket = "bitbucket" @@ -179,7 +179,6 @@ func parseURLRef(ref string) (parsed Ref, fragment string, err error) { refURL.Scheme = refURL.Scheme[5:] // remove file+ parsed.URL = refURL.String() case "git", "git+http", "git+https", "git+ssh", "git+git", "git+file": - parsed.Type = TypeGit q := refURL.Query() parsed.Dir = q.Get("dir") parsed.Ref = q.Get("ref") @@ -195,6 +194,12 @@ func parseURLRef(ref string) (parsed Ref, fragment string, err error) { refURL.Scheme = refURL.Scheme[4:] // remove git+ } + if strings.HasPrefix(refURL.Scheme, TypeSSH) { + parsed.Type = TypeSSH + } else if strings.HasPrefix(refURL.Scheme, TypeFile) { + parsed.Type = TypeFile + } + parsed.URL = refURL.String() if err := parseGitRef(refURL, &parsed); err != nil { @@ -266,58 +271,6 @@ func parseGitRef(refURL *url.URL, parsed *Ref) error { return nil } -func parseGitLabRef(refURL *url.URL, parsed *Ref) error { - // github:/(/)?(\?)? - - parsed.Type = TypeGitLab - - // Only split up to 3 times (owner, repo, ref/rev) so that we handle - // refs that have slashes in them. For example, - // "github:jetify-com/devbox/gcurtis/flakeref" parses as "gcurtis/flakeref". - split, err := splitPathOrOpaque(refURL, 3) - - if err != nil { - return err - } - - parsed.Owner = split[0] - parsed.Repo = split[1] - - if len(split) > 2 { - if revOrRef := split[2]; isGitHash(revOrRef) { - parsed.Rev = revOrRef - } else { - parsed.Ref = revOrRef - } - } - - parsed.Host = refURL.Query().Get("host") - parsed.Dir = refURL.Query().Get("dir") - - if qRef := refURL.Query().Get("ref"); qRef != "" { - if parsed.Rev != "" { - return redact.Errorf("gitlab flake reference has a ref and a rev") - } - if parsed.Ref != "" && qRef != parsed.Ref { - return redact.Errorf("gitlab flake reference has a ref in the path (%q) and a ref query parameter (%q)", parsed.Ref, qRef) - } - parsed.Ref = qRef - } - - if qRev := refURL.Query().Get("rev"); qRev != "" { - if parsed.Ref != "" { - return redact.Errorf("gitlab flake reference has a ref and a rev") - } - if parsed.Rev != "" && qRev != parsed.Rev { - return redact.Errorf("gitlab flake reference has a rev in the path (%q) and a rev query parameter (%q)", parsed.Rev, qRev) - } - parsed.Rev = qRev - } - - parsed.Dir = refURL.Query().Get("dir") - return nil -} - // String encodes the flake reference as a URL-like string. It normalizes the // result such that if two Ref values are equal, then their strings will also be // equal. @@ -341,7 +294,7 @@ func (r Ref) String() string { return "" } return "file+" + r.URL - case TypeGit: + case TypeSSH: if r.URL == "" { return "" } From 8234390260b108d340cbc044a8a27cd5c9b96496 Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Sun, 30 Jun 2024 23:24:45 -0400 Subject: [PATCH 06/42] basic retrieval of private plugins. need to clean it up --- internal/plugin/git.go | 77 +++++++++++++++++++++++++++++++++++------- 1 file changed, 65 insertions(+), 12 deletions(-) diff --git a/internal/plugin/git.go b/internal/plugin/git.go index ee82eea0e14..91222157f6b 100644 --- a/internal/plugin/git.go +++ b/internal/plugin/git.go @@ -4,10 +4,12 @@ import ( "cmp" "fmt" "io" + "log/slog" "net/http" "net/url" "os" "os/exec" + "path/filepath" "regexp" "strings" "time" @@ -16,6 +18,7 @@ import ( "github.com/samber/lo" "go.jetpack.io/devbox/internal/boxcli/usererr" "go.jetpack.io/devbox/internal/cachehash" + "go.jetpack.io/devbox/internal/fileutil" "go.jetpack.io/devbox/nix/flake" "go.jetpack.io/pkg/filecache" ) @@ -70,13 +73,34 @@ func (p *gitPlugin) Hash() string { func (p *gitPlugin) FileContent(subpath string) ([]byte, error) { contentURL, err := p.url(subpath) + slog.Debug("CONTENT URL: " + contentURL) + slog.Debug("SUBPATH: " + subpath + "") if err != nil { return nil, err } + readFile := func() ([]byte, time.Duration, error) { + file, err := os.Open(contentURL) + info, err := file.Stat() + + if err != nil || info.Size() == 0 { + return nil, 0, err + } + + defer file.Close() + body, err := io.ReadAll(file) + slog.Debug(string(body)) + + if err != nil { + return nil, 0, err + } + + return body, 24 * time.Hour, nil + } + retrieve := func() ([]byte, time.Duration, error) { - req, err := p.request(contentURL) // TODO: adjust this function to handle private repos + req, err := p.request(contentURL) if err != nil { return nil, 0, err @@ -102,6 +126,7 @@ func (p *gitPlugin) FileContent(subpath string) ([]byte, error) { } body, err := io.ReadAll(res.Body) + slog.Debug(string(body)) if err != nil { return nil, 0, err @@ -115,7 +140,8 @@ func (p *gitPlugin) FileContent(subpath string) ([]byte, error) { switch p.ref.Type { case flake.TypeSSH: - return sshCache.GetOrSet(contentURL, retrieve) + slog.Debug("TYPE SSH: " + contentURL) + return sshCache.GetOrSet(contentURL, readFile) case flake.TypeGitHub: return githubCache.GetOrSet(contentURL, retrieve) case flake.TypeGitLab: @@ -123,6 +149,7 @@ func (p *gitPlugin) FileContent(subpath string) ([]byte, error) { case flake.TypeBitBucket: return bitbucketCache.GetOrSet(contentURL, retrieve) default: + slog.Debug("HERE") return nil, err } } @@ -130,6 +157,7 @@ func (p *gitPlugin) FileContent(subpath string) ([]byte, error) { func (p *gitPlugin) url(subpath string) (string, error) { switch p.ref.Type { case flake.TypeSSH: + slog.Debug("TYPE SSH in url func: " + subpath) return p.sshGitUrl() case flake.TypeGitLab: return p.gitlabUrl(subpath) @@ -157,18 +185,48 @@ func (p *gitPlugin) sshGitUrl() (string, error) { } branch := cmp.Or(p.ref.Rev, p.ref.Ref, defaultBranch) - baseCommand := "git archive --format=tar --remote=git@" + format := "tar.gz" + baseCommand := fmt.Sprintf("git archive --format=%s --remote=git@", format) path, err := url.JoinPath(p.ref.Owner, p.ref.Repo) + host := address.Query().Get("host") + archive := filepath.Join("/", "tmp", p.ref.Dir+"."+format) + + // TODO: try to use the Devbox file hashing mechanism to make sure it's stored properly + command := fmt.Sprintf("%s%s:%s %s %s -o %s", baseCommand, host, path, branch, p.ref.Dir, archive) - formattedCommand := fmt.Sprintf("%s%s:%s %s %s -o %s.tar ", baseCommand, address.Query().Get("host"), path, branch, p.ref.Dir, p.ref.Dir) + slog.Debug("Generated git archive command: " + command) - if err == nil { + args := strings.Fields(command) + archiveInfo, err := os.Stat(archive) + + if err != nil { return "", err } - // TODO: need to store the git archive in a temporary file...or something. Basically need to figure out how to handle this lol + currentTime := time.Now() + threshold := 24 * time.Hour // 24 hours is currently when files are considered "expired" + oldTime := currentTime.Add(-threshold) + + if archiveInfo.ModTime().Before(oldTime) { + // TODO: make this async + cmd := exec.Command(args[0], args[1:]...) + + _, err := cmd.Output() + + if err != nil { + slog.Error("Error executing git archive: ", err) + return "", err + } - return formattedCommand, nil + reader, err := os.Open(archive) + err = fileutil.Untar(reader, "/tmp") // TODO: add UUID? + + if err == nil { + return "", err + } + } + + return filepath.Join("/", "tmp", p.ref.Dir, "plugin.json"), nil } func (p *gitPlugin) githubUrl(subpath string) (string, error) { @@ -243,11 +301,6 @@ func (p *gitPlugin) request(contentURL string) (*http.Request, error) { // TODO: Determine if private repo. Maybe use `git archive`? // git archive --format=tar --remote=git@gitlab.com:astro-tec/devbox-plugin-test HEAD plugin -o plugin.tar - if p.ref.Type == flake.TypeSSH { - command := exec.Command(contentURL) - command.Wait() - } - req, err := http.NewRequest(http.MethodGet, contentURL, nil) if err != nil { From e855a1c336ec60ce8a29378c18155b2024dae575 Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Sun, 30 Jun 2024 23:26:43 -0400 Subject: [PATCH 07/42] removed bad debug stmt --- internal/plugin/git.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/internal/plugin/git.go b/internal/plugin/git.go index 91222157f6b..8fc8ffd7849 100644 --- a/internal/plugin/git.go +++ b/internal/plugin/git.go @@ -73,8 +73,6 @@ func (p *gitPlugin) Hash() string { func (p *gitPlugin) FileContent(subpath string) ([]byte, error) { contentURL, err := p.url(subpath) - slog.Debug("CONTENT URL: " + contentURL) - slog.Debug("SUBPATH: " + subpath + "") if err != nil { return nil, err @@ -90,7 +88,6 @@ func (p *gitPlugin) FileContent(subpath string) ([]byte, error) { defer file.Close() body, err := io.ReadAll(file) - slog.Debug(string(body)) if err != nil { return nil, 0, err @@ -126,7 +123,6 @@ func (p *gitPlugin) FileContent(subpath string) ([]byte, error) { } body, err := io.ReadAll(res.Body) - slog.Debug(string(body)) if err != nil { return nil, 0, err @@ -140,7 +136,6 @@ func (p *gitPlugin) FileContent(subpath string) ([]byte, error) { switch p.ref.Type { case flake.TypeSSH: - slog.Debug("TYPE SSH: " + contentURL) return sshCache.GetOrSet(contentURL, readFile) case flake.TypeGitHub: return githubCache.GetOrSet(contentURL, retrieve) @@ -149,7 +144,6 @@ func (p *gitPlugin) FileContent(subpath string) ([]byte, error) { case flake.TypeBitBucket: return bitbucketCache.GetOrSet(contentURL, retrieve) default: - slog.Debug("HERE") return nil, err } } @@ -157,7 +151,6 @@ func (p *gitPlugin) FileContent(subpath string) ([]byte, error) { func (p *gitPlugin) url(subpath string) (string, error) { switch p.ref.Type { case flake.TypeSSH: - slog.Debug("TYPE SSH in url func: " + subpath) return p.sshGitUrl() case flake.TypeGitLab: return p.gitlabUrl(subpath) @@ -193,7 +186,6 @@ func (p *gitPlugin) sshGitUrl() (string, error) { // TODO: try to use the Devbox file hashing mechanism to make sure it's stored properly command := fmt.Sprintf("%s%s:%s %s %s -o %s", baseCommand, host, path, branch, p.ref.Dir, archive) - slog.Debug("Generated git archive command: " + command) args := strings.Fields(command) From 2ddf616e739e1a0daa8d80af662ba297014d057a Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Tue, 2 Jul 2024 23:21:18 -0400 Subject: [PATCH 08/42] added port and subgroup arguments to handle more edge cases of private repos --- internal/plugin/git.go | 22 +++++++++++++++------- nix/flake/flakeref.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/internal/plugin/git.go b/internal/plugin/git.go index 8fc8ffd7849..4ef4b990e49 100644 --- a/internal/plugin/git.go +++ b/internal/plugin/git.go @@ -177,12 +177,19 @@ func (p *gitPlugin) sshGitUrl() (string, error) { defaultBranch = "master" } + fileFormat := "tar.gz" + baseCommand := fmt.Sprintf("git archive --format=%s --remote=git@", fileFormat) + + path, _ := url.JoinPath(p.ref.Owner, p.ref.Subgroup, p.ref.Repo) + + archive := filepath.Join("/", "tmp", p.ref.Dir+"."+fileFormat) branch := cmp.Or(p.ref.Rev, p.ref.Ref, defaultBranch) - format := "tar.gz" - baseCommand := fmt.Sprintf("git archive --format=%s --remote=git@", format) - path, err := url.JoinPath(p.ref.Owner, p.ref.Repo) - host := address.Query().Get("host") - archive := filepath.Join("/", "tmp", p.ref.Dir+"."+format) + + host := p.ref.Host + + if p.ref.Port != "" { + host += ":" + p.ref.Port + } // TODO: try to use the Devbox file hashing mechanism to make sure it's stored properly command := fmt.Sprintf("%s%s:%s %s %s -o %s", baseCommand, host, path, branch, p.ref.Dir, archive) @@ -195,8 +202,9 @@ func (p *gitPlugin) sshGitUrl() (string, error) { return "", err } + // 24 hours is currently when files are considered "expired" in other FileContent function currentTime := time.Now() - threshold := 24 * time.Hour // 24 hours is currently when files are considered "expired" + threshold := 24 * time.Hour oldTime := currentTime.Add(-threshold) if archiveInfo.ModTime().Before(oldTime) { @@ -249,7 +257,7 @@ func (p *gitPlugin) bitbucketUrl(subpath string) (string, error) { } func (p *gitPlugin) gitlabUrl(subpath string) (string, error) { - project, err := url.JoinPath(p.ref.Owner, p.ref.Repo) + project, err := url.JoinPath(p.ref.Owner, p.ref.Subgroup, p.ref.Repo) if err != nil { return "", err diff --git a/nix/flake/flakeref.go b/nix/flake/flakeref.go index 965eeb67d0e..12827a2107e 100644 --- a/nix/flake/flakeref.go +++ b/nix/flake/flakeref.go @@ -69,6 +69,13 @@ type Ref struct { // or "git". Note that the URL is not the same as the raw unparsed // flake ref. URL string `json:"url,omitempty"` + + // Port of the server git server, to support privately hosted git servers or tunnels + Port string `json:port,omitempty` + + // Subgroup pertains to GitLab. GitHub and Bitbucket don't support multi-level + // hierarchy, and this allows the subgroup to exist without breaking the parsing logic already in place + Subgroup string `json:subgroup,omitempty` } // ParseRef parses a raw flake reference. Nix supports a variety of flake ref @@ -230,6 +237,26 @@ func parseURLRef(ref string) (parsed Ref, fragment string, err error) { func parseGitRef(refURL *url.URL, parsed *Ref) error { // github:/(/)?(\?)? + // NOTE: this currently doesn't handle subgroups with GitLab, and will + // continue to cause problems restructuring plugins to use JSON objects + // will make this much easier in the long run. GitHub and Bitbucket don't support + // subgroups, so this won't be an issue with those repos. + // something akin to the example below can help eliminate a vast majority + // of this URL parsing logic, and make things more flexible + + /* + "include": [ + "username/subgroup/repo": { + "type": "ssh", + "host": "gitlab", + "port": 9999, + "dir": "my-plugins", + "ref": "myref", + "branch": "mybranch" + } + ] + */ + // Only split up to 3 times (owner, repo, ref/rev) so that we handle // refs that have slashes in them. For example, // "github:jetify-com/devbox/gcurtis/flakeref" parses as "gcurtis/flakeref". @@ -237,8 +264,10 @@ func parseGitRef(refURL *url.URL, parsed *Ref) error { if err != nil { return err } + parsed.Owner = split[0] parsed.Repo = split[1] + if len(split) > 2 { if revOrRef := split[2]; isGitHash(revOrRef) { parsed.Rev = revOrRef @@ -249,6 +278,9 @@ func parseGitRef(refURL *url.URL, parsed *Ref) error { parsed.Host = refURL.Query().Get("host") parsed.Dir = refURL.Query().Get("dir") + parsed.Subgroup = refURL.Query().Get("subgroup") + parsed.Port = refURL.Query().Get("port") + if qRef := refURL.Query().Get("ref"); qRef != "" { if parsed.Rev != "" { return redact.Errorf("%s flake reference has a ref and a rev", parsed.Type) From 9c4bc59987ca7044b3e448b9d2edc5c691a827b1 Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Wed, 3 Jul 2024 12:35:31 -0400 Subject: [PATCH 09/42] changed to use explicit ssh protocol --- internal/plugin/git.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/plugin/git.go b/internal/plugin/git.go index 4ef4b990e49..4ec1da7f719 100644 --- a/internal/plugin/git.go +++ b/internal/plugin/git.go @@ -178,7 +178,7 @@ func (p *gitPlugin) sshGitUrl() (string, error) { } fileFormat := "tar.gz" - baseCommand := fmt.Sprintf("git archive --format=%s --remote=git@", fileFormat) + baseCommand := fmt.Sprintf("git archive --format=%s --remote=ssh://git@", fileFormat) path, _ := url.JoinPath(p.ref.Owner, p.ref.Subgroup, p.ref.Repo) @@ -192,7 +192,7 @@ func (p *gitPlugin) sshGitUrl() (string, error) { } // TODO: try to use the Devbox file hashing mechanism to make sure it's stored properly - command := fmt.Sprintf("%s%s:%s %s %s -o %s", baseCommand, host, path, branch, p.ref.Dir, archive) + command := fmt.Sprintf("%s%s/%s %s %s -o %s", baseCommand, host, path, branch, p.ref.Dir, archive) slog.Debug("Generated git archive command: " + command) args := strings.Fields(command) From 6448a7ba0cdf9a5b3ee80a05d29e1e100a701c61 Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Wed, 3 Jul 2024 13:28:17 -0400 Subject: [PATCH 10/42] added check to see if the plugin file exists --- internal/plugin/git.go | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/internal/plugin/git.go b/internal/plugin/git.go index 4ec1da7f719..b8334fa579d 100644 --- a/internal/plugin/git.go +++ b/internal/plugin/git.go @@ -195,21 +195,16 @@ func (p *gitPlugin) sshGitUrl() (string, error) { command := fmt.Sprintf("%s%s/%s %s %s -o %s", baseCommand, host, path, branch, p.ref.Dir, archive) slog.Debug("Generated git archive command: " + command) - args := strings.Fields(command) - archiveInfo, err := os.Stat(archive) - - if err != nil { - return "", err - } - // 24 hours is currently when files are considered "expired" in other FileContent function currentTime := time.Now() threshold := 24 * time.Hour - oldTime := currentTime.Add(-threshold) + expiration := currentTime.Add(-threshold) + + args := strings.Fields(command) + archiveInfo, err := os.Stat(archive) - if archiveInfo.ModTime().Before(oldTime) { - // TODO: make this async - cmd := exec.Command(args[0], args[1:]...) + if os.IsNotExist(err) || archiveInfo.ModTime().Before(expiration) { + cmd := exec.Command(args[0], args[1:]...) // Maybe make async? _, err := cmd.Output() From 23bcc4cb46ae995e1a0bdd3e5edf5dda38fde440 Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Sat, 13 Jul 2024 14:30:09 -0400 Subject: [PATCH 11/42] devbox schema for plugin layout --- .schema/devbox.schema.json | 327 ++++++++++++++++++++++--------------- 1 file changed, 199 insertions(+), 128 deletions(-) diff --git a/.schema/devbox.schema.json b/.schema/devbox.schema.json index 41c3ffd3534..99d97f01b54 100644 --- a/.schema/devbox.schema.json +++ b/.schema/devbox.schema.json @@ -1,145 +1,216 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", - "$id": "https://github.com/jetify-com/devbox", - "title": "Devbox json definition", - "description": "Defines fields and acceptable values of devbox.json", - "type": "object", - "properties": { - "$schema": { - "description": "The schema version of this devbox.json file.", - "type": "string" - }, - "name": { - "description": "The name of the Devbox development environment.", - "type": "string" - }, - "description": { - "description": "A description of the Devbox development environment.", + "$schema": "http://json-schema.org/draft-04/schema#", + "$id": "https://github.com/jetify-com/devbox", + "title": "Devbox json definition", + "description": "Defines fields and acceptable values of devbox.json", + "type": "object", + "properties": { + "$schema": { + "description": "The schema version of this devbox.json file.", + "type": "string" + }, + "name": { + "description": "The name of the Devbox development environment.", + "type": "string" + }, + "description": { + "description": "A description of the Devbox development environment.", + "type": "string" + }, + "packages": { + "description": "Collection of packages to install", + "oneOf": [ + { + "type": "array", + "items": { + "description": "Name and version of each package in name@version format.", "type": "string" + } }, - "packages": { - "description": "Collection of packages to install", - "oneOf": [ + { + "type": "object", + "description": "Name of each package in {\"name\": {\"version\": \"1.2.3\"}} format.", + "patternProperties": { + ".*": { + "oneOf": [ { - "type": "array", - "items": { - "description": "Name and version of each package in name@version format.", - "type": "string" + "type": "object", + "description": "Version number of the specified package in {\"version\": \"1.2.3\"} format.", + "properties": { + "version": { + "type": "string", + "description": "Version of the package" + }, + "platforms": { + "type": "array", + "description": "Names of platforms to install the package on. This package will be skipped for any platforms not on this list", + "items": { + "enum": [ + "i686-linux", + "aarch64-linux", + "aarch64-darwin", + "x86_64-darwin", + "x86_64-linux", + "armv7l-linux" + ] + } + }, + "excluded_platforms": { + "type": "array", + "description": "Names of platforms to exclude the package on", + "items": { + "enum": [ + "i686-linux", + "aarch64-linux", + "aarch64-darwin", + "x86_64-darwin", + "x86_64-linux", + "armv7l-linux" + ] + } + }, + "glibc_patch": { + "type": "boolean", + "description": "Whether to patch glibc to the latest available version for this package" } + } }, { - "type": "object", - "description": "Name of each package in {\"name\": {\"version\": \"1.2.3\"}} format.", - "patternProperties": { - ".*": { - "oneOf": [ - { - "type": "object", - "description": "Version number of the specified package in {\"version\": \"1.2.3\"} format.", - "properties": { - "version": { - "type": "string", - "description": "Version of the package" - }, - "platforms": { - "type": "array", - "description": "Names of platforms to install the package on. This package will be skipped for any platforms not on this list", - "items": { - "enum": [ - "i686-linux", - "aarch64-linux", - "aarch64-darwin", - "x86_64-darwin", - "x86_64-linux", - "armv7l-linux" - ] - } - }, - "excluded_platforms": { - "type": "array", - "description": "Names of platforms to exclude the package on", - "items": { - "enum": [ - "i686-linux", - "aarch64-linux", - "aarch64-darwin", - "x86_64-darwin", - "x86_64-linux", - "armv7l-linux" - ] - } - }, - "glibc_patch": { - "type": "boolean", - "description": "Whether to patch glibc to the latest available version for this package" - } - } - }, - { - "type": "string", - "description": "Version of the package to install." - } - ] - } - } - } - ] - }, - "env": { - "description": "List of additional environment variables to be set in the Devbox environment. Values containing $PATH or $PWD will be expanded. No other variable expansion or command substitution will occur.", - "type": "object", - "patternProperties": { - ".*": { - "type": "string", - "description": "Value of the environment variable." + "type": "string", + "description": "Version of the package to install." } + ] } + } + } + ] + }, + "env": { + "description": "List of additional environment variables to be set in the Devbox environment. Values containing $PATH or $PWD will be expanded. No other variable expansion or command substitution will occur.", + "type": "object", + "patternProperties": { + ".*": { + "type": "string", + "description": "Value of the environment variable." + } + } + }, + "shell": { + "description": "Definitions of scripts and actions to take when in devbox shell.", + "type": "object", + "properties": { + "init_hook": { + "type": [ + "array", + "string" + ], + "items": { + "description": "List of shell commands/scripts to run right after devbox shell starts.", + "type": "string" + } }, - "shell": { - "description": "Definitions of scripts and actions to take when in devbox shell.", - "type": "object", - "properties": { - "init_hook": { - "type": [ - "array", - "string" - ], - "items": { - "description": "List of shell commands/scripts to run right after devbox shell starts.", - "type": "string" + "scripts": { + "description": "List of command/script definitions to run with `devbox run `.", + "type": "object", + "patternProperties": { + ".*": { + "description": "Alias name for the script.", + "type": [ + "array", + "string" + ], + "items": { + "type": "string", + "description": "The script's shell commands." + } + } + } + } + }, + "additionalProperties": false + }, + "include": { + "description": "List of additional plugins to activate within your devbox shell", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9/_.-]+$": { + "description": "Configuration for each plugin specified by its path", + "type": "object", + "properties": { + "protocol": { + "description": "Protocol to use (https, ssh, or file)", + "type": "string", + "enum": [ + "https", + "ssh", + "file" + ] + }, + "host": { + "description": "Host of the repository (e.g., gitlab, github, bitbucket, localhost)", + "type": "string" + }, + "port": { + "description": "Port to use for the connection", + "type": "integer", + "minimum": 1, + "maximum": 65535 + }, + "directory": { + "description": "Subdirectory that should be accessed in the repo. Defaults to the path key if not provided", + "type": "string" + }, + "ref": { + "description": "Git reference", + "type": "string" + }, + "branch": { + "description": "Git branch (only relevant for https or ssh protocols)", + "type": "string" + } + }, + "required": [ + "protocol", + "host" + ], + "dependencies": { + "branch": { + "oneOf": [ + { + "properties": { + "protocol": { + "enum": [ + "https", + "ssh" + ] } + } }, - "scripts": { - "description": "List of command/script definitions to run with `devbox run `.", - "type": "object", - "patternProperties": { - ".*": { - "description": "Alias name for the script.", - "type": [ - "array", - "string" - ], - "items": { - "type": "string", - "description": "The script's shell commands." - } - } + { + "properties": { + "protocol": { + "enum": [ + "file" + ] } + }, + "not": { + "required": [ + "branch" + ] + } } - }, - "additionalProperties": false - }, - "include": { - "description": "List of additional plugins to activate within your devbox shell", - "type": "array", - "items": { - "description": "Name of the plugin to activate.", - "type": "string" + ] } - }, - "env_from": { - "type": "string" + }, + "additionalProperties": false } + }, + "additionalProperties": false }, - "additionalProperties": false -} \ No newline at end of file + "env_from": { + "type": "string" + } + }, + "additionalProperties": false +} From 97ee2c7e69525285146ece37d631635e207a4a14 Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Sat, 13 Jul 2024 14:30:35 -0400 Subject: [PATCH 12/42] update draft version --- .schema/devbox.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.schema/devbox.schema.json b/.schema/devbox.schema.json index 99d97f01b54..4eb9b7f7d6d 100644 --- a/.schema/devbox.schema.json +++ b/.schema/devbox.schema.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", + "$schema": "http://json-schema.org/draft-05/schema#", "$id": "https://github.com/jetify-com/devbox", "title": "Devbox json definition", "description": "Defines fields and acceptable values of devbox.json", From 1db728a5c8363e1d26a1b37ffb7e30558d57b107 Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Sun, 14 Jul 2024 17:38:35 -0400 Subject: [PATCH 13/42] wip; --- .schema/devbox-plugin.schema.json | 98 ++++++++++++++++++++++++--- .schema/devbox.schema.json | 3 +- internal/devconfig/config.go | 7 +- internal/devconfig/configfile/file.go | 30 +++++++- internal/plugin/git.go | 64 +++++++++++------ internal/plugin/includable.go | 16 +++-- internal/plugin/includes.go | 15 ++-- internal/plugin/local.go | 1 + 8 files changed, 184 insertions(+), 50 deletions(-) diff --git a/.schema/devbox-plugin.schema.json b/.schema/devbox-plugin.schema.json index fda6b446304..e1a485dc64a 100644 --- a/.schema/devbox-plugin.schema.json +++ b/.schema/devbox-plugin.schema.json @@ -114,7 +114,10 @@ "description": "Shell specific options and hooks for the plugin.", "items": { "init_hook": { - "type": ["array", "string"], + "type": [ + "array", + "string" + ], "description": "Shell command to run right before initializing the user's shell, running a script, or starting a service" }, "scripts": { @@ -123,7 +126,10 @@ "patternProperties": { ".*": { "description": "Alias name for the script.", - "type": ["array", "string"], + "type": [ + "array", + "string" + ], "items": { "type": "string", "description": "The script's shell commands." @@ -135,12 +141,88 @@ }, "include": { "description": "List of additional plugins to activate within your devbox shell", - "type": "array", - "items": { - "description": "Name of the plugin to activate.", - "type": "string" - } + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9/_.-]+$": { + "description": "Configuration for each plugin specified by its path", + "type": "object", + "properties": { + "protocol": { + "description": "Protocol to use (https, ssh, or file)", + "type": "string", + "enum": [ + "https", + "ssh", + "file", + "builtin" + ] + }, + "host": { + "description": "Host of the repository (e.g., gitlab, github, bitbucket, localhost)", + "type": "string" + }, + "port": { + "description": "Port to use for the connection", + "type": "integer", + "minimum": 1, + "maximum": 65535 + }, + "directory": { + "description": "Subdirectory that should be accessed in the repo. Defaults to the path key if not provided", + "type": "string" + }, + "ref": { + "description": "Git reference", + "type": "string" + }, + "branch": { + "description": "Git branch (only relevant for https or ssh protocols)", + "type": "string" + } + }, + "required": [ + "protocol", + "host" + ], + "dependencies": { + "branch": { + "oneOf": [ + { + "properties": { + "protocol": { + "enum": [ + "https", + "ssh" + ] + } + } + }, + { + "properties": { + "protocol": { + "enum": [ + "file" + ] + } + }, + "not": { + "required": [ + "branch" + ] + } + } + ] + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false } }, - "required": ["name", "version", "readme"] + "required": [ + "name", + "version", + "readme" + ] } diff --git a/.schema/devbox.schema.json b/.schema/devbox.schema.json index 4eb9b7f7d6d..9853cd1e8f4 100644 --- a/.schema/devbox.schema.json +++ b/.schema/devbox.schema.json @@ -143,7 +143,8 @@ "enum": [ "https", "ssh", - "file" + "file", + "builtin" ] }, "host": { diff --git a/internal/devconfig/config.go b/internal/devconfig/config.go index 2250a50f4e5..1fc62948364 100644 --- a/internal/devconfig/config.go +++ b/internal/devconfig/config.go @@ -123,14 +123,15 @@ func (c *Config) loadRecursive( ) error { included := make([]*Config, 0, len(c.Root.Include)) - for _, includeRef := range c.Root.Include { + // TODO UPDATEME + for path, includeRef := range c.Root.Include { pluginConfig, err := plugin.LoadConfigFromInclude( - includeRef, lockfile, filepath.Dir(c.Root.AbsRootPath)) + path, includeRef, lockfile, filepath.Dir(c.Root.AbsRootPath)) if err != nil { return errors.WithStack(err) } - newCyclePath := fmt.Sprintf("%s -> %s", cyclePath, includeRef) + newCyclePath := fmt.Sprintf("%s -> %s", cyclePath, path) if seen[pluginConfig.Source.Hash()] { // Note that duplicate includes are allowed if they are in different paths // e.g. 2 different plugins can include the same plugin. diff --git a/internal/devconfig/configfile/file.go b/internal/devconfig/configfile/file.go index 0f6d9b79857..2c4b06b518d 100644 --- a/internal/devconfig/configfile/file.go +++ b/internal/devconfig/configfile/file.go @@ -23,6 +23,33 @@ const ( DefaultName = "devbox.json" ) +type Plugin struct { + // Reserved to allow including other config files. Proposed format is: + // file: for local files + // https: for remote files + // ssh: for remote files + // plugin: for built-in plugins + // protocol to use (https, ssh, file, builtin); required + Protocol string `json:"protocol,omitempty"` + + // where the plugin is hosted (github.com, gitlab.com, localhost, etc); required + Host string `json:"host,omitempty"` + + // port, 1 to 65535; optional + Port uint16 `json:"port,omitempty"` + + // Subdirectory of the plugin's repo to retrieve; optional + // if empty, the plugin key is the assumed path + Directory string `json:"directory,omitempty"` + + // The Git ref; optional + Ref string `json:"ref,omitempty"` + + // The branch to access; optional + // if omitted, master/main is assumed + Branch string `json:"branch,omitempty"` +} + // ConfigFile defines a devbox environment as JSON. type ConfigFile struct { // AbsRootPath is the absolute path to the devbox.json or plugin.json file @@ -53,7 +80,8 @@ type ConfigFile struct { // https:// for remote files // plugin: for built-in plugins // This is a similar format to nix inputs - Include []string `json:"include,omitempty"` + //Include []string `json:"include,omitempty"` + Include map[string]Plugin `json:"include,omitempty"` ast *configAST } diff --git a/internal/plugin/git.go b/internal/plugin/git.go index b8334fa579d..a7fadda28d7 100644 --- a/internal/plugin/git.go +++ b/internal/plugin/git.go @@ -79,6 +79,21 @@ func (p *gitPlugin) FileContent(subpath string) ([]byte, error) { } readFile := func() ([]byte, time.Duration, error) { + archive := filepath.Join("/", "tmp", p.ref.Dir+".tar.gz") + args := strings.Fields(contentURL) + cmd := exec.Command(args[0], args[1:]...) // Maybe make async? + + _, err := cmd.Output() + + if err != nil { + slog.Error("Error executing git archive: ", err) + return nil, 24 * time.Hour, err + } + + reader, err := os.Open(archive) + io.ReadAll(reader) + err = fileutil.Untar(reader, "/tmp") // TODO: add UUID? + file, err := os.Open(contentURL) info, err := file.Stat() @@ -193,35 +208,43 @@ func (p *gitPlugin) sshGitUrl() (string, error) { // TODO: try to use the Devbox file hashing mechanism to make sure it's stored properly command := fmt.Sprintf("%s%s/%s %s %s -o %s", baseCommand, host, path, branch, p.ref.Dir, archive) + slog.Debug("Generated git archive command: " + command) + return command, nil + + //sshCache.GetOrSet(command, func() ([]byte, time.Duration, error) { + + //}) + // 24 hours is currently when files are considered "expired" in other FileContent function - currentTime := time.Now() - threshold := 24 * time.Hour - expiration := currentTime.Add(-threshold) + //currentTime := time.Now() + //threshold := 24 * time.Hour + //expiration := currentTime.Add(-threshold) - args := strings.Fields(command) - archiveInfo, err := os.Stat(archive) + //args := strings.Fields(command) + //archiveInfo, err := os.Stat(archive) - if os.IsNotExist(err) || archiveInfo.ModTime().Before(expiration) { - cmd := exec.Command(args[0], args[1:]...) // Maybe make async? + //if os.IsNotExist(err) || archiveInfo.ModTime().Before(expiration) { + // cmd := exec.Command(args[0], args[1:]...) // Maybe make async? - _, err := cmd.Output() + // _, err := cmd.Output() - if err != nil { - slog.Error("Error executing git archive: ", err) - return "", err - } + // if err != nil { + // slog.Error("Error executing git archive: ", err) + // return "", err + // } - reader, err := os.Open(archive) - err = fileutil.Untar(reader, "/tmp") // TODO: add UUID? + // reader, err := os.Open(archive) + // io.ReadAll(reader) + // err = fileutil.Untar(reader, "/tmp") // TODO: add UUID? - if err == nil { - return "", err - } - } + // if err == nil { + // return "", err + // } + //} - return filepath.Join("/", "tmp", p.ref.Dir, "plugin.json"), nil + //return filepath.Join("/", "tmp", p.ref.Dir, "plugin.json"), nil } func (p *gitPlugin) githubUrl(subpath string) (string, error) { @@ -293,9 +316,6 @@ func (p *gitPlugin) gitlabUrl(subpath string) (string, error) { } func (p *gitPlugin) request(contentURL string) (*http.Request, error) { - // TODO: Determine if private repo. Maybe use `git archive`? - // git archive --format=tar --remote=git@gitlab.com:astro-tec/devbox-plugin-test HEAD plugin -o plugin.tar - req, err := http.NewRequest(http.MethodGet, contentURL, nil) if err != nil { diff --git a/internal/plugin/includable.go b/internal/plugin/includable.go index 5074cae0ab3..149608a47d5 100644 --- a/internal/plugin/includable.go +++ b/internal/plugin/includable.go @@ -6,6 +6,7 @@ import ( "regexp" "go.jetpack.io/devbox/internal/boxcli/usererr" + "go.jetpack.io/devbox/internal/devconfig/configfile" "go.jetpack.io/devbox/nix/flake" ) @@ -16,13 +17,16 @@ type Includable interface { LockfileKey() string } -func parseIncludable(includableRef, workingDir string) (Includable, error) { - ref, err := flake.ParseRef(includableRef) - if err != nil { - return nil, err - } +// TODO UPDATEME +func parseIncludable(path string, plugin configfile.Plugin, workingDir string) (Includable, error) { + + //ref, err := flake.ParseRef(path) + + //if err != nil { + // return nil, err + //} - switch ref.Type { + switch plugin.Protocol { case flake.TypePath: return newLocalPlugin(ref, workingDir) case flake.TypeSSH: diff --git a/internal/plugin/includes.go b/internal/plugin/includes.go index 678e1aeb790..55c6af7d434 100644 --- a/internal/plugin/includes.go +++ b/internal/plugin/includes.go @@ -1,22 +1,19 @@ package plugin import ( - "strings" - + "go.jetpack.io/devbox/internal/devconfig/configfile" "go.jetpack.io/devbox/internal/devpkg" "go.jetpack.io/devbox/internal/lock" ) -func LoadConfigFromInclude(include string, lockfile *lock.File, workingDir string) (*Config, error) { +func LoadConfigFromInclude(path string, plugin configfile.Plugin, lockfile *lock.File, workingDir string) (*Config, error) { var includable Includable var err error - if t, name, _ := strings.Cut(include, ":"); t == "plugin" { - includable = devpkg.PackageFromStringWithDefaults( - name, - lockfile, - ) + + if plugin.Protocol == "builtin" { + includable = devpkg.PackageFromStringWithDefaults(path, lockfile) } else { - includable, err = parseIncludable(include, workingDir) + includable, err = parseIncludable(path, plugin, workingDir) if err != nil { return nil, err } diff --git a/internal/plugin/local.go b/internal/plugin/local.go index 996f5905804..25b6e881bcb 100644 --- a/internal/plugin/local.go +++ b/internal/plugin/local.go @@ -16,6 +16,7 @@ type LocalPlugin struct { pluginDir string } +// TODO UPDATEME func newLocalPlugin(ref flake.Ref, pluginDir string) (*LocalPlugin, error) { plugin := &LocalPlugin{ref: ref, pluginDir: pluginDir} name, err := getPluginNameFromContent(plugin) From bd2b547e9a6470943771cb673ecca86931d7a097 Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Sun, 14 Jul 2024 22:10:33 -0400 Subject: [PATCH 14/42] reworking things to work with the flake.Ref struct --- .schema/devbox-plugin.schema.json | 2 +- .schema/devbox.schema.json | 2 +- internal/devconfig/configfile/file.go | 31 ++------------------------- internal/plugin/includable.go | 5 ++--- internal/plugin/includes.go | 8 +++---- nix/flake/.flakeref.go.~undo-tree~ | 7 ++++++ nix/flake/flakeref.go | 1 + 7 files changed, 18 insertions(+), 38 deletions(-) create mode 100644 nix/flake/.flakeref.go.~undo-tree~ diff --git a/.schema/devbox-plugin.schema.json b/.schema/devbox-plugin.schema.json index e1a485dc64a..2b6598dd279 100644 --- a/.schema/devbox-plugin.schema.json +++ b/.schema/devbox-plugin.schema.json @@ -167,7 +167,7 @@ "minimum": 1, "maximum": 65535 }, - "directory": { + "dir": { "description": "Subdirectory that should be accessed in the repo. Defaults to the path key if not provided", "type": "string" }, diff --git a/.schema/devbox.schema.json b/.schema/devbox.schema.json index 9853cd1e8f4..0398d911630 100644 --- a/.schema/devbox.schema.json +++ b/.schema/devbox.schema.json @@ -157,7 +157,7 @@ "minimum": 1, "maximum": 65535 }, - "directory": { + "dir": { "description": "Subdirectory that should be accessed in the repo. Defaults to the path key if not provided", "type": "string" }, diff --git a/internal/devconfig/configfile/file.go b/internal/devconfig/configfile/file.go index 2c4b06b518d..ce749a46362 100644 --- a/internal/devconfig/configfile/file.go +++ b/internal/devconfig/configfile/file.go @@ -17,39 +17,13 @@ import ( "go.jetpack.io/devbox/internal/boxcli/usererr" "go.jetpack.io/devbox/internal/cachehash" "go.jetpack.io/devbox/internal/devbox/shellcmd" + "go.jetpack.io/devbox/nix/flake" ) const ( DefaultName = "devbox.json" ) -type Plugin struct { - // Reserved to allow including other config files. Proposed format is: - // file: for local files - // https: for remote files - // ssh: for remote files - // plugin: for built-in plugins - // protocol to use (https, ssh, file, builtin); required - Protocol string `json:"protocol,omitempty"` - - // where the plugin is hosted (github.com, gitlab.com, localhost, etc); required - Host string `json:"host,omitempty"` - - // port, 1 to 65535; optional - Port uint16 `json:"port,omitempty"` - - // Subdirectory of the plugin's repo to retrieve; optional - // if empty, the plugin key is the assumed path - Directory string `json:"directory,omitempty"` - - // The Git ref; optional - Ref string `json:"ref,omitempty"` - - // The branch to access; optional - // if omitted, master/main is assumed - Branch string `json:"branch,omitempty"` -} - // ConfigFile defines a devbox environment as JSON. type ConfigFile struct { // AbsRootPath is the absolute path to the devbox.json or plugin.json file @@ -80,8 +54,7 @@ type ConfigFile struct { // https:// for remote files // plugin: for built-in plugins // This is a similar format to nix inputs - //Include []string `json:"include,omitempty"` - Include map[string]Plugin `json:"include,omitempty"` + Include map[string]flake.Ref `json:"include,omitempty"` ast *configAST } diff --git a/internal/plugin/includable.go b/internal/plugin/includable.go index 149608a47d5..a66e95dff8b 100644 --- a/internal/plugin/includable.go +++ b/internal/plugin/includable.go @@ -6,7 +6,6 @@ import ( "regexp" "go.jetpack.io/devbox/internal/boxcli/usererr" - "go.jetpack.io/devbox/internal/devconfig/configfile" "go.jetpack.io/devbox/nix/flake" ) @@ -18,7 +17,7 @@ type Includable interface { } // TODO UPDATEME -func parseIncludable(path string, plugin configfile.Plugin, workingDir string) (Includable, error) { +func parseIncludable(path string, ref flake.Ref, workingDir string) (Includable, error) { //ref, err := flake.ParseRef(path) @@ -26,7 +25,7 @@ func parseIncludable(path string, plugin configfile.Plugin, workingDir string) ( // return nil, err //} - switch plugin.Protocol { + switch ref.Type { case flake.TypePath: return newLocalPlugin(ref, workingDir) case flake.TypeSSH: diff --git a/internal/plugin/includes.go b/internal/plugin/includes.go index 55c6af7d434..2c7cf3197e4 100644 --- a/internal/plugin/includes.go +++ b/internal/plugin/includes.go @@ -1,19 +1,19 @@ package plugin import ( - "go.jetpack.io/devbox/internal/devconfig/configfile" "go.jetpack.io/devbox/internal/devpkg" "go.jetpack.io/devbox/internal/lock" + "go.jetpack.io/devbox/nix/flake" ) -func LoadConfigFromInclude(path string, plugin configfile.Plugin, lockfile *lock.File, workingDir string) (*Config, error) { +func LoadConfigFromInclude(path string, ref flake.Ref, lockfile *lock.File, workingDir string) (*Config, error) { var includable Includable var err error - if plugin.Protocol == "builtin" { + if ref.Type == "builtin" { includable = devpkg.PackageFromStringWithDefaults(path, lockfile) } else { - includable, err = parseIncludable(path, plugin, workingDir) + includable, err = parseIncludable(path, ref, workingDir) if err != nil { return nil, err } diff --git a/nix/flake/.flakeref.go.~undo-tree~ b/nix/flake/.flakeref.go.~undo-tree~ new file mode 100644 index 00000000000..26ddb7a89a9 --- /dev/null +++ b/nix/flake/.flakeref.go.~undo-tree~ @@ -0,0 +1,7 @@ +(undo-tree-save-format-version . 1) +"0a4b9df52b174d1a98526c3152fc92aecc40fc35" +[nil nil nil nil (26257 9689 231551 618000) 0 nil] +([nil nil ((173 . 175) (172 . 174) ("\"" . -172) (172 . 173) (163 . 172) (t 26244 48638 196107 765000)) nil (26257 9689 231550 611000) 0 nil]) +([nil current ((" + " . 163) (undo-tree-id0 . 9) (undo-tree-id1 . -9) (undo-tree-id2 . -9) (undo-tree-id3 . -9) (undo-tree-id4 . -9) (undo-tree-id5 . -9) (undo-tree-id6 . -9) (undo-tree-id7 . -9) (undo-tree-id8 . -9) (undo-tree-id9 . -9) (undo-tree-id10 . -9) (undo-tree-id11 . -9) (undo-tree-id12 . -9) (undo-tree-id13 . -9) (undo-tree-id14 . -9) (undo-tree-id15 . -9) (undo-tree-id16 . -9) (undo-tree-id17 . -9) (undo-tree-id18 . -9) (undo-tree-id19 . -9) (undo-tree-id20 . -9) (undo-tree-id21 . -9) (undo-tree-id22 . -9) (undo-tree-id23 . -9) (undo-tree-id24 . -9) (undo-tree-id25 . -9) (undo-tree-id26 . -9) (undo-tree-id27 . -9) (undo-tree-id28 . -9) (undo-tree-id29 . -9) (undo-tree-id30 . -9) (undo-tree-id31 . -9) (undo-tree-id32 . -9) (undo-tree-id33 . -9) (undo-tree-id34 . -9) (undo-tree-id35 . -9) (undo-tree-id36 . -9) (undo-tree-id37 . -9) (undo-tree-id38 . -9) (undo-tree-id39 . -9) (undo-tree-id40 . -9) (undo-tree-id41 . -9) (undo-tree-id42 . -9) (undo-tree-id43 . -9) (undo-tree-id44 . -9) (undo-tree-id45 . -9) (undo-tree-id46 . -9) (undo-tree-id47 . -9) (undo-tree-id48 . -9) (undo-tree-id49 . -9) (undo-tree-id50 . -9) (undo-tree-id51 . -9) (undo-tree-id52 . -9) (undo-tree-id53 . -9) (undo-tree-id54 . -9) (undo-tree-id55 . -9) (undo-tree-id56 . -9) (undo-tree-id57 . -9) (undo-tree-id58 . -9) (undo-tree-id59 . -9) (undo-tree-id60 . -9) (undo-tree-id61 . -9) (undo-tree-id62 . -9) (undo-tree-id63 . -9) (undo-tree-id64 . -9) (undo-tree-id65 . -9) (undo-tree-id66 . -9) (undo-tree-id67 . -9) (undo-tree-id68 . -9) (undo-tree-id69 . -9) (undo-tree-id70 . -9) (undo-tree-id71 . -9) (undo-tree-id72 . -9) (undo-tree-id73 . -9) (undo-tree-id74 . -9) (undo-tree-id75 . -9) (undo-tree-id76 . -9) (undo-tree-id77 . -9) (undo-tree-id78 . -9) (undo-tree-id79 . -9) (undo-tree-id80 . -9) (undo-tree-id81 . -9) (undo-tree-id82 . -9) (undo-tree-id83 . -9) (undo-tree-id84 . -9) (undo-tree-id85 . -9) (undo-tree-id86 . -9) (undo-tree-id87 . -9) (undo-tree-id88 . -9) (undo-tree-id89 . -9) (undo-tree-id90 . -9) (undo-tree-id91 . -9) (undo-tree-id92 . -9) (undo-tree-id93 . -9) (undo-tree-id94 . -9) (undo-tree-id95 . -9) (undo-tree-id96 . -9) (undo-tree-id97 . -9) (undo-tree-id98 . -9) (undo-tree-id99 . -9) (undo-tree-id100 . -9) (undo-tree-id101 . -9) (undo-tree-id102 . -9) (undo-tree-id103 . -9) (undo-tree-id104 . -9) (undo-tree-id105 . -9) (undo-tree-id106 . -9) (undo-tree-id107 . -9) (undo-tree-id108 . -9) (undo-tree-id109 . -9) (undo-tree-id110 . -9) (undo-tree-id111 . -9) (undo-tree-id112 . -9) (undo-tree-id113 . -9) (undo-tree-id114 . -9) (undo-tree-id115 . -9) (undo-tree-id116 . -9) (undo-tree-id117 . -9) (undo-tree-id118 . -9) (undo-tree-id119 . -9) (undo-tree-id120 . -9) (undo-tree-id121 . -9) (undo-tree-id122 . -9) (undo-tree-id123 . -9) (undo-tree-id124 . -9) (undo-tree-id125 . -9) ("\"" . 172) (172 . 173) ("\"\"" . 172) (undo-tree-id126 . -1) (undo-tree-id127 . -1) (undo-tree-id128 . -1) (undo-tree-id129 . -1) (undo-tree-id130 . -1) (undo-tree-id131 . -1) (undo-tree-id132 . -1) (undo-tree-id133 . -1) (undo-tree-id134 . -1) (undo-tree-id135 . -1) (undo-tree-id136 . -1) (undo-tree-id137 . -1) (undo-tree-id138 . -1) (undo-tree-id139 . -1) (undo-tree-id140 . -1) (undo-tree-id141 . -1) (undo-tree-id142 . -1) (undo-tree-id143 . -1) (undo-tree-id144 . -1) (undo-tree-id145 . -1) (undo-tree-id146 . -1) (undo-tree-id147 . -1) (undo-tree-id148 . -1) (undo-tree-id149 . -1) (undo-tree-id150 . -1) (undo-tree-id151 . -1) (undo-tree-id152 . -1) (undo-tree-id153 . -1) (undo-tree-id154 . -1) (undo-tree-id155 . -1) (undo-tree-id156 . -1) (undo-tree-id157 . -1) (undo-tree-id158 . -1) (undo-tree-id159 . -1) (undo-tree-id160 . -1) (undo-tree-id161 . -1) (undo-tree-id162 . -1) (undo-tree-id163 . -1) (undo-tree-id164 . -1) (undo-tree-id165 . -1) (undo-tree-id166 . -1) (undo-tree-id167 . -1) (undo-tree-id168 . -1) (undo-tree-id169 . -1) (undo-tree-id170 . -1) (undo-tree-id171 . -1) (undo-tree-id172 . -1) (undo-tree-id173 . -1) (undo-tree-id174 . -1) (undo-tree-id175 . -1) (undo-tree-id176 . -1) (undo-tree-id177 . -1) (undo-tree-id178 . -1) (undo-tree-id179 . -1) (undo-tree-id180 . -1) (undo-tree-id181 . -1) (undo-tree-id182 . -1) (undo-tree-id183 . -1) (undo-tree-id184 . -1) (undo-tree-id185 . -1) (undo-tree-id186 . -1) (undo-tree-id187 . -1) (undo-tree-id188 . -1) (undo-tree-id189 . -1) (undo-tree-id190 . -1) (undo-tree-id191 . -1) (undo-tree-id192 . -1) (undo-tree-id193 . -1) (undo-tree-id194 . -1) (undo-tree-id195 . -1) (undo-tree-id196 . -1) (undo-tree-id197 . -1) (undo-tree-id198 . 1) (undo-tree-id199 . -1) (undo-tree-id200 . -1) (undo-tree-id201 . -1) (undo-tree-id202 . -1) (undo-tree-id203 . -1) (undo-tree-id204 . -1) (undo-tree-id205 . -1) (undo-tree-id206 . -1) (undo-tree-id207 . -1) (undo-tree-id208 . -1) (undo-tree-id209 . -1) (undo-tree-id210 . -1) (undo-tree-id211 . -1) (undo-tree-id212 . -1) (undo-tree-id213 . -1) (undo-tree-id214 . -1) (undo-tree-id215 . -1) (undo-tree-id216 . -1) (undo-tree-id217 . -1) (undo-tree-id218 . -1) (undo-tree-id219 . -1) (undo-tree-id220 . -1) (undo-tree-id221 . -1) (undo-tree-id222 . -1) (undo-tree-id223 . -1) (undo-tree-id224 . -1) (undo-tree-id225 . -1) (undo-tree-id226 . -1) (undo-tree-id227 . -1) (undo-tree-id228 . -1) (undo-tree-id229 . -1) (undo-tree-id230 . -1) ("go" . 173) (undo-tree-id231 . -1) (undo-tree-id232 . -1) (undo-tree-id233 . -1) (undo-tree-id234 . -1) (undo-tree-id235 . -1) (undo-tree-id236 . -1) (undo-tree-id237 . -1) (undo-tree-id238 . -1) (undo-tree-id239 . -1) (undo-tree-id240 . -1) (undo-tree-id241 . -1) (undo-tree-id242 . -1) (undo-tree-id243 . -1) (undo-tree-id244 . -1) (undo-tree-id245 . -1) (undo-tree-id246 . -1) (undo-tree-id247 . -1) (undo-tree-id248 . -1) (undo-tree-id249 . -1) (undo-tree-id250 . -1) (undo-tree-id251 . -1) (undo-tree-id252 . -1) (undo-tree-id253 . -1) (undo-tree-id254 . -1) (undo-tree-id255 . -1) (undo-tree-id256 . -2) (undo-tree-id257 . -2) (undo-tree-id258 . -2) (undo-tree-id259 . -2) (undo-tree-id260 . -2) (undo-tree-id261 . -2) (undo-tree-id262 . -2) (undo-tree-id263 . -2) (undo-tree-id264 . -2) (undo-tree-id265 . -2) (undo-tree-id266 . -2) (undo-tree-id267 . -2) (undo-tree-id268 . -2) (undo-tree-id269 . -2) (undo-tree-id270 . -2) (undo-tree-id271 . -2) (undo-tree-id272 . -2) (undo-tree-id273 . -2) (undo-tree-id274 . -2) (undo-tree-id275 . -2) (undo-tree-id276 . -2) (undo-tree-id277 . -2) (undo-tree-id278 . -2) (undo-tree-id279 . -2) (undo-tree-id280 . -2) (undo-tree-id281 . -2) (undo-tree-id282 . -2) (undo-tree-id283 . -1) (undo-tree-id284 . -1) (undo-tree-id285 . -1) (undo-tree-id286 . -1) (undo-tree-id287 . -1) (undo-tree-id288 . -1) (undo-tree-id289 . -1) (undo-tree-id290 . -1) (undo-tree-id291 . -1) (undo-tree-id292 . -1) (undo-tree-id293 . -1) (undo-tree-id294 . -1) (undo-tree-id295 . -1) (undo-tree-id296 . -1) (undo-tree-id297 . -1) (undo-tree-id298 . -1) (undo-tree-id299 . -1) (undo-tree-id300 . -1) (undo-tree-id301 . -1) (undo-tree-id302 . -1) (undo-tree-id303 . -1) (undo-tree-id304 . -1) (undo-tree-id305 . -1) (undo-tree-id306 . -1) (undo-tree-id307 . -1) (undo-tree-id308 . -1) (undo-tree-id309 . -1) (undo-tree-id310 . -1) (undo-tree-id311 . -1) (undo-tree-id312 . -1) (undo-tree-id313 . -1) (undo-tree-id314 . -1)) nil (26257 9689 231541 913000) 0 nil]) +nil diff --git a/nix/flake/flakeref.go b/nix/flake/flakeref.go index 12827a2107e..84baa601c64 100644 --- a/nix/flake/flakeref.go +++ b/nix/flake/flakeref.go @@ -73,6 +73,7 @@ type Ref struct { // Port of the server git server, to support privately hosted git servers or tunnels Port string `json:port,omitempty` + // TODO UPDATEME // Subgroup pertains to GitLab. GitHub and Bitbucket don't support multi-level // hierarchy, and this allows the subgroup to exist without breaking the parsing logic already in place Subgroup string `json:subgroup,omitempty` From e17c8308a9bbd7c8a3852e80d40315d34aad1da4 Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Sun, 14 Jul 2024 22:10:56 -0400 Subject: [PATCH 15/42] removing undo tree file --- nix/flake/.flakeref.go.~undo-tree~ | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 nix/flake/.flakeref.go.~undo-tree~ diff --git a/nix/flake/.flakeref.go.~undo-tree~ b/nix/flake/.flakeref.go.~undo-tree~ deleted file mode 100644 index 26ddb7a89a9..00000000000 --- a/nix/flake/.flakeref.go.~undo-tree~ +++ /dev/null @@ -1,7 +0,0 @@ -(undo-tree-save-format-version . 1) -"0a4b9df52b174d1a98526c3152fc92aecc40fc35" -[nil nil nil nil (26257 9689 231551 618000) 0 nil] -([nil nil ((173 . 175) (172 . 174) ("\"" . -172) (172 . 173) (163 . 172) (t 26244 48638 196107 765000)) nil (26257 9689 231550 611000) 0 nil]) -([nil current ((" - " . 163) (undo-tree-id0 . 9) (undo-tree-id1 . -9) (undo-tree-id2 . -9) (undo-tree-id3 . -9) (undo-tree-id4 . -9) (undo-tree-id5 . -9) (undo-tree-id6 . -9) (undo-tree-id7 . -9) (undo-tree-id8 . -9) (undo-tree-id9 . -9) (undo-tree-id10 . -9) (undo-tree-id11 . -9) (undo-tree-id12 . -9) (undo-tree-id13 . -9) (undo-tree-id14 . -9) (undo-tree-id15 . -9) (undo-tree-id16 . -9) (undo-tree-id17 . -9) (undo-tree-id18 . -9) (undo-tree-id19 . -9) (undo-tree-id20 . -9) (undo-tree-id21 . -9) (undo-tree-id22 . -9) (undo-tree-id23 . -9) (undo-tree-id24 . -9) (undo-tree-id25 . -9) (undo-tree-id26 . -9) (undo-tree-id27 . -9) (undo-tree-id28 . -9) (undo-tree-id29 . -9) (undo-tree-id30 . -9) (undo-tree-id31 . -9) (undo-tree-id32 . -9) (undo-tree-id33 . -9) (undo-tree-id34 . -9) (undo-tree-id35 . -9) (undo-tree-id36 . -9) (undo-tree-id37 . -9) (undo-tree-id38 . -9) (undo-tree-id39 . -9) (undo-tree-id40 . -9) (undo-tree-id41 . -9) (undo-tree-id42 . -9) (undo-tree-id43 . -9) (undo-tree-id44 . -9) (undo-tree-id45 . -9) (undo-tree-id46 . -9) (undo-tree-id47 . -9) (undo-tree-id48 . -9) (undo-tree-id49 . -9) (undo-tree-id50 . -9) (undo-tree-id51 . -9) (undo-tree-id52 . -9) (undo-tree-id53 . -9) (undo-tree-id54 . -9) (undo-tree-id55 . -9) (undo-tree-id56 . -9) (undo-tree-id57 . -9) (undo-tree-id58 . -9) (undo-tree-id59 . -9) (undo-tree-id60 . -9) (undo-tree-id61 . -9) (undo-tree-id62 . -9) (undo-tree-id63 . -9) (undo-tree-id64 . -9) (undo-tree-id65 . -9) (undo-tree-id66 . -9) (undo-tree-id67 . -9) (undo-tree-id68 . -9) (undo-tree-id69 . -9) (undo-tree-id70 . -9) (undo-tree-id71 . -9) (undo-tree-id72 . -9) (undo-tree-id73 . -9) (undo-tree-id74 . -9) (undo-tree-id75 . -9) (undo-tree-id76 . -9) (undo-tree-id77 . -9) (undo-tree-id78 . -9) (undo-tree-id79 . -9) (undo-tree-id80 . -9) (undo-tree-id81 . -9) (undo-tree-id82 . -9) (undo-tree-id83 . -9) (undo-tree-id84 . -9) (undo-tree-id85 . -9) (undo-tree-id86 . -9) (undo-tree-id87 . -9) (undo-tree-id88 . -9) (undo-tree-id89 . -9) (undo-tree-id90 . -9) (undo-tree-id91 . -9) (undo-tree-id92 . -9) (undo-tree-id93 . -9) (undo-tree-id94 . -9) (undo-tree-id95 . -9) (undo-tree-id96 . -9) (undo-tree-id97 . -9) (undo-tree-id98 . -9) (undo-tree-id99 . -9) (undo-tree-id100 . -9) (undo-tree-id101 . -9) (undo-tree-id102 . -9) (undo-tree-id103 . -9) (undo-tree-id104 . -9) (undo-tree-id105 . -9) (undo-tree-id106 . -9) (undo-tree-id107 . -9) (undo-tree-id108 . -9) (undo-tree-id109 . -9) (undo-tree-id110 . -9) (undo-tree-id111 . -9) (undo-tree-id112 . -9) (undo-tree-id113 . -9) (undo-tree-id114 . -9) (undo-tree-id115 . -9) (undo-tree-id116 . -9) (undo-tree-id117 . -9) (undo-tree-id118 . -9) (undo-tree-id119 . -9) (undo-tree-id120 . -9) (undo-tree-id121 . -9) (undo-tree-id122 . -9) (undo-tree-id123 . -9) (undo-tree-id124 . -9) (undo-tree-id125 . -9) ("\"" . 172) (172 . 173) ("\"\"" . 172) (undo-tree-id126 . -1) (undo-tree-id127 . -1) (undo-tree-id128 . -1) (undo-tree-id129 . -1) (undo-tree-id130 . -1) (undo-tree-id131 . -1) (undo-tree-id132 . -1) (undo-tree-id133 . -1) (undo-tree-id134 . -1) (undo-tree-id135 . -1) (undo-tree-id136 . -1) (undo-tree-id137 . -1) (undo-tree-id138 . -1) (undo-tree-id139 . -1) (undo-tree-id140 . -1) (undo-tree-id141 . -1) (undo-tree-id142 . -1) (undo-tree-id143 . -1) (undo-tree-id144 . -1) (undo-tree-id145 . -1) (undo-tree-id146 . -1) (undo-tree-id147 . -1) (undo-tree-id148 . -1) (undo-tree-id149 . -1) (undo-tree-id150 . -1) (undo-tree-id151 . -1) (undo-tree-id152 . -1) (undo-tree-id153 . -1) (undo-tree-id154 . -1) (undo-tree-id155 . -1) (undo-tree-id156 . -1) (undo-tree-id157 . -1) (undo-tree-id158 . -1) (undo-tree-id159 . -1) (undo-tree-id160 . -1) (undo-tree-id161 . -1) (undo-tree-id162 . -1) (undo-tree-id163 . -1) (undo-tree-id164 . -1) (undo-tree-id165 . -1) (undo-tree-id166 . -1) (undo-tree-id167 . -1) (undo-tree-id168 . -1) (undo-tree-id169 . -1) (undo-tree-id170 . -1) (undo-tree-id171 . -1) (undo-tree-id172 . -1) (undo-tree-id173 . -1) (undo-tree-id174 . -1) (undo-tree-id175 . -1) (undo-tree-id176 . -1) (undo-tree-id177 . -1) (undo-tree-id178 . -1) (undo-tree-id179 . -1) (undo-tree-id180 . -1) (undo-tree-id181 . -1) (undo-tree-id182 . -1) (undo-tree-id183 . -1) (undo-tree-id184 . -1) (undo-tree-id185 . -1) (undo-tree-id186 . -1) (undo-tree-id187 . -1) (undo-tree-id188 . -1) (undo-tree-id189 . -1) (undo-tree-id190 . -1) (undo-tree-id191 . -1) (undo-tree-id192 . -1) (undo-tree-id193 . -1) (undo-tree-id194 . -1) (undo-tree-id195 . -1) (undo-tree-id196 . -1) (undo-tree-id197 . -1) (undo-tree-id198 . 1) (undo-tree-id199 . -1) (undo-tree-id200 . -1) (undo-tree-id201 . -1) (undo-tree-id202 . -1) (undo-tree-id203 . -1) (undo-tree-id204 . -1) (undo-tree-id205 . -1) (undo-tree-id206 . -1) (undo-tree-id207 . -1) (undo-tree-id208 . -1) (undo-tree-id209 . -1) (undo-tree-id210 . -1) (undo-tree-id211 . -1) (undo-tree-id212 . -1) (undo-tree-id213 . -1) (undo-tree-id214 . -1) (undo-tree-id215 . -1) (undo-tree-id216 . -1) (undo-tree-id217 . -1) (undo-tree-id218 . -1) (undo-tree-id219 . -1) (undo-tree-id220 . -1) (undo-tree-id221 . -1) (undo-tree-id222 . -1) (undo-tree-id223 . -1) (undo-tree-id224 . -1) (undo-tree-id225 . -1) (undo-tree-id226 . -1) (undo-tree-id227 . -1) (undo-tree-id228 . -1) (undo-tree-id229 . -1) (undo-tree-id230 . -1) ("go" . 173) (undo-tree-id231 . -1) (undo-tree-id232 . -1) (undo-tree-id233 . -1) (undo-tree-id234 . -1) (undo-tree-id235 . -1) (undo-tree-id236 . -1) (undo-tree-id237 . -1) (undo-tree-id238 . -1) (undo-tree-id239 . -1) (undo-tree-id240 . -1) (undo-tree-id241 . -1) (undo-tree-id242 . -1) (undo-tree-id243 . -1) (undo-tree-id244 . -1) (undo-tree-id245 . -1) (undo-tree-id246 . -1) (undo-tree-id247 . -1) (undo-tree-id248 . -1) (undo-tree-id249 . -1) (undo-tree-id250 . -1) (undo-tree-id251 . -1) (undo-tree-id252 . -1) (undo-tree-id253 . -1) (undo-tree-id254 . -1) (undo-tree-id255 . -1) (undo-tree-id256 . -2) (undo-tree-id257 . -2) (undo-tree-id258 . -2) (undo-tree-id259 . -2) (undo-tree-id260 . -2) (undo-tree-id261 . -2) (undo-tree-id262 . -2) (undo-tree-id263 . -2) (undo-tree-id264 . -2) (undo-tree-id265 . -2) (undo-tree-id266 . -2) (undo-tree-id267 . -2) (undo-tree-id268 . -2) (undo-tree-id269 . -2) (undo-tree-id270 . -2) (undo-tree-id271 . -2) (undo-tree-id272 . -2) (undo-tree-id273 . -2) (undo-tree-id274 . -2) (undo-tree-id275 . -2) (undo-tree-id276 . -2) (undo-tree-id277 . -2) (undo-tree-id278 . -2) (undo-tree-id279 . -2) (undo-tree-id280 . -2) (undo-tree-id281 . -2) (undo-tree-id282 . -2) (undo-tree-id283 . -1) (undo-tree-id284 . -1) (undo-tree-id285 . -1) (undo-tree-id286 . -1) (undo-tree-id287 . -1) (undo-tree-id288 . -1) (undo-tree-id289 . -1) (undo-tree-id290 . -1) (undo-tree-id291 . -1) (undo-tree-id292 . -1) (undo-tree-id293 . -1) (undo-tree-id294 . -1) (undo-tree-id295 . -1) (undo-tree-id296 . -1) (undo-tree-id297 . -1) (undo-tree-id298 . -1) (undo-tree-id299 . -1) (undo-tree-id300 . -1) (undo-tree-id301 . -1) (undo-tree-id302 . -1) (undo-tree-id303 . -1) (undo-tree-id304 . -1) (undo-tree-id305 . -1) (undo-tree-id306 . -1) (undo-tree-id307 . -1) (undo-tree-id308 . -1) (undo-tree-id309 . -1) (undo-tree-id310 . -1) (undo-tree-id311 . -1) (undo-tree-id312 . -1) (undo-tree-id313 . -1) (undo-tree-id314 . -1)) nil (26257 9689 231541 913000) 0 nil]) -nil From de013d9a027ea91f1a2706071d7aa958eed03f77 Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Tue, 16 Jul 2024 00:08:56 -0400 Subject: [PATCH 16/42] reworking things to use a json structure as input for plugins --- internal/devconfig/config.go | 5 +++- internal/plugin/git.go | 53 +++++++++++++++++------------------ internal/plugin/includable.go | 9 ++---- internal/plugin/includes.go | 6 ++-- internal/plugin/local.go | 2 ++ nix/flake/flakeref.go | 27 +++--------------- 6 files changed, 41 insertions(+), 61 deletions(-) diff --git a/internal/devconfig/config.go b/internal/devconfig/config.go index 1fc62948364..3c2ee98b7d5 100644 --- a/internal/devconfig/config.go +++ b/internal/devconfig/config.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "log/slog" "maps" "net/http" "os" @@ -125,8 +126,10 @@ func (c *Config) loadRecursive( // TODO UPDATEME for path, includeRef := range c.Root.Include { + includeRef.Path = path + pluginConfig, err := plugin.LoadConfigFromInclude( - path, includeRef, lockfile, filepath.Dir(c.Root.AbsRootPath)) + includeRef, lockfile, filepath.Dir(c.Root.AbsRootPath)) if err != nil { return errors.WithStack(err) } diff --git a/internal/plugin/git.go b/internal/plugin/git.go index a7fadda28d7..def83b1e7cd 100644 --- a/internal/plugin/git.go +++ b/internal/plugin/git.go @@ -42,16 +42,20 @@ func newGitPlugin(ref flake.Ref) (*gitPlugin, error) { // For backward compatibility, we don't strictly require name to be present // in github plugins. If it's missing, we just use the directory as the name. name, err := getPluginNameFromContent(plugin) + if err != nil && !errors.Is(err, errNameMissing) { return nil, err } + if name == "" { name = strings.ReplaceAll(ref.Dir, "/", "-") } + plugin.name = githubNameRegexp.ReplaceAllString( strings.Join(lo.Compact([]string{ref.Owner, ref.Repo, name}), "."), " ", ) + return plugin, nil } @@ -167,27 +171,20 @@ func (p *gitPlugin) url(subpath string) (string, error) { switch p.ref.Type { case flake.TypeSSH: return p.sshGitUrl() - case flake.TypeGitLab: - return p.gitlabUrl(subpath) - case flake.TypeGitHub: - return p.githubUrl(subpath) - case flake.TypeBitBucket: - return p.bitbucketUrl(subpath) + case flake.TypeHttps: + u, err := p.repoUrl(subpath) + slog.Debug(u) + return u, err + //return p.repoUrl(subpath) default: - return "", nil + return "", errors.New("Unsupported plugin type: " + p.ref.Type) } } func (p *gitPlugin) sshGitUrl() (string, error) { - address, err := url.Parse(p.ref.URL) - - if err != nil { - return "", err - } - defaultBranch := "main" - if address.Host == flake.TypeGitHub { + if p.ref.Host == flake.TypeGitHub { // using master for GitHub repos for the same reasoning established in `githubUrl` defaultBranch = "master" } @@ -195,8 +192,6 @@ func (p *gitPlugin) sshGitUrl() (string, error) { fileFormat := "tar.gz" baseCommand := fmt.Sprintf("git archive --format=%s --remote=ssh://git@", fileFormat) - path, _ := url.JoinPath(p.ref.Owner, p.ref.Subgroup, p.ref.Repo) - archive := filepath.Join("/", "tmp", p.ref.Dir+"."+fileFormat) branch := cmp.Or(p.ref.Rev, p.ref.Ref, defaultBranch) @@ -207,7 +202,7 @@ func (p *gitPlugin) sshGitUrl() (string, error) { } // TODO: try to use the Devbox file hashing mechanism to make sure it's stored properly - command := fmt.Sprintf("%s%s/%s %s %s -o %s", baseCommand, host, path, branch, p.ref.Dir, archive) + command := fmt.Sprintf("%s%s/%s %s %s -o %s", baseCommand, host, p.ref.Path, branch, p.ref.Dir, archive) slog.Debug("Generated git archive command: " + command) @@ -252,8 +247,7 @@ func (p *gitPlugin) githubUrl(subpath string) (string, error) { // so setting master here is better. return url.JoinPath( "https://raw.githubusercontent.com/", - p.ref.Owner, - p.ref.Repo, + p.ref.Path, cmp.Or(p.ref.Rev, p.ref.Ref, "master"), p.ref.Dir, subpath, @@ -265,8 +259,7 @@ func (p *gitPlugin) bitbucketUrl(subpath string) (string, error) { // as the default in this case return url.JoinPath( "https://api.bitbucket.org/2.0/repositories", - p.ref.Owner, - p.ref.Repo, + p.ref.Path, "src", cmp.Or(p.ref.Rev, p.ref.Ref, "main"), p.ref.Dir, @@ -274,13 +267,19 @@ func (p *gitPlugin) bitbucketUrl(subpath string) (string, error) { ) } -func (p *gitPlugin) gitlabUrl(subpath string) (string, error) { - project, err := url.JoinPath(p.ref.Owner, p.ref.Subgroup, p.ref.Repo) - - if err != nil { - return "", err +func (p *gitPlugin) repoUrl(subpath string) (string, error) { + if p.ref.Host == "github.com" { + return p.githubUrl(subpath) + } else if p.ref.Host == "gitlab.com" { + return p.gitlabUrl(subpath) + } else if p.ref.Host == "bitbucket.com" { + return p.bitbucketUrl(subpath) } + return "", errors.New("Unknown hostname provided in plugin: " + p.ref.Host) +} + +func (p *gitPlugin) gitlabUrl(subpath string) (string, error) { file, err := url.JoinPath(p.ref.Dir, subpath) if err != nil { @@ -289,7 +288,7 @@ func (p *gitPlugin) gitlabUrl(subpath string) (string, error) { path, err := url.JoinPath( "https://gitlab.com/api/v4/projects", - url.PathEscape(project), + p.ref.Path, "repository", "files", url.PathEscape(file), diff --git a/internal/plugin/includable.go b/internal/plugin/includable.go index a66e95dff8b..60c61ca71d4 100644 --- a/internal/plugin/includable.go +++ b/internal/plugin/includable.go @@ -17,8 +17,7 @@ type Includable interface { } // TODO UPDATEME -func parseIncludable(path string, ref flake.Ref, workingDir string) (Includable, error) { - +func parseIncludable(ref flake.Ref, workingDir string) (Includable, error) { //ref, err := flake.ParseRef(path) //if err != nil { @@ -30,11 +29,7 @@ func parseIncludable(path string, ref flake.Ref, workingDir string) (Includable, return newLocalPlugin(ref, workingDir) case flake.TypeSSH: fallthrough - case flake.TypeBitBucket: - fallthrough - case flake.TypeGitLab: - fallthrough - case flake.TypeGitHub: + case flake.TypeHttps: return newGitPlugin(ref) default: return nil, fmt.Errorf("unsupported ref type %q", ref.Type) diff --git a/internal/plugin/includes.go b/internal/plugin/includes.go index 2c7cf3197e4..00e3a9bb677 100644 --- a/internal/plugin/includes.go +++ b/internal/plugin/includes.go @@ -6,14 +6,14 @@ import ( "go.jetpack.io/devbox/nix/flake" ) -func LoadConfigFromInclude(path string, ref flake.Ref, lockfile *lock.File, workingDir string) (*Config, error) { +func LoadConfigFromInclude(ref flake.Ref, lockfile *lock.File, workingDir string) (*Config, error) { var includable Includable var err error if ref.Type == "builtin" { - includable = devpkg.PackageFromStringWithDefaults(path, lockfile) + includable = devpkg.PackageFromStringWithDefaults(ref.Path, lockfile) } else { - includable, err = parseIncludable(path, ref, workingDir) + includable, err = parseIncludable(ref, workingDir) if err != nil { return nil, err } diff --git a/internal/plugin/local.go b/internal/plugin/local.go index 25b6e881bcb..3ab9bd24e76 100644 --- a/internal/plugin/local.go +++ b/internal/plugin/local.go @@ -20,9 +20,11 @@ type LocalPlugin struct { func newLocalPlugin(ref flake.Ref, pluginDir string) (*LocalPlugin, error) { plugin := &LocalPlugin{ref: ref, pluginDir: pluginDir} name, err := getPluginNameFromContent(plugin) + if err != nil { return nil, err } + plugin.name = name return plugin, nil } diff --git a/nix/flake/flakeref.go b/nix/flake/flakeref.go index 84baa601c64..736c4291719 100644 --- a/nix/flake/flakeref.go +++ b/nix/flake/flakeref.go @@ -16,9 +16,10 @@ const ( TypePath = "path" TypeFile = "file" TypeSSH = "ssh" - TypeGitHub = "github" - TypeGitLab = "gitlab" - TypeBitBucket = "bitbucket" + TypeGitHub = "github" // TODO UPDATEME delete + TypeGitLab = "gitlab" // TODO UPDATEME delete + TypeBitBucket = "bitbucket" // TODO UPDATEME delete + TypeHttps = "https" // TODO UPDATEME, this should take place of Github, GitLab, and Bitbucket types TypeTarball = "tarball" ) @@ -238,26 +239,6 @@ func parseURLRef(ref string) (parsed Ref, fragment string, err error) { func parseGitRef(refURL *url.URL, parsed *Ref) error { // github:/(/)?(\?)? - // NOTE: this currently doesn't handle subgroups with GitLab, and will - // continue to cause problems restructuring plugins to use JSON objects - // will make this much easier in the long run. GitHub and Bitbucket don't support - // subgroups, so this won't be an issue with those repos. - // something akin to the example below can help eliminate a vast majority - // of this URL parsing logic, and make things more flexible - - /* - "include": [ - "username/subgroup/repo": { - "type": "ssh", - "host": "gitlab", - "port": 9999, - "dir": "my-plugins", - "ref": "myref", - "branch": "mybranch" - } - ] - */ - // Only split up to 3 times (owner, repo, ref/rev) so that we handle // refs that have slashes in them. For example, // "github:jetify-com/devbox/gcurtis/flakeref" parses as "gcurtis/flakeref". From e1c50ac8f1508fe79781e7c2fcadc3214b67f89d Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Thu, 18 Jul 2024 22:35:27 -0400 Subject: [PATCH 17/42] placeholder --- .schema/devbox-plugin.schema.json | 170 +++++++++++++++----------- .schema/devbox.schema.json | 170 +++++++++++++++----------- internal/devconfig/config.go | 15 ++- internal/devconfig/configfile/file.go | 2 +- internal/plugin/git.go | 2 +- internal/plugin/git_test.go | 83 ++++++++++--- 6 files changed, 275 insertions(+), 167 deletions(-) diff --git a/.schema/devbox-plugin.schema.json b/.schema/devbox-plugin.schema.json index 2b6598dd279..cfb32d0c8ff 100644 --- a/.schema/devbox-plugin.schema.json +++ b/.schema/devbox-plugin.schema.json @@ -141,83 +141,111 @@ }, "include": { "description": "List of additional plugins to activate within your devbox shell", - "type": "object", - "patternProperties": { - "^[a-zA-Z0-9/_.-]+$": { - "description": "Configuration for each plugin specified by its path", - "type": "object", - "properties": { - "protocol": { - "description": "Protocol to use (https, ssh, or file)", - "type": "string", - "enum": [ - "https", - "ssh", - "file", - "builtin" - ] - }, - "host": { - "description": "Host of the repository (e.g., gitlab, github, bitbucket, localhost)", - "type": "string" - }, - "port": { - "description": "Port to use for the connection", - "type": "integer", - "minimum": 1, - "maximum": 65535 - }, - "dir": { - "description": "Subdirectory that should be accessed in the repo. Defaults to the path key if not provided", - "type": "string" - }, - "ref": { - "description": "Git reference", - "type": "string" - }, - "branch": { - "description": "Git branch (only relevant for https or ssh protocols)", - "type": "string" + "type": "array", + "items": { + "type": "object", + "properties": { + "owner": { + "description": "Username or organization name", + "type": "string" + }, + "repo": { + "description": "Repository name", + "type": "string" + }, + "type": { + "description": "Type of connection (https, ssh, file, builtin)", + "type": "string", + "enum": [ + "https", + "ssh", + "file", + "builtin" + ], + "default": "https" + }, + "host": { + "description": "Host of the repository (e.g., gitlab, github, bitbucket, localhost)", + "type": "string" + }, + "port": { + "description": "Port to use for the connection", + "type": "integer", + "minimum": 1, + "maximum": 65535 + }, + "dir": { + "description": "Subdirectory that should be accessed in the repo. Defaults to the path key if not provided", + "type": "string" + }, + "ref": { + "description": "Git reference", + "type": "string" + }, + "rev": { + "description": "Git revision (only relevant for https or ssh types)", + "type": "string" + } + }, + "required": [ + "owner", + "repo" + ], + "dependencies": { + "host": { + "not": { + "properties": { + "type": { + "enum": [ + "builtin" + ] + } + } } }, - "required": [ - "protocol", - "host" - ], - "dependencies": { - "branch": { - "oneOf": [ - { - "properties": { - "protocol": { - "enum": [ - "https", - "ssh" - ] - } + "owner": { + "not": { + "properties": { + "type": { + "enum": [ + "builtin" + ] + } + } + } + }, + "rev": { + "oneOf": [ + { + "properties": { + "type": { + "enum": [ + "https", + "ssh" + ] } - }, - { - "properties": { - "protocol": { - "enum": [ - "file" - ] - } - }, - "not": { - "required": [ - "branch" + } + }, + { + "properties": { + "type": { + "enum": [ + "file", + "builtin" ] } + }, + "not": { + "required": [ + "rev" + ] } - ] - } - }, - "additionalProperties": false - } - }, - "additionalProperties": false + } + ] + } + }, + "additionalProperties": false + } } }, "required": [ diff --git a/.schema/devbox.schema.json b/.schema/devbox.schema.json index 0398d911630..c332924e3ee 100644 --- a/.schema/devbox.schema.json +++ b/.schema/devbox.schema.json @@ -131,83 +131,111 @@ }, "include": { "description": "List of additional plugins to activate within your devbox shell", - "type": "object", - "patternProperties": { - "^[a-zA-Z0-9/_.-]+$": { - "description": "Configuration for each plugin specified by its path", - "type": "object", - "properties": { - "protocol": { - "description": "Protocol to use (https, ssh, or file)", - "type": "string", - "enum": [ - "https", - "ssh", - "file", - "builtin" - ] - }, - "host": { - "description": "Host of the repository (e.g., gitlab, github, bitbucket, localhost)", - "type": "string" - }, - "port": { - "description": "Port to use for the connection", - "type": "integer", - "minimum": 1, - "maximum": 65535 - }, - "dir": { - "description": "Subdirectory that should be accessed in the repo. Defaults to the path key if not provided", - "type": "string" - }, - "ref": { - "description": "Git reference", - "type": "string" - }, - "branch": { - "description": "Git branch (only relevant for https or ssh protocols)", - "type": "string" + "type": "array", + "items": { + "type": "object", + "properties": { + "owner": { + "description": "Username or organization name", + "type": "string" + }, + "repo": { + "description": "Repository name", + "type": "string" + }, + "type": { + "description": "Type of connection (https, ssh, file, builtin)", + "type": "string", + "enum": [ + "https", + "ssh", + "file", + "builtin" + ], + "default": "https" + }, + "host": { + "description": "Host of the repository (e.g., gitlab, github, bitbucket, localhost)", + "type": "string" + }, + "port": { + "description": "Port to use for the connection", + "type": "integer", + "minimum": 1, + "maximum": 65535 + }, + "dir": { + "description": "Subdirectory that should be accessed in the repo. Defaults to the path key if not provided", + "type": "string" + }, + "ref": { + "description": "Git reference", + "type": "string" + }, + "rev": { + "description": "Git revision (only relevant for https or ssh types)", + "type": "string" + } + }, + "required": [ + "owner", + "repo" + ], + "dependencies": { + "host": { + "not": { + "properties": { + "type": { + "enum": [ + "builtin" + ] + } + } } }, - "required": [ - "protocol", - "host" - ], - "dependencies": { - "branch": { - "oneOf": [ - { - "properties": { - "protocol": { - "enum": [ - "https", - "ssh" - ] - } + "owner": { + "not": { + "properties": { + "type": { + "enum": [ + "builtin" + ] + } + } + } + }, + "rev": { + "oneOf": [ + { + "properties": { + "type": { + "enum": [ + "https", + "ssh" + ] } - }, - { - "properties": { - "protocol": { - "enum": [ - "file" - ] - } - }, - "not": { - "required": [ - "branch" + } + }, + { + "properties": { + "type": { + "enum": [ + "file", + "builtin" ] } + }, + "not": { + "required": [ + "rev" + ] } - ] - } - }, - "additionalProperties": false - } - }, - "additionalProperties": false + } + ] + } + }, + "additionalProperties": false + } }, "env_from": { "type": "string" diff --git a/internal/devconfig/config.go b/internal/devconfig/config.go index 3c2ee98b7d5..64feecf666d 100644 --- a/internal/devconfig/config.go +++ b/internal/devconfig/config.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "io" - "log/slog" "maps" "net/http" "os" @@ -125,16 +124,22 @@ func (c *Config) loadRecursive( included := make([]*Config, 0, len(c.Root.Include)) // TODO UPDATEME - for path, includeRef := range c.Root.Include { - includeRef.Path = path + for _, includeRef := range c.Root.Include { + if includeRef.Type == "" { + includeRef.Type = "https" // default + } pluginConfig, err := plugin.LoadConfigFromInclude( - includeRef, lockfile, filepath.Dir(c.Root.AbsRootPath)) + includeRef, + lockfile, + filepath.Dir(c.Root.AbsRootPath), + ) + if err != nil { return errors.WithStack(err) } - newCyclePath := fmt.Sprintf("%s -> %s", cyclePath, path) + newCyclePath := fmt.Sprintf("%s -> %s", cyclePath, includeRef) if seen[pluginConfig.Source.Hash()] { // Note that duplicate includes are allowed if they are in different paths // e.g. 2 different plugins can include the same plugin. diff --git a/internal/devconfig/configfile/file.go b/internal/devconfig/configfile/file.go index ce749a46362..5fb8ef10c1c 100644 --- a/internal/devconfig/configfile/file.go +++ b/internal/devconfig/configfile/file.go @@ -54,7 +54,7 @@ type ConfigFile struct { // https:// for remote files // plugin: for built-in plugins // This is a similar format to nix inputs - Include map[string]flake.Ref `json:"include,omitempty"` + Include []flake.Ref `json:"include,omitempty"` ast *configAST } diff --git a/internal/plugin/git.go b/internal/plugin/git.go index def83b1e7cd..0a035555d02 100644 --- a/internal/plugin/git.go +++ b/internal/plugin/git.go @@ -202,7 +202,7 @@ func (p *gitPlugin) sshGitUrl() (string, error) { } // TODO: try to use the Devbox file hashing mechanism to make sure it's stored properly - command := fmt.Sprintf("%s%s/%s %s %s -o %s", baseCommand, host, p.ref.Path, branch, p.ref.Dir, archive) + command := fmt.Sprintf("%s%s/%s/%s %s %s -o %s", baseCommand, host, p.ref.Owner, p.ref.Repo, branch, p.ref.Dir, archive) slog.Debug("Generated git archive command: " + command) diff --git a/internal/plugin/git_test.go b/internal/plugin/git_test.go index 5e87a6e6aa1..50a2bc98c17 100644 --- a/internal/plugin/git_test.go +++ b/internal/plugin/git_test.go @@ -12,13 +12,19 @@ import ( func TestNewGitPlugin(t *testing.T) { testCases := []struct { name string - Include string + Include []flake.Ref expected gitPlugin expectedURL string }{ { - name: "parse basic github plugin", - Include: "github:jetify-com/devbox-plugins", + name: "parse basic github plugin", + Include: []flake.Ref{ + { + Host: "github.com", + Owner: "jetify-com", + Repo: "devbox-plugins", + }, + }, expected: gitPlugin{ ref: flake.Ref{ Type: "github", @@ -30,8 +36,15 @@ func TestNewGitPlugin(t *testing.T) { expectedURL: "https://raw.githubusercontent.com/jetify-com/devbox-plugins/master", }, { - name: "parse github plugin with dir param", - Include: "github:jetify-com/devbox-plugins?dir=mongodb", + name: "parse github plugin with dir param", + Include: []flake.Ref{ + { + Host: "github.com", + Owner: "jetify-com", + Repo: "devbox-plugins", + Dir: "monogodb", + }, + }, expected: gitPlugin{ ref: flake.Ref{ Type: "github", @@ -44,11 +57,19 @@ func TestNewGitPlugin(t *testing.T) { expectedURL: "https://raw.githubusercontent.com/jetify-com/devbox-plugins/master/mongodb", }, { - name: "parse github plugin with dir param and rev", - Include: "github:jetify-com/devbox-plugins/my-branch?dir=mongodb", + name: "parse github plugin with dir param and rev", + Include: []flake.Ref{ + { + Host: "github.com", + Owner: "jetify-com", + Repo: "devbox-plugins", + Dir: "monogodb", + Ref: "my-branch", + }, + }, expected: gitPlugin{ ref: flake.Ref{ - Type: "github", + Type: "https", Owner: "jetify-com", Repo: "devbox-plugins", Ref: "my-branch", @@ -59,11 +80,21 @@ func TestNewGitPlugin(t *testing.T) { expectedURL: "https://raw.githubusercontent.com/jetify-com/devbox-plugins/my-branch/mongodb", }, { - name: "parse github plugin with dir param and rev", - Include: "github:jetify-com/devbox-plugins/initials/my-branch?dir=mongodb", + name: "parse github plugin with dir param and rev", + Include: []flake.Ref{ + { + Host: "github.com", + Owner: "jetify-com", + Repo: "devbox-plugins", + Dir: "monogodb", + Ref: "my-branch", + Rev: "initials", + }, + }, expected: gitPlugin{ ref: flake.Ref{ - Type: "github", + Type: "https", + Host: "github.com", Owner: "jetify-com", Repo: "devbox-plugins", Ref: "initials/my-branch", @@ -73,11 +104,32 @@ func TestNewGitPlugin(t *testing.T) { }, expectedURL: "https://raw.githubusercontent.com/jetify-com/devbox-plugins/initials/my-branch/mongodb", }, + { + name: "parse gitlab plugin", + Include: []flake.Ref{ + { + Host: "gitlab.com", + Owner: "username", + Repo: "my-repo", + }, + }, + + expected: gitPlugin{ + ref: flake.Ref{ + Type: "https", + Owner: "username", + Repo: "my-repo", + Host: "gitlab.com", + }, + name: "username.my-repo", + }, + expectedURL: "https://gitlab.com/api/v4/projects/username/my-repo/files/plugin.json/raw", + }, } for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { - actual, err := newGitPluginForTest(testCase.Include) + actual, err := newGitPluginForTest(testCase.Include[0]) assert.NoError(t, err) assert.Equal(t, &testCase.expected, actual) u, err := testCase.expected.url("") @@ -88,12 +140,7 @@ func TestNewGitPlugin(t *testing.T) { } // keep in sync with newGithubPlugin -func newGitPluginForTest(include string) (*gitPlugin, error) { - ref, err := flake.ParseRef(include) - if err != nil { - return nil, err - } - +func newGitPluginForTest(ref flake.Ref) (*gitPlugin, error) { plugin := &gitPlugin{ref: ref} name := strings.ReplaceAll(ref.Dir, "/", "-") plugin.name = githubNameRegexp.ReplaceAllString( From ea649568173da65c6900476a321e26075934be59 Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Tue, 23 Jul 2024 23:10:38 -0400 Subject: [PATCH 18/42] revisiting the json schema stuff --- internal/plugin/git.go | 119 +++++++++++++++------------------- internal/plugin/includable.go | 12 ++-- nix/flake/flakeref.go | 63 ++++++++++++------ 3 files changed, 100 insertions(+), 94 deletions(-) diff --git a/internal/plugin/git.go b/internal/plugin/git.go index 0a035555d02..a9b4fd6074f 100644 --- a/internal/plugin/git.go +++ b/internal/plugin/git.go @@ -76,36 +76,57 @@ func (p *gitPlugin) Hash() string { } func (p *gitPlugin) FileContent(subpath string) ([]byte, error) { - contentURL, err := p.url(subpath) + pluginLocation, err := p.url(subpath) if err != nil { return nil, err } - readFile := func() ([]byte, time.Duration, error) { - archive := filepath.Join("/", "tmp", p.ref.Dir+".tar.gz") - args := strings.Fields(contentURL) - cmd := exec.Command(args[0], args[1:]...) // Maybe make async? + retrieveArchive := func() ([]byte, time.Duration, error) { + archiveDir, _ := os.MkdirTemp("", p.ref.Owner) + archive := filepath.Join(archiveDir, p.ref.Owner+".tar.gz") + args := strings.Fields(pluginLocation + archive) // this is really just the base git archive command + file + // TODO get this working properly + //defer func() { + // slog.Debug("Removing archive " + archive) + // os.RemoveAll(archive) + // slog.Debug("Removing archive directory " + archiveDir) + // os.RemoveAll(archiveDir) + //}() + + cmd := exec.Command(args[0], args[1:]...) _, err := cmd.Output() if err != nil { slog.Error("Error executing git archive: ", err) - return nil, 24 * time.Hour, err + return nil, 0, err } reader, err := os.Open(archive) - io.ReadAll(reader) - err = fileutil.Untar(reader, "/tmp") // TODO: add UUID? + err = fileutil.Untar(reader, archiveDir) + + if err != nil { + slog.Error("Encountered error while trying to extract "+archive+": ", err) + return nil, 0, err + } + + pluginJson := filepath.Join(archiveDir, p.ref.Dir, "plugin.json") + file, err := os.Open(pluginJson) + defer file.Close() - file, err := os.Open(contentURL) info, err := file.Stat() - if err != nil || info.Size() == 0 { + if err != nil { + slog.Error("Error extracting file " + file.Name() + ". Cannot process plugin.") + return nil, 0, err + } + + if info.Size() == 0 { + slog.Error("Extracted file " + file.Name() + " is empty. Cannot process plugin.") return nil, 0, err } - defer file.Close() body, err := io.ReadAll(file) if err != nil { @@ -115,8 +136,8 @@ func (p *gitPlugin) FileContent(subpath string) ([]byte, error) { return body, 24 * time.Hour, nil } - retrieve := func() ([]byte, time.Duration, error) { - req, err := p.request(contentURL) + retrieveHttp := func() ([]byte, time.Duration, error) { + req, err := p.request(pluginLocation) if err != nil { return nil, 0, err @@ -155,13 +176,13 @@ func (p *gitPlugin) FileContent(subpath string) ([]byte, error) { switch p.ref.Type { case flake.TypeSSH: - return sshCache.GetOrSet(contentURL, readFile) + return sshCache.GetOrSet(pluginLocation, retrieveArchive) case flake.TypeGitHub: - return githubCache.GetOrSet(contentURL, retrieve) + return githubCache.GetOrSet(pluginLocation, retrieveHttp) case flake.TypeGitLab: - return gitlabCache.GetOrSet(contentURL, retrieve) + return gitlabCache.GetOrSet(pluginLocation, retrieveHttp) case flake.TypeBitBucket: - return bitbucketCache.GetOrSet(contentURL, retrieve) + return bitbucketCache.GetOrSet(pluginLocation, retrieveHttp) default: return nil, err } @@ -170,76 +191,40 @@ func (p *gitPlugin) FileContent(subpath string) ([]byte, error) { func (p *gitPlugin) url(subpath string) (string, error) { switch p.ref.Type { case flake.TypeSSH: - return p.sshGitUrl() - case flake.TypeHttps: - u, err := p.repoUrl(subpath) - slog.Debug(u) - return u, err - //return p.repoUrl(subpath) + return p.sshBaseGitCommand() + case flake.TypeBitBucket: + fallthrough + case flake.TypeGitHub: + fallthrough + case flake.TypeGitLab: + return p.repoUrl(subpath) default: return "", errors.New("Unsupported plugin type: " + p.ref.Type) } } -func (p *gitPlugin) sshGitUrl() (string, error) { +func (p *gitPlugin) sshBaseGitCommand() (string, error) { defaultBranch := "main" - if p.ref.Host == flake.TypeGitHub { + if p.ref.Host == flake.TypeGitHub+".com" { // using master for GitHub repos for the same reasoning established in `githubUrl` defaultBranch = "master" } - fileFormat := "tar.gz" - baseCommand := fmt.Sprintf("git archive --format=%s --remote=ssh://git@", fileFormat) + p.ref.Ref = defaultBranch - archive := filepath.Join("/", "tmp", p.ref.Dir+"."+fileFormat) + prefix := "git archive --format=tar.gz --remote=ssh://git@" + path, _ := url.JoinPath(p.ref.Owner, p.ref.Repo) branch := cmp.Or(p.ref.Rev, p.ref.Ref, defaultBranch) host := p.ref.Host - if p.ref.Port != "" { host += ":" + p.ref.Port } - // TODO: try to use the Devbox file hashing mechanism to make sure it's stored properly - command := fmt.Sprintf("%s%s/%s/%s %s %s -o %s", baseCommand, host, p.ref.Owner, p.ref.Repo, branch, p.ref.Dir, archive) - - slog.Debug("Generated git archive command: " + command) - + command := fmt.Sprintf("%s%s/%s %s %s -o", prefix, host, path, branch, p.ref.Dir) + slog.Debug("Generated base git archive command: " + command) return command, nil - - //sshCache.GetOrSet(command, func() ([]byte, time.Duration, error) { - - //}) - - // 24 hours is currently when files are considered "expired" in other FileContent function - //currentTime := time.Now() - //threshold := 24 * time.Hour - //expiration := currentTime.Add(-threshold) - - //args := strings.Fields(command) - //archiveInfo, err := os.Stat(archive) - - //if os.IsNotExist(err) || archiveInfo.ModTime().Before(expiration) { - // cmd := exec.Command(args[0], args[1:]...) // Maybe make async? - - // _, err := cmd.Output() - - // if err != nil { - // slog.Error("Error executing git archive: ", err) - // return "", err - // } - - // reader, err := os.Open(archive) - // io.ReadAll(reader) - // err = fileutil.Untar(reader, "/tmp") // TODO: add UUID? - - // if err == nil { - // return "", err - // } - //} - - //return filepath.Join("/", "tmp", p.ref.Dir, "plugin.json"), nil } func (p *gitPlugin) githubUrl(subpath string) (string, error) { diff --git a/internal/plugin/includable.go b/internal/plugin/includable.go index 60c61ca71d4..e54a135a274 100644 --- a/internal/plugin/includable.go +++ b/internal/plugin/includable.go @@ -18,18 +18,16 @@ type Includable interface { // TODO UPDATEME func parseIncludable(ref flake.Ref, workingDir string) (Includable, error) { - //ref, err := flake.ParseRef(path) - - //if err != nil { - // return nil, err - //} - switch ref.Type { case flake.TypePath: return newLocalPlugin(ref, workingDir) case flake.TypeSSH: fallthrough - case flake.TypeHttps: + case flake.TypeBitBucket: + fallthrough + case flake.TypeGitHub: + fallthrough + case flake.TypeGitLab: return newGitPlugin(ref) default: return nil, fmt.Errorf("unsupported ref type %q", ref.Type) diff --git a/nix/flake/flakeref.go b/nix/flake/flakeref.go index 736c4291719..d7da40cf0ef 100644 --- a/nix/flake/flakeref.go +++ b/nix/flake/flakeref.go @@ -2,6 +2,8 @@ package flake import ( + //"log/slog" + "log/slog" "net/url" "path" "slices" @@ -16,10 +18,10 @@ const ( TypePath = "path" TypeFile = "file" TypeSSH = "ssh" - TypeGitHub = "github" // TODO UPDATEME delete - TypeGitLab = "gitlab" // TODO UPDATEME delete - TypeBitBucket = "bitbucket" // TODO UPDATEME delete - TypeHttps = "https" // TODO UPDATEME, this should take place of Github, GitLab, and Bitbucket types + TypeGitHub = "github" + TypeGitLab = "gitlab" + TypeBitBucket = "bitbucket" + TypeHttps = "https" TypeTarball = "tarball" ) @@ -73,11 +75,6 @@ type Ref struct { // Port of the server git server, to support privately hosted git servers or tunnels Port string `json:port,omitempty` - - // TODO UPDATEME - // Subgroup pertains to GitLab. GitHub and Bitbucket don't support multi-level - // hierarchy, and this allows the subgroup to exist without breaking the parsing logic already in place - Subgroup string `json:subgroup,omitempty` } // ParseRef parses a raw flake reference. Nix supports a variety of flake ref @@ -239,16 +236,20 @@ func parseURLRef(ref string) (parsed Ref, fragment string, err error) { func parseGitRef(refURL *url.URL, parsed *Ref) error { // github:/(/)?(\?)? - // Only split up to 3 times (owner, repo, ref/rev) so that we handle - // refs that have slashes in them. For example, - // "github:jetify-com/devbox/gcurtis/flakeref" parses as "gcurtis/flakeref". - split, err := splitPathOrOpaque(refURL, 3) + // gitlab supports up to 20 levels of nesting: https://docs.gitlab.com/ee/user/group/subgroups/ + // using an object structure allows the subgroup(s) to be easily identified by just getting the splits + + slog.Debug("REEEEEEEEEEEPO: ", parsed.Repo) + split, err := splitRepoString(parsed.Repo, 20) if err != nil { + slog.Debug("FAILED TO DO THE SPLITS") return err } - parsed.Owner = split[0] - parsed.Repo = split[1] + parsed.Owner = strings.Join(split[0:len(split)-2], "/") + parsed.Repo = split[len(split)-1] + + slog.Debug("THINGGGGGGS", parsed.Owner, parsed.Repo) if len(split) > 2 { if revOrRef := split[2]; isGitHash(revOrRef) { @@ -258,11 +259,6 @@ func parseGitRef(refURL *url.URL, parsed *Ref) error { } } - parsed.Host = refURL.Query().Get("host") - parsed.Dir = refURL.Query().Get("dir") - parsed.Subgroup = refURL.Query().Get("subgroup") - parsed.Port = refURL.Query().Get("port") - if qRef := refURL.Query().Get("ref"); qRef != "" { if parsed.Rev != "" { return redact.Errorf("%s flake reference has a ref and a rev", parsed.Type) @@ -423,6 +419,8 @@ func isArchive(path string) bool { // ensuring that path elements with an encoded '/' (%2F) are not split. // For example, "/dir/file%2Fname" becomes the elements "dir" and "file/name". // The count limits the number of substrings per [strings.SplitN] + +// TODO git rid of this func splitPathOrOpaque(u *url.URL, n int) ([]string, error) { upath := u.EscapedPath() if upath == "" { @@ -450,6 +448,31 @@ func splitPathOrOpaque(u *url.URL, n int) ([]string, error) { return split, nil } +// TODO maybe use this? +func splitRepoString(repo string, n int) ([]string, error) { + repo = strings.TrimSpace(repo) + + if repo == "" { + return nil, nil + } + + // We don't want an empty element if the path is rooted. + if repo[0] == '/' { + repo = repo[1:] + } + repo = path.Clean(repo) + + var err error + split := strings.SplitN(repo, "/", n) + for i := range split { + split[i], err = url.PathUnescape(split[i]) + if err != nil { + return nil, err + } + } + return split, nil +} + // buildEscapedPath escapes and joins path elements for a URL flake ref. The // resulting path is cleaned according to url.JoinPath. func buildEscapedPath(elem ...string) string { From 158880a7d2741c1f60a68485d3153e24ef9af57b Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Sun, 28 Jul 2024 12:25:53 -0400 Subject: [PATCH 19/42] basic gitlab repo working --- internal/plugin/git.go | 28 +++++++++++++++++----------- internal/plugin/includable.go | 5 ++++- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/internal/plugin/git.go b/internal/plugin/git.go index a9b4fd6074f..6d1cdbdbea8 100644 --- a/internal/plugin/git.go +++ b/internal/plugin/git.go @@ -87,13 +87,13 @@ func (p *gitPlugin) FileContent(subpath string) ([]byte, error) { archive := filepath.Join(archiveDir, p.ref.Owner+".tar.gz") args := strings.Fields(pluginLocation + archive) // this is really just the base git archive command + file - // TODO get this working properly - //defer func() { - // slog.Debug("Removing archive " + archive) - // os.RemoveAll(archive) - // slog.Debug("Removing archive directory " + archiveDir) - // os.RemoveAll(archiveDir) - //}() + defer func() { + slog.Debug("Cleaning up retrieved files related to privately hosted plugin") + slog.Debug("Removing archive " + archive) + os.RemoveAll(archive) + slog.Debug("Removing archive directory " + archiveDir) + os.RemoveAll(archiveDir) + }() cmd := exec.Command(args[0], args[1:]...) _, err := cmd.Output() @@ -253,11 +253,11 @@ func (p *gitPlugin) bitbucketUrl(subpath string) (string, error) { } func (p *gitPlugin) repoUrl(subpath string) (string, error) { - if p.ref.Host == "github.com" { + if p.ref.Type == flake.TypeGitHub { return p.githubUrl(subpath) - } else if p.ref.Host == "gitlab.com" { + } else if p.ref.Type == flake.TypeGitLab { return p.gitlabUrl(subpath) - } else if p.ref.Host == "bitbucket.com" { + } else if p.ref.Type == flake.TypeBitBucket { return p.bitbucketUrl(subpath) } @@ -271,9 +271,15 @@ func (p *gitPlugin) gitlabUrl(subpath string) (string, error) { return "", err } + repoPath, err := url.JoinPath(p.ref.Owner, p.ref.Repo) + + if err != nil { + return "", err + } + path, err := url.JoinPath( "https://gitlab.com/api/v4/projects", - p.ref.Path, + url.PathEscape(repoPath), "repository", "files", url.PathEscape(file), diff --git a/internal/plugin/includable.go b/internal/plugin/includable.go index e54a135a274..697252496b0 100644 --- a/internal/plugin/includable.go +++ b/internal/plugin/includable.go @@ -20,7 +20,7 @@ type Includable interface { func parseIncludable(ref flake.Ref, workingDir string) (Includable, error) { switch ref.Type { case flake.TypePath: - return newLocalPlugin(ref, workingDir) + return newLocalPlugin(ref, workingDir) // TODO case flake.TypeSSH: fallthrough case flake.TypeBitBucket: @@ -28,6 +28,9 @@ func parseIncludable(ref flake.Ref, workingDir string) (Includable, error) { case flake.TypeGitHub: fallthrough case flake.TypeGitLab: + if ref.Host == "" { + ref.Host = ref.Type + ".com" + } return newGitPlugin(ref) default: return nil, fmt.Errorf("unsupported ref type %q", ref.Type) From 23bd74147b67911db787d84fada5a594cc36865d Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Sun, 28 Jul 2024 12:36:13 -0400 Subject: [PATCH 20/42] bitbucket function at basic level --- internal/plugin/git.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/plugin/git.go b/internal/plugin/git.go index 6d1cdbdbea8..b873b7ca486 100644 --- a/internal/plugin/git.go +++ b/internal/plugin/git.go @@ -242,9 +242,11 @@ func (p *gitPlugin) githubUrl(subpath string) (string, error) { func (p *gitPlugin) bitbucketUrl(subpath string) (string, error) { // bitbucket doesn't redirect master -> main or main -> master, so using "main" // as the default in this case + return url.JoinPath( "https://api.bitbucket.org/2.0/repositories", - p.ref.Path, + p.ref.Owner, + p.ref.Repo, "src", cmp.Or(p.ref.Rev, p.ref.Ref, "main"), p.ref.Dir, From d07086fca8ed2cb320758c26d0c2af4c4e542c87 Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Sun, 28 Jul 2024 12:56:40 -0400 Subject: [PATCH 21/42] fixed github url structure --- internal/plugin/git.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/plugin/git.go b/internal/plugin/git.go index b873b7ca486..9855a791c39 100644 --- a/internal/plugin/git.go +++ b/internal/plugin/git.go @@ -232,7 +232,8 @@ func (p *gitPlugin) githubUrl(subpath string) (string, error) { // so setting master here is better. return url.JoinPath( "https://raw.githubusercontent.com/", - p.ref.Path, + p.ref.Owner, + p.ref.Repo, cmp.Or(p.ref.Rev, p.ref.Ref, "master"), p.ref.Dir, subpath, From 3372b096106651b69e785e18bdd0d32a0be8733b Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Sun, 28 Jul 2024 13:39:52 -0400 Subject: [PATCH 22/42] substitute slashes for periods when using repo name in plugin.name to handle subgroups from gitlab --- internal/plugin/git.go | 5 ++++- internal/plugin/git_test.go | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/plugin/git.go b/internal/plugin/git.go index 9855a791c39..ceda56f5550 100644 --- a/internal/plugin/git.go +++ b/internal/plugin/git.go @@ -51,8 +51,11 @@ func newGitPlugin(ref flake.Ref) (*gitPlugin, error) { name = strings.ReplaceAll(ref.Dir, "/", "-") } + // gitlab repos can have up to 20 subgroups. We need to capture the subgroups in the plugin name + repoDotted := strings.ReplaceAll(ref.Repo, "/", ".") + plugin.name = githubNameRegexp.ReplaceAllString( - strings.Join(lo.Compact([]string{ref.Owner, ref.Repo, name}), "."), + strings.Join(lo.Compact([]string{ref.Owner, repoDotted, name}), "."), " ", ) diff --git a/internal/plugin/git_test.go b/internal/plugin/git_test.go index 50a2bc98c17..f0edc9f87cb 100644 --- a/internal/plugin/git_test.go +++ b/internal/plugin/git_test.go @@ -143,8 +143,9 @@ func TestNewGitPlugin(t *testing.T) { func newGitPluginForTest(ref flake.Ref) (*gitPlugin, error) { plugin := &gitPlugin{ref: ref} name := strings.ReplaceAll(ref.Dir, "/", "-") + repoDotted := strings.ReplaceAll(ref.Repo, "/", ".") plugin.name = githubNameRegexp.ReplaceAllString( - strings.Join(lo.Compact([]string{ref.Owner, ref.Repo, name}), "."), + strings.Join(lo.Compact([]string{ref.Owner, repoDotted, name}), "."), " ", ) return plugin, nil From b47c544e3f6da75ed87a776491bcce74fb7b3944 Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Sun, 28 Jul 2024 18:39:54 -0400 Subject: [PATCH 23/42] some cleanup --- internal/devconfig/config.go | 5 +---- internal/plugin/includable.go | 3 +-- internal/plugin/includes.go | 2 +- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/internal/devconfig/config.go b/internal/devconfig/config.go index 4f0ec7f9159..aa082832098 100644 --- a/internal/devconfig/config.go +++ b/internal/devconfig/config.go @@ -238,10 +238,6 @@ func (c *Config) loadRecursive( // TODO UPDATEME for _, includeRef := range c.Root.Include { - if includeRef.Type == "" { - includeRef.Type = "https" // default - } - pluginConfig, err := plugin.LoadConfigFromInclude( includeRef, lockfile, @@ -276,6 +272,7 @@ func (c *Config) loadRecursive( c.Root.TopLevelPackages(), lockfile, ) + if err != nil { return errors.WithStack(err) } diff --git a/internal/plugin/includable.go b/internal/plugin/includable.go index 697252496b0..7a627d83469 100644 --- a/internal/plugin/includable.go +++ b/internal/plugin/includable.go @@ -16,11 +16,10 @@ type Includable interface { LockfileKey() string } -// TODO UPDATEME func parseIncludable(ref flake.Ref, workingDir string) (Includable, error) { switch ref.Type { case flake.TypePath: - return newLocalPlugin(ref, workingDir) // TODO + return newLocalPlugin(ref, workingDir) case flake.TypeSSH: fallthrough case flake.TypeBitBucket: diff --git a/internal/plugin/includes.go b/internal/plugin/includes.go index 00e3a9bb677..8143864ad22 100644 --- a/internal/plugin/includes.go +++ b/internal/plugin/includes.go @@ -10,7 +10,7 @@ func LoadConfigFromInclude(ref flake.Ref, lockfile *lock.File, workingDir string var includable Includable var err error - if ref.Type == "builtin" { + if ref.Type == flake.TypeBuiltin { includable = devpkg.PackageFromStringWithDefaults(ref.Path, lockfile) } else { includable, err = parseIncludable(ref, workingDir) From c9a7554a0989606dec4c04954d61f9f36ce8e537 Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Wed, 30 Oct 2024 18:44:08 -0400 Subject: [PATCH 24/42] include all plugin caches in `devbox update` --- internal/plugin/github.go | 157 -------------------------------------- internal/plugin/update.go | 15 +++- nix/flake/flakeref.go | 2 +- 3 files changed, 15 insertions(+), 159 deletions(-) delete mode 100644 internal/plugin/github.go diff --git a/internal/plugin/github.go b/internal/plugin/github.go deleted file mode 100644 index f47dd3b7a75..00000000000 --- a/internal/plugin/github.go +++ /dev/null @@ -1,157 +0,0 @@ -package plugin - -import ( - "cmp" - "fmt" - "io" - "net/http" - "net/url" - "os" - "regexp" - "strings" - "time" - - "github.com/pkg/errors" - "github.com/samber/lo" - "go.jetpack.io/devbox/internal/boxcli/usererr" - "go.jetpack.io/devbox/internal/cachehash" - "go.jetpack.io/devbox/nix/flake" - "go.jetpack.io/pkg/filecache" -) - -var githubCache = filecache.New[[]byte]("devbox/plugin/github") - -type githubPlugin struct { - ref flake.Ref - name string -} - -// Github only allows alphanumeric, hyphen, underscore, and period in repo names. -// but we clean up just in case. -var githubNameRegexp = regexp.MustCompile("[^a-zA-Z0-9-_.]+") - -func newGithubPlugin(ref flake.Ref) (*githubPlugin, error) { - plugin := &githubPlugin{ref: ref} - // For backward compatibility, we don't strictly require name to be present - // in github plugins. If it's missing, we just use the directory as the name. - name, err := getPluginNameFromContent(plugin) - if err != nil && !errors.Is(err, errNameMissing) { - return nil, err - } - if name == "" { - name = strings.ReplaceAll(ref.Dir, "/", "-") - } - plugin.name = githubNameRegexp.ReplaceAllString( - strings.Join(lo.Compact([]string{ref.Owner, ref.Repo, name}), "."), - " ", - ) - return plugin, nil -} - -func (p *githubPlugin) Fetch() ([]byte, error) { - content, err := p.FileContent(pluginConfigName) - if err != nil { - return nil, err - } - return jsonPurifyPluginContent(content) -} - -func (p *githubPlugin) CanonicalName() string { - return p.name -} - -func (p *githubPlugin) Hash() string { - return cachehash.Bytes([]byte(p.ref.String())) -} - -func (p *githubPlugin) FileContent(subpath string) ([]byte, error) { - contentURL, err := p.url(subpath) - if err != nil { - return nil, err - } - - // Cache for 24 hours. Once we store the plugin in the lockfile, we - // should cache this indefinitely and only invalidate if the plugin - // is updated. - ttl := 24 * time.Hour - - // This is a stopgap until plugin is stored in lockfile. - // DEVBOX_X indicates this is an experimental env var. - // Use DEVBOX_X_GITHUB_PLUGIN_CACHE_TTL to override the default TTL. - // e.g. DEVBOX_X_GITHUB_PLUGIN_CACHE_TTL=1h will cache the plugin for 1 hour. - // Note: If you want to disable cache, we recommend using a low second value instead of zero to - // ensure only one network request is made. - ttlStr := os.Getenv("DEVBOX_X_GITHUB_PLUGIN_CACHE_TTL") - if ttlStr != "" { - ttl, err = time.ParseDuration(ttlStr) - if err != nil { - return nil, err - } - } - - return githubCache.GetOrSet( - contentURL+ttlStr, - func() ([]byte, time.Duration, error) { - req, err := p.request(contentURL) - if err != nil { - return nil, 0, err - } - - client := &http.Client{} - res, err := client.Do(req) - if err != nil { - return nil, 0, err - } - defer res.Body.Close() - if res.StatusCode != http.StatusOK { - return nil, 0, usererr.New( - "failed to get plugin %s @ %s (Status code %d). \nPlease make "+ - "sure a plugin.json file exists in plugin directory.", - p.LockfileKey(), - req.URL.String(), - res.StatusCode, - ) - } - body, err := io.ReadAll(res.Body) - if err != nil { - return nil, 0, err - } - - return body, ttl, nil - }, - ) -} - -func (p *githubPlugin) url(subpath string) (string, error) { - // Github redirects "master" to "main" in new repos. They don't do the reverse - // so setting master here is better. - return url.JoinPath( - "https://raw.githubusercontent.com/", - p.ref.Owner, - p.ref.Repo, - cmp.Or(p.ref.Rev, p.ref.Ref, "master"), - p.ref.Dir, - subpath, - ) -} - -func (p *githubPlugin) request(contentURL string) (*http.Request, error) { - req, err := http.NewRequest(http.MethodGet, contentURL, nil) - if err != nil { - return nil, err - } - - // Add github token to request if available - ghToken := os.Getenv("GITHUB_TOKEN") - - if ghToken != "" { - authValue := fmt.Sprintf("token %s", ghToken) - req.Header.Add("Authorization", authValue) - } - - return req, nil -} - -func (p *githubPlugin) LockfileKey() string { - return p.ref.String() -} diff --git a/internal/plugin/update.go b/internal/plugin/update.go index 7d7e521ee2e..3ba3f376c60 100644 --- a/internal/plugin/update.go +++ b/internal/plugin/update.go @@ -1,5 +1,18 @@ package plugin +import "go.jetpack.io/pkg/filecache" + func Update() error { - return githubCache.Clear() + pluginCaches := []*filecache.Cache[[]byte]{githubCache, sshCache, gitlabCache, bitbucketCache} + + for _, cache := range pluginCaches { + err := cache.Clear() + + if err != nil { + return err + } + + } + + return nil } diff --git a/nix/flake/flakeref.go b/nix/flake/flakeref.go index b4f7d986c79..3854785b0d7 100644 --- a/nix/flake/flakeref.go +++ b/nix/flake/flakeref.go @@ -20,7 +20,7 @@ const ( TypeGitLab = "gitlab" TypeBitBucket = "bitbucket" TypeTarball = "tarball" - TypeBuiltin = "plugin" + TypeBuiltin = "builtin" ) // Ref is a parsed Nix flake reference. A flake reference is a subset of the From 665eca9fd87e19d24ca0eb02095aff98c344cc0a Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Wed, 30 Oct 2024 19:06:50 -0400 Subject: [PATCH 25/42] corrected arg provided to slog error func --- internal/plugin/git.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/plugin/git.go b/internal/plugin/git.go index ceda56f5550..e44cf9b9195 100644 --- a/internal/plugin/git.go +++ b/internal/plugin/git.go @@ -102,7 +102,7 @@ func (p *gitPlugin) FileContent(subpath string) ([]byte, error) { _, err := cmd.Output() if err != nil { - slog.Error("Error executing git archive: ", err) + slog.Error("Error executing git archive: " + err.Error()) return nil, 0, err } @@ -110,7 +110,7 @@ func (p *gitPlugin) FileContent(subpath string) ([]byte, error) { err = fileutil.Untar(reader, archiveDir) if err != nil { - slog.Error("Encountered error while trying to extract "+archive+": ", err) + slog.Error("Encountered error while trying to extract " + archive + ": " + err.Error()) return nil, 0, err } From d16d1af5d7c69ce6c6a7d7a92a80db410e7be789 Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Fri, 1 Nov 2024 10:33:46 -0400 Subject: [PATCH 26/42] update port to be int in ref struct (silly mistake) --- .schema/devbox-plugin.schema.json | 3 ++- internal/plugin/git.go | 6 +++--- nix/flake/flakeref.go | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.schema/devbox-plugin.schema.json b/.schema/devbox-plugin.schema.json index 3c9b73aa644..c80446d91d5 100644 --- a/.schema/devbox-plugin.schema.json +++ b/.schema/devbox-plugin.schema.json @@ -172,7 +172,8 @@ "description": "Port to use for the connection", "type": "integer", "minimum": 1, - "maximum": 65535 + "maximum": 65535, + "default": -1 }, "dir": { "description": "Subdirectory that should be accessed in the repo. Defaults to the path key if not provided", diff --git a/internal/plugin/git.go b/internal/plugin/git.go index e44cf9b9195..a619f441c55 100644 --- a/internal/plugin/git.go +++ b/internal/plugin/git.go @@ -86,7 +86,7 @@ func (p *gitPlugin) FileContent(subpath string) ([]byte, error) { } retrieveArchive := func() ([]byte, time.Duration, error) { - archiveDir, _ := os.MkdirTemp("", p.ref.Owner) + archiveDir, _ := os.MkdirTemp("", p.ref.Repo) archive := filepath.Join(archiveDir, p.ref.Owner+".tar.gz") args := strings.Fields(pluginLocation + archive) // this is really just the base git archive command + file @@ -221,8 +221,8 @@ func (p *gitPlugin) sshBaseGitCommand() (string, error) { branch := cmp.Or(p.ref.Rev, p.ref.Ref, defaultBranch) host := p.ref.Host - if p.ref.Port != "" { - host += ":" + p.ref.Port + if p.ref.Port != -1 { + host += ":" + fmt.Sprintf("%d", p.ref.Port) } command := fmt.Sprintf("%s%s/%s %s %s -o", prefix, host, path, branch, p.ref.Dir) diff --git a/nix/flake/flakeref.go b/nix/flake/flakeref.go index 3854785b0d7..290b2100d00 100644 --- a/nix/flake/flakeref.go +++ b/nix/flake/flakeref.go @@ -72,7 +72,7 @@ type Ref struct { URL string `json:"url,omitempty"` // Port of the server git server, to support privately hosted git servers or tunnels - Port string `json:port,omitempty` + Port int32 `json:port,omitempty` } // ParseRef parses a raw flake reference. Nix supports a variety of flake ref From c93d661b0b5e06a0c78d2b095e2fe8556b7816cb Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Fri, 1 Nov 2024 14:25:18 -0400 Subject: [PATCH 27/42] private plugins with hashing check done --- .schema/devbox-plugin.schema.json | 2 +- internal/plugin/git.go | 4 ++- internal/plugin/includable.go | 11 +------ nix/flake/flakeref.go | 50 ++++++++++++++++++------------- 4 files changed, 35 insertions(+), 32 deletions(-) diff --git a/.schema/devbox-plugin.schema.json b/.schema/devbox-plugin.schema.json index c80446d91d5..d78be0b6ca9 100644 --- a/.schema/devbox-plugin.schema.json +++ b/.schema/devbox-plugin.schema.json @@ -173,7 +173,7 @@ "type": "integer", "minimum": 1, "maximum": 65535, - "default": -1 + "default": 0 }, "dir": { "description": "Subdirectory that should be accessed in the repo. Defaults to the path key if not provided", diff --git a/internal/plugin/git.go b/internal/plugin/git.go index a619f441c55..c6959c277ef 100644 --- a/internal/plugin/git.go +++ b/internal/plugin/git.go @@ -221,7 +221,9 @@ func (p *gitPlugin) sshBaseGitCommand() (string, error) { branch := cmp.Or(p.ref.Rev, p.ref.Ref, defaultBranch) host := p.ref.Host - if p.ref.Port != -1 { + + // the Ref struct defaults the field to 0. This technically a valid port for UDP, but we aren't using UDP + if p.ref.Port > 0 { host += ":" + fmt.Sprintf("%d", p.ref.Port) } diff --git a/internal/plugin/includable.go b/internal/plugin/includable.go index c164d028048..d32f974ce69 100644 --- a/internal/plugin/includable.go +++ b/internal/plugin/includable.go @@ -20,16 +20,7 @@ func parseIncludable(ref flake.Ref, workingDir string) (Includable, error) { switch ref.Type { case flake.TypePath: return newLocalPlugin(ref, workingDir) - case flake.TypeSSH: - fallthrough - case flake.TypeBitBucket: - fallthrough - case flake.TypeBuiltin: - // TODO FIXME - fallthrough - case flake.TypeGitHub: - fallthrough - case flake.TypeGitLab: + case flake.TypeSSH, flake.TypeBuiltin, flake.TypeGitHub, flake.TypeGitLab, flake.TypeBitBucket: if ref.Host == "" { ref.Host = ref.Type + ".com" } diff --git a/nix/flake/flakeref.go b/nix/flake/flakeref.go index 290b2100d00..774309fecf3 100644 --- a/nix/flake/flakeref.go +++ b/nix/flake/flakeref.go @@ -2,6 +2,7 @@ package flake import ( + "fmt" "net/url" "path" "slices" @@ -295,44 +296,53 @@ func parseGitRef(refURL *url.URL, parsed *Ref) error { // string. func (r Ref) String() string { switch r.Type { + case TypeFile: if r.URL == "" { return "" } return "file+" + r.URL case TypeSSH: - if r.URL == "" { - return "" - } - if !strings.HasPrefix(r.URL, "git") { - r.URL = "git+" + r.URL + base := fmt.Sprintf("git+ssh://git@%s", r.Host) + if r.Port > 0 { + base = fmt.Sprintf("%s:%d", base, r.Port) } - // Nix removes "ref" and "rev" from the query string - // (but not other parameters) after parsing. If they're empty, - // we can skip parsing the URL. Otherwise, we need to add them - // back. - if r.Ref == "" && r.Rev == "" { - return r.URL + queryParams := url.Values{} + + if r.Rev != "" { + queryParams.Add("rev", r.Rev) } - url, err := url.Parse(r.URL) - if err != nil { - // This should be rare and only happen if the caller - // messed with the parsed URL. - return "" + if r.Ref != "" { + queryParams.Add("ref", r.Ref) } - url.RawQuery = buildQueryString("ref", r.Ref, "rev", r.Rev, "dir", r.Dir) - return url.String() - case TypeGitHub: + if r.Dir != "" { + queryParams.Add("dir", r.Dir) + } + + return fmt.Sprintf("%s/%s/%s?%s", base, r.Owner, r.Repo, queryParams.Encode()) + + case TypeGitLab, TypeBitBucket, TypeGitHub: if r.Owner == "" || r.Repo == "" { return "" } + + scheme := "github" // using as default + if r.Type == TypeGitLab { + scheme = "gitlab" + } + if r.Type == TypeBitBucket { + scheme = "bitbucket" + } + url := &url.URL{ - Scheme: "github", + Scheme: scheme, Opaque: buildEscapedPath(r.Owner, r.Repo, r.Rev, r.Ref), RawQuery: buildQueryString("host", r.Host, "dir", r.Dir), } + return url.String() + case TypeIndirect: if r.ID == "" { return "" From a721ad43ae80d337c4ff823e317f8ecd25636a8d Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Fri, 1 Nov 2024 17:04:25 -0400 Subject: [PATCH 28/42] update example devbox.json files for unit tests --- examples/plugins/builtin/devbox.json | 5 ++++- .../plugins/github-with-revision/devbox.json | 7 ++++++- examples/plugins/github/devbox.json | 20 ++++++++++++++++--- examples/plugins/local/devbox.json | 5 ++++- examples/plugins/v2-github/devbox.json | 13 ++++++++++-- examples/plugins/v2-local/devbox.json | 19 ++++++++++++++---- examples/plugins/v2-local/plugin1/plugin.json | 9 +++++++-- .../v2-local/plugin1/plugin1a/plugin.json | 9 +++++++-- 8 files changed, 71 insertions(+), 16 deletions(-) diff --git a/examples/plugins/builtin/devbox.json b/examples/plugins/builtin/devbox.json index 5718936fe66..232881c81c1 100644 --- a/examples/plugins/builtin/devbox.json +++ b/examples/plugins/builtin/devbox.json @@ -12,6 +12,9 @@ }, "include": [ // Installs the php plugin using php82 as trigger package - "plugin:php82" + { + "type": "builtin", + "path": "php82" + } ] } diff --git a/examples/plugins/github-with-revision/devbox.json b/examples/plugins/github-with-revision/devbox.json index 201fcaa9e33..59dc5cc793d 100644 --- a/examples/plugins/github-with-revision/devbox.json +++ b/examples/plugins/github-with-revision/devbox.json @@ -11,6 +11,11 @@ } }, "include": [ - "github:jetify-com/devbox-plugin-example/d9c00334353c9b1294c7bd5dbea128c149b2eb3a" + { + "type": "github", + "owner": "jetify-com", + "repo": "devbox-plugin-example", + "rev": "d9c00334353c9b1294c7bd5dbea128c149b2eb3a" + } ] } diff --git a/examples/plugins/github/devbox.json b/examples/plugins/github/devbox.json index 4711986c7b9..e896f907f3b 100644 --- a/examples/plugins/github/devbox.json +++ b/examples/plugins/github/devbox.json @@ -11,8 +11,22 @@ } }, "include": [ - "github:jetify-com/devbox-plugin-example", - "github:jetify-com/devbox-plugin-example?dir=custom-dir", - "github:jetify-com/devbox-plugin-example/test/branch", + { + "type": "github", + "owner": "jetify-com", + "repo": "devbox-plugin-example" + }, + { + "type": "github", + "owner": "jetify-com", + "repo": "devbox-plugin-example", + "dir": "custom-dir" + }, + { + "type": "github", + "owner": "jetify-com", + "repo": "devbox-plugin-example/test", + "ref": "branch" + } ] } diff --git a/examples/plugins/local/devbox.json b/examples/plugins/local/devbox.json index b537e30e930..d17db910c29 100644 --- a/examples/plugins/local/devbox.json +++ b/examples/plugins/local/devbox.json @@ -11,6 +11,9 @@ } }, "include": [ - "path:my-plugin/plugin.json" + { + "type": "path", + "path": "my-plugin/plugin.json" + } ] } diff --git a/examples/plugins/v2-github/devbox.json b/examples/plugins/v2-github/devbox.json index dbcea18f16a..e4466cd5fae 100644 --- a/examples/plugins/v2-github/devbox.json +++ b/examples/plugins/v2-github/devbox.json @@ -12,7 +12,16 @@ }, "include": [ // TODO, make these more interesting by adding v2 capabilities - "github:jetify-com/devbox-plugin-example", - "github:jetify-com/devbox-plugin-example?dir=custom-dir" + { + "type": "github", + "owner": "jetify-com", + "repo": "devbox-plugin-example" + }, + { + "type": "github", + "owner": "jetify-com", + "repo": "devbox-plugin-example", + "dir": "custom-dir" + } ] } diff --git a/examples/plugins/v2-local/devbox.json b/examples/plugins/v2-local/devbox.json index 9ebd1ae46c1..83d373024df 100644 --- a/examples/plugins/v2-local/devbox.json +++ b/examples/plugins/v2-local/devbox.json @@ -1,5 +1,4 @@ { - "$schema": "https://raw.githubusercontent.com/jetify-com/devbox/main/.schema/devbox.schema.json", "packages": [], "env": { "PLUGIN1_ENV2": "override" @@ -28,8 +27,20 @@ } }, "include": [ - "./plugin1", - "path:plugin2", - "./plugin3/plugin.json", + { + "type": "path", + "path": "./plugin1" + }, + { + "type": "path", + "path": "plugin2" + }, + { + "type": "path", + "path": "./plugin3/plugin.json" + } + //"./plugin1", + //"path:plugin2", + //"./plugin3/plugin.json", ] } diff --git a/examples/plugins/v2-local/plugin1/plugin.json b/examples/plugins/v2-local/plugin1/plugin.json index 54dda78bffc..1ba583e39b7 100644 --- a/examples/plugins/v2-local/plugin1/plugin.json +++ b/examples/plugins/v2-local/plugin1/plugin.json @@ -1,6 +1,8 @@ { "name": "plugin1", - "packages": ["hello@latest"], + "packages": [ + "hello@latest" + ], "env": { "PLUGIN1_ENV": "success", "PLUGIN1_ENV2": "success" @@ -20,6 +22,9 @@ "{{ .Virtenv }}/process-compose.yaml": "process-compose.yaml" }, "include": [ - "./plugin1a" + { + "type": "path", + "path": "./plugin1a" + } ] } diff --git a/examples/plugins/v2-local/plugin1/plugin1a/plugin.json b/examples/plugins/v2-local/plugin1/plugin1a/plugin.json index bbe1fa2bda0..7b951201a17 100644 --- a/examples/plugins/v2-local/plugin1/plugin1a/plugin.json +++ b/examples/plugins/v2-local/plugin1/plugin1a/plugin.json @@ -1,6 +1,8 @@ { "name": "plugin1a", - "packages": ["cowsay@latest"], + "packages": [ + "cowsay@latest" + ], "env": { "PLUGIN1A_ENV": "success" }, @@ -15,6 +17,9 @@ } }, "include": [ - "../../plugin2" + { + "type": "path", + "path": "../../plugin2" + } ] } From 352be83b3142c219177c2b15d0ffdf81c1d1532d Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Fri, 1 Nov 2024 18:07:27 -0400 Subject: [PATCH 29/42] some cleanup --- internal/plugin/git.go | 195 ++++++++++++++++++++-------------- internal/plugin/includable.go | 2 +- nix/flake/flakeref.go | 4 + 3 files changed, 122 insertions(+), 79 deletions(-) diff --git a/internal/plugin/git.go b/internal/plugin/git.go index c6959c277ef..cdb09d459ff 100644 --- a/internal/plugin/git.go +++ b/internal/plugin/git.go @@ -24,6 +24,7 @@ import ( ) var sshCache = filecache.New[[]byte]("devbox/plugin/ssh") +var gitCache = filecache.New[[]byte]("devbox/plugin/git") var githubCache = filecache.New[[]byte]("devbox/plugin/github") var gitlabCache = filecache.New[[]byte]("devbox/plugin/gitlab") var bitbucketCache = filecache.New[[]byte]("devbox/plugin/bitbucket") @@ -78,115 +79,116 @@ func (p *gitPlugin) Hash() string { return cachehash.Bytes([]byte(p.ref.String())) } -func (p *gitPlugin) FileContent(subpath string) ([]byte, error) { - pluginLocation, err := p.url(subpath) +func (p *gitPlugin) fetchSSHArchive(location string) ([]byte, error) { + archiveDir, _ := os.MkdirTemp("", p.ref.Repo) + archive := filepath.Join(archiveDir, p.ref.Owner+".tar.gz") + args := strings.Fields(location + archive) // this is really just the base git archive command + file + + defer os.RemoveAll(archiveDir) + + cmd := exec.Command(args[0], args[1:]...) + _, err := cmd.Output() if err != nil { + slog.Error("Error executing git archive: " + err.Error()) return nil, err } - retrieveArchive := func() ([]byte, time.Duration, error) { - archiveDir, _ := os.MkdirTemp("", p.ref.Repo) - archive := filepath.Join(archiveDir, p.ref.Owner+".tar.gz") - args := strings.Fields(pluginLocation + archive) // this is really just the base git archive command + file - - defer func() { - slog.Debug("Cleaning up retrieved files related to privately hosted plugin") - slog.Debug("Removing archive " + archive) - os.RemoveAll(archive) - slog.Debug("Removing archive directory " + archiveDir) - os.RemoveAll(archiveDir) - }() + reader, err := os.Open(archive) + err = fileutil.Untar(reader, archiveDir) - cmd := exec.Command(args[0], args[1:]...) - _, err := cmd.Output() - - if err != nil { - slog.Error("Error executing git archive: " + err.Error()) - return nil, 0, err - } + if err != nil { + slog.Error("Encountered error while trying to extract " + archive + ": " + err.Error()) + return nil, err + } - reader, err := os.Open(archive) - err = fileutil.Untar(reader, archiveDir) + pluginJson := filepath.Join(archiveDir, p.ref.Dir, "plugin.json") + file, err := os.Open(pluginJson) - if err != nil { - slog.Error("Encountered error while trying to extract " + archive + ": " + err.Error()) - return nil, 0, err - } + defer file.Close() + info, err := file.Stat() - pluginJson := filepath.Join(archiveDir, p.ref.Dir, "plugin.json") - file, err := os.Open(pluginJson) - defer file.Close() + if err != nil { + slog.Error("Error extracting file " + file.Name() + ". Cannot process plugin.") + return nil, err + } - info, err := file.Stat() + if info.Size() == 0 { + slog.Error("Extracted file " + file.Name() + " is empty. Cannot process plugin.") + return nil, err + } - if err != nil { - slog.Error("Error extracting file " + file.Name() + ". Cannot process plugin.") - return nil, 0, err - } + return io.ReadAll(file) +} - if info.Size() == 0 { - slog.Error("Extracted file " + file.Name() + " is empty. Cannot process plugin.") - return nil, 0, err - } +func (p *gitPlugin) fetchHttp(location string) ([]byte, error) { + req, err := p.request(location) - body, err := io.ReadAll(file) + if err != nil { + return nil, err + } - if err != nil { - return nil, 0, err - } + client := &http.Client{} + res, err := client.Do(req) - return body, 24 * time.Hour, nil + if err != nil { + return nil, err } - retrieveHttp := func() ([]byte, time.Duration, error) { - req, err := p.request(pluginLocation) + defer res.Body.Close() - if err != nil { - return nil, 0, err - } + if res.StatusCode != http.StatusOK { + return nil, usererr.New( + "failed to get plugin %s @ %s (Status code %d). \nPlease make "+ + "sure a plugin.json file exists in plugin directory.", + p.LockfileKey(), + req.URL.String(), + res.StatusCode, + ) + } - client := &http.Client{} - res, err := client.Do(req) + return io.ReadAll(res.Body) +} - if err != nil { - return nil, 0, err - } +func (p *gitPlugin) FileContent(subpath string) ([]byte, error) { + location, err := p.url(subpath) - defer res.Body.Close() + if err != nil { + return nil, err + } - if res.StatusCode != http.StatusOK { - return nil, 0, usererr.New( - "failed to get plugin %s @ %s (Status code %d). \nPlease make "+ - "sure a plugin.json file exists in plugin directory.", - p.LockfileKey(), - req.URL.String(), - res.StatusCode, - ) - } + var bytes []byte - body, err := io.ReadAll(res.Body) + if p.ref.Type == flake.TypeSSH { + bytes, err = p.fetchSSHArchive(location) + } else { + bytes, err = p.fetchHttp(location) + } - if err != nil { - return nil, 0, err - } + if err != nil { + return nil, err + } + process := func() ([]byte, time.Duration, error) { // Cache for 24 hours. Once we store the plugin in the lockfile, we // should cache this indefinitely and only invalidate if the plugin // is updated. - return body, 24 * time.Hour, nil + return bytes, 24 * time.Hour, nil } switch p.ref.Type { case flake.TypeSSH: - return sshCache.GetOrSet(pluginLocation, retrieveArchive) + return sshCache.GetOrSet(location, process) case flake.TypeGitHub: - return githubCache.GetOrSet(pluginLocation, retrieveHttp) + return githubCache.GetOrSet(location, process) case flake.TypeGitLab: - return gitlabCache.GetOrSet(pluginLocation, retrieveHttp) + return gitlabCache.GetOrSet(location, process) case flake.TypeBitBucket: - return bitbucketCache.GetOrSet(pluginLocation, retrieveHttp) + return bitbucketCache.GetOrSet(location, process) + case flake.TypeGit: + return gitCache.GetOrSet(location, process) default: + slog.Error("Unable to handle flake ref type: " + p.ref.Type) return nil, err } } @@ -195,11 +197,7 @@ func (p *gitPlugin) url(subpath string) (string, error) { switch p.ref.Type { case flake.TypeSSH: return p.sshBaseGitCommand() - case flake.TypeBitBucket: - fallthrough - case flake.TypeGitHub: - fallthrough - case flake.TypeGitLab: + case flake.TypeGit, flake.TypeGitHub, flake.TypeGitLab, flake.TypeBitBucket: return p.repoUrl(subpath) default: return "", errors.New("Unsupported plugin type: " + p.ref.Type) @@ -260,6 +258,45 @@ func (p *gitPlugin) bitbucketUrl(subpath string) (string, error) { ) } +func (p *gitPlugin) genericGitUrl(subpath string) (string, error) { + address, err := url.JoinPath( + p.ref.Host, + p.ref.Repo, + cmp.Or(p.ref.Rev, p.ref.Ref, "main"), + p.ref.Dir, + subpath, + ) + + if err != nil { + return "", err + } + + parsed, err := url.Parse(address) + + if err != nil { + return "", err + } + + // gitlab doesn't redirect master -> main or main -> master, so using "main" + // as the default in this case + query := parsed.Query() + query.Add("ref", cmp.Or(p.ref.Rev, p.ref.Ref, "main")) + + if p.ref.Dir != "" { + query.Add("dir", p.ref.Dir) + } + + if p.ref.Port != 0 { + query.Add("port", fmt.Sprintf("%d", p.ref.Port)) + } + + parsed.RawQuery = query.Encode() + query.Add("ref", cmp.Or(p.ref.Rev, p.ref.Ref, "main")) + parsed.RawQuery = query.Encode() + + return parsed.String(), nil +} + func (p *gitPlugin) repoUrl(subpath string) (string, error) { if p.ref.Type == flake.TypeGitHub { return p.githubUrl(subpath) @@ -267,6 +304,8 @@ func (p *gitPlugin) repoUrl(subpath string) (string, error) { return p.gitlabUrl(subpath) } else if p.ref.Type == flake.TypeBitBucket { return p.bitbucketUrl(subpath) + } else if p.ref.Type == flake.TypeGit { + return p.genericGitUrl(subpath) } return "", errors.New("Unknown hostname provided in plugin: " + p.ref.Host) diff --git a/internal/plugin/includable.go b/internal/plugin/includable.go index d32f974ce69..c9ac88731a0 100644 --- a/internal/plugin/includable.go +++ b/internal/plugin/includable.go @@ -20,7 +20,7 @@ func parseIncludable(ref flake.Ref, workingDir string) (Includable, error) { switch ref.Type { case flake.TypePath: return newLocalPlugin(ref, workingDir) - case flake.TypeSSH, flake.TypeBuiltin, flake.TypeGitHub, flake.TypeGitLab, flake.TypeBitBucket: + case flake.TypeSSH, flake.TypeBuiltin, flake.TypeGitHub, flake.TypeGitLab, flake.TypeBitBucket, flake.TypeGit: if ref.Host == "" { ref.Host = ref.Type + ".com" } diff --git a/nix/flake/flakeref.go b/nix/flake/flakeref.go index 774309fecf3..5576764176a 100644 --- a/nix/flake/flakeref.go +++ b/nix/flake/flakeref.go @@ -19,6 +19,7 @@ const ( TypeSSH = "ssh" TypeGitHub = "github" TypeGitLab = "gitlab" + TypeGit = "git" TypeBitBucket = "bitbucket" TypeTarball = "tarball" TypeBuiltin = "builtin" @@ -203,6 +204,8 @@ func parseURLRef(ref string) (parsed Ref, fragment string, err error) { parsed.Type = TypeSSH } else if strings.HasPrefix(refURL.Scheme, TypeFile) { parsed.Type = TypeFile + } else { + parsed.Type = TypeGit } parsed.URL = refURL.String() @@ -256,6 +259,7 @@ func parseGitRef(refURL *url.URL, parsed *Ref) error { parsed.Host = refURL.Query().Get("host") parsed.Dir = refURL.Query().Get("dir") + if qRef := refURL.Query().Get("ref"); qRef != "" { if parsed.Rev != "" { return redact.Errorf("github flake reference has a ref and a rev") From ab5053bf245ab1c8e90599a9d854909890e13e02 Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Sun, 10 Nov 2024 10:38:18 -0500 Subject: [PATCH 30/42] cleaning up some tests --- internal/devconfig/config.go | 20 ++++++++------ internal/plugin/git_test.go | 51 ++++++++++++++--------------------- internal/plugin/includable.go | 3 --- 3 files changed, 32 insertions(+), 42 deletions(-) diff --git a/internal/devconfig/config.go b/internal/devconfig/config.go index aa082832098..8bf7029d2be 100644 --- a/internal/devconfig/config.go +++ b/internal/devconfig/config.go @@ -20,6 +20,7 @@ import ( "go.jetpack.io/devbox/internal/devconfig/configfile" "go.jetpack.io/devbox/internal/lock" "go.jetpack.io/devbox/internal/plugin" + "go.jetpack.io/devbox/nix/flake" ) // ErrNotFound occurs when [Open] or [Find] cannot find a devbox config file @@ -237,18 +238,21 @@ func (c *Config) loadRecursive( included := make([]*Config, 0, len(c.Root.Include)) // TODO UPDATEME - for _, includeRef := range c.Root.Include { - pluginConfig, err := plugin.LoadConfigFromInclude( - includeRef, - lockfile, - filepath.Dir(c.Root.AbsRootPath), - ) + for _, ref := range c.Root.Include { + + switch ref.Type { + case flake.TypeGitHub, flake.TypeGitLab, flake.TypeBitBucket: + ref.Host = fmt.Sprintf("%s.com", ref.Type) + } + + pluginConfig, err := plugin.LoadConfigFromInclude(ref, lockfile, filepath.Dir(c.Root.AbsRootPath)) if err != nil { return errors.WithStack(err) } - newCyclePath := fmt.Sprintf("%s -> %s", cyclePath, includeRef) + newCyclePath := fmt.Sprintf("%s -> %s", cyclePath, ref) + if seen[pluginConfig.Source.Hash()] { // Note that duplicate includes are allowed if they are in different paths // e.g. 2 different plugins can include the same plugin. @@ -256,8 +260,8 @@ func (c *Config) loadRecursive( return errors.Errorf( "circular or duplicate include detected:\n%s", newCyclePath) } - seen[pluginConfig.Source.Hash()] = true + seen[pluginConfig.Source.Hash()] = true includable := createIncludableFromPluginConfig(pluginConfig) if err := includable.loadRecursive( diff --git a/internal/plugin/git_test.go b/internal/plugin/git_test.go index f0edc9f87cb..2a5fe3a95cf 100644 --- a/internal/plugin/git_test.go +++ b/internal/plugin/git_test.go @@ -1,10 +1,8 @@ package plugin import ( - "strings" "testing" - "github.com/samber/lo" "github.com/stretchr/testify/assert" "go.jetpack.io/devbox/nix/flake" ) @@ -20,7 +18,7 @@ func TestNewGitPlugin(t *testing.T) { name: "parse basic github plugin", Include: []flake.Ref{ { - Host: "github.com", + Type: "github", Owner: "jetify-com", Repo: "devbox-plugins", }, @@ -28,6 +26,7 @@ func TestNewGitPlugin(t *testing.T) { expected: gitPlugin{ ref: flake.Ref{ Type: "github", + Host: "github.com", Owner: "jetify-com", Repo: "devbox-plugins", }, @@ -39,15 +38,16 @@ func TestNewGitPlugin(t *testing.T) { name: "parse github plugin with dir param", Include: []flake.Ref{ { - Host: "github.com", + Type: "github", Owner: "jetify-com", Repo: "devbox-plugins", - Dir: "monogodb", + Dir: "mongodb", }, }, expected: gitPlugin{ ref: flake.Ref{ Type: "github", + Host: "github.com", Owner: "jetify-com", Repo: "devbox-plugins", Dir: "mongodb", @@ -60,16 +60,17 @@ func TestNewGitPlugin(t *testing.T) { name: "parse github plugin with dir param and rev", Include: []flake.Ref{ { - Host: "github.com", + Type: "github", Owner: "jetify-com", Repo: "devbox-plugins", - Dir: "monogodb", Ref: "my-branch", + Dir: "mongodb", }, }, expected: gitPlugin{ ref: flake.Ref{ - Type: "https", + Type: "github", + Host: "github.com", Owner: "jetify-com", Repo: "devbox-plugins", Ref: "my-branch", @@ -83,22 +84,22 @@ func TestNewGitPlugin(t *testing.T) { name: "parse github plugin with dir param and rev", Include: []flake.Ref{ { - Host: "github.com", + Type: "github", Owner: "jetify-com", Repo: "devbox-plugins", - Dir: "monogodb", - Ref: "my-branch", + Dir: "mongodb", + Ref: "initials/my-branch", Rev: "initials", }, }, expected: gitPlugin{ ref: flake.Ref{ - Type: "https", - Host: "github.com", + Type: "github", Owner: "jetify-com", Repo: "devbox-plugins", - Ref: "initials/my-branch", Dir: "mongodb", + Ref: "initials/my-branch", // FIXME + Rev: "initials", }, name: "jetify-com.devbox-plugins.mongodb", }, @@ -108,7 +109,7 @@ func TestNewGitPlugin(t *testing.T) { name: "parse gitlab plugin", Include: []flake.Ref{ { - Host: "gitlab.com", + Type: "gitlab", Owner: "username", Repo: "my-repo", }, @@ -116,20 +117,20 @@ func TestNewGitPlugin(t *testing.T) { expected: gitPlugin{ ref: flake.Ref{ - Type: "https", + Type: "gitlab", + Host: "gitlab.com", Owner: "username", Repo: "my-repo", - Host: "gitlab.com", }, name: "username.my-repo", }, - expectedURL: "https://gitlab.com/api/v4/projects/username/my-repo/files/plugin.json/raw", + expectedURL: "https://gitlab.com/api/v4/projects/username%2Fmy-repo/repository/files/raw?ref=main", }, } for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { - actual, err := newGitPluginForTest(testCase.Include[0]) + actual, err := newGitPlugin(testCase.Include[0]) // FIXME: need to evaluate URL assert.NoError(t, err) assert.Equal(t, &testCase.expected, actual) u, err := testCase.expected.url("") @@ -139,18 +140,6 @@ func TestNewGitPlugin(t *testing.T) { } } -// keep in sync with newGithubPlugin -func newGitPluginForTest(ref flake.Ref) (*gitPlugin, error) { - plugin := &gitPlugin{ref: ref} - name := strings.ReplaceAll(ref.Dir, "/", "-") - repoDotted := strings.ReplaceAll(ref.Repo, "/", ".") - plugin.name = githubNameRegexp.ReplaceAllString( - strings.Join(lo.Compact([]string{ref.Owner, repoDotted, name}), "."), - " ", - ) - return plugin, nil -} - func TestGitPluginAuth(t *testing.T) { gitPlugin := gitPlugin{ ref: flake.Ref{ diff --git a/internal/plugin/includable.go b/internal/plugin/includable.go index c9ac88731a0..483281f0122 100644 --- a/internal/plugin/includable.go +++ b/internal/plugin/includable.go @@ -21,9 +21,6 @@ func parseIncludable(ref flake.Ref, workingDir string) (Includable, error) { case flake.TypePath: return newLocalPlugin(ref, workingDir) case flake.TypeSSH, flake.TypeBuiltin, flake.TypeGitHub, flake.TypeGitLab, flake.TypeBitBucket, flake.TypeGit: - if ref.Host == "" { - ref.Host = ref.Type + ".com" - } return newGitPlugin(ref) default: return nil, fmt.Errorf("unsupported ref type %q", ref.Type) From 01220b1c14b0392cac4c327b2e1dce06bcfbb810 Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Sun, 10 Nov 2024 18:22:35 -0500 Subject: [PATCH 31/42] basic tests for gitlab and bitbucket --- internal/devconfig/config.go | 1 - internal/plugin/git.go | 6 +- internal/plugin/git_test.go | 280 +++++++++++++++++++++++++++++++++-- 3 files changed, 273 insertions(+), 14 deletions(-) diff --git a/internal/devconfig/config.go b/internal/devconfig/config.go index 8bf7029d2be..45fa7a712a4 100644 --- a/internal/devconfig/config.go +++ b/internal/devconfig/config.go @@ -239,7 +239,6 @@ func (c *Config) loadRecursive( // TODO UPDATEME for _, ref := range c.Root.Include { - switch ref.Type { case flake.TypeGitHub, flake.TypeGitLab, flake.TypeBitBucket: ref.Host = fmt.Sprintf("%s.com", ref.Type) diff --git a/internal/plugin/git.go b/internal/plugin/git.go index cdb09d459ff..93a42f89408 100644 --- a/internal/plugin/git.go +++ b/internal/plugin/git.go @@ -277,10 +277,7 @@ func (p *gitPlugin) genericGitUrl(subpath string) (string, error) { return "", err } - // gitlab doesn't redirect master -> main or main -> master, so using "main" - // as the default in this case query := parsed.Query() - query.Add("ref", cmp.Or(p.ref.Rev, p.ref.Ref, "main")) if p.ref.Dir != "" { query.Add("dir", p.ref.Dir) @@ -290,7 +287,8 @@ func (p *gitPlugin) genericGitUrl(subpath string) (string, error) { query.Add("port", fmt.Sprintf("%d", p.ref.Port)) } - parsed.RawQuery = query.Encode() + // gitlab doesn't redirect master -> main or main -> master, so using "main" + // as the default in this case query.Add("ref", cmp.Or(p.ref.Rev, p.ref.Ref, "main")) parsed.RawQuery = query.Encode() diff --git a/internal/plugin/git_test.go b/internal/plugin/git_test.go index 2a5fe3a95cf..414ff5c8a9f 100644 --- a/internal/plugin/git_test.go +++ b/internal/plugin/git_test.go @@ -1,8 +1,11 @@ package plugin import ( + "fmt" + "strings" "testing" + "github.com/samber/lo" "github.com/stretchr/testify/assert" "go.jetpack.io/devbox/nix/flake" ) @@ -81,7 +84,31 @@ func TestNewGitPlugin(t *testing.T) { expectedURL: "https://raw.githubusercontent.com/jetify-com/devbox-plugins/my-branch/mongodb", }, { - name: "parse github plugin with dir param and rev", + name: "parse github plugin with dir param and ref", + Include: []flake.Ref{ + { + Type: "github", + Owner: "jetify-com", + Repo: "devbox-plugins", + Dir: "mongodb", + Ref: "initials/my-branch", + }, + }, + expected: gitPlugin{ + ref: flake.Ref{ + Type: "github", + Host: "github.com", + Owner: "jetify-com", + Repo: "devbox-plugins", + Dir: "mongodb", + Ref: "initials/my-branch", + }, + name: "jetify-com.devbox-plugins.mongodb", + }, + expectedURL: "https://raw.githubusercontent.com/jetify-com/devbox-plugins/initials/my-branch/mongodb", + }, + { + name: "parse github plugin with dir param, rev, and ref", Include: []flake.Ref{ { Type: "github", @@ -95,23 +122,24 @@ func TestNewGitPlugin(t *testing.T) { expected: gitPlugin{ ref: flake.Ref{ Type: "github", + Host: "github.com", Owner: "jetify-com", Repo: "devbox-plugins", Dir: "mongodb", - Ref: "initials/my-branch", // FIXME + Ref: "initials/my-branch", // Rev takes precendence over Ref; we exclude the Ref in the URL based on original useage of cmp.Or Rev: "initials", }, name: "jetify-com.devbox-plugins.mongodb", }, - expectedURL: "https://raw.githubusercontent.com/jetify-com/devbox-plugins/initials/my-branch/mongodb", + expectedURL: "https://raw.githubusercontent.com/jetify-com/devbox-plugins/initials/mongodb", }, { - name: "parse gitlab plugin", + name: "parse basic gitlab plugin", Include: []flake.Ref{ { Type: "gitlab", Owner: "username", - Repo: "my-repo", + Repo: "my-plugin", }, }, @@ -120,17 +148,234 @@ func TestNewGitPlugin(t *testing.T) { Type: "gitlab", Host: "gitlab.com", Owner: "username", - Repo: "my-repo", + Repo: "my-plugin", + }, + name: "username.my-plugin", + }, + expectedURL: "https://gitlab.com/api/v4/projects/username%2Fmy-plugin/repository/files/raw?ref=main", + }, + { + name: "parse gitlab plugin with dir param", + Include: []flake.Ref{ + { + Type: "gitlab", + Owner: "username", + Repo: "my-plugin", + Dir: "mongodb", + }, + }, + expected: gitPlugin{ + ref: flake.Ref{ + Type: "gitlab", + Host: "gitlab.com", + Owner: "username", + Repo: "my-plugin", + Dir: "mongodb", }, - name: "username.my-repo", + name: "username.my-plugin.mongodb", }, - expectedURL: "https://gitlab.com/api/v4/projects/username%2Fmy-repo/repository/files/raw?ref=main", + expectedURL: "https://gitlab.com/api/v4/projects/username%2Fmy-plugin/repository/files/mongodb/raw?ref=main", + }, + { + name: "parse gitlab plugin with dir param and ref", + Include: []flake.Ref{ + { + Type: "gitlab", + Owner: "username", + Repo: "my-plugin", + Dir: "mongodb", + Ref: "some/branch", + }, + }, + expected: gitPlugin{ + ref: flake.Ref{ + Type: "gitlab", + Host: "gitlab.com", + Owner: "username", + Repo: "my-plugin", + Dir: "mongodb", + Ref: "some/branch", + }, + name: "username.my-plugin.mongodb", + }, + expectedURL: "https://gitlab.com/api/v4/projects/username%2Fmy-plugin/repository/files/mongodb/raw?ref=some%2Fbranch", + }, + { + name: "parse gitlab plugin with dir param and rev", + Include: []flake.Ref{ + { + Type: "gitlab", + Owner: "username", + Repo: "my-plugin", + Dir: "mongodb", + Rev: "1234567", + }, + }, + expected: gitPlugin{ + ref: flake.Ref{ + Type: "gitlab", + Host: "gitlab.com", + Owner: "username", + Repo: "my-plugin", + Dir: "mongodb", + Rev: "1234567", + }, + name: "username.my-plugin.mongodb", + }, + expectedURL: "https://gitlab.com/api/v4/projects/username%2Fmy-plugin/repository/files/mongodb/raw?ref=1234567", + }, + { + name: "parse gitlab plugin with dir param and rev", + Include: []flake.Ref{ + { + Type: "gitlab", + Owner: "username", + Repo: "my-plugin", + Dir: "mongodb", + Ref: "some/branch", + Rev: "1234567", + }, + }, + expected: gitPlugin{ + ref: flake.Ref{ + Type: "gitlab", + Host: "gitlab.com", + Owner: "username", + Repo: "my-plugin", + Dir: "mongodb", + Ref: "some/branch", + Rev: "1234567", + }, + name: "username.my-plugin.mongodb", + }, + expectedURL: "https://gitlab.com/api/v4/projects/username%2Fmy-plugin/repository/files/mongodb/raw?ref=1234567", + }, + { + name: "parse basic bitbucket plugin", + Include: []flake.Ref{ + { + Type: "bitbucket", + Owner: "username", + Repo: "my-plugin", + }, + }, + + expected: gitPlugin{ + ref: flake.Ref{ + Type: "bitbucket", + Host: "bitbucket.com", + Owner: "username", + Repo: "my-plugin", + }, + name: "username.my-plugin", + }, + expectedURL: "https://api.bitbucket.org/2.0/repositories/username/my-plugin/src/main", + }, + { + name: "parse bitbucket plugin with dir param", + Include: []flake.Ref{ + { + Type: "bitbucket", + Owner: "username", + Repo: "my-plugin", + Dir: "subdir", + }, + }, + + expected: gitPlugin{ + ref: flake.Ref{ + Type: "bitbucket", + Host: "bitbucket.com", + Owner: "username", + Repo: "my-plugin", + Dir: "subdir", + }, + name: "username.my-plugin.subdir", + }, + expectedURL: "https://api.bitbucket.org/2.0/repositories/username/my-plugin/src/main/subdir", + }, + { + name: "parse bitbucket plugin with dir param and ref", + Include: []flake.Ref{ + { + Type: "bitbucket", + Owner: "username", + Repo: "my-plugin", + Dir: "subdir", + Ref: "some/branch", + }, + }, + + expected: gitPlugin{ + ref: flake.Ref{ + Type: "bitbucket", + Host: "bitbucket.com", + Owner: "username", + Repo: "my-plugin", + Dir: "subdir", + Ref: "some/branch", + }, + name: "username.my-plugin.subdir", + }, + expectedURL: "https://api.bitbucket.org/2.0/repositories/username/my-plugin/src/some/branch/subdir", + }, + { + name: "parse bitbucket plugin with dir param and rev", + Include: []flake.Ref{ + { + Type: "bitbucket", + Owner: "username", + Repo: "my-plugin", + Dir: "subdir", + Rev: "1234567", + }, + }, + + expected: gitPlugin{ + ref: flake.Ref{ + Type: "bitbucket", + Host: "bitbucket.com", + Owner: "username", + Repo: "my-plugin", + Dir: "subdir", + Rev: "1234567", + }, + name: "username.my-plugin.subdir", + }, + expectedURL: "https://api.bitbucket.org/2.0/repositories/username/my-plugin/src/1234567/subdir", + }, + { + name: "parse bitbucket plugin with dir param, ref and rev", + Include: []flake.Ref{ + { + Type: "bitbucket", + Owner: "username", + Repo: "my-plugin", + Dir: "subdir", + Ref: "some/branch", + Rev: "1234567", + }, + }, + + expected: gitPlugin{ + ref: flake.Ref{ + Type: "bitbucket", + Host: "bitbucket.com", + Owner: "username", + Repo: "my-plugin", + Dir: "subdir", + Ref: "some/branch", + Rev: "1234567", + }, + name: "username.my-plugin.subdir", + }, + expectedURL: "https://api.bitbucket.org/2.0/repositories/username/my-plugin/src/1234567/subdir", }, } for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { - actual, err := newGitPlugin(testCase.Include[0]) // FIXME: need to evaluate URL + actual, err := newGitPluginForTest(testCase.Include[0]) // FIXME: need to evaluate URL assert.NoError(t, err) assert.Equal(t, &testCase.expected, actual) u, err := testCase.expected.url("") @@ -140,6 +385,23 @@ func TestNewGitPlugin(t *testing.T) { } } +func newGitPluginForTest(ref flake.Ref) (*gitPlugin, error) { + // added because this occurs much earlier in processing within `internal/devconfig/config.go` + switch ref.Type { + case flake.TypeGitHub, flake.TypeGitLab, flake.TypeBitBucket: + ref.Host = fmt.Sprintf("%s.com", ref.Type) + } + + plugin := &gitPlugin{ref: ref} + name := strings.ReplaceAll(ref.Dir, "/", "-") + repoDotted := strings.ReplaceAll(ref.Repo, "/", ".") + plugin.name = githubNameRegexp.ReplaceAllString( + strings.Join(lo.Compact([]string{ref.Owner, repoDotted, name}), "."), + " ", + ) + return plugin, nil +} + func TestGitPluginAuth(t *testing.T) { gitPlugin := gitPlugin{ ref: flake.Ref{ From 089acf9897beb0643a4f97b538062054f0fc487b Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Sun, 10 Nov 2024 18:43:01 -0500 Subject: [PATCH 32/42] added tests for ssh archives --- internal/plugin/git.go | 10 ++- internal/plugin/git_test.go | 158 ++++++++++++++++++++++++++++++++++++ 2 files changed, 164 insertions(+), 4 deletions(-) diff --git a/internal/plugin/git.go b/internal/plugin/git.go index 93a42f89408..e37355f0406 100644 --- a/internal/plugin/git.go +++ b/internal/plugin/git.go @@ -212,12 +212,9 @@ func (p *gitPlugin) sshBaseGitCommand() (string, error) { defaultBranch = "master" } - p.ref.Ref = defaultBranch - prefix := "git archive --format=tar.gz --remote=ssh://git@" path, _ := url.JoinPath(p.ref.Owner, p.ref.Repo) branch := cmp.Or(p.ref.Rev, p.ref.Ref, defaultBranch) - host := p.ref.Host // the Ref struct defaults the field to 0. This technically a valid port for UDP, but we aren't using UDP @@ -225,7 +222,12 @@ func (p *gitPlugin) sshBaseGitCommand() (string, error) { host += ":" + fmt.Sprintf("%d", p.ref.Port) } - command := fmt.Sprintf("%s%s/%s %s %s -o", prefix, host, path, branch, p.ref.Dir) + command := fmt.Sprintf("%s%s/%s %s", prefix, host, path, branch) + if p.ref.Dir != "" { + command += fmt.Sprintf(" %s", p.ref.Dir) + } + command += " -o" + slog.Debug("Generated base git archive command: " + command) return command, nil } diff --git a/internal/plugin/git_test.go b/internal/plugin/git_test.go index 414ff5c8a9f..f4a005ab9e7 100644 --- a/internal/plugin/git_test.go +++ b/internal/plugin/git_test.go @@ -371,6 +371,164 @@ func TestNewGitPlugin(t *testing.T) { }, expectedURL: "https://api.bitbucket.org/2.0/repositories/username/my-plugin/src/1234567/subdir", }, + { + name: "parse basic ssh plugin", + Include: []flake.Ref{ + { + Type: "ssh", + Host: "localhost", + Owner: "username", + Repo: "my-plugin", + }, + }, + + expected: gitPlugin{ + ref: flake.Ref{ + Type: "ssh", + Host: "localhost", + Owner: "username", + Repo: "my-plugin", + }, + name: "username.my-plugin", + }, + expectedURL: "git archive --format=tar.gz --remote=ssh://git@localhost/username/my-plugin main -o", + }, + { + name: "parse ssh plugin with port", + Include: []flake.Ref{ + { + Type: "ssh", + Host: "localhost", + Port: 9999, + Owner: "username", + Repo: "my-plugin", + }, + }, + + expected: gitPlugin{ + ref: flake.Ref{ + Type: "ssh", + Host: "localhost", + Port: 9999, + Owner: "username", + Repo: "my-plugin", + }, + name: "username.my-plugin", + }, + expectedURL: "git archive --format=tar.gz --remote=ssh://git@localhost:9999/username/my-plugin main -o", + }, + { + name: "parse ssh plugin with port and dir", + Include: []flake.Ref{ + { + Type: "ssh", + Host: "localhost", + Port: 9999, + Owner: "username", + Repo: "my-plugin", + Dir: "subdir", + }, + }, + + expected: gitPlugin{ + ref: flake.Ref{ + Type: "ssh", + Host: "localhost", + Port: 9999, + Owner: "username", + Repo: "my-plugin", + Dir: "subdir", + }, + name: "username.my-plugin.subdir", + }, + expectedURL: "git archive --format=tar.gz --remote=ssh://git@localhost:9999/username/my-plugin main subdir -o", + }, + { + name: "parse ssh plugin with port, dir and rev", + Include: []flake.Ref{ + { + Type: "ssh", + Host: "localhost", + Port: 9999, + Owner: "username", + Repo: "my-plugin", + Dir: "subdir", + Rev: "1234567", + }, + }, + + expected: gitPlugin{ + ref: flake.Ref{ + Type: "ssh", + Host: "localhost", + Port: 9999, + Owner: "username", + Repo: "my-plugin", + Dir: "subdir", + Rev: "1234567", + }, + name: "username.my-plugin.subdir", + }, + expectedURL: "git archive --format=tar.gz --remote=ssh://git@localhost:9999/username/my-plugin 1234567 subdir -o", + }, + { + name: "parse ssh plugin with port, dir and ref", + Include: []flake.Ref{ + { + Type: "ssh", + Host: "localhost", + Port: 9999, + Owner: "username", + Repo: "my-plugin", + Dir: "subdir", + Ref: "some/branch", + }, + }, + + expected: gitPlugin{ + ref: flake.Ref{ + Type: "ssh", + Host: "localhost", + Port: 9999, + Owner: "username", + Repo: "my-plugin", + Dir: "subdir", + Ref: "some/branch", + }, + name: "username.my-plugin.subdir", + }, + expectedURL: "git archive --format=tar.gz --remote=ssh://git@localhost:9999/username/my-plugin some/branch subdir -o", + }, + { + name: "parse ssh plugin with port, dir, ref and ref", + Include: []flake.Ref{ + { + Type: "ssh", + Host: "localhost", + Port: 9999, + Owner: "username", + Repo: "my-plugin", + Dir: "subdir", + Ref: "some/branch", + Rev: "1234567", + }, + }, + + expected: gitPlugin{ + ref: flake.Ref{ + Type: "ssh", + Host: "localhost", + Port: 9999, + Owner: "username", + Repo: "my-plugin", + Dir: "subdir", + Ref: "some/branch", + Rev: "1234567", + }, + name: "username.my-plugin.subdir", + }, + expectedURL: "git archive --format=tar.gz --remote=ssh://git@localhost:9999/username/my-plugin 1234567 subdir -o", + }, } for _, testCase := range testCases { From 174ba55896ce1855116f6b1cc8f2070338cd26b1 Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Sat, 23 Nov 2024 21:04:43 -0500 Subject: [PATCH 33/42] moved call to fetch plugin inside lambda --- internal/plugin/git.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/internal/plugin/git.go b/internal/plugin/git.go index e37355f0406..512bd0c832a 100644 --- a/internal/plugin/git.go +++ b/internal/plugin/git.go @@ -157,19 +157,19 @@ func (p *gitPlugin) FileContent(subpath string) ([]byte, error) { return nil, err } - var bytes []byte + process := func() ([]byte, time.Duration, error) { + var bytes []byte - if p.ref.Type == flake.TypeSSH { - bytes, err = p.fetchSSHArchive(location) - } else { - bytes, err = p.fetchHttp(location) - } + if p.ref.Type == flake.TypeSSH { + bytes, err = p.fetchSSHArchive(location) + } else { + bytes, err = p.fetchHttp(location) + } - if err != nil { - return nil, err - } + if err != nil { + return nil, 0, err + } - process := func() ([]byte, time.Duration, error) { // Cache for 24 hours. Once we store the plugin in the lockfile, we // should cache this indefinitely and only invalidate if the plugin // is updated. From 2d9f7a126db3bd8f33127d4d4d812a40b6a31d25 Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Sat, 23 Nov 2024 22:57:13 -0500 Subject: [PATCH 34/42] removed overriding of GitHub flake type --- nix/flake/flakeref.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/nix/flake/flakeref.go b/nix/flake/flakeref.go index 5576764176a..fca4259d3ee 100644 --- a/nix/flake/flakeref.go +++ b/nix/flake/flakeref.go @@ -238,8 +238,6 @@ func parseURLRef(ref string) (parsed Ref, fragment string, err error) { func parseGitRef(refURL *url.URL, parsed *Ref) error { // github:/(/)?(\?)? - parsed.Type = TypeGitHub - // Only split up to 3 times (owner, repo, ref/rev) so that we handle // refs that have slashes in them. For example, // "github:jetify-com/devbox/gcurtis/flakeref" parses as "gcurtis/flakeref". From ce63909c09715efd402f80e076b6edbb8afc3a4f Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Sat, 23 Nov 2024 22:58:32 -0500 Subject: [PATCH 35/42] added comment --- nix/flake/flakeref.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nix/flake/flakeref.go b/nix/flake/flakeref.go index fca4259d3ee..5c36ad65250 100644 --- a/nix/flake/flakeref.go +++ b/nix/flake/flakeref.go @@ -77,6 +77,8 @@ type Ref struct { Port int32 `json:port,omitempty` } +// TODO move `ParseRef` to the unit test file. It isn't used anywhere else + // ParseRef parses a raw flake reference. Nix supports a variety of flake ref // formats, and isn't entirely consistent about how it parses them. ParseRef // attempts to mimic how Nix parses flake refs on the command line. The raw ref From e85646001a18e34543164f40cd20c2e809d5f83a Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Sun, 22 Dec 2024 12:23:04 -0500 Subject: [PATCH 36/42] wip --- internal/plugin/git.go | 14 +-- internal/plugin/update.go | 14 +-- nix/flake/flakeref.go | 44 +------ nix/flake/flakeref_test.go | 229 +++---------------------------------- 4 files changed, 20 insertions(+), 281 deletions(-) diff --git a/internal/plugin/git.go b/internal/plugin/git.go index 512bd0c832a..cae834af75c 100644 --- a/internal/plugin/git.go +++ b/internal/plugin/git.go @@ -23,11 +23,7 @@ import ( "go.jetpack.io/pkg/filecache" ) -var sshCache = filecache.New[[]byte]("devbox/plugin/ssh") var gitCache = filecache.New[[]byte]("devbox/plugin/git") -var githubCache = filecache.New[[]byte]("devbox/plugin/github") -var gitlabCache = filecache.New[[]byte]("devbox/plugin/gitlab") -var bitbucketCache = filecache.New[[]byte]("devbox/plugin/bitbucket") type gitPlugin struct { ref flake.Ref @@ -177,15 +173,7 @@ func (p *gitPlugin) FileContent(subpath string) ([]byte, error) { } switch p.ref.Type { - case flake.TypeSSH: - return sshCache.GetOrSet(location, process) - case flake.TypeGitHub: - return githubCache.GetOrSet(location, process) - case flake.TypeGitLab: - return gitlabCache.GetOrSet(location, process) - case flake.TypeBitBucket: - return bitbucketCache.GetOrSet(location, process) - case flake.TypeGit: + case flake.TypeSSH, flake.TypeGitHub, flake.TypeGitLab, flake.TypeBitBucket, flake.TypeGit: return gitCache.GetOrSet(location, process) default: slog.Error("Unable to handle flake ref type: " + p.ref.Type) diff --git a/internal/plugin/update.go b/internal/plugin/update.go index 3ba3f376c60..6778298a5e1 100644 --- a/internal/plugin/update.go +++ b/internal/plugin/update.go @@ -1,18 +1,6 @@ package plugin -import "go.jetpack.io/pkg/filecache" - func Update() error { - pluginCaches := []*filecache.Cache[[]byte]{githubCache, sshCache, gitlabCache, bitbucketCache} - - for _, cache := range pluginCaches { - err := cache.Clear() - - if err != nil { - return err - } - - } - + gitCache.Clear() return nil } diff --git a/nix/flake/flakeref.go b/nix/flake/flakeref.go index 5c36ad65250..f767aed0c3f 100644 --- a/nix/flake/flakeref.go +++ b/nix/flake/flakeref.go @@ -15,6 +15,7 @@ import ( const ( TypeIndirect = "indirect" TypePath = "path" + TypeHttps = "https" TypeFile = "file" TypeSSH = "ssh" TypeGitHub = "github" @@ -77,49 +78,6 @@ type Ref struct { Port int32 `json:port,omitempty` } -// TODO move `ParseRef` to the unit test file. It isn't used anywhere else - -// ParseRef parses a raw flake reference. Nix supports a variety of flake ref -// formats, and isn't entirely consistent about how it parses them. ParseRef -// attempts to mimic how Nix parses flake refs on the command line. The raw ref -// can be one of the following: -// -// - Indirect reference such as "nixpkgs" or "nixpkgs/unstable". -// - Path-like reference such as "./flake" or "/path/to/flake". They must -// start with a '.' or '/' and not contain a '#' or '?'. -// - URL-like reference which must be a valid URL with any special characters -// encoded. The scheme can be any valid flake ref type except for mercurial, -// gitlab, and sourcehut. -// -// ParseRef does not guarantee that a parsed flake ref is valid or that an -// error indicates an invalid flake ref. Use the "nix flake metadata" command or -// the builtins.parseFlakeRef Nix function to validate a flake ref. -func ParseRef(ref string) (Ref, error) { - if ref == "" { - return Ref{}, redact.Errorf("empty flake reference") - } - - // Handle path-style references first. - parsed := Ref{} - if ref[0] == '.' || ref[0] == '/' { - if strings.ContainsAny(ref, "?#") { - // The Nix CLI does seem to allow paths with a '?' - // (contrary to the manual) but ignores everything that - // comes after it. This is a bit surprising, so we just - // don't allow it at all. - return Ref{}, redact.Errorf("path-style flake reference %q contains a '?' or '#'", ref) - } - parsed.Type = TypePath - parsed.Path = ref - return parsed, nil - } - parsed, fragment, err := parseURLRef(ref) - if fragment != "" { - return Ref{}, redact.Errorf("flake reference %q contains a URL fragment", ref) - } - return parsed, err -} - func parseURLRef(ref string) (parsed Ref, fragment string, err error) { // A good way to test how Nix parses a flake reference is to run: // diff --git a/nix/flake/flakeref_test.go b/nix/flake/flakeref_test.go index e38572cabc4..84104ee2196 100644 --- a/nix/flake/flakeref_test.go +++ b/nix/flake/flakeref_test.go @@ -6,201 +6,6 @@ import ( "github.com/google/go-cmp/cmp" ) -func TestParseFlakeRef(t *testing.T) { - cases := map[string]Ref{ - // Path-like references start with a '.' or '/'. - // This distinguishes them from indirect references - // (./nixpkgs is a directory; nixpkgs is an indirect). - ".": {Type: TypePath, Path: "."}, - "./": {Type: TypePath, Path: "./"}, - "./flake": {Type: TypePath, Path: "./flake"}, - "./relative/flake": {Type: TypePath, Path: "./relative/flake"}, - "/": {Type: TypePath, Path: "/"}, - "/flake": {Type: TypePath, Path: "/flake"}, - "/absolute/flake": {Type: TypePath, Path: "/absolute/flake"}, - - // Path-like references can have raw unicode characters unlike - // path: URL references. - "./Ûñî©ôδ€/flake\n": {Type: TypePath, Path: "./Ûñî©ôδ€/flake\n"}, - "/Ûñî©ôδ€/flake\n": {Type: TypePath, Path: "/Ûñî©ôδ€/flake\n"}, - - // URL-like path references. - "path:": {Type: TypePath, Path: ""}, - "path:.": {Type: TypePath, Path: "."}, - "path:./": {Type: TypePath, Path: "./"}, - "path:./flake": {Type: TypePath, Path: "./flake"}, - "path:./relative/flake": {Type: TypePath, Path: "./relative/flake"}, - "path:./relative/my%20flake": {Type: TypePath, Path: "./relative/my flake"}, - "path:/": {Type: TypePath, Path: "/"}, - "path:/flake": {Type: TypePath, Path: "/flake"}, - "path:/absolute/flake": {Type: TypePath, Path: "/absolute/flake"}, - - // URL-like paths can omit the "./" prefix for relative - // directories. - "path:flake": {Type: TypePath, Path: "flake"}, - "path:relative/flake": {Type: TypePath, Path: "relative/flake"}, - - // Indirect references. - "flake:indirect": {Type: TypeIndirect, ID: "indirect"}, - "flake:indirect/ref": {Type: TypeIndirect, ID: "indirect", Ref: "ref"}, - "flake:indirect/my%2Fref": {Type: TypeIndirect, ID: "indirect", Ref: "my/ref"}, - "flake:indirect/5233fd2ba76a3accb5aaa999c00509a11fd0793c": {Type: TypeIndirect, ID: "indirect", Rev: "5233fd2ba76a3accb5aaa999c00509a11fd0793c"}, - "flake:indirect/ref/5233fd2ba76a3accb5aaa999c00509a11fd0793c": {Type: TypeIndirect, ID: "indirect", Ref: "ref", Rev: "5233fd2ba76a3accb5aaa999c00509a11fd0793c"}, - - // Indirect references can omit their "indirect:" type prefix. - "indirect": {Type: TypeIndirect, ID: "indirect"}, - "indirect/ref": {Type: TypeIndirect, ID: "indirect", Ref: "ref"}, - "indirect/5233fd2ba76a3accb5aaa999c00509a11fd0793c": {Type: TypeIndirect, ID: "indirect", Rev: "5233fd2ba76a3accb5aaa999c00509a11fd0793c"}, - "indirect/ref/5233fd2ba76a3accb5aaa999c00509a11fd0793c": {Type: TypeIndirect, ID: "indirect", Ref: "ref", Rev: "5233fd2ba76a3accb5aaa999c00509a11fd0793c"}, - - // GitHub references. - "github:NixOS/nix": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix"}, - "github:NixOS/nix/v1.2.3": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Ref: "v1.2.3"}, - "github:NixOS/nix?ref=v1.2.3": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Ref: "v1.2.3"}, - "github:NixOS/nix?ref=5233fd2ba76a3accb5aaa999c00509a11fd0793c": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Ref: "5233fd2ba76a3accb5aaa999c00509a11fd0793c"}, - "github:NixOS/nix/main": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Ref: "main"}, - "github:NixOS/nix/main/5233fd2ba76a3accb5aaa999c00509a11fd0793c": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Ref: "main/5233fd2ba76a3accb5aaa999c00509a11fd0793c"}, - "github:NixOS/nix/5233fd2bb76a3accb5aaa999c00509a11fd0793z": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Ref: "5233fd2bb76a3accb5aaa999c00509a11fd0793z"}, - "github:NixOS/nix/5233fd2ba76a3accb5aaa999c00509a11fd0793c": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Rev: "5233fd2ba76a3accb5aaa999c00509a11fd0793c"}, - "github:NixOS/nix?rev=5233fd2ba76a3accb5aaa999c00509a11fd0793c": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Rev: "5233fd2ba76a3accb5aaa999c00509a11fd0793c"}, - "github:NixOS/nix?host=example.com": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Host: "example.com"}, - "github:NixOS/nix?host=example.com&dir=subdir": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Host: "example.com", Dir: "subdir"}, - - // The github type allows clone-style URLs. The username and - // host are ignored. - "github://git@github.com/NixOS/nix": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix"}, - "github://git@github.com/NixOS/nix/v1.2.3": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Ref: "v1.2.3"}, - "github://git@github.com/NixOS/nix?ref=v1.2.3": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Ref: "v1.2.3"}, - "github://git@github.com/NixOS/nix?ref=5233fd2ba76a3accb5aaa999c00509a11fd0793c": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Ref: "5233fd2ba76a3accb5aaa999c00509a11fd0793c"}, - "github://git@github.com/NixOS/nix?rev=5233fd2ba76a3accb5aaa999c00509a11fd0793c": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Rev: "5233fd2ba76a3accb5aaa999c00509a11fd0793c"}, - "github://git@github.com/NixOS/nix?host=example.com": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Host: "example.com"}, - - // Git references. - "git://example.com/repo/flake": {Type: TypeGit, URL: "git://example.com/repo/flake"}, - "git+https://example.com/repo/flake": {Type: TypeGit, URL: "https://example.com/repo/flake"}, - "git+ssh://git@example.com/repo/flake": {Type: TypeGit, URL: "ssh://git@example.com/repo/flake"}, - "git:/repo/flake": {Type: TypeGit, URL: "git:/repo/flake"}, - "git+file:///repo/flake": {Type: TypeGit, URL: "file:///repo/flake"}, - "git://example.com/repo/flake?ref=unstable&rev=e486d8d40e626a20e06d792db8cc5ac5aba9a5b4&dir=subdir": {Type: TypeGit, URL: "git://example.com/repo/flake?dir=subdir", Ref: "unstable", Rev: "e486d8d40e626a20e06d792db8cc5ac5aba9a5b4", Dir: "subdir"}, - - // Tarball references. - "tarball+http://example.com/flake": {Type: TypeTarball, URL: "http://example.com/flake"}, - "tarball+https://example.com/flake": {Type: TypeTarball, URL: "https://example.com/flake"}, - "tarball+file:///home/flake": {Type: TypeTarball, URL: "file:///home/flake"}, - - // Regular URLs have the tarball type if they have a known - // archive extension: - // .zip, .tar, .tgz, .tar.gz, .tar.xz, .tar.bz2 or .tar.zst - "http://example.com/flake.zip": {Type: TypeTarball, URL: "http://example.com/flake.zip"}, - "http://example.com/flake.tar": {Type: TypeTarball, URL: "http://example.com/flake.tar"}, - "http://example.com/flake.tgz": {Type: TypeTarball, URL: "http://example.com/flake.tgz"}, - "http://example.com/flake.tar.gz": {Type: TypeTarball, URL: "http://example.com/flake.tar.gz"}, - "http://example.com/flake.tar.xz": {Type: TypeTarball, URL: "http://example.com/flake.tar.xz"}, - "http://example.com/flake.tar.bz2": {Type: TypeTarball, URL: "http://example.com/flake.tar.bz2"}, - "http://example.com/flake.tar.zst": {Type: TypeTarball, URL: "http://example.com/flake.tar.zst"}, - "http://example.com/flake.tar?dir=subdir": {Type: TypeTarball, URL: "http://example.com/flake.tar?dir=subdir", Dir: "subdir"}, - "file:///flake.zip": {Type: TypeTarball, URL: "file:///flake.zip"}, - "file:///flake.tar": {Type: TypeTarball, URL: "file:///flake.tar"}, - "file:///flake.tgz": {Type: TypeTarball, URL: "file:///flake.tgz"}, - "file:///flake.tar.gz": {Type: TypeTarball, URL: "file:///flake.tar.gz"}, - "file:///flake.tar.xz": {Type: TypeTarball, URL: "file:///flake.tar.xz"}, - "file:///flake.tar.bz2": {Type: TypeTarball, URL: "file:///flake.tar.bz2"}, - "file:///flake.tar.zst": {Type: TypeTarball, URL: "file:///flake.tar.zst"}, - "file:///flake.tar?dir=subdir": {Type: TypeTarball, URL: "file:///flake.tar?dir=subdir", Dir: "subdir"}, - - // File URL references. - "file+file:///flake": {Type: TypeFile, URL: "file:///flake"}, - "file+http://example.com/flake": {Type: TypeFile, URL: "http://example.com/flake"}, - "file+http://example.com/flake.git": {Type: TypeFile, URL: "http://example.com/flake.git"}, - "file+http://example.com/flake.tar?dir=subdir": {Type: TypeFile, URL: "http://example.com/flake.tar?dir=subdir", Dir: "subdir"}, - - // Regular URLs have the file type if they don't have a known - // archive extension. - "http://example.com/flake": {Type: TypeFile, URL: "http://example.com/flake"}, - "http://example.com/flake.git": {Type: TypeFile, URL: "http://example.com/flake.git"}, - "http://example.com/flake?dir=subdir": {Type: TypeFile, URL: "http://example.com/flake?dir=subdir", Dir: "subdir"}, - } - for ref, want := range cases { - t.Run(ref, func(t *testing.T) { - got, err := ParseRef(ref) - if diff := cmp.Diff(want, got); diff != "" { - if err != nil { - t.Errorf("got error: %s", err) - } - t.Errorf("wrong flakeref (-want +got):\n%s", diff) - } - }) - } -} - -func TestParseFlakeRefError(t *testing.T) { - t.Run("EmptyString", func(t *testing.T) { - ref := "" - _, err := ParseRef(ref) - if err == nil { - t.Error("got nil error for bad flakeref:", ref) - } - }) - t.Run("InvalidURL", func(t *testing.T) { - ref := "://bad/url" - _, err := ParseRef(ref) - if err == nil { - t.Error("got nil error for bad flakeref:", ref) - } - }) - t.Run("InvalidURLEscape", func(t *testing.T) { - ref := "path:./relative/my%flake" - _, err := ParseRef(ref) - if err == nil { - t.Error("got nil error for bad flakeref:", ref) - } - }) - t.Run("UnsupportedURLScheme", func(t *testing.T) { - ref := "runx:mvdan/gofumpt@latest" - _, err := ParseRef(ref) - if err == nil { - t.Error("got nil error for bad flakeref:", ref) - } - }) - t.Run("PathLikeWith?#", func(t *testing.T) { - in := []string{ - "./invalid#path", - "./invalid?path", - "/invalid#path", - "/invalid?path", - "/#", - "/?", - } - for _, ref := range in { - _, err := ParseRef(ref) - if err == nil { - t.Error("got nil error for bad flakeref:", ref) - } - } - }) - t.Run("GitHubInvalidRefRevCombo", func(t *testing.T) { - in := []string{ - "github:NixOS/nix?ref=v1.2.3&rev=5233fd2ba76a3accb5aaa999c00509a11fd0793c", - "github:NixOS/nix/v1.2.3?ref=v4.5.6", - "github:NixOS/nix/5233fd2ba76a3accb5aaa999c00509a11fd0793c?rev=e486d8d40e626a20e06d792db8cc5ac5aba9a5b4", - "github:NixOS/nix/5233fd2ba76a3accb5aaa999c00509a11fd0793c?ref=v1.2.3", - } - for _, ref := range in { - _, err := ParseRef(ref) - if err == nil { - t.Error("got nil error for bad flakeref:", ref) - } - } - }) - t.Run("URLFragment", func(t *testing.T) { - ref := "https://github.com/NixOS/patchelf/archive/master.tar.gz#patchelf" - _, err := ParseRef(ref) - if err == nil { - t.Error("got nil error for flakeref with fragment:", ref) - } - }) -} - func TestFlakeRefString(t *testing.T) { cases := map[Ref]string{ {}: "", @@ -238,26 +43,26 @@ func TestFlakeRefString(t *testing.T) { {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Dir: "sub/dir", Host: "example.com"}: "github:NixOS/nix?dir=sub%2Fdir&host=example.com", // Git references. - {Type: TypeGit, URL: "git://example.com/repo/flake"}: "git://example.com/repo/flake", - {Type: TypeGit, URL: "https://example.com/repo/flake"}: "git+https://example.com/repo/flake", - {Type: TypeGit, URL: "ssh://git@example.com/repo/flake"}: "git+ssh://git@example.com/repo/flake", - {Type: TypeGit, URL: "git:/repo/flake"}: "git:/repo/flake", - {Type: TypeGit, URL: "file:///repo/flake"}: "git+file:///repo/flake", - {Type: TypeGit, URL: "ssh://git@example.com/repo/flake", Ref: "my/ref", Rev: "e486d8d40e626a20e06d792db8cc5ac5aba9a5b4"}: "git+ssh://git@example.com/repo/flake?ref=my%2Fref&rev=e486d8d40e626a20e06d792db8cc5ac5aba9a5b4", - {Type: TypeGit, URL: "ssh://git@example.com/repo/flake?dir=sub%2Fdir", Ref: "my/ref", Rev: "e486d8d40e626a20e06d792db8cc5ac5aba9a5b4", Dir: "sub/dir"}: "git+ssh://git@example.com/repo/flake?dir=sub%2Fdir&ref=my%2Fref&rev=e486d8d40e626a20e06d792db8cc5ac5aba9a5b4", - {Type: TypeGit, URL: "git:repo/flake?dir=sub%2Fdir", Ref: "my/ref", Rev: "e486d8d40e626a20e06d792db8cc5ac5aba9a5b4", Dir: "sub/dir"}: "git:repo/flake?dir=sub%2Fdir&ref=my%2Fref&rev=e486d8d40e626a20e06d792db8cc5ac5aba9a5b4", + {Type: TypeGit, Host: "example.com", Owner: "repo", Repo: "flake"}: "git://example.com/repo/flake", + {Type: TypeHttps, Host: "example.com", Owner: "repo", Repo: "flake"}: "git+https://example.com/repo/flake", + {Type: TypeSSH, Host: "example.com", Owner: "repo", Repo: "flake"}: "git+ssh://git@example.com/repo/flake", + {Type: TypeGit, Owner: "repo", Repo: "flake"}: "git:/repo/flake", + {Type: TypeFile, Owner: "repo", Repo: "flake"}: "git+file:///repo/flake", + {Type: TypeSSH, Host: "example.com", Owner: "repo", Repo: "flake", Ref: "my/ref", Rev: "e486d8d40e626a20e06d792db8cc5ac5aba9a5b4"}: "git+ssh://git@example.com/repo/flake?ref=my%2Fref&rev=e486d8d40e626a20e06d792db8cc5ac5aba9a5b4", + {Type: TypeSSH, Host: "example.com", Owner: "repo", Repo: "flake", Ref: "my/ref", Rev: "e486d8d40e626a20e06d792db8cc5ac5aba9a5b4", Dir: "sub/dir"}: "git+ssh://git@example.com/repo/flake?dir=sub%2Fdir&ref=my%2Fref&rev=e486d8d40e626a20e06d792db8cc5ac5aba9a5b4", + {Type: TypeGit, Owner: "repo", Repo: "flake", Ref: "my/ref", Rev: "e486d8d40e626a20e06d792db8cc5ac5aba9a5b4", Dir: "sub/dir"}: "", // "git:/repo/flake?dir=sub%2Fdir&ref=my%2Fref&rev=e486d8d40e626a20e06d792db8cc5ac5aba9a5b4", // how is this supposed to be a valid URL? There isn't a hostname in it... // Tarball references. - {Type: TypeTarball, URL: "http://example.com/flake"}: "tarball+http://example.com/flake", - {Type: TypeTarball, URL: "https://example.com/flake"}: "tarball+https://example.com/flake", - {Type: TypeTarball, URL: "https://example.com/flake", Dir: "sub/dir"}: "tarball+https://example.com/flake?dir=sub%2Fdir", - {Type: TypeTarball, URL: "file:///home/flake"}: "tarball+file:///home/flake", + {Type: TypeTarball, Host: "example.com", Owner: "flake", URL: "http://example.com/flake"}: "tarball+http://example.com/flake", + {Type: TypeTarball, Host: "example.com", Owner: "flake", URL: "https://example.com/flake"}: "tarball+https://example.com/flake", + {Type: TypeTarball, Host: "example.com", Owner: "flake", URL: "https://example.com/flake", Dir: "sub/dir"}: "tarball+https://example.com/flake?dir=sub%2Fdir", + {Type: TypeTarball, URL: "file:///home/flake"}: "tarball+file:///home/flake", // File URL references. - {Type: TypeFile, URL: "file:///flake"}: "file+file:///flake", - {Type: TypeFile, URL: "http://example.com/flake"}: "file+http://example.com/flake", - {Type: TypeFile, URL: "http://example.com/flake.git"}: "file+http://example.com/flake.git", - {Type: TypeFile, URL: "http://example.com/flake.tar?dir=sub%2Fdir", Dir: "sub/dir"}: "file+http://example.com/flake.tar?dir=sub%2Fdir", + {Type: TypePath, URL: "file:///flake"}: "file+file:///flake", + {Type: TypePath, URL: "http://example.com/flake"}: "file+http://example.com/flake", + {Type: TypePath, URL: "http://example.com/flake.git"}: "file+http://example.com/flake.git", + {Type: TypePath, URL: "http://example.com/flake.tar?dir=sub%2Fdir", Dir: "sub/dir"}: "file+http://example.com/flake.tar?dir=sub%2Fdir", } for ref, want := range cases { @@ -371,7 +176,7 @@ func TestFlakeInstallableString(t *testing.T) { {AttrPath: "app", Outputs: "%2F", Ref: Ref{Type: TypeIndirect, ID: "nixpkgs"}}: "flake:nixpkgs#app^%2F", // Missing or invalid fields. - {AttrPath: "app", Ref: Ref{Type: TypeFile, URL: ""}}: "", + {AttrPath: "app", Ref: Ref{Type: TypePath, URL: ""}}: "", {AttrPath: "app", Ref: Ref{Type: TypeGit, URL: ""}}: "", {AttrPath: "app", Ref: Ref{Type: TypeGitHub, Owner: ""}}: "", {AttrPath: "app", Ref: Ref{Type: TypeIndirect, ID: ""}}: "", From b05fc25481c27b913c6a95eccf1476e3ba562aa2 Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Sun, 22 Dec 2024 17:14:39 -0500 Subject: [PATCH 37/42] truly...no idea why i went down that rabbit hole in the first place --- nix/flake/flakeref.go | 150 ++++++++++++++---------- nix/flake/flakeref_test.go | 233 ++++++++++++++++++++++++++++++++++--- 2 files changed, 302 insertions(+), 81 deletions(-) diff --git a/nix/flake/flakeref.go b/nix/flake/flakeref.go index a2f58f9b6ae..811a417d776 100644 --- a/nix/flake/flakeref.go +++ b/nix/flake/flakeref.go @@ -3,7 +3,6 @@ package flake import ( "cmp" - "fmt" "net/url" "path" "slices" @@ -88,6 +87,47 @@ type Ref struct { Port int32 `json:port,omitempty` } +// ParseRef parses a raw flake reference. Nix supports a variety of flake ref +// formats, and isn't entirely consistent about how it parses them. ParseRef +// attempts to mimic how Nix parses flake refs on the command line. The raw ref +// can be one of the following: +// +// - Indirect reference such as "nixpkgs" or "nixpkgs/unstable". +// - Path-like reference such as "./flake" or "/path/to/flake". They must +// start with a '.' or '/' and not contain a '#' or '?'. +// - URL-like reference which must be a valid URL with any special characters +// encoded. The scheme can be any valid flake ref type except for mercurial, +// gitlab, and sourcehut. +// +// ParseRef does not guarantee that a parsed flake ref is valid or that an +// error indicates an invalid flake ref. Use the "nix flake metadata" command or +// the builtins.parseFlakeRef Nix function to validate a flake ref. +func ParseRef(ref string) (Ref, error) { + if ref == "" { + return Ref{}, redact.Errorf("empty flake reference") + } + + // Handle path-style references first. + parsed := Ref{} + if ref[0] == '.' || ref[0] == '/' { + if strings.ContainsAny(ref, "?#") { + // The Nix CLI does seem to allow paths with a '?' + // (contrary to the manual) but ignores everything that + // comes after it. This is a bit surprising, so we just + // don't allow it at all. + return Ref{}, redact.Errorf("path-style flake reference %q contains a '?' or '#'", ref) + } + parsed.Type = TypePath + parsed.Path = ref + return parsed, nil + } + parsed, fragment, err := parseURLRef(ref) + if fragment != "" { + return Ref{}, redact.Errorf("flake reference %q contains a URL fragment", ref) + } + return parsed, err +} + func parseURLRef(ref string) (parsed Ref, fragment string, err error) { // A good way to test how Nix parses a flake reference is to run: // @@ -196,6 +236,7 @@ func parseURLRef(ref string) (parsed Ref, fragment string, err error) { refURL.Scheme = refURL.Scheme[5:] // remove file+ parsed.URL = refURL.String() case "git", "git+http", "git+https", "git+ssh", "git+git", "git+file": + parsed.Type = TypeGit query := refURL.Query() parsed.Dir = query.Get("dir") parsed.Ref = query.Get("ref") @@ -206,49 +247,25 @@ func parseURLRef(ref string) (parsed Ref, fragment string, err error) { query.Del("ref") query.Del("rev") refURL.RawQuery = query.Encode() - if len(refURL.Scheme) > 3 { refURL.Scheme = refURL.Scheme[4:] // remove git+ } - - if strings.HasPrefix(refURL.Scheme, TypeSSH) { - parsed.Type = TypeSSH - } else if strings.HasPrefix(refURL.Scheme, TypeFile) { - parsed.Type = TypeFile - } else { - parsed.Type = TypeGit - } - parsed.URL = refURL.String() - - if err := parseGitRef(refURL, &parsed); err != nil { - return Ref{}, "", err - } - case "bitbucket": - parsed.Type = TypeBitBucket - if err := parseGitRef(refURL, &parsed); err != nil { - return Ref{}, "", err - } - case "gitlab": - parsed.Type = TypeGitLab - if err := parseGitRef(refURL, &parsed); err != nil { - return Ref{}, "", err - } case "github": - parsed.Type = TypeGitHub - if err := parseGitRef(refURL, &parsed); err != nil { + if err := parseGitHubRef(refURL, &parsed); err != nil { return Ref{}, "", err } default: return Ref{}, "", redact.Errorf("unsupported flake reference URL scheme: %s", redact.Safe(refURL.Scheme)) } - return parsed, fragment, nil } -func parseGitRef(refURL *url.URL, parsed *Ref) error { +func parseGitHubRef(refURL *url.URL, parsed *Ref) error { // github:/(/)?(\?)? + parsed.Type = TypeGitHub + // Only split up to 3 times (owner, repo, ref/rev) so that we handle // refs that have slashes in them. For example, // "github:jetify-com/devbox/gcurtis/flakeref" parses as "gcurtis/flakeref". @@ -268,7 +285,6 @@ func parseGitRef(refURL *url.URL, parsed *Ref) error { parsed.Host = refURL.Query().Get("host") parsed.Dir = refURL.Query().Get("dir") - if qRef := refURL.Query().Get("ref"); qRef != "" { if parsed.Rev != "" { return redact.Errorf("github flake reference has a ref and a rev") @@ -342,50 +358,52 @@ func (r Ref) Locked() bool { // string. func (r Ref) String() string { switch r.Type { - case TypeFile: if r.URL == "" { return "" } - return "file+" + r.URL - case TypeSSH: - base := fmt.Sprintf("git+ssh://git@%s", r.Host) - if r.Port > 0 { - base = fmt.Sprintf("%s:%d", base, r.Port) - } - queryParams := url.Values{} - - if r.Rev != "" { - queryParams.Add("rev", r.Rev) + url, err := url.Parse("file+" + r.URL) + if err != nil { + // This should be rare and only happen if the caller + // messed with the parsed URL. + return "" } - if r.Ref != "" { - queryParams.Add("ref", r.Ref) + url.RawQuery = appendQueryString(url.Query(), + "lastModified", itoaOmitZero(r.LastModified), + "narHash", r.NARHash, + ) + return url.String() + case TypeGit: + if r.URL == "" { + return "" } - - if r.Dir != "" { - queryParams.Add("dir", r.Dir) + if !strings.HasPrefix(r.URL, "git") { + r.URL = "git+" + r.URL } - return fmt.Sprintf("%s/%s/%s?%s", base, r.Owner, r.Repo, queryParams.Encode()) - - case TypeGitLab, TypeBitBucket, TypeGitHub: - if r.Owner == "" || r.Repo == "" { - return "" + // Nix removes "ref" and "rev" from the query string + // (but not other parameters) after parsing. If they're empty, + // we can skip parsing the URL. Otherwise, we need to add them + // back. + if r.Ref == "" && r.Rev == "" { + return r.URL } - - scheme := "github" // using as default - if r.Type == TypeGitLab { - scheme = "gitlab" + url, err := url.Parse(r.URL) + if err != nil { + // This should be rare and only happen if the caller + // messed with the parsed URL. + return "" } - if r.Type == TypeBitBucket { - scheme = "bitbucket" + url.RawQuery = appendQueryString(url.Query(), "ref", r.Ref, "rev", r.Rev, "dir", r.Dir) + return url.String() + case TypeGitHub: + if r.Owner == "" || r.Repo == "" { + return "" } - url := &url.URL{ - Scheme: scheme, - Opaque: buildEscapedPath(r.Owner, r.Repo, r.Rev, r.Ref), - //RawQuery: buildQueryString("host", r.Host, "dir", r.Dir), + Scheme: "github", + Opaque: buildEscapedPath(r.Owner, r.Repo, cmp.Or(r.Rev, r.Ref)), RawQuery: appendQueryString(nil, "host", r.Host, "dir", r.Dir, @@ -393,9 +411,7 @@ func (r Ref) String() string { "narHash", r.NARHash, ), } - return url.String() - case TypeIndirect: if r.ID == "" { return "" @@ -660,7 +676,13 @@ func ParseInstallable(raw string) (Installable, error) { // Interpret installables with path-style flake refs as URLs to extract // the attribute path (fragment). This means that path-style flake refs - // cannot point to files with a '#' or '?' in their name, since those + // + // + // + // + // + // + //// cannot point to files with a '#' or '?' in their name, since those // would be parsed as the URL fragment or query string. This mimic's // Nix's CLI behavior. if raw[0] == '.' || raw[0] == '/' { diff --git a/nix/flake/flakeref_test.go b/nix/flake/flakeref_test.go index 3b8b1193bd2..803980f8bd9 100644 --- a/nix/flake/flakeref_test.go +++ b/nix/flake/flakeref_test.go @@ -6,6 +6,205 @@ import ( "github.com/google/go-cmp/cmp" ) +func TestParseFlakeRef(t *testing.T) { + cases := map[string]Ref{ + // Path-like references start with a '.' or '/'. + // This distinguishes them from indirect references + // (./nixpkgs is a directory; nixpkgs is an indirect). + ".": {Type: TypePath, Path: "."}, + "./": {Type: TypePath, Path: "./"}, + "./flake": {Type: TypePath, Path: "./flake"}, + "./relative/flake": {Type: TypePath, Path: "./relative/flake"}, + "/": {Type: TypePath, Path: "/"}, + "/flake": {Type: TypePath, Path: "/flake"}, + "/absolute/flake": {Type: TypePath, Path: "/absolute/flake"}, + + // Path-like references can have raw unicode characters unlike + // path: URL references. + "./Ûñî©ôδ€/flake\n": {Type: TypePath, Path: "./Ûñî©ôδ€/flake\n"}, + "/Ûñî©ôδ€/flake\n": {Type: TypePath, Path: "/Ûñî©ôδ€/flake\n"}, + + // URL-like path references. + "path:": {Type: TypePath, Path: ""}, + "path:.": {Type: TypePath, Path: "."}, + "path:./": {Type: TypePath, Path: "./"}, + "path:./flake": {Type: TypePath, Path: "./flake"}, + "path:./relative/flake": {Type: TypePath, Path: "./relative/flake"}, + "path:./relative/my%20flake": {Type: TypePath, Path: "./relative/my flake"}, + "path:/": {Type: TypePath, Path: "/"}, + "path:/flake": {Type: TypePath, Path: "/flake"}, + "path:/absolute/flake": {Type: TypePath, Path: "/absolute/flake"}, + "path:/absolute/flake?lastModified=1734435836&narHash=sha256-kMBQ5PRiFLagltK0sH%2B08aiNt3zGERC2297iB6vrvlU%3D": {Type: TypePath, Path: "/absolute/flake", LastModified: 1734435836, NARHash: "sha256-kMBQ5PRiFLagltK0sH+08aiNt3zGERC2297iB6vrvlU="}, + + // URL-like paths can omit the "./" prefix for relative + // directories. + "path:flake": {Type: TypePath, Path: "flake"}, + "path:relative/flake": {Type: TypePath, Path: "relative/flake"}, + + // Indirect references. + "flake:indirect": {Type: TypeIndirect, ID: "indirect"}, + "flake:indirect/ref": {Type: TypeIndirect, ID: "indirect", Ref: "ref"}, + "flake:indirect/my%2Fref": {Type: TypeIndirect, ID: "indirect", Ref: "my/ref"}, + "flake:indirect/5233fd2ba76a3accb5aaa999c00509a11fd0793c": {Type: TypeIndirect, ID: "indirect", Rev: "5233fd2ba76a3accb5aaa999c00509a11fd0793c"}, + "flake:indirect/ref/5233fd2ba76a3accb5aaa999c00509a11fd0793c": {Type: TypeIndirect, ID: "indirect", Ref: "ref", Rev: "5233fd2ba76a3accb5aaa999c00509a11fd0793c"}, + + // Indirect references can omit their "indirect:" type prefix. + "indirect": {Type: TypeIndirect, ID: "indirect"}, + "indirect/ref": {Type: TypeIndirect, ID: "indirect", Ref: "ref"}, + "indirect/5233fd2ba76a3accb5aaa999c00509a11fd0793c": {Type: TypeIndirect, ID: "indirect", Rev: "5233fd2ba76a3accb5aaa999c00509a11fd0793c"}, + "indirect/ref/5233fd2ba76a3accb5aaa999c00509a11fd0793c": {Type: TypeIndirect, ID: "indirect", Ref: "ref", Rev: "5233fd2ba76a3accb5aaa999c00509a11fd0793c"}, + + // GitHub references. + "github:NixOS/nix": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix"}, + "github:NixOS/nix/v1.2.3": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Ref: "v1.2.3"}, + "github:NixOS/nix?ref=v1.2.3": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Ref: "v1.2.3"}, + "github:NixOS/nix?ref=5233fd2ba76a3accb5aaa999c00509a11fd0793c": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Ref: "5233fd2ba76a3accb5aaa999c00509a11fd0793c"}, + "github:NixOS/nix/main": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Ref: "main"}, + "github:NixOS/nix/main/5233fd2ba76a3accb5aaa999c00509a11fd0793c": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Ref: "main/5233fd2ba76a3accb5aaa999c00509a11fd0793c"}, + "github:NixOS/nix/5233fd2bb76a3accb5aaa999c00509a11fd0793z": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Ref: "5233fd2bb76a3accb5aaa999c00509a11fd0793z"}, + "github:NixOS/nix/5233fd2ba76a3accb5aaa999c00509a11fd0793c": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Rev: "5233fd2ba76a3accb5aaa999c00509a11fd0793c"}, + "github:NixOS/nix?rev=5233fd2ba76a3accb5aaa999c00509a11fd0793c": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Rev: "5233fd2ba76a3accb5aaa999c00509a11fd0793c"}, + "github:NixOS/nix?host=example.com": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Host: "example.com"}, + "github:NixOS/nix?host=example.com&dir=subdir": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Host: "example.com", Dir: "subdir"}, + "github:NixOS/nix?host=example.com&dir=subdir&lastModified=1734435836&narHash=sha256-kMBQ5PRiFLagltK0sH%2B08aiNt3zGERC2297iB6vrvlU%3D": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Host: "example.com", Dir: "subdir", NARHash: "sha256-kMBQ5PRiFLagltK0sH+08aiNt3zGERC2297iB6vrvlU="}, + + // The github type allows clone-style URLs. The username and + // host are ignored. + "github://git@github.com/NixOS/nix": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix"}, + "github://git@github.com/NixOS/nix/v1.2.3": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Ref: "v1.2.3"}, + "github://git@github.com/NixOS/nix?ref=v1.2.3": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Ref: "v1.2.3"}, + "github://git@github.com/NixOS/nix?ref=5233fd2ba76a3accb5aaa999c00509a11fd0793c": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Ref: "5233fd2ba76a3accb5aaa999c00509a11fd0793c"}, + "github://git@github.com/NixOS/nix?rev=5233fd2ba76a3accb5aaa999c00509a11fd0793c": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Rev: "5233fd2ba76a3accb5aaa999c00509a11fd0793c"}, + "github://git@github.com/NixOS/nix?host=example.com": {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Host: "example.com"}, + + // Git references. + "git://example.com/repo/flake": {Type: TypeGit, URL: "git://example.com/repo/flake"}, + "git+https://example.com/repo/flake": {Type: TypeGit, URL: "https://example.com/repo/flake"}, + "git+ssh://git@example.com/repo/flake": {Type: TypeGit, URL: "ssh://git@example.com/repo/flake"}, + "git:/repo/flake": {Type: TypeGit, URL: "git:/repo/flake"}, + "git+file:///repo/flake": {Type: TypeGit, URL: "file:///repo/flake"}, + "git://example.com/repo/flake?ref=unstable&rev=e486d8d40e626a20e06d792db8cc5ac5aba9a5b4&dir=subdir": {Type: TypeGit, URL: "git://example.com/repo/flake?dir=subdir", Ref: "unstable", Rev: "e486d8d40e626a20e06d792db8cc5ac5aba9a5b4", Dir: "subdir"}, + "git://example.com/repo/flake?ref=unstable&rev=e486d8d40e626a20e06d792db8cc5ac5aba9a5b4&dir=subdir&lastModified=1734435836&narHash=sha256-kMBQ5PRiFLagltK0sH%2B08aiNt3zGERC2297iB6vrvlU%3D": {Type: TypeGit, URL: "git://example.com/repo/flake?dir=subdir&lastModified=1734435836&narHash=sha256-kMBQ5PRiFLagltK0sH%2B08aiNt3zGERC2297iB6vrvlU%3D", Ref: "unstable", Rev: "e486d8d40e626a20e06d792db8cc5ac5aba9a5b4", Dir: "subdir"}, + + // Tarball references. + "tarball+http://example.com/flake": {Type: TypeTarball, URL: "http://example.com/flake"}, + "tarball+https://example.com/flake": {Type: TypeTarball, URL: "https://example.com/flake"}, + "tarball+file:///home/flake": {Type: TypeTarball, URL: "file:///home/flake"}, + + // Regular URLs have the tarball type if they have a known + // archive extension: + // .zip, .tar, .tgz, .tar.gz, .tar.xz, .tar.bz2 or .tar.zst + "http://example.com/flake.zip": {Type: TypeTarball, URL: "http://example.com/flake.zip"}, + "http://example.com/flake.tar": {Type: TypeTarball, URL: "http://example.com/flake.tar"}, + "http://example.com/flake.tgz": {Type: TypeTarball, URL: "http://example.com/flake.tgz"}, + "http://example.com/flake.tar.gz": {Type: TypeTarball, URL: "http://example.com/flake.tar.gz"}, + "http://example.com/flake.tar.xz": {Type: TypeTarball, URL: "http://example.com/flake.tar.xz"}, + "http://example.com/flake.tar.bz2": {Type: TypeTarball, URL: "http://example.com/flake.tar.bz2"}, + "http://example.com/flake.tar.zst": {Type: TypeTarball, URL: "http://example.com/flake.tar.zst"}, + "http://example.com/flake.tar?dir=subdir": {Type: TypeTarball, URL: "http://example.com/flake.tar?dir=subdir", Dir: "subdir"}, + "http://example.com/flake.tar?dir=subdir&lastModified=1734435836&narHash=sha256-kMBQ5PRiFLagltK0sH%2B08aiNt3zGERC2297iB6vrvlU%3D": {Type: TypeTarball, URL: "http://example.com/flake.tar?dir=subdir", Dir: "subdir", LastModified: 1734435836, NARHash: "sha256-kMBQ5PRiFLagltK0sH+08aiNt3zGERC2297iB6vrvlU="}, + "file:///flake.zip": {Type: TypeTarball, URL: "file:///flake.zip"}, + "file:///flake.tar": {Type: TypeTarball, URL: "file:///flake.tar"}, + "file:///flake.tgz": {Type: TypeTarball, URL: "file:///flake.tgz"}, + "file:///flake.tar.gz": {Type: TypeTarball, URL: "file:///flake.tar.gz"}, + "file:///flake.tar.xz": {Type: TypeTarball, URL: "file:///flake.tar.xz"}, + "file:///flake.tar.bz2": {Type: TypeTarball, URL: "file:///flake.tar.bz2"}, + "file:///flake.tar.zst": {Type: TypeTarball, URL: "file:///flake.tar.zst"}, + "file:///flake.tar?dir=subdir": {Type: TypeTarball, URL: "file:///flake.tar?dir=subdir", Dir: "subdir"}, + + // File URL references. + "file+file:///flake": {Type: TypeFile, URL: "file:///flake"}, + "file+http://example.com/flake": {Type: TypeFile, URL: "http://example.com/flake"}, + "file+http://example.com/flake.git": {Type: TypeFile, URL: "http://example.com/flake.git"}, + "file+http://example.com/flake.tar?dir=subdir": {Type: TypeFile, URL: "http://example.com/flake.tar?dir=subdir", Dir: "subdir"}, + + // Regular URLs have the file type if they don't have a known + // archive extension. + "http://example.com/flake": {Type: TypeFile, URL: "http://example.com/flake"}, + "http://example.com/flake.git": {Type: TypeFile, URL: "http://example.com/flake.git"}, + "http://example.com/flake?dir=subdir": {Type: TypeFile, URL: "http://example.com/flake?dir=subdir", Dir: "subdir"}, + } + for ref, want := range cases { + t.Run(ref, func(t *testing.T) { + got, err := ParseRef(ref) + if diff := cmp.Diff(want, got); diff != "" { + if err != nil { + t.Errorf("got error: %s", err) + } + t.Errorf("wrong flakeref (-want +got):\n%s", diff) + } + }) + } +} + +func TestParseFlakeRefError(t *testing.T) { + t.Run("EmptyString", func(t *testing.T) { + ref := "" + _, err := ParseRef(ref) + if err == nil { + t.Error("got nil error for bad flakeref:", ref) + } + }) + t.Run("InvalidURL", func(t *testing.T) { + ref := "://bad/url" + _, err := ParseRef(ref) + if err == nil { + t.Error("got nil error for bad flakeref:", ref) + } + }) + t.Run("InvalidURLEscape", func(t *testing.T) { + ref := "path:./relative/my%flake" + _, err := ParseRef(ref) + if err == nil { + t.Error("got nil error for bad flakeref:", ref) + } + }) + t.Run("UnsupportedURLScheme", func(t *testing.T) { + ref := "runx:mvdan/gofumpt@latest" + _, err := ParseRef(ref) + if err == nil { + t.Error("got nil error for bad flakeref:", ref) + } + }) + t.Run("PathLikeWith?#", func(t *testing.T) { + in := []string{ + "./invalid#path", + "./invalid?path", + "/invalid#path", + "/invalid?path", + "/#", + "/?", + } + for _, ref := range in { + _, err := ParseRef(ref) + if err == nil { + t.Error("got nil error for bad flakeref:", ref) + } + } + }) + t.Run("GitHubInvalidRefRevCombo", func(t *testing.T) { + in := []string{ + "github:NixOS/nix?ref=v1.2.3&rev=5233fd2ba76a3accb5aaa999c00509a11fd0793c", + "github:NixOS/nix/v1.2.3?ref=v4.5.6", + "github:NixOS/nix/5233fd2ba76a3accb5aaa999c00509a11fd0793c?rev=e486d8d40e626a20e06d792db8cc5ac5aba9a5b4", + "github:NixOS/nix/5233fd2ba76a3accb5aaa999c00509a11fd0793c?ref=v1.2.3", + } + for _, ref := range in { + _, err := ParseRef(ref) + if err == nil { + t.Error("got nil error for bad flakeref:", ref) + } + } + }) + t.Run("URLFragment", func(t *testing.T) { + ref := "https://github.com/NixOS/patchelf/archive/master.tar.gz#patchelf" + _, err := ParseRef(ref) + if err == nil { + t.Error("got nil error for flakeref with fragment:", ref) + } + }) +} + func TestFlakeRefString(t *testing.T) { cases := map[Ref]string{ {}: "", @@ -44,26 +243,26 @@ func TestFlakeRefString(t *testing.T) { {Type: TypeGitHub, Owner: "NixOS", Repo: "nix", Dir: "sub/dir", Host: "example.com"}: "github:NixOS/nix?dir=sub%2Fdir&host=example.com", // Git references. - {Type: TypeGit, Host: "example.com", Owner: "repo", Repo: "flake"}: "git://example.com/repo/flake", - {Type: TypeHttps, Host: "example.com", Owner: "repo", Repo: "flake"}: "git+https://example.com/repo/flake", - {Type: TypeSSH, Host: "example.com", Owner: "repo", Repo: "flake"}: "git+ssh://git@example.com/repo/flake", - {Type: TypeGit, Owner: "repo", Repo: "flake"}: "git:/repo/flake", - {Type: TypeFile, Owner: "repo", Repo: "flake"}: "git+file:///repo/flake", - {Type: TypeSSH, Host: "example.com", Owner: "repo", Repo: "flake", Ref: "my/ref", Rev: "e486d8d40e626a20e06d792db8cc5ac5aba9a5b4"}: "git+ssh://git@example.com/repo/flake?ref=my%2Fref&rev=e486d8d40e626a20e06d792db8cc5ac5aba9a5b4", - {Type: TypeSSH, Host: "example.com", Owner: "repo", Repo: "flake", Ref: "my/ref", Rev: "e486d8d40e626a20e06d792db8cc5ac5aba9a5b4", Dir: "sub/dir"}: "git+ssh://git@example.com/repo/flake?dir=sub%2Fdir&ref=my%2Fref&rev=e486d8d40e626a20e06d792db8cc5ac5aba9a5b4", - {Type: TypeGit, Owner: "repo", Repo: "flake", Ref: "my/ref", Rev: "e486d8d40e626a20e06d792db8cc5ac5aba9a5b4", Dir: "sub/dir"}: "", // "git:/repo/flake?dir=sub%2Fdir&ref=my%2Fref&rev=e486d8d40e626a20e06d792db8cc5ac5aba9a5b4", // how is this supposed to be a valid URL? There isn't a hostname in it... + {Type: TypeGit, URL: "git://example.com/repo/flake"}: "git://example.com/repo/flake", + {Type: TypeGit, URL: "https://example.com/repo/flake"}: "git+https://example.com/repo/flake", + {Type: TypeGit, URL: "ssh://git@example.com/repo/flake"}: "git+ssh://git@example.com/repo/flake", + {Type: TypeGit, URL: "git:/repo/flake"}: "git:/repo/flake", + {Type: TypeGit, URL: "file:///repo/flake"}: "git+file:///repo/flake", + {Type: TypeGit, URL: "ssh://git@example.com/repo/flake", Ref: "my/ref", Rev: "e486d8d40e626a20e06d792db8cc5ac5aba9a5b4"}: "git+ssh://git@example.com/repo/flake?ref=my%2Fref&rev=e486d8d40e626a20e06d792db8cc5ac5aba9a5b4", + {Type: TypeGit, URL: "ssh://git@example.com/repo/flake?dir=sub%2Fdir", Ref: "my/ref", Rev: "e486d8d40e626a20e06d792db8cc5ac5aba9a5b4", Dir: "sub/dir"}: "git+ssh://git@example.com/repo/flake?dir=sub%2Fdir&ref=my%2Fref&rev=e486d8d40e626a20e06d792db8cc5ac5aba9a5b4", + {Type: TypeGit, URL: "git:repo/flake?dir=sub%2Fdir", Ref: "my/ref", Rev: "e486d8d40e626a20e06d792db8cc5ac5aba9a5b4", Dir: "sub/dir"}: "git:repo/flake?dir=sub%2Fdir&ref=my%2Fref&rev=e486d8d40e626a20e06d792db8cc5ac5aba9a5b4", // Tarball references. - {Type: TypeTarball, Host: "example.com", Owner: "flake", URL: "http://example.com/flake"}: "tarball+http://example.com/flake", - {Type: TypeTarball, Host: "example.com", Owner: "flake", URL: "https://example.com/flake"}: "tarball+https://example.com/flake", - {Type: TypeTarball, Host: "example.com", Owner: "flake", URL: "https://example.com/flake", Dir: "sub/dir"}: "tarball+https://example.com/flake?dir=sub%2Fdir", - {Type: TypeTarball, URL: "file:///home/flake"}: "tarball+file:///home/flake", + {Type: TypeTarball, URL: "http://example.com/flake"}: "tarball+http://example.com/flake", + {Type: TypeTarball, URL: "https://example.com/flake"}: "tarball+https://example.com/flake", + {Type: TypeTarball, URL: "https://example.com/flake", Dir: "sub/dir"}: "tarball+https://example.com/flake?dir=sub%2Fdir", + {Type: TypeTarball, URL: "file:///home/flake"}: "tarball+file:///home/flake", // File URL references. - {Type: TypePath, URL: "file:///flake"}: "file+file:///flake", - {Type: TypePath, URL: "http://example.com/flake"}: "file+http://example.com/flake", - {Type: TypePath, URL: "http://example.com/flake.git"}: "file+http://example.com/flake.git", - {Type: TypePath, URL: "http://example.com/flake.tar?dir=sub%2Fdir", Dir: "sub/dir"}: "file+http://example.com/flake.tar?dir=sub%2Fdir", + {Type: TypeFile, URL: "file:///flake"}: "file+file:///flake", + {Type: TypeFile, URL: "http://example.com/flake"}: "file+http://example.com/flake", + {Type: TypeFile, URL: "http://example.com/flake.git"}: "file+http://example.com/flake.git", + {Type: TypeFile, URL: "http://example.com/flake.tar?dir=sub%2Fdir", Dir: "sub/dir"}: "file+http://example.com/flake.tar?dir=sub%2Fdir", } for ref, want := range cases { @@ -177,7 +376,7 @@ func TestFlakeInstallableString(t *testing.T) { {AttrPath: "app", Outputs: "%2F", Ref: Ref{Type: TypeIndirect, ID: "nixpkgs"}}: "flake:nixpkgs#app^%2F", // Missing or invalid fields. - {AttrPath: "app", Ref: Ref{Type: TypePath, URL: ""}}: "", + {AttrPath: "app", Ref: Ref{Type: TypeFile, URL: ""}}: "", {AttrPath: "app", Ref: Ref{Type: TypeGit, URL: ""}}: "", {AttrPath: "app", Ref: Ref{Type: TypeGitHub, Owner: ""}}: "", {AttrPath: "app", Ref: Ref{Type: TypeIndirect, ID: ""}}: "", From c93d43e6d2992023f264d0a763b2ba43140330df Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Mon, 23 Dec 2024 10:52:23 -0500 Subject: [PATCH 38/42] Update update.go Signed-off-by: Brandon Marlowe --- internal/plugin/update.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/plugin/update.go b/internal/plugin/update.go index 6778298a5e1..a3e824db385 100644 --- a/internal/plugin/update.go +++ b/internal/plugin/update.go @@ -1,6 +1,5 @@ package plugin func Update() error { - gitCache.Clear() - return nil + return gitCache.Clear() } From fce2ea94d7d2c5e891f35a90f9eb3eb88dfcc906 Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Tue, 24 Dec 2024 14:19:21 -0500 Subject: [PATCH 39/42] removed old updateme comments --- internal/devconfig/config.go | 1 - internal/plugin/local.go | 1 - 2 files changed, 2 deletions(-) diff --git a/internal/devconfig/config.go b/internal/devconfig/config.go index 48270045776..7cb4aa2ff90 100644 --- a/internal/devconfig/config.go +++ b/internal/devconfig/config.go @@ -237,7 +237,6 @@ func (c *Config) loadRecursive( ) error { included := make([]*Config, 0, len(c.Root.Include)) - // TODO UPDATEME for _, ref := range c.Root.Include { switch ref.Type { case flake.TypeGitHub, flake.TypeGitLab, flake.TypeBitBucket: diff --git a/internal/plugin/local.go b/internal/plugin/local.go index 3ab9bd24e76..c6f4a79c185 100644 --- a/internal/plugin/local.go +++ b/internal/plugin/local.go @@ -16,7 +16,6 @@ type LocalPlugin struct { pluginDir string } -// TODO UPDATEME func newLocalPlugin(ref flake.Ref, pluginDir string) (*LocalPlugin, error) { plugin := &LocalPlugin{ref: ref, pluginDir: pluginDir} name, err := getPluginNameFromContent(plugin) From d4ab61be1e18b555f5ce45e4d81f3c2e4b74a10d Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Sat, 11 Jan 2025 18:02:30 -0500 Subject: [PATCH 40/42] update version --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index bc183affa02..43e09908375 100644 --- a/flake.nix +++ b/flake.nix @@ -11,7 +11,7 @@ let pkgs = nixpkgs.legacyPackages.${system}; - lastTag = "0.13.7"; + lastTag = "0.14.0"; revision = if (self ? shortRev) From 18d23de292ce7d15f7b9cf6d6aa8e4606cbd4b2d Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Sat, 1 Feb 2025 20:33:33 -0500 Subject: [PATCH 41/42] updated include section examples --- devbox.json | 24 ++++++------------------ devbox.lock | 3 +++ 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/devbox.json b/devbox.json index db401fdeba9..592f8b598f4 100644 --- a/devbox.json +++ b/devbox.json @@ -8,33 +8,21 @@ "gopls": "latest", }, "include": [ + // examples: //{ - // "type": "ssh", - // "host": "gitlab.com", - // "owner": "astro-tec", - // "repo": "devbox-plugin-test", - // "dir": "plugin", - //}, - //, - //{ - // "type": "path", - // "path": "/home/bmarlowe/Development/bitbucket/devbox-plugin-test/plugin" + // "type": "github", + // "owner": "jetify-com", + // "repo": "devbox-plugins", + // "dir": "tmux" //} //, //{ // "type": "github", // "owner": "jetify-com", // "repo": "devbox-plugins", - // "dir": "tmux" + // "dir": "mongodb" //} //, - { - "type": "github", - "owner": "jetify-com", - "repo": "devbox-plugins", - "dir": "mongodb" - } - //, //{ // "type": "builtin", // "path": "nginx" diff --git a/devbox.lock b/devbox.lock index bb568dca9b7..e864b5c72d2 100644 --- a/devbox.lock +++ b/devbox.lock @@ -1,6 +1,9 @@ { "lockfile_version": "1", "packages": { + "github:NixOS/nixpkgs/nixpkgs-unstable": { + "resolved": "github:NixOS/nixpkgs/9189ac18287c599860e878e905da550aa6dec1cd?lastModified=1738297584&narHash=sha256-AYvaFBzt8dU0fcSK2jKD0Vg23K2eIRxfsVXIPCW9a0E%3D" + }, "go@latest": { "last_modified": "2024-08-14T11:41:26Z", "resolved": "github:NixOS/nixpkgs/0cb2fd7c59fed0cd82ef858cbcbdb552b9a33465#go_1_23", From eee6c039bfdb843730bba861ffe521f952cd66b4 Mon Sep 17 00:00:00 2001 From: Brandon Marlowe Date: Fri, 14 Mar 2025 18:47:46 -0400 Subject: [PATCH 42/42] update devbox.lock --- devbox.lock | 51 ++++++++++++++++++++++++--------------------------- 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/devbox.lock b/devbox.lock index e864b5c72d2..5e571f9a532 100644 --- a/devbox.lock +++ b/devbox.lock @@ -1,108 +1,105 @@ { "lockfile_version": "1", "packages": { - "github:NixOS/nixpkgs/nixpkgs-unstable": { - "resolved": "github:NixOS/nixpkgs/9189ac18287c599860e878e905da550aa6dec1cd?lastModified=1738297584&narHash=sha256-AYvaFBzt8dU0fcSK2jKD0Vg23K2eIRxfsVXIPCW9a0E%3D" - }, "go@latest": { - "last_modified": "2024-08-14T11:41:26Z", - "resolved": "github:NixOS/nixpkgs/0cb2fd7c59fed0cd82ef858cbcbdb552b9a33465#go_1_23", + "last_modified": "2025-03-11T17:52:14Z", + "resolved": "github:NixOS/nixpkgs/0d534853a55b5d02a4ababa1d71921ce8f0aee4c#go", "source": "devbox-search", - "version": "1.23.0", + "version": "1.24.1", "systems": { "aarch64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/x097y3rdlvw6ax1id52xx2didgzxjg34-go-1.23.0", + "path": "/nix/store/ja4jxx60lh1qfqfl4z4p2rff56ia1c3c-go-1.24.1", "default": true } ], - "store_path": "/nix/store/x097y3rdlvw6ax1id52xx2didgzxjg34-go-1.23.0" + "store_path": "/nix/store/ja4jxx60lh1qfqfl4z4p2rff56ia1c3c-go-1.24.1" }, "aarch64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/znns5qk7fbhjb89bqhabxjxpgiq2ag1a-go-1.23.0", + "path": "/nix/store/8ply43gnxk1xwichr81mpgbjcd9a1y5w-go-1.24.1", "default": true } ], - "store_path": "/nix/store/znns5qk7fbhjb89bqhabxjxpgiq2ag1a-go-1.23.0" + "store_path": "/nix/store/8ply43gnxk1xwichr81mpgbjcd9a1y5w-go-1.24.1" }, "x86_64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/psym5ggdincwihwd81kssc4gbw1k3p3h-go-1.23.0", + "path": "/nix/store/87yxrfx5lh78bdz393i33cr5z23x06q4-go-1.24.1", "default": true } ], - "store_path": "/nix/store/psym5ggdincwihwd81kssc4gbw1k3p3h-go-1.23.0" + "store_path": "/nix/store/87yxrfx5lh78bdz393i33cr5z23x06q4-go-1.24.1" }, "x86_64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/0k144pnsvy0wa3dcl1y3df7d4zskylc4-go-1.23.0", + "path": "/nix/store/cfjhl0kn7xc65466pha9fkrvigw3g72n-go-1.24.1", "default": true } ], - "store_path": "/nix/store/0k144pnsvy0wa3dcl1y3df7d4zskylc4-go-1.23.0" + "store_path": "/nix/store/cfjhl0kn7xc65466pha9fkrvigw3g72n-go-1.24.1" } } }, "gopls@latest": { - "last_modified": "2024-09-12T02:28:40Z", - "resolved": "github:NixOS/nixpkgs/111ed8812c10d7dc3017de46cbf509600c93f551#gopls", + "last_modified": "2025-03-11T17:52:14Z", + "resolved": "github:NixOS/nixpkgs/0d534853a55b5d02a4ababa1d71921ce8f0aee4c#gopls", "source": "devbox-search", - "version": "0.16.2", + "version": "0.18.1", "systems": { "aarch64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/62algcwigrdx7ak14dwppjdsdy9aziva-gopls-0.16.2", + "path": "/nix/store/wphphjsknh3n3rnfazbcskvb077chb3n-gopls-0.18.1", "default": true } ], - "store_path": "/nix/store/62algcwigrdx7ak14dwppjdsdy9aziva-gopls-0.16.2" + "store_path": "/nix/store/wphphjsknh3n3rnfazbcskvb077chb3n-gopls-0.18.1" }, "aarch64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/l4xb6niqplxri2ziwajjm87apdwgld21-gopls-0.16.2", + "path": "/nix/store/arsvk8nxa38j5hr7k7i7bybrcz2ln5cd-gopls-0.18.1", "default": true } ], - "store_path": "/nix/store/l4xb6niqplxri2ziwajjm87apdwgld21-gopls-0.16.2" + "store_path": "/nix/store/arsvk8nxa38j5hr7k7i7bybrcz2ln5cd-gopls-0.18.1" }, "x86_64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/8ffc2jqw59mkhzj7s06rnwgh7i16qric-gopls-0.16.2", + "path": "/nix/store/ankj7f3zma8dkhzlpvqxh4c82wyhl9ap-gopls-0.18.1", "default": true } ], - "store_path": "/nix/store/8ffc2jqw59mkhzj7s06rnwgh7i16qric-gopls-0.16.2" + "store_path": "/nix/store/ankj7f3zma8dkhzlpvqxh4c82wyhl9ap-gopls-0.18.1" }, "x86_64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/6nysag23irns7ldbsdrjwc1ckap1hqm6-gopls-0.16.2", + "path": "/nix/store/yw5i9bhlilril5y89x16i5fnmj8qm7np-gopls-0.18.1", "default": true } ], - "store_path": "/nix/store/6nysag23irns7ldbsdrjwc1ckap1hqm6-gopls-0.16.2" + "store_path": "/nix/store/yw5i9bhlilril5y89x16i5fnmj8qm7np-gopls-0.18.1" } } }, "runx:golangci/golangci-lint@latest": { - "resolved": "golangci/golangci-lint@v1.60.2", - "version": "v1.60.2" + "resolved": "golangci/golangci-lint@v1.64.7", + "version": "v1.64.7" }, "runx:mvdan/gofumpt@latest": { "resolved": "mvdan/gofumpt@v0.7.0",