diff --git a/cmd/image-builder/repos.go b/cmd/image-builder/repos.go index 36edc20c..2c888930 100644 --- a/cmd/image-builder/repos.go +++ b/cmd/image-builder/repos.go @@ -7,9 +7,11 @@ import ( "os" "path/filepath" - "github.com/osbuild/images/data/repositories" + repos "github.com/osbuild/images/data/repositories" "github.com/osbuild/images/pkg/reporegistry" "github.com/osbuild/images/pkg/rpmmd" + + "github.com/osbuild/image-builder-cli/internal/dnfparse" ) // defaultRepoDirs contains the default search paths to look for @@ -38,6 +40,40 @@ func parseRepoURLs(repoURLs []string, what string) ([]rpmmd.RepoConfig, error) { return nil, fmt.Errorf(`scheme missing in %q, please prefix with e.g. file:// or https://`, repoURL) } + // file:// path to a .repo file: parse as DNF repo file and append each repo + if baseURL.Scheme == "file" && baseURL.Host == "" { + path := baseURL.Path + if path == "" { + return nil, fmt.Errorf("file URL has no path: %q", repoURL) + } + info, err := os.Stat(path) + if err == nil && info.Mode().IsRegular() { + parsed, err := dnfparse.ParseFile(path) + if err != nil { + return nil, fmt.Errorf("parse repo file %q: %w", repoURL, err) + } + for _, repo := range parsed { + name := repo.Name + if name == "" { + name = repo.Id + } + repoConf = append(repoConf, rpmmd.RepoConfig{ + Id: fmt.Sprintf("%s-repo-%d-%s", what, i, repo.Id), + Name: name, + BaseURLs: repo.BaseURLs, + CheckGPG: repo.CheckGPG, + CheckRepoGPG: repo.CheckRepoGPG, + GPGKeys: repo.GPGKeys, + SSLCACert: repo.SSLCACert, + SSLClientCert: repo.SSLClientCert, + SSLClientKey: repo.SSLClientKey, + IgnoreSSL: repo.IgnoreSSL, + }) + } + continue + } + } + // TODO: to support gpg checking we will need to add signing keys. // We will eventually add support for our own "repo.json" format // which is rich enough to contain gpg keys (and more). diff --git a/cmd/image-builder/repos_test.go b/cmd/image-builder/repos_test.go index 23bfc191..824320c4 100644 --- a/cmd/image-builder/repos_test.go +++ b/cmd/image-builder/repos_test.go @@ -45,6 +45,59 @@ func TestParseExtraRepoSad(t *testing.T) { assert.EqualError(t, err, `scheme missing in "/just/a/path", please prefix with e.g. file:// or https://`) } +func TestParseRepoURLsFileRepo(t *testing.T) { + dir := t.TempDir() + repoPath := filepath.Join(dir, "test.repo") + err := os.WriteFile(repoPath, []byte(`[fedora] +name=Fedora +baseurl=https://download.fedoraproject.org/pub/fedora/linux/releases/42/Everything/x86_64/os/ +gpgcheck=1 +repo_gpgcheck=0 +`), 0644) + require.NoError(t, err) + + cfg, err := parseRepoURLs([]string{"file://" + repoPath}, "extra") + require.NoError(t, err) + require.Len(t, cfg, 1) + assert.Equal(t, "extra-repo-0-fedora", cfg[0].Id) + assert.Equal(t, "Fedora", cfg[0].Name) + assert.Equal(t, []string{"https://download.fedoraproject.org/pub/fedora/linux/releases/42/Everything/x86_64/os/"}, cfg[0].BaseURLs) + assert.True(t, cfg[0].CheckGPG != nil && *cfg[0].CheckGPG) + assert.True(t, cfg[0].CheckRepoGPG != nil && !*cfg[0].CheckRepoGPG) +} + +func TestParseRepoURLsFileRepoWithGPGKey(t *testing.T) { + dir := t.TempDir() + keyPath := filepath.Join(dir, "RPM-GPG-KEY-test") + keyContent := []byte("-----BEGIN PGP PUBLIC KEY BLOCK-----\nxyz\n-----END PGP PUBLIC KEY BLOCK-----") + require.NoError(t, os.WriteFile(keyPath, keyContent, 0644)) + + repoPath := filepath.Join(dir, "test.repo") + err := os.WriteFile(repoPath, []byte(`[myrepo] +name=My Repo +baseurl=https://example.com/repo +gpgcheck=1 +gpgkey=file://`+keyPath+` +`), 0644) + require.NoError(t, err) + + cfg, err := parseRepoURLs([]string{"file://" + repoPath}, "extra") + require.NoError(t, err) + require.Len(t, cfg, 1) + require.Len(t, cfg[0].GPGKeys, 1) + assert.Equal(t, string(keyContent), cfg[0].GPGKeys[0]) +} + +func TestParseRepoURLsFileDirectoryTreatedAsURL(t *testing.T) { + // file:// to a directory is not a .repo file: treat as single base-URL repo + dir := t.TempDir() + cfg, err := parseRepoURLs([]string{"file://" + dir}, "extra") + require.NoError(t, err) + require.Len(t, cfg, 1) + assert.Equal(t, "extra-repo-0", cfg[0].Id) + assert.Equal(t, []string{"file://" + dir}, cfg[0].BaseURLs) +} + func TestNewRepoRegistryImplSmoke(t *testing.T) { registry, err := newRepoRegistryImpl("", nil) require.NoError(t, err) diff --git a/internal/dnfparse/dnfparse.go b/internal/dnfparse/dnfparse.go new file mode 100644 index 00000000..77040a93 --- /dev/null +++ b/internal/dnfparse/dnfparse.go @@ -0,0 +1,146 @@ +// Package dnfparse provides a trivial parser for DNF/Yum repository files (.repo). +// It supports: id, name, baseurl, gpgcheck, repo_gpgcheck, gpgkey, sslcacert, sslclientcert, sslclientkey, sslverify. +package dnfparse + +import ( + "bufio" + "fmt" + "io" + "net/url" + "os" + "strconv" + "strings" +) + +// Repo holds the parsed repository configuration for a single [section]. +type Repo struct { + Id string + Name string + BaseURLs []string + CheckGPG *bool + CheckRepoGPG *bool + GPGKeys []string + SSLCACert string + SSLClientCert string + SSLClientKey string + // IgnoreSSL corresponds to sslverify=0 (true = ignore SSL verification). sslverify=1 means false. + IgnoreSSL *bool +} + +// ParseFile reads a DNF repo file and returns all repository sections +// that have at least one baseurl. Sections without baseurl are skipped. +func ParseFile(path string) ([]Repo, error) { + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("open repo file: %w", err) + } + defer f.Close() + return Parse(f, path) +} + +// Parse reads from r (e.g. an open .repo file) and returns all repository +// sections that have at least one baseurl. The name argument is used in errors. +func Parse(r io.Reader, name string) ([]Repo, error) { + var repos []Repo + var cur *Repo + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + // Comment or empty + if line == "" || line[0] == '#' || line[0] == ';' { + continue + } + if len(line) >= 2 && line[0] == '[' && line[len(line)-1] == ']' { + // Flush current section if it has baseurls + if cur != nil && len(cur.BaseURLs) > 0 { + repos = append(repos, *cur) + } + id := strings.TrimSpace(line[1 : len(line)-1]) + cur = &Repo{Id: id} + continue + } + eq := strings.Index(line, "=") + if eq <= 0 || cur == nil { + continue + } + key := strings.TrimSpace(strings.ToLower(line[:eq])) + val := strings.TrimSpace(line[eq+1:]) + // Remove optional quotes around value + if len(val) >= 2 && (val[0] == '"' && val[len(val)-1] == '"' || val[0] == '\'' && val[len(val)-1] == '\'') { + val = val[1 : len(val)-1] + } + switch key { + case "name": + cur.Name = val + case "baseurl": + cur.BaseURLs = append(cur.BaseURLs, val) + case "gpgcheck": + b := parseBool(val) + cur.CheckGPG = &b + case "repo_gpgcheck": + b := parseBool(val) + cur.CheckRepoGPG = &b + case "gpgkey": + cur.GPGKeys = append(cur.GPGKeys, val) + case "sslcacert": + cur.SSLCACert = val + case "sslclientcert": + cur.SSLClientCert = val + case "sslclientkey": + cur.SSLClientKey = val + case "sslverify": + // sslverify=1 -> verify (IgnoreSSL=false), sslverify=0 -> ignore (IgnoreSSL=true) + verify := parseBool(val) + ignoreSSL := !verify + cur.IgnoreSSL = &ignoreSSL + } + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("read %s: %w", name, err) + } + if cur != nil && len(cur.BaseURLs) > 0 { + repos = append(repos, *cur) + } + return repos, nil +} + +func parseBool(s string) bool { + s = strings.TrimSpace(strings.ToLower(s)) + if s == "1" || s == "yes" || s == "true" { + return true + } + n, err := strconv.ParseInt(s, 10, 64) + if err == nil && n != 0 { + return true + } + return false +} + +// GPGKeyContents resolves GPG key URIs and returns the concatenated contents +// of the key files. Only file:// URIs are supported; any other scheme returns an error. +func (r *Repo) GPGKeyContents() ([]byte, error) { + var out []byte + for i, uri := range r.GPGKeys { + u, err := url.Parse(uri) + if err != nil { + return nil, fmt.Errorf("gpgkey[%d] invalid URI %q: %w", i, uri, err) + } + if u.Scheme != "file" { + return nil, fmt.Errorf("gpgkey[%d] only file:// URIs are supported, got %q", i, uri) + } + path := u.Path + if path == "" { + return nil, fmt.Errorf("gpgkey[%d] file URI has no path: %q", i, uri) + } + + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("gpgkey[%d] read %q: %w", i, path, err) + } + if len(out) > 0 { + out = append(out, '\n') + } + out = append(out, data...) + } + return out, nil +} diff --git a/internal/dnfparse/dnfparse_test.go b/internal/dnfparse/dnfparse_test.go new file mode 100644 index 00000000..000c230a --- /dev/null +++ b/internal/dnfparse/dnfparse_test.go @@ -0,0 +1,313 @@ +package dnfparse + +import ( + "fmt" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var ( + trueVal = true + falseVal = false +) + +// wantRepo describes expected Repo fields for table-driven tests. All booleans are pointers to match RepoConfig. +type wantRepo struct { + id string + name string + baseURLs []string + checkGPG *bool + checkRepoGPG *bool + ignoreSSL *bool + gpgKeys []string + sslCACert string + sslClientCert string + sslClientKey string +} + +func assertOptionalBool(t *testing.T, want, got *bool, label string) { + t.Helper() + if want == nil { + assert.Nil(t, got, "%s (expected nil)", label) + return + } + require.NotNil(t, got, "%s", label) + assert.Equal(t, *want, *got, "%s", label) +} + +func TestParse(t *testing.T) { + tests := []struct { + name string + input string + want []wantRepo + }{ + { + name: "fedora", + input: ` +[main] +# global defaults ignored + +[fedora] +name=Fedora $releasever - $basearch +baseurl=https://download.fedoraproject.org/pub/fedora/linux/releases/$releasever/Everything/$basearch/os/ +gpgcheck=1 +repo_gpgcheck=1 + +[updates] +name=Fedora $releasever - $basearch - Updates +baseurl=https://download.fedoraproject.org/pub/fedora/linux/updates/$releasever/Everything/$basearch/ +gpgcheck=1 +repo_gpgcheck=0 +`, + want: []wantRepo{ + { + id: "fedora", + name: "Fedora $releasever - $basearch", + baseURLs: []string{"https://download.fedoraproject.org/pub/fedora/linux/releases/$releasever/Everything/$basearch/os/"}, + checkGPG: &trueVal, + checkRepoGPG: &trueVal, + ignoreSSL: nil, + gpgKeys: nil, + }, + { + id: "updates", + name: "Fedora $releasever - $basearch - Updates", + baseURLs: []string{"https://download.fedoraproject.org/pub/fedora/linux/updates/$releasever/Everything/$basearch/"}, + checkGPG: &trueVal, + checkRepoGPG: &falseVal, + ignoreSSL: nil, + gpgKeys: nil, + }, + }, + }, + { + name: "rhel", + input: ` +# Managed by (rhsm) subscription-manager + +[rhel-10-for-x86_64-baseos-rpms] +name = Red Hat Enterprise Linux 10 for x86_64 - BaseOS (RPMs) +baseurl = https://satellite.example.com/pulp/content/Default_Organization/Library/content/dist/rhel10/$releasever/x86_64/baseos/os +enabled = 1 +gpgcheck = 1 +gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release +sslverify = 1 +sslcacert = /etc/rhsm/ca/katello-server-ca.pem +sslclientkey = /etc/pki/entitlement/3619656444745875922-key.pem +sslclientcert = /etc/pki/entitlement/3619656444745875922.pem +metadata_expire = 1 +enabled_metadata = 1 + +[rhel-10-for-x86_64-appstream-rpms] +name = Red Hat Enterprise Linux 10 for x86_64 - AppStream (RPMs) +baseurl = https://satellite.example.com/pulp/content/Default_Organization/Library/content/dist/rhel10/$releasever/x86_64/appstream/os +enabled = 1 +gpgcheck = 1 +gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release +sslverify = 1 +sslcacert = /etc/rhsm/ca/katello-server-ca.pem +sslclientkey = /etc/pki/entitlement/3619656444745875922-key.pem +sslclientcert = /etc/pki/entitlement/3619656444745875922.pem +metadata_expire = 1 +enabled_metadata = 1`, + want: []wantRepo{ + { + id: "rhel-10-for-x86_64-baseos-rpms", + name: "Red Hat Enterprise Linux 10 for x86_64 - BaseOS (RPMs)", + baseURLs: []string{"https://satellite.example.com/pulp/content/Default_Organization/Library/content/dist/rhel10/$releasever/x86_64/baseos/os"}, + checkGPG: &trueVal, + checkRepoGPG: nil, + ignoreSSL: &falseVal, + gpgKeys: []string{"file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release"}, + sslCACert: "/etc/rhsm/ca/katello-server-ca.pem", + sslClientCert: "/etc/pki/entitlement/3619656444745875922.pem", + sslClientKey: "/etc/pki/entitlement/3619656444745875922-key.pem", + }, + { + id: "rhel-10-for-x86_64-appstream-rpms", + name: "Red Hat Enterprise Linux 10 for x86_64 - AppStream (RPMs)", + baseURLs: []string{"https://satellite.example.com/pulp/content/Default_Organization/Library/content/dist/rhel10/$releasever/x86_64/appstream/os"}, + checkGPG: &trueVal, + checkRepoGPG: nil, + ignoreSSL: &falseVal, + gpgKeys: []string{"file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release"}, + sslCACert: "/etc/rhsm/ca/katello-server-ca.pem", + sslClientCert: "/etc/pki/entitlement/3619656444745875922.pem", + sslClientKey: "/etc/pki/entitlement/3619656444745875922-key.pem", + }, + }, + }, + { + name: "multiple baseurls", + input: ` +[myrepo] +name=My Repo +baseurl=https://one.com/repo +baseurl=https://two.com/repo +gpgcheck=0 +`, + want: []wantRepo{ + { + id: "myrepo", + name: "My Repo", + baseURLs: []string{"https://one.com/repo", "https://two.com/repo"}, + checkGPG: &falseVal, + checkRepoGPG: nil, + ignoreSSL: nil, + gpgKeys: nil, + }, + }, + }, + { + name: "sslverify=0", + input: ` +[insecure] +name=Insecure Repo +baseurl=https://insecure.example.com/repo +sslverify=0 +`, + want: []wantRepo{ + { + id: "insecure", + name: "Insecure Repo", + baseURLs: []string{"https://insecure.example.com/repo"}, + checkGPG: nil, + checkRepoGPG: nil, + ignoreSSL: &trueVal, + gpgKeys: nil, + }, + }, + }, + { + name: "skips section without baseurl", + input: ` +[main] +name=Main config + +[hasbase] +name=Has baseurl +baseurl=https://example.com/repo +`, + want: []wantRepo{ + { + id: "hasbase", + name: "Has baseurl", + baseURLs: []string{"https://example.com/repo"}, + checkGPG: nil, + checkRepoGPG: nil, + ignoreSSL: nil, + gpgKeys: nil, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repos, err := Parse(strings.NewReader(tt.input), "test.repo") + require.NoError(t, err) + require.Len(t, repos, len(tt.want), "number of repos") + for i, w := range tt.want { + r := repos[i] + assert.Equal(t, w.id, r.Id, "repo[%d].Id", i) + assert.Equal(t, w.name, r.Name, "repo[%d].Name", i) + assert.Equal(t, w.baseURLs, r.BaseURLs, "repo[%d].BaseURLs", i) + assertOptionalBool(t, w.checkGPG, r.CheckGPG, fmt.Sprintf("repo[%d].CheckGPG", i)) + assertOptionalBool(t, w.checkRepoGPG, r.CheckRepoGPG, fmt.Sprintf("repo[%d].CheckRepoGPG", i)) + assertOptionalBool(t, w.ignoreSSL, r.IgnoreSSL, fmt.Sprintf("repo[%d].IgnoreSSL", i)) + if w.gpgKeys != nil { + assert.Equal(t, w.gpgKeys, r.GPGKeys, "repo[%d].GPGKeys", i) + } + assert.Equal(t, w.sslCACert, r.SSLCACert, "repo[%d].SSLCACert", i) + assert.Equal(t, w.sslClientCert, r.SSLClientCert, "repo[%d].SSLClientCert", i) + assert.Equal(t, w.sslClientKey, r.SSLClientKey, "repo[%d].SSLClientKey", i) + } + }) + } +} + +func TestParseBool(t *testing.T) { + tests := []struct { + name string + input string + want bool + }{ + {"1", "1", true}, + {"yes", "yes", true}, + {"true", "true", true}, + {"0", "0", false}, + {"no", "no", false}, + {"false", "false", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := parseBool(tt.input) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestGPGKeyContents(t *testing.T) { + tests := []struct { + name string + repo func(t *testing.T) *Repo + wantContent []byte + wantErr string + }{ + { + name: "single file", + repo: func(t *testing.T) *Repo { + dir := t.TempDir() + path := dir + "/RPM-GPG-KEY-test" + content := []byte("-----BEGIN PGP PUBLIC KEY BLOCK-----\nxyz\n-----END PGP PUBLIC KEY BLOCK-----") + require.NoError(t, os.WriteFile(path, content, 0644)) + return &Repo{GPGKeys: []string{"file://" + path}} + }, + wantContent: []byte("-----BEGIN PGP PUBLIC KEY BLOCK-----\nxyz\n-----END PGP PUBLIC KEY BLOCK-----"), + wantErr: "", + }, + { + name: "multiple files", + repo: func(t *testing.T) *Repo { + dir := t.TempDir() + k1, k2 := dir+"/key1", dir+"/key2" + require.NoError(t, os.WriteFile(k1, []byte("key1"), 0644)) + require.NoError(t, os.WriteFile(k2, []byte("key2"), 0644)) + return &Repo{GPGKeys: []string{"file://" + k1, "file://" + k2}} + }, + wantContent: []byte("key1\nkey2"), + wantErr: "", + }, + { + name: "non-file URI", + repo: func(t *testing.T) *Repo { + return &Repo{GPGKeys: []string{"https://example.com/key.asc"}} + }, + wantErr: "only file:// URIs are supported", + }, + { + name: "empty GPGKeys", + repo: func(t *testing.T) *Repo { + return &Repo{GPGKeys: nil} + }, + wantContent: nil, + wantErr: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repo := tt.repo(t) + got, err := repo.GPGKeyContents() + if tt.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantContent, got) + }) + } +}