Merge pull request #1537 from tonobo/reproducible-publish

Add SOURCE_DATE_EPOCH support for reproducible builds

Author: neolynx

AUTHORS

@@ -80,3 +80,4 @@ List of contributors, in chronological order:
 * Roman Lebedev (https://github.com/LebedevRI)
 * Brian Witt (https://github.com/bwitt)
 * Ales Bregar (https://github.com/abregar)
+* Tim Foerster (https://github.com/tonobo)

deb/publish.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -1136,8 +1137,14 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 	release["Suite"] = p.GetSuite()
 	release["Codename"] = p.GetCodename()
 	datetime_format := "Mon, 2 Jan 2006 15:04:05 MST"
-	date_now := time.Now().UTC()
-	release["Date"] = date_now.Format(datetime_format)
+
+	publishDate := time.Now().UTC()
+	if epoch := os.Getenv("SOURCE_DATE_EPOCH"); epoch != "" {
+		if sec, err := strconv.ParseInt(epoch, 10, 64); err == nil {
+			publishDate = time.Unix(sec, 0).UTC()
+		}
+	}
+	release["Date"] = publishDate.Format(datetime_format)
 	release["Architectures"] = strings.Join(utils.StrSlicesSubstract(p.Architectures, []string{ArchitectureSource}), " ")
 	if p.AcquireByHash {
 		release["Acquire-By-Hash"] = "yes"
@@ -1149,7 +1156,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 		// is not present or if it is expired."
 		release["Signed-By"] = p.SignedBy
 		// Let's use a century as a "forever" value.
-		release["Valid-Until"] = date_now.AddDate(100, 0, 0).Format(datetime_format)
+		release["Valid-Until"] = publishDate.AddDate(100, 0, 0).Format(datetime_format)
 	}
 	release["Description"] = " Generated by aptly\n"
 	release["MD5Sum"] = ""

deb/publish_test.go

@@ -433,6 +433,47 @@ func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) {
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Release"), PathExists)
 }
 
+func (s *PublishedRepoSuite) TestPublishSourceDateEpoch(c *C) {
+	// Test with SOURCE_DATE_EPOCH set
+	_ = os.Setenv("SOURCE_DATE_EPOCH", "1234567890")
+	defer os.Unsetenv("SOURCE_DATE_EPOCH")
+
+	err := s.repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false, "")
+	c.Assert(err, IsNil)
+
+	rf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"))
+	c.Assert(err, IsNil)
+	defer rf.Close()
+
+	cfr := NewControlFileReader(rf, true, false)
+	st, err := cfr.ReadStanza()
+	c.Assert(err, IsNil)
+
+	// Expected date for Unix timestamp 1234567890: Fri, 13 Feb 2009 23:31:30 UTC
+	c.Check(st["Date"], Equals, "Fri, 13 Feb 2009 23:31:30 UTC")
+}
+
+func (s *PublishedRepoSuite) TestPublishSourceDateEpochInvalid(c *C) {
+	// Test with invalid SOURCE_DATE_EPOCH (should fallback to current time)
+	_ = os.Setenv("SOURCE_DATE_EPOCH", "invalid")
+	defer os.Unsetenv("SOURCE_DATE_EPOCH")
+
+	err := s.repo2.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, "")
+	c.Assert(err, IsNil)
+
+	rf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/Release"))
+	c.Assert(err, IsNil)
+	defer rf.Close()
+
+	cfr := NewControlFileReader(rf, true, false)
+	st, err := cfr.ReadStanza()
+	c.Assert(err, IsNil)
+
+	// Should have a valid Date field (not empty, not the fixed date from SOURCE_DATE_EPOCH)
+	c.Check(st["Date"], Not(Equals), "")
+	c.Check(st["Date"], Not(Equals), "Fri, 13 Feb 2009 23:31:30 UTC")
+}
+
 func (s *PublishedRepoSuite) TestPublishLocalRepo(c *C) {
 	err := s.repo2.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, "")
 	c.Assert(err, IsNil)

man/aptly.1

@@ -2472,6 +2472,9 @@ show yaml config
 .SH "ENVIRONMENT"
 If environment variable \fBHTTP_PROXY\fR is set \fBaptly\fR would use its value to proxy all HTTP requests\.
 .
+.P
+If environment variable \fBSOURCE_DATE_EPOCH\fR is set to a Unix timestamp, \fBaptly\fR would use that timestamp for the \fBDate\fR and \fBValid\-Until\fR fields in the \fBRelease\fR file when publishing\. This enables reproducible builds as specified by \fIhttps://reproducible\-builds\.org/specs/source\-date\-epoch/\fR\.
+.
 .SH "RETURN VALUES"
 \fBaptly\fR exists with:
 .

man/aptly.1.ronn.tmpl

@@ -533,6 +533,11 @@ For example, default aptly display format could be presented with the following
 If environment variable `HTTP_PROXY` is set `aptly` would use its value
 to proxy all HTTP requests.
 
+If environment variable `SOURCE_DATE_EPOCH` is set to a Unix timestamp,
+`aptly` would use that timestamp for the `Date` and `Valid-Until` fields
+in the `Release` file when publishing. This enables reproducible builds
+as specified by https://reproducible-builds.org/specs/source-date-epoch/.
+
 ## RETURN VALUES
 
 `aptly` exists with:

system/t06_publish/PublishRepo36Test_gold

@@ -0,0 +1,12 @@
+Loading packages...
+Generating metadata files and linking package files...
+Finalizing metadata files...
+
+Local repo local-repo has been successfully published.
+Please setup your webserver to serve directory '${HOME}/.aptly/public' with autoindexing.
+Now you can add following line to apt sources:
+  deb http://your-server/ maverick main
+  deb-src http://your-server/ maverick main
+Don't forget to add your GPG key to apt with apt-key.
+
+You can also use `aptly serve` to publish your repositories over HTTP quickly.

system/t06_publish/PublishRepo36Test_release

@@ -0,0 +1,12 @@
+Origin: . maverick
+Label: . maverick
+Suite: maverick
+Codename: maverick
+Date: Fri, 13 Feb 2009 23:31:30 UTC
+Architectures: i386
+Components: main
+Description: Generated by aptly
+MD5Sum:
+SHA1:
+SHA256:
+SHA512:

system/t06_publish/repo.py

@@ -9,6 +9,10 @@ def strip_processor(output):
     return "\n".join([l for l in output.split("\n") if not l.startswith(' ') and not l.startswith('Date:') and not l.startswith('Valid-Until:')])
 
 
+def strip_processor_keep_date(output):
+    return "\n".join([l for l in output.split("\n") if not l.startswith(' ')])
+
+
 class PublishRepo1Test(BaseTest):
     """
     publish repo: default
@@ -970,3 +974,34 @@ def check(self):
         # verify contents except of sums
         self.check_file_contents(
             'public/dists/maverick/Release', 'release', match_prepare=strip_processor)
+
+
+class PublishRepo36Test(BaseTest):
+    """
+    publish repo: SOURCE_DATE_EPOCH produces byte-identical output
+    """
+    fixtureCmds = [
+        "aptly repo create local-repo",
+        "aptly repo add local-repo ${files}",
+    ]
+    runCmd = "aptly publish repo -skip-signing -distribution=maverick local-repo"
+    gold_processor = BaseTest.expand_environ
+    environmentOverride = {"SOURCE_DATE_EPOCH": "1234567890"}
+
+    def check(self):
+        super(PublishRepo36Test, self).check()
+
+        # verify Release file includes the expected date from SOURCE_DATE_EPOCH
+        self.check_file_contents(
+            'public/dists/maverick/Release', 'release', match_prepare=strip_processor_keep_date)
+
+        # save Release file from first publish
+        first_release = self.read_file('public/dists/maverick/Release')
+
+        # drop and republish with same SOURCE_DATE_EPOCH
+        self.run_cmd("aptly publish drop maverick")
+        self.run_cmd("aptly publish repo -skip-signing -distribution=maverick local-repo")
+
+        # verify byte-identical output
+        second_release = self.read_file('public/dists/maverick/Release')
+        self.check_equal(first_release, second_release)