Merge pull request #196 from gitpod-io/wv/fix-transitive-dependency-cache-miss

Build dependencies of cached packages

Author: WVerlaek

pkg/leeway/build.go

@@ -605,15 +605,6 @@ func (p *Package) buildDependencies(buildctx *buildContext) (err error) {
 }
 
 func (p *Package) build(buildctx *buildContext) (err error) {
-	_, alreadyBuilt := buildctx.LocalCache.Location(p)
-
-	if p.Ephemeral {
-		// ephemeral packages always require a rebuild
-	} else if alreadyBuilt {
-		log.WithField("package", p.FullName()).Debug("already built")
-		return nil
-	}
-
 	doBuild := buildctx.ObtainBuildLock(p)
 	if !doBuild {
 		return nil
@@ -630,6 +621,16 @@ func (p *Package) build(buildctx *buildContext) (err error) {
 		return err
 	}
 
+	// Return early if the package is already built. We're explicitly performing this check after having built all the dependencies.
+	//  Previously we had it before, but that resulted in failed builds as there's no guarantee that the cache will contain transitive dependencies; in our case they were sometimes evicted from the cache due to S3 lifecycle rules
+	_, alreadyBuilt := buildctx.LocalCache.Location(p)
+	if p.Ephemeral {
+		// ephemeral packages always require a rebuild
+	} else if alreadyBuilt {
+		log.WithField("package", p.FullName()).Debug("already built")
+		return nil
+	}
+
 	pkgRep := &PackageBuildReport{
 		phaseEnter: make(map[PackageBuildPhase]time.Time),
 		phaseDone:  make(map[PackageBuildPhase]time.Time),

pkg/leeway/version.go

@@ -1,4 +1,4 @@
 package leeway
 
 // Version is the version of this leeway build
-var Version string = "unkown"
+var Version string = "unknown"

pkg/leeway/workspace.go

@@ -230,7 +230,13 @@ func loadWorkspace(ctx context.Context, path string, args Arguments, variant str
 		if err != nil {
 			return Workspace{}, err
 		}
-		ignores = strings.Split(string(fc), "\n")
+		split := strings.Split(string(fc), "\n")
+		for _, s := range split {
+			if s == "" {
+				continue
+			}
+			ignores = append(ignores, s)
+		}
 	}
 	otherWS, err := doublestar.Glob(workspace.Origin, "**/WORKSPACE.yaml", workspace.ShouldIgnoreSource)
 	if err != nil {