Rustc binary hash (#14)

* add rustc binary hash to artifact key

* extra env vars to artifact metadata

* fix tests

Author: AndWeHaveAPlan

.github/workflows/build-cargo.yml

@@ -57,7 +57,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: rust-lang/cargo
-          ref: rust-1.77.0
+          ref: rust-1.88.0
           path: ./cargo-1
       - run: cp -r .forklift ./cargo-1
       - name: build cargo
@@ -68,7 +68,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: rust-lang/cargo
-          ref: rust-1.77.0
+          ref: rust-1.88.0
           path: ./cargo-2
       - run: cp -r .forklift ./cargo-2
       - name: build cargo

Commands/Wrapper/Wrapper.go

@@ -30,7 +30,7 @@ func Run(args []string) {
 
 	WorkDir = wd
 
-	var wrapperTool = Rustc.NewWrapperToolFromArgs(WorkDir, &rustcArgsOnly)
+	var wrapperTool = Rustc.NewWrapperToolFromArgs(WorkDir, args)
 
 	logger := Logging.CreateLogger("Wrapper", 4, log.Fields{
 		"crate": wrapperTool.CrateName,

FileManager/Models/Models.go

@@ -19,4 +19,5 @@ type CacheItem struct {
 	RustCArgsHash           string
 	CrateExternDepsChecksum string
 	CrateNativeDepsChecksum string
+	RustCBinHash            string
 }

Lib/Config/Config.go

@@ -31,7 +31,8 @@ var AppConfig = ForkliftConfig{
 		ExtraLabels: map[string]string{},
 	},
 	Cache: ForkliftCache{
-		ExtraEnv: []string{},
+		ExtraEnv:      []string{},
+		ExtraMetadata: map[string]string{},
 	},
 }
 

Lib/Config/Models.go

@@ -14,7 +14,8 @@ type ForkliftStorage struct {
 }
 
 type ForkliftCache struct {
-	ExtraEnv []string
+	ExtraEnv      []string
+	ExtraMetadata map[string]string
 }
 
 type ForkliftCompression struct {

Lib/Rustc/Rustc_test.go

@@ -30,7 +30,7 @@ func TestCmdTool_GetExternDeps(t *testing.T) {
 
 func TestWrapperTool_WriteStderrFile(t *testing.T) {
 	var wd, _ = os.Getwd()
-	var wrapper = Rustc.NewWrapperToolFromArgs(wd, &[]string{})
+	var wrapper = Rustc.NewWrapperToolFromArgs(wd, []string{"aaaa", "bbbb"})
 
 	var data = "{\"artifact\":\"deps/base64-a62ed92405ecbfa1.d\",\"emit\":\"dep-info\"}"
 	var expectedData = "{\"artifact\":\"base64-a62ed92405ecbfa1.d\",\"emit\":\"dep-info\"}\n"
@@ -59,7 +59,7 @@ func TestWrapperTool_WriteStderrFile(t *testing.T) {
 
 func TestWrapperTool_ReadStderrFile(t *testing.T) {
 	var wd, _ = os.Getwd()
-	var wrapper = Rustc.NewWrapperToolFromArgs(wd, &[]string{"-a", "b"})
+	var wrapper = Rustc.NewWrapperToolFromArgs(wd, []string{"-a", "b"})
 
 	var dataBytes, _ = json.Marshal(Rustc.Artifact{
 		Artifact: filepath.Join("target", "deps", "base64-a62ed92405ecbfa1.d"),

Lib/Rustc/WrapperTool.go

@@ -23,7 +23,7 @@ import (
 	"strings"
 )
 
-const CachePackageVersion = "3"
+const CachePackageVersion = "4"
 
 var cargoHashRegex = regexp.MustCompile("^metadata=([0-9a-f]{16})$")
 
@@ -40,18 +40,22 @@ type WrapperTool struct {
 	osWorkDir               string
 	CrateExternDepsChecksum string
 	CrateNativeDepsChecksum string
+	RustCBinHash            string
 
 	cachePackageName string
 }
 
-func NewWrapperToolFromArgs(workDir string, rustArgs *[]string) *WrapperTool {
+func NewWrapperToolFromArgs(workDir string, rustcCommand []string) *WrapperTool {
 	var wrapper = WrapperTool{}
-	wrapper.rustcArgs = rustArgs
+	var rustcArgs = rustcCommand[1:]
 
-	wrapper.CrateName, wrapper.CargoCrateHash, wrapper.OutDir = wrapper.extractNameMetaHashDir(rustArgs)
+	wrapper.rustcArgs = &rustcArgs
+
+	wrapper.CrateName, wrapper.CargoCrateHash, wrapper.OutDir = wrapper.extractNameMetaHashDir(wrapper.rustcArgs)
 	wrapper.workDir = workDir
 	wrapper.OutDir = FileManager.GetTrueRelFilePath(wrapper.workDir, wrapper.OutDir)
-	wrapper.RustCArgsHash = GetArgsHash(rustArgs, wrapper.workDir)
+	wrapper.RustCArgsHash = GetArgsHash(wrapper.rustcArgs, wrapper.workDir)
+	wrapper.RustCBinHash = GetRustcBinHash(rustcCommand[0])
 
 	wrapper.ExternDepsChecksum()
 
@@ -69,13 +73,30 @@ func NewWrapperToolFromCacheItem(workDir string, item Models.CacheItem) *Wrapper
 	wrapper.workDir = workDir
 	wrapper.CrateSourceChecksum = item.CrateSourceChecksum
 	wrapper.RustCArgsHash = item.RustCArgsHash
+	wrapper.RustCBinHash = item.RustCBinHash
 
 	wrapper.CrateExternDepsChecksum = item.CrateExternDepsChecksum
 	wrapper.CrateNativeDepsChecksum = item.CrateNativeDepsChecksum
 
 	return &wrapper
 }
 
+func GetRustcBinHash(path string) string {
+	f, err := os.Open(path)
+	if err != nil {
+		log.Fatalf(err.Error())
+	}
+
+	var sha = sha1.New()
+	_, _ = io.Copy(sha, f)
+
+	err = f.Close()
+	if err != nil {
+		log.Fatalf(err.Error())
+	}
+	return fmt.Sprintf("%x", sha.Sum(nil))
+}
+
 func GetArgsHash(args *[]string, toRemove string) string {
 	var sha = sha1.New()
 	for _, arg := range *args {
@@ -238,6 +259,7 @@ func (wrapperTool *WrapperTool) GetCachePackageName() string {
 	sha.Write([]byte(wrapperTool.CrateSourceChecksum))
 	sha.Write([]byte(wrapperTool.OutDir))
 	sha.Write([]byte(wrapperTool.RustCArgsHash))
+	sha.Write([]byte(wrapperTool.RustCBinHash))
 	sha.Write([]byte(wrapperTool.ExternDepsChecksum()))
 
 	sha.Write(wrapperTool.ExtraEnvVarsChecksum())
@@ -263,6 +285,7 @@ func (wrapperTool *WrapperTool) ToCacheItem() Models.CacheItem {
 		OutDir:              wrapperTool.OutDir,
 		CrateSourceChecksum: wrapperTool.CrateSourceChecksum,
 		RustCArgsHash:       wrapperTool.RustCArgsHash,
+		RustCBinHash:        wrapperTool.RustCBinHash,
 
 		CrateExternDepsChecksum: wrapperTool.CrateExternDepsChecksum,
 		CrateNativeDepsChecksum: wrapperTool.CrateNativeDepsChecksum,
@@ -396,6 +419,7 @@ func (wrapperTool *WrapperTool) CreateMetadata() map[string]*string {
 		"cargo-hash":        &wrapperTool.CargoCrateHash,
 		"sha1-source-files": &wrapperTool.CrateSourceChecksum,
 		"sha1-rustc-args":   &wrapperTool.RustCArgsHash,
+		"sha1-rustc-bin":    &wrapperTool.RustCBinHash,
 		"sha1-extern-deps":  &wrapperTool.CrateExternDepsChecksum,
 		"sha1-native-deps":  &wrapperTool.CrateNativeDepsChecksum,
 	}

Rpc/Uploader.go

@@ -8,11 +8,13 @@ import (
 	"forklift/CacheStorage/Storages"
 	"forklift/FileManager/Models"
 	"forklift/FileManager/Tar"
+	"forklift/Lib/Config"
 	"forklift/Lib/Diagnostic/Time"
 	"forklift/Lib/Logging"
 	log "forklift/Lib/Logging/ConsoleLogger"
 	"forklift/Lib/Rustc"
 	"forklift/Rpc/Models/CacheUpload"
+	"os"
 	"path"
 	"path/filepath"
 	"strings"
@@ -132,6 +134,16 @@ func (uploader *Uploader) TryUpload(
 		var shaLocal = fmt.Sprintf("%x", sha.Sum(nil))
 		metaMap["sha1-artifact"] = &shaLocal
 
+		for key, value := range Config.AppConfig.Cache.ExtraMetadata {
+			if value[0] == '$' {
+				if data, ok := os.LookupEnv(value[1:]); ok {
+					metaMap[key] = &data
+				}
+			} else {
+				metaMap[key] = &value
+			}
+		}
+
 		timer.Start("Compress time")
 		compressed, err := uploader.compressor.Compress(reader)
 		statusReport.CompressTime += timer.Stop("Compress time")