support sha check in download, require during Pkg.build

base/download.jl

@@ -1,21 +1,23 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
+##############################################################################
 # file downloading
 
 downloadcmd = nothing
 if Sys.iswindows()
     downloadcmd = :powershell
-    function download(url::AbstractString, filename::AbstractString)
+    function download(url::AbstractString, filename::AbstractString; sha=nothing)
         ps = "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe"
         tls12 = "[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12"
         client = "New-Object System.Net.Webclient"
         # in the following we escape ' with '' (see https://ss64.com/ps/syntax-esc.html)
         downloadfile = "($client).DownloadFile('$(replace(url, "'" => "''"))', '$(replace(filename, "'" => "''"))')"
         run(`$ps -NoProfile -Command "$tls12; $downloadfile"`)
+        shacheck(filename, sha)
         filename
     end
 else
-    function download(url::AbstractString, filename::AbstractString)
+    function download(url::AbstractString, filename::AbstractString; sha=nothing)
         global downloadcmd
         if downloadcmd === nothing
             for checkcmd in (:curl, :wget, :fetch)
@@ -39,21 +41,74 @@ else
         else
             error("no download agent available; install curl, wget, or fetch")
         end
+        shacheck(filename, sha)
         filename
     end
 end
-function download(url::AbstractString)
+function download(url::AbstractString; sha=nothing)
     filename = tempname()
-    download(url, filename)
+    download(url, filename, sha=sha)
 end
 
 """
-    download(url::AbstractString, [localfile::AbstractString])
+    download(url::AbstractString, [localfile::AbstractString]; sha=nothing)
 
 Download a file from the given url, optionally renaming it to the given local file name.
+
+If the optional `sha` keyword is specified, it should be the expected SHA-256
+hash of the downloaded file (a 64-character hexadecimal string). An error is
+thrown if the downloaded file does not have this hash.
+
 Note that this function relies on the availability of external tools such as `curl`, `wget`
 or `fetch` to download the file and is provided for convenience. For production use or
 situations in which more options are needed, please use a package that provides the desired
 functionality instead.
 """
 download(url, filename)
+
+##############################################################################
+# SHA-256 hash validation, via mbedtls.
+
+# If sha==nothing is supplied, then no check is performed.  However,
+# if Base._downloadsecurity[] is set to a non-empty string, a
+# deprecation message is emitted.  This is so that in certain
+# security-sensitive contexts, future versions of Julia can disallow
+# non-validated downloads.
+
+const _downloadsecurity = Ref("") # non-empty in secure download contexts
+
+function shacheck(filename::AbstractString, sha::Nothing)
+    if !isempty(_downloadsecurity[])
+        depwarn("Calling download without an sha=\"...\" argument for validation is deprecated in the security context: $(_downloadsecurity[])", :shacheck)
+    end
+end
+
+_updatehash!(ctx, data, nb) =
+    ccall((:mbedtls_sha256_update,:libmbedtls), Cvoid, (Ptr{UInt8},Ptr{UInt8},Csize_t), ctx,data,nb%Csize_t)
+
+# check that contents of filename match the given SHA-256 hash `sha`
+function shacheck(filename::AbstractString, sha::AbstractString)
+    (length(sha) == 64 && all(c -> c in '0':'9' || lowercase(c) in 'a':'f', sha)) ||
+        throw(ArgumentError("invalid SHA-256 hash $sha"))
+    ctx = Vector{UInt8}(undef, 10*sizeof(UInt32) + 64 + sizeof(Cint)) # mbedtls_sha256_context
+    ccall((:mbedtls_sha256_init,:libmbedtls), Cvoid, (Ptr{UInt8},), ctx)
+    try
+        # todo: switch to mbedtls_sha256_starts_ret etcetera for mbedtls ≥ v2.7
+        ccall((:mbedtls_sha256_starts,:libmbedtls), Cvoid, (Ptr{UInt8},Cint), ctx,0)
+        open(filename, "r") do io
+            nb = filesize(io)-position(io)
+            buf = Vector{UInt8}(undef, min(nb, 32768))
+            while !eof(io) && nb > 32768
+                n = readbytes!(io, buf)
+                _updatehash!(ctx, buf, n)
+                nb -= n
+            end
+            _updatehash!(ctx, buf, readbytes!(io, buf, min(nb, length(buf))))
+        end
+        h = Vector{UInt8}(undef, 32)
+        ccall((:mbedtls_sha256_finish,:libmbedtls), Cvoid, (Ptr{UInt8},Ptr{UInt8}), ctx,h)
+        sha == bytes2hex(h) || error("downloaded file ", filename, " had incorrect SHA256 hash")
+    finally
+        ccall((:mbedtls_sha256_free,:libmbedtls), Cvoid, (Ptr{UInt8},), ctx)
+    end
+end

stdlib/Pkg/src/entry.jl

@@ -590,6 +590,7 @@ function build(pkg::AbstractString, build_file::AbstractString, errfile::Abstrac
         append!(Base.DEPOT_PATH, $(repr(DEPOT_PATH)))
         empty!(Base.DL_LOAD_PATH)
         append!(Base.DL_LOAD_PATH, $(repr(Base.DL_LOAD_PATH)))
+        Base._downloadsecurity[] = "package build"
         open("$(escape_string(errfile))", "a") do f
             pkg, build_file = "$pkg", "$(escape_string(build_file))"
             try
@@ -615,7 +616,6 @@ function build(pkg::AbstractString, build_file::AbstractString, errfile::Abstrac
         --startup-file=$(Base.JLOptions().startupfile != 2 ? "yes" : "no")
         --eval $code
         ```
-
     success(pipeline(cmd, stdout=stdout, stderr=stderr))
 end
 

stdlib/Pkg3/src/Operations.jl

@@ -799,6 +799,7 @@ function build_versions(ctx::Context, uuids::Vector{UUID}; might_need_to_resolve
             append!(Base.DEPOT_PATH, $(repr(map(abspath, DEPOT_PATH))))
             empty!(Base.DL_LOAD_PATH)
             append!(Base.DL_LOAD_PATH, $(repr(map(abspath, Base.DL_LOAD_PATH))))
+            Base._downloadsecurity[] = "package build"
             cd($(repr(dirname(build_file))))
             include($(repr(build_file)))
             """
@@ -1070,4 +1071,3 @@ function init(ctx::Context)
 end
 
 end # module
-

test/download.jl

@@ -7,6 +7,17 @@ mktempdir() do temp_dir
     @test isfile(file)
     @test !isempty(read(file))
 
+    # Download a file and check its SHA-256 hash
+    file = joinpath(temp_dir, "html")
+    @test file == download("http://httpbin.org/html", file,
+          sha="3f324f9914742e62cf082861ba03b207282dba781c3349bee9d7c1b5ef8e0bfe")
+    @test_throws ErrorException file == download("http://httpbin.org/html", file,
+        sha="3f324f9914742e62cf082861ba03b207282dba781c3349bee9d7c1b5ef8e0000")
+    @test_throws ArgumentError file == download("http://httpbin.org/html", file,
+        sha="3f324f9914742e62cf082861ba03b207282dba781c3349bee9d7c1b5ef8e0xxx")
+    @test_throws ArgumentError file == download("http://httpbin.org/html", file,
+        sha="3f324f9914742e62cf082861ba03b207282dba781c3349bee9d7c1b5ef8e0")
+
     # Download an empty file
     empty_file = joinpath(temp_dir, "empty")
     @test download("http://httpbin.org/status/200", empty_file) == empty_file