diff --git a/base/download.jl b/base/download.jl index f4d564d93f0f4..9e93405c0ac21 100644 --- a/base/download.jl +++ b/base/download.jl @@ -1,21 +1,23 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license +############################################################################## # file downloading downloadcmd = nothing if Sys.iswindows() downloadcmd = :powershell - function download(url::AbstractString, filename::AbstractString) + function download(url::AbstractString, filename::AbstractString; sha=nothing) ps = "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe" tls12 = "[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12" client = "New-Object System.Net.Webclient" # in the following we escape ' with '' (see https://ss64.com/ps/syntax-esc.html) downloadfile = "($client).DownloadFile('$(replace(url, "'" => "''"))', '$(replace(filename, "'" => "''"))')" run(`$ps -NoProfile -Command "$tls12; $downloadfile"`) + shacheck(filename, sha) filename end else - function download(url::AbstractString, filename::AbstractString) + function download(url::AbstractString, filename::AbstractString; sha=nothing) global downloadcmd if downloadcmd === nothing for checkcmd in (:curl, :wget, :fetch) @@ -39,21 +41,74 @@ else else error("no download agent available; install curl, wget, or fetch") end + shacheck(filename, sha) filename end end -function download(url::AbstractString) +function download(url::AbstractString; sha=nothing) filename = tempname() - download(url, filename) + download(url, filename, sha=sha) end """ - download(url::AbstractString, [localfile::AbstractString]) + download(url::AbstractString, [localfile::AbstractString]; sha=nothing) Download a file from the given url, optionally renaming it to the given local file name. + +If the optional `sha` keyword is specified, it should be the expected SHA-256 +hash of the downloaded file (a 64-character hexadecimal string). An error is +thrown if the downloaded file does not have this hash. + Note that this function relies on the availability of external tools such as `curl`, `wget` or `fetch` to download the file and is provided for convenience. For production use or situations in which more options are needed, please use a package that provides the desired functionality instead. """ download(url, filename) + +############################################################################## +# SHA-256 hash validation, via mbedtls. + +# If sha==nothing is supplied, then no check is performed. However, +# if Base._downloadsecurity[] is set to a non-empty string, a +# deprecation message is emitted. This is so that in certain +# security-sensitive contexts, future versions of Julia can disallow +# non-validated downloads. + +const _downloadsecurity = Ref("") # non-empty in secure download contexts + +function shacheck(filename::AbstractString, sha::Nothing) + if !isempty(_downloadsecurity[]) + depwarn("Calling download without an sha=\"...\" argument for validation is deprecated in the security context: $(_downloadsecurity[])", :shacheck) + end +end + +_updatehash!(ctx, data, nb) = + ccall((:mbedtls_sha256_update,:libmbedtls), Cvoid, (Ptr{UInt8},Ptr{UInt8},Csize_t), ctx,data,nb%Csize_t) + +# check that contents of filename match the given SHA-256 hash `sha` +function shacheck(filename::AbstractString, sha::AbstractString) + (length(sha) == 64 && all(c -> c in '0':'9' || lowercase(c) in 'a':'f', sha)) || + throw(ArgumentError("invalid SHA-256 hash $sha")) + ctx = Vector{UInt8}(undef, 10*sizeof(UInt32) + 64 + sizeof(Cint)) # mbedtls_sha256_context + ccall((:mbedtls_sha256_init,:libmbedtls), Cvoid, (Ptr{UInt8},), ctx) + try + # todo: switch to mbedtls_sha256_starts_ret etcetera for mbedtls ≥ v2.7 + ccall((:mbedtls_sha256_starts,:libmbedtls), Cvoid, (Ptr{UInt8},Cint), ctx,0) + open(filename, "r") do io + nb = filesize(io)-position(io) + buf = Vector{UInt8}(undef, min(nb, 32768)) + while !eof(io) && nb > 32768 + n = readbytes!(io, buf) + _updatehash!(ctx, buf, n) + nb -= n + end + _updatehash!(ctx, buf, readbytes!(io, buf, min(nb, length(buf)))) + end + h = Vector{UInt8}(undef, 32) + ccall((:mbedtls_sha256_finish,:libmbedtls), Cvoid, (Ptr{UInt8},Ptr{UInt8}), ctx,h) + sha == bytes2hex(h) || error("downloaded file ", filename, " had incorrect SHA256 hash") + finally + ccall((:mbedtls_sha256_free,:libmbedtls), Cvoid, (Ptr{UInt8},), ctx) + end +end diff --git a/stdlib/Pkg/src/entry.jl b/stdlib/Pkg/src/entry.jl index d0175f1790f99..36b8a2951c3cb 100644 --- a/stdlib/Pkg/src/entry.jl +++ b/stdlib/Pkg/src/entry.jl @@ -590,6 +590,7 @@ function build(pkg::AbstractString, build_file::AbstractString, errfile::Abstrac append!(Base.DEPOT_PATH, $(repr(DEPOT_PATH))) empty!(Base.DL_LOAD_PATH) append!(Base.DL_LOAD_PATH, $(repr(Base.DL_LOAD_PATH))) + Base._downloadsecurity[] = "package build" open("$(escape_string(errfile))", "a") do f pkg, build_file = "$pkg", "$(escape_string(build_file))" try @@ -615,7 +616,6 @@ function build(pkg::AbstractString, build_file::AbstractString, errfile::Abstrac --startup-file=$(Base.JLOptions().startupfile != 2 ? "yes" : "no") --eval $code ``` - success(pipeline(cmd, stdout=stdout, stderr=stderr)) end diff --git a/stdlib/Pkg3/src/Operations.jl b/stdlib/Pkg3/src/Operations.jl index 3bd23ce9b2c9a..58a5df0f97d68 100644 --- a/stdlib/Pkg3/src/Operations.jl +++ b/stdlib/Pkg3/src/Operations.jl @@ -799,6 +799,7 @@ function build_versions(ctx::Context, uuids::Vector{UUID}; might_need_to_resolve append!(Base.DEPOT_PATH, $(repr(map(abspath, DEPOT_PATH)))) empty!(Base.DL_LOAD_PATH) append!(Base.DL_LOAD_PATH, $(repr(map(abspath, Base.DL_LOAD_PATH)))) + Base._downloadsecurity[] = "package build" cd($(repr(dirname(build_file)))) include($(repr(build_file))) """ @@ -1070,4 +1071,3 @@ function init(ctx::Context) end end # module - diff --git a/test/download.jl b/test/download.jl index 5fa3eef9597b3..50db191379e1b 100644 --- a/test/download.jl +++ b/test/download.jl @@ -7,6 +7,17 @@ mktempdir() do temp_dir @test isfile(file) @test !isempty(read(file)) + # Download a file and check its SHA-256 hash + file = joinpath(temp_dir, "html") + @test file == download("http://httpbin.org/html", file, + sha="3f324f9914742e62cf082861ba03b207282dba781c3349bee9d7c1b5ef8e0bfe") + @test_throws ErrorException file == download("http://httpbin.org/html", file, + sha="3f324f9914742e62cf082861ba03b207282dba781c3349bee9d7c1b5ef8e0000") + @test_throws ArgumentError file == download("http://httpbin.org/html", file, + sha="3f324f9914742e62cf082861ba03b207282dba781c3349bee9d7c1b5ef8e0xxx") + @test_throws ArgumentError file == download("http://httpbin.org/html", file, + sha="3f324f9914742e62cf082861ba03b207282dba781c3349bee9d7c1b5ef8e0") + # Download an empty file empty_file = joinpath(temp_dir, "empty") @test download("http://httpbin.org/status/200", empty_file) == empty_file