LibGit2: consult known hosts files to verify SSH server identity

stdlib/LibGit2/Project.toml

@@ -2,8 +2,10 @@ name = "LibGit2"
 uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
 
 [deps]
+Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 NetworkOptions = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
+SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 
 [extras]
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"

stdlib/LibGit2/src/LibGit2.jl

@@ -7,8 +7,10 @@ module LibGit2
 
 import Base: ==
 using Base: something, notnothing
+using Base64: base64decode
+using NetworkOptions
 using Printf: @printf
-import NetworkOptions
+using SHA: sha1, sha256
 
 export with, GitRepo, GitConfig
 

stdlib/LibGit2/src/callbacks.jl

@@ -359,19 +359,180 @@ function fetchhead_foreach_callback(ref_name::Cstring, remote_url::Cstring,
  return Cint(0)
 end
 
+struct CertHostKey
+ parent :: Cint
+ mask :: Cint
+ md5 :: NTuple{16,UInt8}
+ sha1 :: NTuple{20,UInt8}
+ sha256 :: NTuple{32,UInt8}
+end
+
+struct KeyHashes
+ sha1 :: Union{NTuple{20,UInt8}, Nothing}
+ sha256 :: Union{NTuple{32,UInt8}, Nothing}
+end
+
+function KeyHashes(cert_p::Ptr{CertHostKey})
+ cert = unsafe_load(cert_p)
+ return KeyHashes(
+ cert.mask & Consts.CERT_SSH_SHA1 != 0 ? cert.sha1 : nothing,
+ cert.mask & Consts.CERT_SSH_SHA256 != 0 ? cert.sha256 : nothing,
+ )
+end
+
+function verify_host_error(message::AbstractString)
+ printstyled(stderr, "$message\n", color = :cyan, bold = true)
+end
+
 function certificate_callback(
- cert_p :: Ptr{Cvoid},
+ cert_p :: Ptr{CertHostKey},
  valid :: Cint,
  host_p :: Ptr{Cchar},
- user_p :: Ptr{Cvoid},
+ data_p :: Ptr{Cvoid},
 )::Cint
  valid != 0 && return Consts.CERT_ACCEPT
  host = unsafe_string(host_p)
  cert_type = unsafe_load(convert(Ptr{Cint}, cert_p))
  transport = cert_type == Consts.CERT_TYPE_TLS ? "TLS" :
  cert_type == Consts.CERT_TYPE_SSH ? "SSH" : nothing
- verify = NetworkOptions.verify_host(host, transport)
- verify ? Consts.PASSTHROUGH : Consts.CERT_ACCEPT
+ if !NetworkOptions.verify_host(host, transport)
+ # user has opted out of host verification
+ return Consts.CERT_ACCEPT
+ end
+ if transport == "TLS"
+ # TLS verification is done before the callback and indicated with the
+ # incoming `valid` flag, so if we get here then host verification failed
+ verify_host_error("TLS host verification: the identity of the server `$host` could not be verified. Someone could be trying to man-in-the-middle your connection. It is also possible that the correct server is using an invalid certificate or that your system's certificate authority root store is misconfigured.")
+ return Consts.CERT_REJECT
+ end
+ if transport == "SSH"
+ # SSH verification has to be done here
+ files = [joinpath(homedir(), ".ssh", "known_hosts")]
+ check = ssh_knownhost_check(files, host, KeyHashes(cert_p))
+ valid = false
+ if check == Consts.SSH_HOST_KNOWN
+ valid = true
+ elseif check == Consts.SSH_HOST_UNKNOWN
+ if Sys.which("ssh-keyscan") !== nothing
+ msg = "Please run `ssh-keyscan $host >> $(files[1])` in order to add the server to your known hosts file and the try again."
+ else
+ msg = "Please connect once using `ssh $host` in order to add the server to your known hosts file and the try again. You may not be allowed to log in (wrong user and/or no login allowed), but ssh will prompt you about adding a host key for the server first, which will allow libgit2 to verify the server."
+ end
+ verify_host_error("SSH host verification: the server `$host` is not a known host. $msg")
+ elseif check == Consts.SSH_HOST_MISMATCH
+ verify_host_error("SSH host verification: the identity of the server `$host` does not match its known hosts record. Someone could be trying to man-in-the-middle your connection. It is also possible that the server has changed its key, in which case you should check with the server administrator and use `ssh $host` to login into the server and update your known hosts file.")
+ elseif check == Consts.SSH_HOST_BAD_HASH
+ verify_host_error("SSH host verification: no secure certificate hash available for `$host`, cannot verify server identity.")
+ else
+ @error("unexpected SSH known host check result", check)
+ end
+ return valid ? Consts.CERT_ACCEPT : Consts.CERT_REJECT
+ end
+ @error("unexpected transport encountered, refusing to validate", cert_type)
+ return Consts.CERT_REJECT
+end
+
+## SSH known host checking
+#
+# We can't use libssh2_knownhost_check because libgit2, for no good reason,
+# doesn't give us a host fingerprint that we can use for that and instead gives
+# us multiple hashes of that fingerprint instead. Moreover, since a host can
+# have multiple fingerprints in the known hosts file with different encryption
+# types (gitlab.com does this, for example), we need to iterate through all the
+# known hosts entries and manually check if any of them is a match.
+#
+# The fact that libgit2 won't give us a fingerprint also means that we cannot,
+# even if we wanted to, prompt the user for whether to add the fingerprint to
+# the known hosts file, since we don't have the fingerprint that should be
+# added. The only option is to instruct the user how to add it themselves.
+#
+# Check logic: if a host appears in a known hosts file at all then one of the
+# keys in that file must match or we declare a mismatch; if the host name
+# doesn't appear in the file at all, however, we will continue searching files.
+#
+# This allows adding a host to the system known hosts file to fully override
+# that host appearing in a bundled known hosts file. It is necessary to allow
+# any of multiple entries in a single file to match, however, to allow for the
+# possiblity that the file contains multiple fingerprints for the same host. If
+# libgit2 gave us the fucking fingerprint then we could search for only an entry
+# with the correct type, but we can't do that without the actual fingerprint.
+
+struct KnownHost
+ magic :: Cuint
+ node :: Ptr{Cvoid}
+ name :: Ptr{Cchar}
+ key :: Ptr{Cchar}
+ type :: Cint
+end
+
+function ssh_knownhost_check(
+ files :: AbstractVector{<:AbstractString},
+ host :: AbstractString,
+ hashes :: KeyHashes,
+)
+ hashes.sha1 === hashes.sha256 === nothing &&
+ return Consts.SSH_HOST_BAD_HASH
+ session = @ccall libssh2_session_init_ex(
+ C_NULL :: Ptr{Cvoid},
+ C_NULL :: Ptr{Cvoid},
+ C_NULL :: Ptr{Cvoid},
+ C_NULL :: Ptr{Cvoid},
+ ) :: Ptr{Cvoid}
+ for file in files
+ ispath(file) || continue
+ hosts = @ccall libssh2_knownhost_init(
+ session :: Ptr{Cvoid},
+ ) :: Ptr{Cvoid}
+ count = @ccall libssh2_knownhost_readfile(
+ hosts :: Ptr{Cvoid},
+ file :: Cstring,
+ 1 :: Cint, # standard OpenSSH format
+ ) :: Cint
+ if count < 0
+ @warn("Error parsing SSH known hosts file `$file`")
+ @ccall libssh2_knownhost_free(hosts::Ptr{Cvoid})::Cvoid
+ continue
+ end
+ name_match = false
+ prev = Ptr{KnownHost}(0)
+ store = Ref{Ptr{KnownHost}}()
+ while true
+ get = @ccall libssh2_knownhost_get(
+ hosts :: Ptr{Cvoid},
+ store :: Ptr{Ptr{KnownHost}},
+ prev :: Ptr{KnownHost},
+ ) :: Cint
+ get < 0 && @warn("Error searching SSH known hosts file `$file`")
+ get == 0 || break # end of file or error
+ # got a known hosts record for host, now check its key hash
+ prev = store[]
+ known_host = unsafe_load(prev)
+ known_host.name == C_NULL && continue
+ host == unsafe_string(known_host.name) || continue
+ name_match = true # we've found some entry in this file
+ key_match = true # all available hashes must match
+ key = base64decode(unsafe_string(known_host.key))
+ if hashes.sha1 !== nothing
+ key_match &= sha1(key) == collect(hashes.sha1)
+ end
+ if hashes.sha256 !== nothing
+ key_match &= sha256(key) == collect(hashes.sha256)
+ end
+ key_match || continue
+ # name and key match found
+ @ccall libssh2_knownhost_free(hosts::Ptr{Cvoid})::Cvoid
+ @assert 0 == @ccall libssh2_session_free(session::Ptr{Cvoid})::Cint
+ return Consts.SSH_HOST_KNOWN
+ end
+ @ccall libssh2_knownhost_free(hosts::Ptr{Cvoid})::Cvoid
+ name_match || continue # no name match, search more files
+ # name match but no key match => host mismatch
+ @assert 0 == @ccall libssh2_session_free(session::Ptr{Cvoid})::Cint
+ return Consts.SSH_HOST_MISMATCH
+ end
+ # name not found in any known hosts files
+ @assert 0 == @ccall libssh2_session_free(session::Ptr{Cvoid})::Cint
+ return Consts.SSH_HOST_UNKNOWN
 end
 
 "C function pointer for `mirror_callback`"
@@ -381,4 +542,4 @@ credentials_cb() = @cfunction(credentials_callback, Cint, (Ptr{Ptr{Cvoid}}, Cstr
 "C function pointer for `fetchhead_foreach_callback`"
 fetchhead_foreach_cb() = @cfunction(fetchhead_foreach_callback, Cint, (Cstring, Cstring, Ptr{GitHash}, Cuint, Any))
 "C function pointer for `certificate_callback`"
-certificate_cb() = @cfunction(certificate_callback, Cint, (Ptr{Cvoid}, Cint, Ptr{Cchar}, Ptr{Cvoid}))
+certificate_cb() = @cfunction(certificate_callback, Cint, (Ptr{CertHostKey}, Cint, Ptr{Cchar}, Ptr{Cvoid}))

stdlib/LibGit2/src/consts.jl

@@ -317,6 +317,25 @@ const PASSTHROUGH = -30
 const CERT_REJECT = -1
 const CERT_ACCEPT = 0
 
+# certificate hash flags
+const CERT_SSH_MD5 = 1 << 0
+const CERT_SSH_SHA1 = 1 << 1
+const CERT_SSH_SHA256 = 1 << 2
+
+# libssh2 known host constants
+const LIBSSH2_KNOWNHOST_TYPE_PLAIN = 1
+const LIBSSH2_KNOWNHOST_TYPE_SHA1 = 2
+const LIBSSH2_KNOWNHOST_TYPE_CUSTOM = 3
+
+const LIBSSH2_KNOWNHOST_KEYENC_RAW = 1 << 16
+const LIBSSH2_KNOWNHOST_KEYENC_BASE64 = 2 << 16
+
+# internal constants for SSH host verification outcomes
+const SSH_HOST_KNOWN = 0
+const SSH_HOST_UNKNOWN = 1
+const SSH_HOST_MISMATCH = 2
+const SSH_HOST_BAD_HASH = 3
+
 @enum(GIT_SUBMODULE_IGNORE, SUBMODULE_IGNORE_UNSPECIFIED = -1, # use the submodule's configuration
  SUBMODULE_IGNORE_NONE = 1, # any change or untracked == dirty
  SUBMODULE_IGNORE_UNTRACKED = 2, # dirty if tracked files change

stdlib/LibGit2/test/known_hosts

@@ -0,0 +1,4 @@
+github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==
+gitlab.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCsj2bNKTBSpIYDEGk9KxsGh3mySTRgMtXL583qmBpzeQ+jqCMRgBqB98u3z++J1sKlXHWfM9dyhSevkMwSbhoR8XIq/U0tCNyokEi/ueaBMCvbcTHhO7FcwzY92WK4Yt0aGROY5qX2UKSeOvuP4D6TPqKF1onrSzH9bx9XUf2lEdWT/ia1NEKjunUqu1xOB/StKDHMoX4/OKyIzuS0q/T1zOATthvasJFoPrAjkohTyaDUz2LN5JoH839hViyEG82yB+MjcFV5MU3N1l1QL3cVUCh93xSaua1N85qivl+siMkPGbO5xR/En4iEY6K2XPASUEMaieWVNTRCtJ4S8H+9
+gitlab.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFSMqzJeV9rUzU4kWitGjeR4PWSa29SPqJ1fVkhtj3Hw9xjLVXVYrU9QlYWrOLXBpQ6KWjbjTDTdDkoohFzgbEY=
+gitlab.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAfuCHKVTjquxvt6CM6tdG4SLp1Btn/nOeHHE5UOzRdf

stdlib/LibGit2/test/libgit2.jl

@@ -2390,6 +2390,103 @@ mktempdir() do dir
  Base.shred!(valid_p_cred)
  end
 
+ @testset "SSH known host checking" begin
+ key_hashes(sha1::String, sha256::String) = LibGit2.KeyHashes(
+ Tuple(hex2bytes(sha1)),
+ Tuple(hex2bytes(sha256)),
+ )
+ # randomly generated hashes matching no hosts
+ random_key_hashes = key_hashes(
+ "a9971372d02a67bdfea82e2b4808b4cf478b49c0",
+ "45aac5c20d5c7f8b998fee12fa9b75086c0d3ed6e33063f7ce940409ff4efbbc"
+ )
+ # hashes of the unique github.com fingerprint
+ github_key_hashes = key_hashes(
+ "bf6b6825d2977c511a475bbefb88aad54a92ac73",
+ "9d385b83a9175292561a5ec4d4818e0aca51a264f17420112ef88ac3a139498f"
+ )
+ # hashes of the middle github.com fingerprint
+ gitlab_key_hashes = key_hashes(
+ "4db6b9ab0209fcde106cbf0fc4560ad063a962ad",
+ "1db5b783ccd48cd4a4b056ea4e25163d683606ad71f3174652b9625c5cd29d4c"
+ )
+
+ # various key hash collections
+ partial_hashes(keys::LibGit2.KeyHashes) = [ keys,
+ LibGit2.KeyHashes(keys.sha1, nothing),
+ LibGit2.KeyHashes(nothing, keys.sha256),
+ ]
+ bad_hashes = LibGit2.KeyHashes(nothing, nothing)
+ random_hashes = partial_hashes(random_key_hashes)
+ github_hashes = partial_hashes(github_key_hashes)
+ gitlab_hashes = partial_hashes(gitlab_key_hashes)
+
+ # various known hosts files
+ no_file = tempname()
+ empty_file = tempname(); touch(empty_file)
+ known_hosts = joinpath(@__DIR__, "known_hosts")
+ wrong_hosts = tempname()
+ open(wrong_hosts, write=true) do io
+ for line in eachline(known_hosts)
+ words = split(line)
+ words[1] = words[1] == "github.com" ? "gitlab.com" :
+ words[1] == "gitlab.com" ? "github.com" :
+ words[1]
+ println(io, join(words, " "))
+ end
+ end
+
+ @testset "bad hash errors" begin
+ hash = bad_hashes
+ for host in ["github.com", "gitlab.com", "unknown.host"],
+ files in [[no_file], [empty_file], [known_hosts]]
+ check = LibGit2.ssh_knownhost_check(files, host, hash)
+ @test check == LibGit2.Consts.SSH_HOST_BAD_HASH
+ end
+ end
+
+ @testset "unknown hosts" begin
+ host = "unknown.host"
+ for hash in [github_hashes; gitlab_hashes; random_hashes],
+ files in [[no_file], [empty_file], [known_hosts]]
+ check = LibGit2.ssh_knownhost_check(files, host, hash)
+ @test check == LibGit2.Consts.SSH_HOST_UNKNOWN
+ end
+ end
+
+ @testset "known hosts" begin
+ for (host, hashes) in [
+ "github.com" => github_hashes,
+ "gitlab.com" => gitlab_hashes,
+ ], hash in hashes
+ for files in [[no_file], [empty_file]]
+ check = LibGit2.ssh_knownhost_check(files, host, hash)
+ @test check == LibGit2.Consts.SSH_HOST_UNKNOWN
+ end
+ for files in [
+ [known_hosts],
+ [empty_file; known_hosts],
+ [known_hosts; empty_file],
+ [known_hosts; wrong_hosts],
+ ]
+ check = LibGit2.ssh_knownhost_check(files, host, hash)
+ @test check == LibGit2.Consts.SSH_HOST_KNOWN
+ end
+ for files in [
+ [wrong_hosts],
+ [empty_file; wrong_hosts],
+ [wrong_hosts; empty_file],
+ [wrong_hosts; known_hosts],
+ ]
+ check = LibGit2.ssh_knownhost_check(files, host, hash)
+ @test check == LibGit2.Consts.SSH_HOST_MISMATCH
+ end
+ end
+ end
+
+ rm(empty_file)
+ end
+
  @testset "HTTPS credential prompt" begin
  url = "https://github.com/test/package.jl"