Skip to content

Commit

Permalink
[BinaryPlatforms]: Flesh out CPUID support (#37722)
Browse files Browse the repository at this point in the history
  • Loading branch information
@staticfloat
staticfloat authored Sep 24, 2020
1 parent a62acdf commit b1e1bb3
Show file tree
Hide file tree
Showing 3 changed files with 137 additions and 84 deletions.
113 changes: 31 additions & 82 deletions base/binaryplatforms.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,71 +8,7 @@ export AbstractPlatform, Platform, HostPlatform, platform_dlext, tags, arch, os,
import .Libc.Libdl

### Submodule with information about CPU features
module CPUID

export cpu_isa

"""
ISA(features::Set{UInt32})
A structure which represents the Instruction Set Architecture (ISA) of a
computer. It holds the `Set` of features of the CPU.
The numerical values of the features are automatically generated from the C
source code of Julia and stored in the `features_h.jl` Julia file.
"""
struct ISA
features::Set{UInt32}
end

Base.:<=(a::ISA, b::ISA) = a.features <= b.features
Base.:<(a::ISA, b::ISA) = a.features < b.features
Base.isless(a::ISA, b::ISA) = a < b

include(string(length(Core.ARGS) >= 2 ? Core.ARGS[2] : "", "features_h.jl")) # include($BUILDROOT/base/features_h.jl)

# Keep in sync with `arch_march_isa_mapping`.
const ISAs_by_family = Dict(
"x86_64" => [
# Source: https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html.
# Implicit in all sets, because always required: mmx, sse, sse2
"x86_64" => ISA(Set{UInt32}()),
"core2" => ISA(Set((JL_X86_sse3, JL_X86_ssse3))),
"nehalem" => ISA(Set((JL_X86_sse3, JL_X86_ssse3, JL_X86_sse41, JL_X86_sse42, JL_X86_popcnt))),
"sandybridge" => ISA(Set((JL_X86_sse3, JL_X86_ssse3, JL_X86_sse41, JL_X86_sse42, JL_X86_popcnt, JL_X86_avx, JL_X86_aes, JL_X86_pclmul))),
"haswell" => ISA(Set((JL_X86_movbe, JL_X86_sse3, JL_X86_ssse3, JL_X86_sse41, JL_X86_sse42, JL_X86_popcnt, JL_X86_avx, JL_X86_avx2, JL_X86_aes, JL_X86_pclmul, JL_X86_fsgsbase, JL_X86_rdrnd, JL_X86_fma, JL_X86_bmi, JL_X86_bmi2, JL_X86_f16c))),
"skylake" => ISA(Set((JL_X86_movbe, JL_X86_sse3, JL_X86_ssse3, JL_X86_sse41, JL_X86_sse42, JL_X86_popcnt, JL_X86_avx, JL_X86_avx2, JL_X86_aes, JL_X86_pclmul, JL_X86_fsgsbase, JL_X86_rdrnd, JL_X86_fma, JL_X86_bmi, JL_X86_bmi2, JL_X86_f16c, JL_X86_rdseed, JL_X86_adx, JL_X86_prfchw, JL_X86_clflushopt, JL_X86_xsavec, JL_X86_xsaves))),
"skylake_avx512" => ISA(Set((JL_X86_movbe, JL_X86_sse3, JL_X86_ssse3, JL_X86_sse41, JL_X86_sse42, JL_X86_popcnt, JL_X86_pku, JL_X86_avx, JL_X86_avx2, JL_X86_aes, JL_X86_pclmul, JL_X86_fsgsbase, JL_X86_rdrnd, JL_X86_fma, JL_X86_bmi, JL_X86_bmi2, JL_X86_f16c, JL_X86_rdseed, JL_X86_adx, JL_X86_prfchw, JL_X86_clflushopt, JL_X86_xsavec, JL_X86_xsaves, JL_X86_avx512f, JL_X86_clwb, JL_X86_avx512vl, JL_X86_avx512bw, JL_X86_avx512dq, JL_X86_avx512cd))),
],
"arm" => [
"armv7l" => ISA(Set{UInt32}()),
"armv7l_neon" => ISA(Set((JL_AArch32_neon,))),
"armv7l_neon_vfp4" => ISA(Set((JL_AArch32_neon, JL_AArch32_vfp4))),
],
"aarch64" => [
# Implicit in all sets, because always required: fp, asimd
"armv8.0_a" => ISA(Set{UInt32}()),
"armv8.1_a" => ISA(Set((JL_AArch64_lse, JL_AArch64_crc, JL_AArch64_rdm))),
"armv8.2_a_crypto" => ISA(Set((JL_AArch64_lse, JL_AArch64_crc, JL_AArch64_rdm, JL_AArch64_aes, JL_AArch64_sha2))),
"armv8.4_a_crypto_sve" => ISA(Set((JL_AArch64_lse, JL_AArch64_crc, JL_AArch64_rdm, JL_AArch64_fp16fml, JL_AArch64_dotprod, JL_AArch64_aes, JL_AArch64_sha2, JL_AArch64_dotprod, JL_AArch64_sve))),
],
)

test_cpu_feature(feature::UInt32) = ccall(:jl_test_cpu_feature, Bool, (UInt32,), feature)
cpu_family() = String(Sys.ARCH)

"""
cpu_isa()
Return the [`ISA`](@ref) (instruction set architecture) of the current CPU.
"""
function cpu_isa()
all_features = last(last(get(ISAs_by_family, cpu_family(), "" => [ISA(Set{UInt32}())]))).features
return ISA(Set{UInt32}(feat for feat in all_features if test_cpu_feature(feat)))
end

end # module CPUID

include("cpuid.jl")
using .CPUID

# This exists to ease compatibility with old-style Platform objects
Expand Down Expand Up @@ -108,16 +44,7 @@ struct Platform <: AbstractPlatform
kwargs...)
# A wee bit of normalization
os = lowercase(os)
arch = lowercase(arch)
if arch ∈ ("amd64",)
arch = "x86_64"
elseif arch ∈ ("i386", "i586")
arch = "i686"
elseif arch ∈ ("arm",)
arch = "armv7l"
elseif arch ∈ ("ppc64le",)
arch = "powerpc64le"
end
arch = CPUID.normalize_arch(arch)

tags = Dict{String,String}(
"arch" => arch,
Expand Down Expand Up @@ -228,6 +155,17 @@ function Base.repr(p::Platform; context=nothing)
)
end

# Make showing the platform a bit more palatable
function Base.show(io::IO, p::Platform)
str = string(platform_name(p), " ", arch(p))
# Add on all the other tags not covered by os/arch:
other_tags = sort(collect(filter(kv -> kv[1] ∉ ("os", "arch"), tags(p))))
if !isempty(other_tags)
str = string(str, " {", join([string(k, "=", v) for (k, v) in other_tags], ", "), "}")
end
print(io, str)
end

# Simple equality definition; for compatibility testing, use `platforms_match()`
Base.:(==)(a::AbstractPlatform, b::AbstractPlatform) = tags(a) == tags(b)

Expand Down Expand Up @@ -569,7 +507,7 @@ function triplet(p::AbstractPlatform)

# Tack on all extra tags
for (tag, val) in tags(p)
if tag ∈ ("os", "arch", "libc", "call_abi", "libgfortran_version", "libstdcxx_version", "cxxstring_abi")
if tag ∈ ("os", "arch", "libc", "call_abi", "libgfortran_version", "libstdcxx_version", "cxxstring_abi", "os_version")
continue
end
str = string(str, "-", tag, "+", val)
Expand Down Expand Up @@ -628,28 +566,39 @@ const arch_mapping = Dict(
"powerpc64le" => "p(ower)?pc64le",
)
# Keep this in sync with `CPUID.ISAs_by_family`
# These are the CPUID side of the microarchitectures targeted by GCC flags in BinaryBuilder.jl
const arch_march_isa_mapping = let
function get_set(arch, name)
all = CPUID.ISAs_by_family[arch]
return all[findfirst(x -> x.first == name, all)].second
end
Dict(
"i686" => [
"i686" => get_set("i686", "i686"),
"prescott" => get_set("i686", "prescott"),
],
"x86_64" => [
"x86_64" => get_set("x86_64", "x86_64"),
"avx" => get_set("x86_64", "sandybridge"),
"avx2" => get_set("x86_64", "haswell"),
"avx512" => get_set("x86_64", "skylake_avx512"),
],
"armv6l" => [
"arm1176jzfs" => get_set("armv6l", "arm1176jzfs"),
],
"armv7l" => [
"armv7l" => get_set("arm", "armv7l"),
"neon" => get_set("arm", "armv7l_neon"),
"vfp4" => get_set("arm", "armv7l_neon_vfp4"),
"armv7l" => get_set("armv7l", "armv7l"),
"neonvfpv4" => get_set("armv7l", "armv7l+neon+vfpv4"),
],
"aarch64" => [
"armv8" => get_set("aarch64", "armv8.0_a"),
"thunderx2" => get_set("aarch64", "armv8.1_a"),
"carmel" => get_set("aarch64", "armv8.2_a_crypto"),
"armv8_0" => get_set("aarch64", "armv8.0-a"),
"armv8_1" => get_set("aarch64", "armv8.1-a"),
"armv8_2_crypto" => get_set("aarch64", "armv8.2-a+crypto"),
"armv8_4_crypto_sve" => get_set("aarch64", "armv8.4-a+crypto+sve"),
],
"powerpc64le" => [
"power8" => get_set("powerpc64le", "power8"),
]
)
end
const os_mapping = Dict(
Expand Down
98 changes: 98 additions & 0 deletions base/cpuid.jl
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
module CPUID

export cpu_isa

"""
ISA(features::Set{UInt32})
A structure which represents the Instruction Set Architecture (ISA) of a
computer. It holds the `Set` of features of the CPU.
The numerical values of the features are automatically generated from the C
source code of Julia and stored in the `features_h.jl` Julia file.
"""
struct ISA
features::Set{UInt32}
end

Base.:<=(a::ISA, b::ISA) = a.features <= b.features
Base.:<(a::ISA, b::ISA) = a.features < b.features
Base.isless(a::ISA, b::ISA) = a < b

include(string(length(Core.ARGS) >= 2 ? Core.ARGS[2] : "", "features_h.jl")) # include($BUILDROOT/base/features_h.jl)

# Keep in sync with `arch_march_isa_mapping`.
const ISAs_by_family = Dict(
"i686" => [
# Implicit in all sets, because always required: mmx, sse, sse2
"i686" => ISA(Set{UInt32}()),
"prescott" => ISA(Set((JL_X86_sse3,))),
],
"x86_64" => [
# Source: https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html.
# Implicit in all sets, because always required: mmx, sse, sse2
"x86_64" => ISA(Set{UInt32}()),
"core2" => ISA(Set((JL_X86_sse3, JL_X86_ssse3))),
"nehalem" => ISA(Set((JL_X86_sse3, JL_X86_ssse3, JL_X86_sse41, JL_X86_sse42, JL_X86_popcnt))),
"sandybridge" => ISA(Set((JL_X86_sse3, JL_X86_ssse3, JL_X86_sse41, JL_X86_sse42, JL_X86_popcnt, JL_X86_avx, JL_X86_aes, JL_X86_pclmul))),
"haswell" => ISA(Set((JL_X86_movbe, JL_X86_sse3, JL_X86_ssse3, JL_X86_sse41, JL_X86_sse42, JL_X86_popcnt, JL_X86_avx, JL_X86_avx2, JL_X86_aes, JL_X86_pclmul, JL_X86_fsgsbase, JL_X86_rdrnd, JL_X86_fma, JL_X86_bmi, JL_X86_bmi2, JL_X86_f16c))),
"skylake" => ISA(Set((JL_X86_movbe, JL_X86_sse3, JL_X86_ssse3, JL_X86_sse41, JL_X86_sse42, JL_X86_popcnt, JL_X86_avx, JL_X86_avx2, JL_X86_aes, JL_X86_pclmul, JL_X86_fsgsbase, JL_X86_rdrnd, JL_X86_fma, JL_X86_bmi, JL_X86_bmi2, JL_X86_f16c, JL_X86_rdseed, JL_X86_adx, JL_X86_prfchw, JL_X86_clflushopt, JL_X86_xsavec, JL_X86_xsaves))),
"skylake_avx512" => ISA(Set((JL_X86_movbe, JL_X86_sse3, JL_X86_ssse3, JL_X86_sse41, JL_X86_sse42, JL_X86_popcnt, JL_X86_pku, JL_X86_avx, JL_X86_avx2, JL_X86_aes, JL_X86_pclmul, JL_X86_fsgsbase, JL_X86_rdrnd, JL_X86_fma, JL_X86_bmi, JL_X86_bmi2, JL_X86_f16c, JL_X86_rdseed, JL_X86_adx, JL_X86_prfchw, JL_X86_clflushopt, JL_X86_xsavec, JL_X86_xsaves, JL_X86_avx512f, JL_X86_clwb, JL_X86_avx512vl, JL_X86_avx512bw, JL_X86_avx512dq, JL_X86_avx512cd))),
],
"armv6l" => [
# The only armv6l processor we know of that runs Julia on armv6l
# We don't have a good way to tell the different armv6l variants apart through features,
# and honestly we don't care much since it's basically this one chip that people want to use with Julia.
"arm1176jzfs" => ISA(Set{UInt32}()),
],
"armv7l" => [
"armv7l" => ISA(Set{UInt32}()),
"armv7l+neon" => ISA(Set((JL_AArch32_neon,))),
"armv7l+neon+vfpv4" => ISA(Set((JL_AArch32_neon, JL_AArch32_vfp4))),
],
"aarch64" => [
# Implicit in all sets, because always required: fp, asimd
"armv8.0-a" => ISA(Set{UInt32}()),
"armv8.1-a" => ISA(Set((JL_AArch64_lse, JL_AArch64_crc, JL_AArch64_rdm))),
"armv8.2-a+crypto" => ISA(Set((JL_AArch64_lse, JL_AArch64_crc, JL_AArch64_rdm, JL_AArch64_aes, JL_AArch64_sha2))),
"armv8.4-a+crypto+sve" => ISA(Set((JL_AArch64_lse, JL_AArch64_crc, JL_AArch64_rdm, JL_AArch64_fp16fml, JL_AArch64_aes, JL_AArch64_sha2, JL_AArch64_dotprod, JL_AArch64_sve))),
],
"powerpc64le" => [
# We have no way to test powerpc64le features yet, so we're only going to declare the lowest ISA:
"power8" => ISA(Set{UInt32}()),
]
)

# Test a CPU feature exists on the currently-running host
test_cpu_feature(feature::UInt32) = ccall(:jl_test_cpu_feature, Bool, (UInt32,), feature)

# Normalize some variation in ARCH values (which typically come from `uname -m`)
function normalize_arch(arch::String)
arch = lowercase(arch)
if arch ∈ ("amd64",)
arch = "x86_64"
elseif arch ∈ ("i386", "i486", "i586")
arch = "i686"
elseif arch ∈ ("armv6",)
arch = "armv6l"
elseif arch ∈ ("arm", "armv7", "armv8", "armv8l")
arch = "armv7l"
elseif arch ∈ ("arm64",)
arch = "aarch64"
elseif arch ∈ ("ppc64le",)
arch = "powerpc64le"
end
return arch
end

"""
cpu_isa()
Return the [`ISA`](@ref) (instruction set architecture) of the current CPU.
"""
function cpu_isa()
all_features = last(last(get(ISAs_by_family, normalize_arch(String(Sys.ARCH)), "" => [ISA(Set{UInt32}())]))).features
return ISA(Set{UInt32}(feat for feat in all_features if test_cpu_feature(feat)))
end

end # module CPUID
10 changes: 8 additions & 2 deletions test/binaryplatforms.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@ end
@test triplet(P("x86_64", "linux")) == "x86_64-linux-gnu"
@test triplet(P("armv6l", "linux")) == "armv6l-linux-gnueabihf"
@test triplet(P("x86_64", "macos")) == "x86_64-apple-darwin"
@test triplet(P("x86_64", "macos"; os_version=v"16")) == "x86_64-apple-darwin16"
@test triplet(P("x86_64", "freebsd")) == "x86_64-unknown-freebsd"
@test triplet(P("i686", "freebsd")) == "i686-unknown-freebsd"

Expand Down Expand Up @@ -182,8 +183,13 @@ end
@test R("x86_64-linux-gnu-march+avx2") == P("x86_64", "linux"; march="avx2")
@test R("x86_64-linux-gnu-march+x86_64-cuda+10.1") == P("x86_64", "linux"; march="x86_64", cuda="10.1")

# Round-trip our little homie through `triplet()`
@test parse(Platform, triplet(HostPlatform())) == HostPlatform()
# Round-trip our little homie through `triplet()`, with some bending
# of the rules for MacOS and FreeBSD, who have incomplete `os_version`
# numbers embedded within their triplets.
p = HostPlatform()
if !Sys.isbsd(p)
@test parse(Platform, triplet(p)) == p
end

# Also test round-tripping through `repr()`:
@test eval(Meta.parse(repr(HostPlatform()))) == HostPlatform()
Expand Down

2 comments on commit b1e1bb3

@nanosoldier
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Executing the daily package evaluation, I will reply here when finished:

@nanosoldier runtests(ALL, isdaily = true)

@nanosoldier
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Your package evaluation job has completed - possible new issues were detected. A full report can be found here. cc @maleadt

Please sign in to comment.