add some intrinsic support for atomic operations

Make.inc

@@ -1243,7 +1243,7 @@ else ifneq ($(USEMSVC), 1)
 endif
 
 ifeq ($(OS), Linux)
-OSLIBS += -Wl,--no-as-needed -ldl -lrt -lpthread -Wl,--export-dynamic,--as-needed,--no-whole-archive
+OSLIBS += -Wl,--no-as-needed -ldl -lrt -lpthread -latomic -Wl,--export-dynamic,--as-needed,--no-whole-archive
 # Detect if ifunc is supported
 IFUNC_DETECT_SRC := 'void (*f0(void))(void) { return (void(*)(void))0L; }; void f(void) __attribute__((ifunc("f0")));'
 ifeq (supported, $(shell echo $(IFUNC_DETECT_SRC) | $(CC) -Werror -x c - -S -o /dev/null > /dev/null 2>&1 && echo supported))
@@ -1269,7 +1269,7 @@ endif
 
 ifeq ($(OS), FreeBSD)
 JLDFLAGS := -Wl,-Bdynamic
-OSLIBS += -lelf -lkvm -lrt -lpthread
+OSLIBS += -lelf -lkvm -lrt -lpthread -latomic
 
 # Tweak order of libgcc_s in DT_NEEDED,
 # make it loaded first to

NEWS.md

@@ -42,10 +42,15 @@ Command-line option changes
 Multi-threading changes
 -----------------------
 
-* If the `JULIA_NUM_THREADS` environment variable is set to `auto`, then the number of threads will be set to the number of CPU threads ([#38952])
-* Every `Task` object has a local random number generator state, providing reproducible (schedule-independent) execution
- of parallel simulation code by default. The default generator is also significantly faster in parallel than in
- previous versions.
+* Intrinsics for atomic pointer operations are now defined for certain byte sizes. ([#37847])
+* Support for declaring and using individual fields of a mutable struct as
+ atomic now available. ([#37847])
+* If the `JULIA_NUM_THREADS` environment variable is set to `auto`, then the
+ number of threads will be set to the number of CPU threads ([#38952])
+* Every `Task` object has a local random number generator state, providing
+ reproducible (schedule-independent) execution of parallel simulation code by
+ default. The default generator is also significantly faster in parallel than
+ in previous versions.
 
 
 Build system changes

base/Base.jl

@@ -20,20 +20,48 @@ include(path::String) = include(Base, path)
 const is_primary_base_module = ccall(:jl_module_parent, Ref{Module}, (Any,), Base) === Core.Main
 ccall(:jl_set_istopmod, Cvoid, (Any, Bool), Base, is_primary_base_module)
 
+# The real @inline macro is not available until after array.jl, so this
+# internal macro splices the meta Expr directly into the function body.
+macro _inline_meta()
+ Expr(:meta, :inline)
+end
+macro _noinline_meta()
+ Expr(:meta, :noinline)
+end
+
 # Try to help prevent users from shooting them-selves in the foot
 # with ambiguities by defining a few common and critical operations
 # (and these don't need the extra convert code)
-getproperty(x::Module, f::Symbol) = getfield(x, f)
-setproperty!(x::Module, f::Symbol, v) = setfield!(x, f, v)
-getproperty(x::Type, f::Symbol) = getfield(x, f)
-setproperty!(x::Type, f::Symbol, v) = setfield!(x, f, v)
-getproperty(x::Tuple, f::Int) = getfield(x, f)
+getproperty(x::Module, f::Symbol) = (@_inline_meta; getfield(x, f))
+setproperty!(x::Module, f::Symbol, v) = setfield!(x, f, v) # to get a decent error
+getproperty(x::Type, f::Symbol) = (@_inline_meta; getfield(x, f))
+setproperty!(x::Type, f::Symbol, v) = error("setfield! fields of Types should not be changed")
+getproperty(x::Tuple, f::Int) = (@_inline_meta; getfield(x, f))
 setproperty!(x::Tuple, f::Int, v) = setfield!(x, f, v) # to get a decent error
 
-getproperty(x, f::Symbol) = getfield(x, f)
-dotgetproperty(x, f) = getproperty(x, f)
+getproperty(x, f::Symbol) = (@_inline_meta; getfield(x, f))
 setproperty!(x, f::Symbol, v) = setfield!(x, f, convert(fieldtype(typeof(x), f), v))
 
+dotgetproperty(x, f) = getproperty(x, f)
+
+getproperty(x::Module, f::Symbol, order::Symbol) = (@_inline_meta; getfield(x, f, order))
+setproperty!(x::Module, f::Symbol, v, order::Symbol) = setfield!(x, f, v, order) # to get a decent error
+getproperty(x::Type, f::Symbol, order::Symbol) = (@_inline_meta; getfield(x, f, order))
+setproperty!(x::Type, f::Symbol, v, order::Symbol) = error("setfield! fields of Types should not be changed")
+getproperty(x::Tuple, f::Int, order::Symbol) = (@_inline_meta; getfield(x, f, order))
+setproperty!(x::Tuple, f::Int, v, order::Symbol) = setfield!(x, f, v, order) # to get a decent error
+
+getproperty(x, f::Symbol, order::Symbol) = (@_inline_meta; getfield(x, f, order))
+setproperty!(x, f::Symbol, v, order::Symbol) = (@_inline_meta; setfield!(x, f, convert(fieldtype(typeof(x), f), v), order))
+
+swapproperty!(x, f::Symbol, v, order::Symbol=:notatomic) =
+ (@_inline_meta; Core.swapfield!(x, f, convert(fieldtype(typeof(x), f), v), order))
+modifyproperty!(x, f::Symbol, op, v, order::Symbol=:notatomic) =
+ (@_inline_meta; Core.modifyfield!(x, f, op, v, order))
+replaceproperty!(x, f::Symbol, expected, desired, success_order::Symbol=:notatomic, fail_order::Symbol=success_order) =
+ (@_inline_meta; Core.replacefield!(x, f, expected, convert(fieldtype(typeof(x), f), desired), success_order, fail_order))
+
+
 include("coreio.jl")
 
 eval(x) = Core.eval(Base, x)

base/boot.jl

@@ -187,11 +187,12 @@ export
  InterruptException, InexactError, OutOfMemoryError, ReadOnlyMemoryError,
  OverflowError, StackOverflowError, SegmentationFault, UndefRefError, UndefVarError,
  TypeError, ArgumentError, MethodError, AssertionError, LoadError, InitError,
- UndefKeywordError,
+ UndefKeywordError, ConcurrencyViolationError,
  # AST representation
  Expr, QuoteNode, LineNumberNode, GlobalRef,
  # object model functions
- fieldtype, getfield, setfield!, nfields, throw, tuple, ===, isdefined, eval, ifelse,
+ fieldtype, getfield, setfield!, swapfield!, modifyfield!, replacefield!,
+ nfields, throw, tuple, ===, isdefined, eval, ifelse,
  # sizeof # not exported, to avoid conflicting with Base.sizeof
  # type reflection
  <:, typeof, isa, typeassert,
@@ -290,6 +291,9 @@ struct UndefRefError <: Exception end
 struct UndefVarError <: Exception
  var::Symbol
 end
+struct ConcurrencyViolationError <: Exception
+ msg::AbstractString
+end
 struct InterruptException <: Exception end
 struct DomainError <: Exception
  val

base/compiler/compiler.jl

@@ -10,6 +10,9 @@ import Core: print, println, show, write, unsafe_write, stdout, stderr,
 
 const getproperty = Core.getfield
 const setproperty! = Core.setfield!
+const swapproperty! = Core.swapfield!
+const modifyproperty! = Core.modifyfield!
+const replaceproperty! = Core.replacefield!
 
 ccall(:jl_set_istopmod, Cvoid, (Any, Bool), Compiler, false)
 
@@ -19,9 +22,14 @@ eval(m, x) = Core.eval(m, x)
 include(x) = Core.include(Compiler, x)
 include(mod, x) = Core.include(mod, x)
 
-#############
-# from Base #
-#############
+# The real @inline macro is not available until after array.jl, so this
+# internal macro splices the meta Expr directly into the function body.
+macro _inline_meta()
+ Expr(:meta, :inline)
+end
+macro _noinline_meta()
+ Expr(:meta, :noinline)
+end
 
 # essential files and libraries
 include("essentials.jl")

base/compiler/ssair/ir.jl

@@ -1,6 +1,5 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-@inline isexpr(@nospecialize(stmt), head::Symbol) = isa(stmt, Expr) && stmt.head === head
 Core.PhiNode() = Core.PhiNode(Int32[], Any[])
 
 isterminator(@nospecialize(stmt)) = isa(stmt, GotoNode) || isa(stmt, GotoIfNot) || isa(stmt, ReturnNode)

base/compiler/ssair/passes.jl

@@ -560,14 +560,24 @@ function getfield_elim_pass!(ir::IRCode)
  #ndone += 1
  result_t = compact_exprtype(compact, SSAValue(idx))
  is_getfield = is_setfield = false
+ field_ordering = :unspecified
  is_ccall = false
  # Step 1: Check whether the statement we're looking at is a getfield/setfield!
  if is_known_call(stmt, setfield!, compact)
  is_setfield = true
  4 <= length(stmt.args) <= 5 || continue
+ if length(stmt.args) == 5
+ field_ordering = compact_exprtype(compact, stmt.args[5])
+ end
  elseif is_known_call(stmt, getfield, compact)
  is_getfield = true
- 3 <= length(stmt.args) <= 4 || continue
+ 3 <= length(stmt.args) <= 5 || continue
+ if length(stmt.args) == 5
+ field_ordering = compact_exprtype(compact, stmt.args[5])
+ elseif length(stmt.args) == 4
+ field_ordering = compact_exprtype(compact, stmt.args[4])
+ widenconst(field_ordering) === Bool && (field_ordering = :unspecified)
+ end
  elseif is_known_call(stmt, isa, compact)
  # TODO
  continue
@@ -660,6 +670,11 @@ function getfield_elim_pass!(ir::IRCode)
  end
  isa(struct_typ, DataType) || continue
 
+ struct_typ.name.atomicfields == C_NULL || continue # TODO: handle more
+ if !(field_ordering === :unspecified || (field_ordering isa Const && field_ordering.val === :not_atomic))
+ continue
+ end
+
  def, typeconstraint = stmt.args[2], struct_typ
 
  if struct_typ.name.mutable

base/compiler/tfuncs.jl

@@ -263,6 +263,7 @@ function isdefined_nothrow(argtypes::Array{Any, 1})
  (argtypes[2] ⊑ Symbol || argtypes[2] ⊑ Int) :
  argtypes[2] ⊑ Symbol
 end
+isdefined_tfunc(arg1, sym, order) = (@nospecialize; isdefined_tfunc(arg1, sym))
 function isdefined_tfunc(@nospecialize(arg1), @nospecialize(sym))
  if isa(arg1, Const)
  a1 = typeof(arg1.val)
@@ -316,7 +317,7 @@ function isdefined_tfunc(@nospecialize(arg1), @nospecialize(sym))
  end
  return Bool
 end
-add_tfunc(isdefined, 2, 2, isdefined_tfunc, 1)
+add_tfunc(isdefined, 2, 3, isdefined_tfunc, 1)
 
 function sizeof_nothrow(@nospecialize(x))
  if isa(x, Const)
@@ -470,22 +471,26 @@ add_tfunc(arraysize, 2, 2, (@nospecialize(a), @nospecialize(d))->Int, 4)
 function pointer_eltype(@nospecialize(ptr))
  a = widenconst(ptr)
  if a <: Ptr
- if isa(a,DataType) && isa(a.parameters[1],Type)
+ if isa(a, DataType) && isa(a.parameters[1], Type)
  return a.parameters[1]
- elseif isa(a,UnionAll) && !has_free_typevars(a)
+ elseif isa(a, UnionAll) && !has_free_typevars(a)
  unw = unwrap_unionall(a)
- if isa(unw,DataType)
+ if isa(unw, DataType)
  return rewrap_unionall(unw.parameters[1], a)
  end
  end
  end
  return Any
 end
-add_tfunc(pointerref, 3, 3,
- function (@nospecialize(a), @nospecialize(i), @nospecialize(align))
- return pointer_eltype(a)
- end, 4)
-add_tfunc(pointerset, 4, 4, (@nospecialize(a), @nospecialize(v), @nospecialize(i), @nospecialize(align)) -> a, 5)
+add_tfunc(pointerref, 3, 3, (a, i, align) -> (@nospecialize; pointer_eltype(a)), 4)
+add_tfunc(pointerset, 4, 4, (a, v, i, align) -> (@nospecialize; a), 5)
+
+add_tfunc(atomic_fence, 1, 1, (order) -> (@nospecialize; Nothing), 4)
+add_tfunc(atomic_pointerref, 2, 2, (a, order) -> (@nospecialize; pointer_eltype(a)), 4)
+add_tfunc(atomic_pointerset, 3, 3, (a, v, order) -> (@nospecialize; a), 5)
+add_tfunc(atomic_pointerswap, 3, 3, (a, v, order) -> (@nospecialize; pointer_eltype(a)), 5)
+add_tfunc(atomic_pointermodify, 4, 4, (a, op, v, order) -> (@nospecialize; T = pointer_eltype(a); Tuple{T, T}), 5)
+add_tfunc(atomic_pointerreplace, 5, 5, (a, x, v, success_order, failure_order) -> (@nospecialize; Tuple{pointer_eltype(a), Bool}), 5)
 
 # more accurate typeof_tfunc for vararg tuples abstract only in length
 function typeof_concrete_vararg(t::DataType)
@@ -675,14 +680,25 @@ function try_compute_fieldidx(typ::DataType, @nospecialize(field))
 end
 
 function getfield_nothrow(argtypes::Vector{Any})
- 2 <= length(argtypes) <= 3 || return false
- length(argtypes) == 2 && return getfield_nothrow(argtypes[1], argtypes[2], Const(true))
- return getfield_nothrow(argtypes[1], argtypes[2], argtypes[3])
+ if length(argtypes) == 2
+ boundscheck = Bool
+ elseif length(argtypes) == 3
+ boundscheck = argtypes[3]
+ if boundscheck === Const(:not_atomic) # TODO: this is assuming not atomic
+ boundscheck = Bool
+ end
+ elseif length(argtypes) == 4
+ boundscheck = argtypes[4]
+ else
+ return false
+ end
+ widenconst(boundscheck) !== Bool && return false
+ bounds_check_disabled = isa(boundscheck, Const) && boundscheck.val === false
+ return getfield_nothrow(argtypes[1], argtypes[2], !bounds_check_disabled)
 end
-function getfield_nothrow(@nospecialize(s00), @nospecialize(name), @nospecialize(inbounds))
- bounds_check_disabled = isa(inbounds, Const) && inbounds.val === false
- # If we don't have invounds and don't know the field, don't even bother
- if !bounds_check_disabled
+function getfield_nothrow(@nospecialize(s00), @nospecialize(name), boundscheck::Bool)
+ # If we don't have boundscheck and don't know the field, don't even bother
+ if boundscheck
  isa(name, Const) || return false
  end
 
@@ -700,7 +716,7 @@ function getfield_nothrow(@nospecialize(s00), @nospecialize(name), @nospecialize
  end
  return isdefined(sv, name.val)
  end
- if bounds_check_disabled && !isa(sv, Module)
+ if !boundscheck && !isa(sv, Module)
  # If bounds checking is disabled and all fields are assigned,
  # we may assume that we don't throw
  for i = 1:fieldcount(typeof(sv))
@@ -714,14 +730,15 @@ function getfield_nothrow(@nospecialize(s00), @nospecialize(name), @nospecialize
  s0 = widenconst(s00)
  s = unwrap_unionall(s0)
  if isa(s, Union)
- return getfield_nothrow(rewrap(s.a, s00), name, inbounds) &&
- getfield_nothrow(rewrap(s.b, s00), name, inbounds)
+ return getfield_nothrow(rewrap(s.a, s00), name, boundscheck) &&
+  getfield_nothrow(rewrap(s.b, s00), name, boundscheck)
  elseif isa(s, DataType)
  # Can't say anything about abstract types
  s.name.abstract && return false
+ s.name.atomicfields == C_NULL || return false # TODO: currently we're only testing for ordering == :not_atomic
  # If all fields are always initialized, and bounds check is disabled, we can assume
  # we don't throw
- if bounds_check_disabled && !isvatuple(s) && s.name !== NamedTuple.body.body.name && fieldcount(s) == s.ninitialized
+ if !boundscheck && !isvatuple(s) && s.name !== NamedTuple.body.body.name && fieldcount(s) == s.ninitialized
  return true
  end
  # Else we need to know what the field is
@@ -736,8 +753,8 @@ function getfield_nothrow(@nospecialize(s00), @nospecialize(name), @nospecialize
  return false
 end
 
-getfield_tfunc(@nospecialize(s00), @nospecialize(name), @nospecialize(inbounds)) =
- getfield_tfunc(s00, name)
+getfield_tfunc(s00, name, boundscheck_or_order) = (@nospecialize; getfield_tfunc(s00, name))
+getfield_tfunc(s00, name, order, boundscheck) = (@nospecialize; getfield_tfunc(s00, name))
 function getfield_tfunc(@nospecialize(s00), @nospecialize(name))
  s = unwrap_unionall(s00)
  if isa(s, Union)
@@ -892,10 +909,25 @@ function getfield_tfunc(@nospecialize(s00), @nospecialize(name))
  end
  return rewrap_unionall(R, s00)
 end
-add_tfunc(getfield, 2, 3, getfield_tfunc, 1)
-add_tfunc(setfield!, 3, 3, (@nospecialize(o), @nospecialize(f), @nospecialize(v)) -> v, 3)
-fieldtype_tfunc(@nospecialize(s0), @nospecialize(name), @nospecialize(inbounds)) =
- fieldtype_tfunc(s0, name)
+
+setfield!_tfunc(o, f, v, order) = (@nospecialize; v)
+setfield!_tfunc(o, f, v) = (@nospecialize; v)
+
+swapfield!_tfunc(o, f, v, order) = (@nospecialize; getfield_tfunc(o, f))
+swapfield!_tfunc(o, f, v) = (@nospecialize; getfield_tfunc(o, f))
+modifyfield!_tfunc(o, f, op, v, order) = (@nospecialize; T = getfield_tfunc(o, f); T === Bottom ? T : Tuple{T, T})
+modifyfield!_tfunc(o, f, op, v) = (@nospecialize; T = getfield_tfunc(o, f); T === Bottom ? T : Tuple{T, T}) # TODO: also model op(o.f, v) call
+replacefield!_tfunc(o, f, x, v, success_order, failure_order) = (@nospecialize; replacefield!_tfunc(o, f, x, v))
+replacefield!_tfunc(o, f, x, v, success_order) = (@nospecialize; replacefield!_tfunc(o, f, x, v))
+replacefield!_tfunc(o, f, x, v) = (@nospecialize; T = getfield_tfunc(o, f); T === Bottom ? T : Tuple{widenconst(T), Bool})
+# we could use tuple_tfunc instead of widenconst, but `o` is mutable, so that is unlikely to be beneficial
+
+add_tfunc(getfield, 2, 4, getfield_tfunc, 1)
+add_tfunc(setfield!, 3, 4, setfield!_tfunc, 3)
+
+add_tfunc(swapfield!, 3, 4, swapfield!_tfunc, 3)
+add_tfunc(modifyfield!, 4, 5, modifyfield!_tfunc, 3)
+add_tfunc(replacefield!, 4, 6, replacefield!_tfunc, 3)
 
 function fieldtype_nothrow(@nospecialize(s0), @nospecialize(name))
  s0 === Bottom && return true # unreachable
@@ -954,6 +986,7 @@ function _fieldtype_nothrow(@nospecialize(s), exact::Bool, name::Const)
  return true
 end
 
+fieldtype_tfunc(s0, name, boundscheck) = (@nospecialize; fieldtype_tfunc(s0, name))
 function fieldtype_tfunc(@nospecialize(s0), @nospecialize(name))
  if s0 === Bottom
  return Bottom

base/condition.jl

@@ -5,7 +5,7 @@
 @noinline function concurrency_violation()
  # can be useful for debugging
  #try; error(); catch; ccall(:jlbacktrace, Cvoid, ()); end
- error("concurrency violation detected")
+ throw(ConcurrencyViolationError("lock must be held"))
 end
 
 """