New select and Mutex implementations (Kotlin#3020)

* New selects are twice as fast as the previous version * The implementation doesn't require an NCAS protocol by giving up lock-freedom and linearizability in corner cases * The overall algorithm is much more straightforward and easier to maintain Co-authored-by: Nikita Koval <[email protected]>
Earlycrusher · Aug 4, 2022 · feea6a5 · feea6a5
1 parent 9e886f1
commit feea6a5
Show file tree

Hide file tree

Showing 43 changed files with 2,334 additions and 1,635 deletions.
diff --git a/benchmarks/src/jmh/kotlin/benchmarks/tailcall/SelectBenchmark.kt b/benchmarks/src/jmh/kotlin/benchmarks/tailcall/SelectBenchmark.kt
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2016-2022 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package benchmarks.tailcall
+
+import kotlinx.coroutines.*
+import kotlinx.coroutines.channels.*
+import kotlinx.coroutines.selects.*
+import org.openjdk.jmh.annotations.*
+import java.util.concurrent.*
+
+@Warmup(iterations = 8, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 8, time = 1, timeUnit = TimeUnit.SECONDS)
+@Fork(value = 1)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
+@State(Scope.Benchmark)
+open class SelectBenchmark {
+ // 450
+ private val iterations = 1000
+
+ @Benchmark
+ fun stressSelect() = runBlocking {
+ val pingPong = Channel<Unit>()
+ launch {
+ repeat(iterations) {
+ select {
+ pingPong.onSend(Unit) {}
+ }
+ }
+ }
+
+ launch {
+ repeat(iterations) {
+ select {
+ pingPong.onReceive() {}
+ }
+ }
+ }
+ }
+}
diff --git a/gradle.properties b/gradle.properties
@@ -13,7 +13,7 @@ junit5_version=5.7.0
 atomicfu_version=0.18.2
 knit_version=0.4.0
 html_version=0.7.2
-lincheck_version=2.14
+lincheck_version=2.14.1
 dokka_version=1.6.21
 byte_buddy_version=1.10.9
 reactor_version=3.4.1

diff --git a/kotlinx-coroutines-core/api/kotlinx-coroutines-core.api b/kotlinx-coroutines-core/api/kotlinx-coroutines-core.api
@@ -423,10 +423,11 @@ public final class kotlinx/coroutines/JobKt {
  public static final fun isActive (Lkotlin/coroutines/CoroutineContext;)Z
 }
 
-public class kotlinx/coroutines/JobSupport : kotlinx/coroutines/ChildJob, kotlinx/coroutines/Job, kotlinx/coroutines/ParentJob, kotlinx/coroutines/selects/SelectClause0 {
+public class kotlinx/coroutines/JobSupport : kotlinx/coroutines/ChildJob, kotlinx/coroutines/Job, kotlinx/coroutines/ParentJob {
  public fun <init> (Z)V
  protected fun afterCompletion (Ljava/lang/Object;)V
  public final fun attachChild (Lkotlinx/coroutines/ChildJob;)Lkotlinx/coroutines/ChildHandle;
+ protected final fun awaitInternal (Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
  public synthetic fun cancel ()V
  public synthetic fun cancel (Ljava/lang/Throwable;)Z
  public fun cancel (Ljava/util/concurrent/CancellationException;)V
@@ -443,6 +444,7 @@ public class kotlinx/coroutines/JobSupport : kotlinx/coroutines/ChildJob, kotlin
  protected final fun getCompletionCauseHandled ()Z
  public final fun getCompletionExceptionOrNull ()Ljava/lang/Throwable;
  public final fun getKey ()Lkotlin/coroutines/CoroutineContext$Key;
+ protected final fun getOnAwaitInternal ()Lkotlinx/coroutines/selects/SelectClause1;
  public final fun getOnJoin ()Lkotlinx/coroutines/selects/SelectClause0;
  public fun getParent ()Lkotlinx/coroutines/Job;
  protected fun handleJobException (Ljava/lang/Throwable;)Z
@@ -462,7 +464,6 @@ public class kotlinx/coroutines/JobSupport : kotlinx/coroutines/ChildJob, kotlin
  public final fun parentCancelled (Lkotlinx/coroutines/ParentJob;)V
  public fun plus (Lkotlin/coroutines/CoroutineContext;)Lkotlin/coroutines/CoroutineContext;
  public fun plus (Lkotlinx/coroutines/Job;)Lkotlinx/coroutines/Job;
- public final fun registerSelectClause0 (Lkotlinx/coroutines/selects/SelectInstance;Lkotlin/jvm/functions/Function1;)V
  public final fun start ()Z
  protected final fun toCancellationException (Ljava/lang/Throwable;Ljava/lang/String;)Ljava/util/concurrent/CancellationException;
  public static synthetic fun toCancellationException$default (Lkotlinx/coroutines/JobSupport;Ljava/lang/Throwable;Ljava/lang/String;ILjava/lang/Object;)Ljava/util/concurrent/CancellationException;
@@ -1194,6 +1195,11 @@ public class kotlinx/coroutines/scheduling/ExperimentalCoroutineDispatcher : kot
  public fun toString ()Ljava/lang/String;
 }
 
+public final class kotlinx/coroutines/selects/OnTimeoutKt {
+ public static final fun onTimeout (Lkotlinx/coroutines/selects/SelectBuilder;JLkotlin/jvm/functions/Function1;)V
+ public static final fun onTimeout-8Mi8wO0 (Lkotlinx/coroutines/selects/SelectBuilder;JLkotlin/jvm/functions/Function1;)V
+}
+
 public abstract interface class kotlinx/coroutines/selects/SelectBuilder {
  public abstract fun invoke (Lkotlinx/coroutines/selects/SelectClause0;Lkotlin/jvm/functions/Function1;)V
  public abstract fun invoke (Lkotlinx/coroutines/selects/SelectClause1;Lkotlin/jvm/functions/Function2;)V
@@ -1204,71 +1210,80 @@ public abstract interface class kotlinx/coroutines/selects/SelectBuilder {
 
 public final class kotlinx/coroutines/selects/SelectBuilder$DefaultImpls {
  public static fun invoke (Lkotlinx/coroutines/selects/SelectBuilder;Lkotlinx/coroutines/selects/SelectClause2;Lkotlin/jvm/functions/Function2;)V
+ public static fun onTimeout (Lkotlinx/coroutines/selects/SelectBuilder;JLkotlin/jvm/functions/Function1;)V
 }
 
-public final class kotlinx/coroutines/selects/SelectBuilderImpl : kotlinx/coroutines/internal/LockFreeLinkedListHead, kotlin/coroutines/Continuation, kotlin/coroutines/jvm/internal/CoroutineStackFrame, kotlinx/coroutines/selects/SelectBuilder, kotlinx/coroutines/selects/SelectInstance {
+public final class kotlinx/coroutines/selects/SelectBuilderImpl : kotlinx/coroutines/selects/SelectImplementation {
  public fun <init> (Lkotlin/coroutines/Continuation;)V
- public fun disposeOnSelect (Lkotlinx/coroutines/DisposableHandle;)V
- public fun getCallerFrame ()Lkotlin/coroutines/jvm/internal/CoroutineStackFrame;
- public fun getCompletion ()Lkotlin/coroutines/Continuation;
- public fun getContext ()Lkotlin/coroutines/CoroutineContext;
  public final fun getResult ()Ljava/lang/Object;
- public fun getStackTraceElement ()Ljava/lang/StackTraceElement;
  public final fun handleBuilderException (Ljava/lang/Throwable;)V
- public fun invoke (Lkotlinx/coroutines/selects/SelectClause0;Lkotlin/jvm/functions/Function1;)V
- public fun invoke (Lkotlinx/coroutines/selects/SelectClause1;Lkotlin/jvm/functions/Function2;)V
- public fun invoke (Lkotlinx/coroutines/selects/SelectClause2;Ljava/lang/Object;Lkotlin/jvm/functions/Function2;)V
- public fun invoke (Lkotlinx/coroutines/selects/SelectClause2;Lkotlin/jvm/functions/Function2;)V
- public fun isSelected ()Z
- public fun onTimeout (JLkotlin/jvm/functions/Function1;)V
- public fun performAtomicTrySelect (Lkotlinx/coroutines/internal/AtomicDesc;)Ljava/lang/Object;
- public fun resumeSelectWithException (Ljava/lang/Throwable;)V
- public fun resumeWith (Ljava/lang/Object;)V
- public fun toString ()Ljava/lang/String;
- public fun trySelect ()Z
- public fun trySelectOther (Lkotlinx/coroutines/internal/LockFreeLinkedListNode$PrepareOp;)Ljava/lang/Object;
 }
 
-public abstract interface class kotlinx/coroutines/selects/SelectClause0 {
- public abstract fun registerSelectClause0 (Lkotlinx/coroutines/selects/SelectInstance;Lkotlin/jvm/functions/Function1;)V
+public abstract interface class kotlinx/coroutines/selects/SelectClause {
+ public abstract fun getClauseObject ()Ljava/lang/Object;
+ public abstract fun getOnCancellationConstructor ()Lkotlin/jvm/functions/Function3;
+ public abstract fun getProcessResFunc ()Lkotlin/jvm/functions/Function3;
+ public abstract fun getRegFunc ()Lkotlin/jvm/functions/Function3;
+}
+
+public abstract interface class kotlinx/coroutines/selects/SelectClause0 : kotlinx/coroutines/selects/SelectClause {
 }
 
-public abstract interface class kotlinx/coroutines/selects/SelectClause1 {
- public abstract fun registerSelectClause1 (Lkotlinx/coroutines/selects/SelectInstance;Lkotlin/jvm/functions/Function2;)V
+public abstract interface class kotlinx/coroutines/selects/SelectClause1 : kotlinx/coroutines/selects/SelectClause {
 }
 
-public abstract interface class kotlinx/coroutines/selects/SelectClause2 {
- public abstract fun registerSelectClause2 (Lkotlinx/coroutines/selects/SelectInstance;Ljava/lang/Object;Lkotlin/jvm/functions/Function2;)V
+public abstract interface class kotlinx/coroutines/selects/SelectClause2 : kotlinx/coroutines/selects/SelectClause {
+}
+
+public class kotlinx/coroutines/selects/SelectImplementation : kotlinx/coroutines/selects/SelectBuilder, kotlinx/coroutines/selects/SelectInstanceInternal {
+ public fun <init> (Lkotlin/coroutines/CoroutineContext;)V
+ public fun disposeOnCompletion (Lkotlinx/coroutines/DisposableHandle;)V
+ public fun doSelect (Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
+ public fun getContext ()Lkotlin/coroutines/CoroutineContext;
+ public synthetic fun invoke (Ljava/lang/Object;)Ljava/lang/Object;
+ public fun invoke (Ljava/lang/Throwable;)V
+ public fun invoke (Lkotlinx/coroutines/selects/SelectClause0;Lkotlin/jvm/functions/Function1;)V
+ public fun invoke (Lkotlinx/coroutines/selects/SelectClause1;Lkotlin/jvm/functions/Function2;)V
+ public fun invoke (Lkotlinx/coroutines/selects/SelectClause2;Ljava/lang/Object;Lkotlin/jvm/functions/Function2;)V
+ public fun invoke (Lkotlinx/coroutines/selects/SelectClause2;Lkotlin/jvm/functions/Function2;)V
+ public fun onTimeout (JLkotlin/jvm/functions/Function1;)V
+ public fun selectInRegistrationPhase (Ljava/lang/Object;)V
+ public fun trySelect (Ljava/lang/Object;Ljava/lang/Object;)Z
+ public final fun trySelectDetailed (Ljava/lang/Object;Ljava/lang/Object;)Lkotlinx/coroutines/selects/TrySelectDetailedResult;
 }
 
 public abstract interface class kotlinx/coroutines/selects/SelectInstance {
- public abstract fun disposeOnSelect (Lkotlinx/coroutines/DisposableHandle;)V
- public abstract fun getCompletion ()Lkotlin/coroutines/Continuation;
- public abstract fun isSelected ()Z
- public abstract fun performAtomicTrySelect (Lkotlinx/coroutines/internal/AtomicDesc;)Ljava/lang/Object;
- public abstract fun resumeSelectWithException (Ljava/lang/Throwable;)V
- public abstract fun trySelect ()Z
- public abstract fun trySelectOther (Lkotlinx/coroutines/internal/LockFreeLinkedListNode$PrepareOp;)Ljava/lang/Object;
+ public abstract fun disposeOnCompletion (Lkotlinx/coroutines/DisposableHandle;)V
+ public abstract fun getContext ()Lkotlin/coroutines/CoroutineContext;
+ public abstract fun selectInRegistrationPhase (Ljava/lang/Object;)V
+ public abstract fun trySelect (Ljava/lang/Object;Ljava/lang/Object;)Z
 }
 
 public final class kotlinx/coroutines/selects/SelectKt {
- public static final fun onTimeout-8Mi8wO0 (Lkotlinx/coroutines/selects/SelectBuilder;JLkotlin/jvm/functions/Function1;)V
  public static final fun select (Lkotlin/jvm/functions/Function1;Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
 }
 
+public final class kotlinx/coroutines/selects/SelectOldKt {
+ public static final fun selectOld (Lkotlin/jvm/functions/Function1;Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
+ public static final fun selectUnbiasedOld (Lkotlin/jvm/functions/Function1;Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
+}
+
 public final class kotlinx/coroutines/selects/SelectUnbiasedKt {
  public static final fun selectUnbiased (Lkotlin/jvm/functions/Function1;Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
 }
 
-public final class kotlinx/coroutines/selects/UnbiasedSelectBuilderImpl : kotlinx/coroutines/selects/SelectBuilder {
+public final class kotlinx/coroutines/selects/UnbiasedSelectBuilderImpl : kotlinx/coroutines/selects/UnbiasedSelectImplementation {
  public fun <init> (Lkotlin/coroutines/Continuation;)V
  public final fun handleBuilderException (Ljava/lang/Throwable;)V
  public final fun initSelectResult ()Ljava/lang/Object;
+}
+
+public class kotlinx/coroutines/selects/UnbiasedSelectImplementation : kotlinx/coroutines/selects/SelectImplementation {
+ public fun <init> (Lkotlin/coroutines/CoroutineContext;)V
+ public fun doSelect (Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
  public fun invoke (Lkotlinx/coroutines/selects/SelectClause0;Lkotlin/jvm/functions/Function1;)V
  public fun invoke (Lkotlinx/coroutines/selects/SelectClause1;Lkotlin/jvm/functions/Function2;)V
  public fun invoke (Lkotlinx/coroutines/selects/SelectClause2;Ljava/lang/Object;Lkotlin/jvm/functions/Function2;)V
- public fun invoke (Lkotlinx/coroutines/selects/SelectClause2;Lkotlin/jvm/functions/Function2;)V
- public fun onTimeout (JLkotlin/jvm/functions/Function1;)V
 }
 
 public final class kotlinx/coroutines/selects/WhileSelectKt {

diff --git a/kotlinx-coroutines-core/common/src/Builders.common.kt b/kotlinx-coroutines-core/common/src/Builders.common.kt
@@ -96,12 +96,10 @@ public fun <T> CoroutineScope.async(
 private open class DeferredCoroutine<T>(
  parentContext: CoroutineContext,
  active: Boolean
-) : AbstractCoroutine<T>(parentContext, true, active = active), Deferred<T>, SelectClause1<T> {
+) : AbstractCoroutine<T>(parentContext, true, active = active), Deferred<T> {
  override fun getCompleted(): T = getCompletedInternal() as T
  override suspend fun await(): T = awaitInternal() as T
- override val onAwait: SelectClause1<T> get() = this
- override fun <R> registerSelectClause1(select: SelectInstance<R>, block: suspend (T) -> R) =
- registerSelectClause1Internal(select, block)
+ override val onAwait: SelectClause1<T> get() = onAwaitInternal as SelectClause1<T>
 }
 
 private class LazyDeferredCoroutine<T>(

diff --git a/kotlinx-coroutines-core/common/src/CompletableDeferred.kt b/kotlinx-coroutines-core/common/src/CompletableDeferred.kt
@@ -79,14 +79,12 @@ public fun <T> CompletableDeferred(value: T): CompletableDeferred<T> = Completab
 @Suppress("UNCHECKED_CAST")
 private class CompletableDeferredImpl<T>(
  parent: Job?
-) : JobSupport(true), CompletableDeferred<T>, SelectClause1<T> {
+) : JobSupport(true), CompletableDeferred<T> {
  init { initParentJob(parent) }
  override val onCancelComplete get() = true
  override fun getCompleted(): T = getCompletedInternal() as T
  override suspend fun await(): T = awaitInternal() as T
- override val onAwait: SelectClause1<T> get() = this
- override fun <R> registerSelectClause1(select: SelectInstance<R>, block: suspend (T) -> R) =
- registerSelectClause1Internal(select, block)
+ override val onAwait: SelectClause1<T> get() = onAwaitInternal as SelectClause1<T>
 
  override fun complete(value: T): Boolean =
  makeCompleting(value)