Kernel/Locking: Add lock rank tracking per thread to find deadlocks

This change adds a static lock hierarchy / ranking to the Kernel with
the goal of reducing / finding deadlocks when running with SMP enabled.

We have seen quite a few lock ordering deadlocks (locks taken in a
different order, on two different code paths). As we properly annotate
locks in the system, then these facilities will find these locking
protocol violations automatically

The `LockRank` enum documents the various locks in the system and their
rank. The implementation guarantees that a thread holding one or more
locks of a lower rank cannot acquire an additional lock with rank that
is greater or equal to any of the currently held locks.

Kernel/CMakeLists.txt

@@ -161,6 +161,7 @@ set(KERNEL_SOURCES
  Memory/VirtualRange.cpp
  Memory/VirtualRangeAllocator.cpp
  MiniStdLib.cpp
+ Locking/LockRank.cpp
  Locking/Mutex.cpp
  Net/E1000ENetworkAdapter.cpp
  Net/E1000NetworkAdapter.cpp

Kernel/Locking/LockRank.cpp

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2021, Brian Gianforcaro <[email protected]>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <Kernel/Locking/LockRank.h>
+#include <Kernel/Thread.h>
+
+// Note: These stubs can't be in LockRank.h as that would create
+// a cyclic dependency in the header include graph of the Kernel.
+
+namespace Kernel {
+
+void track_lock_acquire(LockRank rank)
+{
+ auto thread = Thread::current();
+ if (thread && !thread->is_crashing())
+ thread->track_lock_acquire(rank);
+}
+
+void track_lock_release(LockRank rank)
+{
+ auto thread = Thread::current();
+ if (thread && !thread->is_crashing())
+ thread->track_lock_release(rank);
+}
+
+}

Kernel/Locking/LockRank.h

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2021, Brian Gianforcaro <[email protected]>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/EnumBits.h>
+
+namespace Kernel {
+// To catch bugs where locks are taken out of order, we annotate all locks
+// in the kernel with a rank. The rank describes the order in which locks
+// are allowed to be taken. If a lock is acquired, and it is of an incompatible
+// rank with the lock held by the executing thread then the system can detect
+// the lock order violation and respond appropriately (crash with error).
+//
+// A thread holding a lower ranked lock cannot acquire a lock of a greater or equal rank.
+enum class LockRank : int {
+ // Special marker for locks which haven't been annotated yet.
+ // Note: This should be removed once all locks are annotated.
+ None = 0x000,
+
+ // We need to be able to handle page faults from anywhere, so
+ // memory manager locks are our lowest rank lock.
+ MemoryManager = 0x001,
+
+ Interrupts = 0x002,
+
+ FileSystem = 0x004,
+
+ Thread = 0x008,
+
+ // Process locks are the lowest rank, as they normally are taken
+ // first thing when processing syscalls.
+ Process = 0x010,
+};
+
+AK_ENUM_BITWISE_OPERATORS(LockRank);
+
+void track_lock_acquire(LockRank);
+void track_lock_release(LockRank);
+}

Kernel/Thread.cpp

@@ -1226,6 +1226,56 @@ bool Thread::should_be_stopped() const
  return process().is_stopped();
 }
 
+void Thread::track_lock_acquire(LockRank rank)
+{
+ // Nothing to do for locks without a rank.
+ if (rank == LockRank::None)
+ return;
+
+ if (m_lock_rank_mask != LockRank::None) {
+ // Verify we are only attempting to take a lock of a higher rank.
+ VERIFY(m_lock_rank_mask > rank);
+ }
+
+ m_lock_rank_mask |= rank;
+}
+
+void Thread::track_lock_release(LockRank rank)
+{
+ // Nothing to do for locks without a rank.
+ if (rank == LockRank::None)
+ return;
+
+ using PrimitiveType = UnderlyingType<LockRank>;
+
+ // The rank value from the caller should only contain a single bit, otherwise
+ // we are disabling the tracking for multiple locks at once which will corrupt
+ // the lock tracking mask, and we will assert somewhere else.
+ auto rank_is_a_single_bit = [](auto rank_enum) -> bool {
+ auto rank = static_cast<PrimitiveType>(rank_enum);
+ auto rank_without_least_significant_bit = rank - 1;
+ return (rank & rank_without_least_significant_bit) == 0;
+ };
+
+ // We can't release locks out of order, as that would violate the ranking.
+ // This is validated by toggling the least significant bit of the mask, and
+ // then bit wise or-ing the rank we are trying to release with the resulting
+ // mask. If the rank we are releasing is truly the highest rank then the mask
+ // we get back will be equal to the current mask of stored on the thread.
+ auto rank_is_in_order = [](auto mask_enum, auto rank_enum) -> bool {
+ auto mask = static_cast<PrimitiveType>(mask_enum);
+ auto rank = static_cast<PrimitiveType>(rank_enum);
+ auto mask_without_least_significant_bit = mask - 1;
+ return ((mask & mask_without_least_significant_bit) | rank) == mask;
+ };
+
+ VERIFY(has_flag(m_lock_rank_mask, rank));
+ VERIFY(rank_is_a_single_bit(rank));
+ VERIFY(rank_is_in_order(m_lock_rank_mask, rank));
+
+ m_lock_rank_mask ^= rank;
+}
+
 }
 
 void AK::Formatter<Kernel::Thread>::format(FormatBuilder& builder, const Kernel::Thread& value)

Kernel/Thread.h

@@ -29,6 +29,7 @@
 #include <Kernel/Library/ListedRefCounted.h>
 #include <Kernel/Locking/LockLocation.h>
 #include <Kernel/Locking/LockMode.h>
+#include <Kernel/Locking/LockRank.h>
 #include <Kernel/Locking/SpinlockProtected.h>
 #include <Kernel/Memory/VirtualRange.h>
 #include <Kernel/Scheduler.h>
@@ -1083,6 +1084,9 @@ class Thread
  u32 saved_critical() const { return m_saved_critical; }
  void save_critical(u32 critical) { m_saved_critical = critical; }
 
+ void track_lock_acquire(LockRank rank);
+ void track_lock_release(LockRank rank);
+
  [[nodiscard]] bool is_active() const { return m_is_active; }
 
  [[nodiscard]] bool is_finalizable() const
@@ -1302,6 +1306,7 @@ class Thread
  Kernel::Mutex* m_blocking_lock { nullptr };
  u32 m_lock_requested_count { 0 };
  IntrusiveListNode<Thread> m_blocked_threads_list_node;
+ LockRank m_lock_rank_mask { LockRank::None };
 
 #if LOCK_DEBUG
  struct HoldingLockInfo {