Kernel: Move block condition evaluation out of the Scheduler

This makes the Scheduler a lot leaner by not having to evaluate
block conditions every time it is invoked. Instead evaluate them as
the states change, and unblock threads at that point.

This also implements some more waitid/waitpid/wait features and
behavior. For example, WUNTRACED and WNOWAIT are now supported. And
wait will now not return EINTR when SIGCHLD is delivered at the
same time.

Kernel/CMakeLists.txt

@@ -170,6 +170,7 @@ set(KERNEL_SOURCES
  Tasks/FinalizerTask.cpp
  Tasks/SyncTask.cpp
  Thread.cpp
+ ThreadBlockers.cpp
  ThreadTracer.cpp
  Time/APICTimer.cpp
  Time/HPET.cpp

Kernel/Devices/Device.cpp

@@ -95,6 +95,8 @@ void Device::process_next_queued_request(Badge<AsyncDeviceRequest>, const AsyncD
 
  if (next_request)
  next_request->start();
+
+ evaluate_block_conditions();
 }
 
 }

Kernel/Devices/KeyboardDevice.cpp

@@ -272,6 +272,8 @@ void KeyboardDevice::key_state_changed(u8 scan_code, bool pressed)
  }
 
  m_has_e0_prefix = false;
+
+ evaluate_block_conditions();
 }
 
 void KeyboardDevice::handle_irq(const RegisterState&)

Kernel/Devices/PS2MouseDevice.cpp

@@ -89,8 +89,11 @@ void PS2MouseDevice::irq_handle_byte_read(u8 byte)
  auto mouse_packet = backdoor->receive_mouse_packet();
  if (mouse_packet.has_value()) {
  m_entropy_source.add_random_event(mouse_packet.value());
- ScopedSpinLock lock(m_queue_lock);
- m_queue.enqueue(mouse_packet.value());
+ {
+ ScopedSpinLock lock(m_queue_lock);
+ m_queue.enqueue(mouse_packet.value());
+ }
+ evaluate_block_conditions();
  }
  return;
  }
@@ -102,8 +105,11 @@ void PS2MouseDevice::irq_handle_byte_read(u8 byte)
 #endif
  m_entropy_source.add_random_event(m_data.dword);
 
- ScopedSpinLock lock(m_queue_lock);
- m_queue.enqueue(parse_data_packet(m_data));
+ {
+ ScopedSpinLock lock(m_queue_lock);
+ m_queue.enqueue(parse_data_packet(m_data));
+ }
+ evaluate_block_conditions();
  };
 
  ASSERT(m_data_state < sizeof(m_data.bytes) / sizeof(m_data.bytes[0]));

Kernel/DoubleBuffer.cpp

@@ -70,6 +70,8 @@ ssize_t DoubleBuffer::write(const UserOrKernelBuffer& data, size_t size)
  compute_lockfree_metadata();
  if (!data.read(write_ptr, bytes_to_write))
  return -EFAULT;
+ if (m_unblock_callback && !m_empty)
+ m_unblock_callback();
  return (ssize_t)bytes_to_write;
 }
 
@@ -88,6 +90,8 @@ ssize_t DoubleBuffer::read(UserOrKernelBuffer& data, size_t size)
  return -EFAULT;
  m_read_buffer_index += nread;
  compute_lockfree_metadata();
+ if (m_unblock_callback && m_space_for_writing > 0)
+ m_unblock_callback();
  return (ssize_t)nread;
 }
 

Kernel/DoubleBuffer.h

@@ -29,6 +29,7 @@
 #include <AK/Types.h>
 #include <Kernel/KBuffer.h>
 #include <Kernel/Lock.h>
+#include <Kernel/Thread.h>
 #include <Kernel/UserOrKernelBuffer.h>
 
 namespace Kernel {
@@ -53,6 +54,12 @@ class DoubleBuffer {
 
  size_t space_for_writing() const { return m_space_for_writing; }
 
+ void set_unblock_callback(Function<void()> callback)
+ {
+ ASSERT(!m_unblock_callback);
+ m_unblock_callback = move(callback);
+ }
+
 private:
  void flip();
  void compute_lockfree_metadata();
@@ -68,6 +75,7 @@ class DoubleBuffer {
  InnerBuffer m_buffer2;
 
  KBuffer m_storage;
+ Function<void()> m_unblock_callback;
  size_t m_capacity { 0 };
  size_t m_read_buffer_index { 0 };
  size_t m_space_for_writing { 0 };

Kernel/FileSystem/FIFO.cpp

@@ -95,6 +95,11 @@ FIFO::FIFO(uid_t uid)
  LOCKER(all_fifos().lock());
  all_fifos().resource().set(this);
  m_fifo_id = ++s_next_fifo_id;
+
+ // Use the same block condition for read and write
+ m_buffer.set_unblock_callback([this]() {
+ evaluate_block_conditions();
+ });
 }
 
 FIFO::~FIFO()
@@ -116,6 +121,8 @@ void FIFO::attach(Direction direction)
  klog() << "open writer (" << m_writers << ")";
 #endif
  }
+
+ evaluate_block_conditions();
 }
 
 void FIFO::detach(Direction direction)
@@ -133,6 +140,8 @@ void FIFO::detach(Direction direction)
  ASSERT(m_writers);
  --m_writers;
  }
+
+ evaluate_block_conditions();
 }
 
 bool FIFO::can_read(const FileDescription&, size_t) const

Kernel/FileSystem/File.cpp

@@ -31,6 +31,7 @@
 namespace Kernel {
 
 File::File()
+ : m_block_condition(*this)
 {
 }
 

Kernel/FileSystem/File.h

@@ -39,6 +39,36 @@
 
 namespace Kernel {
 
+class File;
+
+class FileBlockCondition : public Thread::BlockCondition {
+public:
+ FileBlockCondition(File& file)
+ : m_file(file)
+ {
+ }
+
+ virtual bool should_add_blocker(Thread::Blocker& b, void* data) override
+ {
+ ASSERT(b.blocker_type() == Thread::Blocker::Type::File);
+ auto& blocker = static_cast<Thread::FileBlocker&>(b);
+ return !blocker.unblock(true, data);
+ }
+
+ void unblock()
+ {
+ ScopedSpinLock lock(m_lock);
+ do_unblock([&](auto& b, void* data) {
+ ASSERT(b.blocker_type() == Thread::Blocker::Type::File);
+ auto& blocker = static_cast<Thread::FileBlocker&>(b);
+ return blocker.unblock(false, data);
+ });
+ }
+
+private:
+ File& m_file;
+};
+
 // File is the base class for anything that can be referenced by a FileDescription.
 //
 // The most important functions in File are:
@@ -103,8 +133,27 @@ class File
  virtual bool is_character_device() const { return false; }
  virtual bool is_socket() const { return false; }
 
+ virtual FileBlockCondition& block_condition() { return m_block_condition; }
+
 protected:
  File();
+
+ void evaluate_block_conditions()
+ {
+ if (Processor::current().in_irq()) {
+ // If called from an IRQ handler we need to delay evaluation
+ // and unblocking of waiting threads
+ Processor::deferred_call_queue([this]() {
+ ASSERT(!Processor::current().in_irq());
+ evaluate_block_conditions();
+ });
+ } else {
+ block_condition().unblock();
+ }
+ }
+
+private:
+ FileBlockCondition m_block_condition;
 };
 
 }

Kernel/FileSystem/FileDescription.cpp

@@ -75,6 +75,26 @@ FileDescription::~FileDescription()
  m_inode = nullptr;
 }
 
+Thread::FileBlocker::BlockFlags FileDescription::should_unblock(Thread::FileBlocker::BlockFlags block_flags) const
+{
+ u32 unblock_flags = (u32)Thread::FileBlocker::BlockFlags::None;
+ if (((u32)block_flags & (u32)Thread::FileBlocker::BlockFlags::Read) && can_read())
+ unblock_flags |= (u32)Thread::FileBlocker::BlockFlags::Read;
+ if (((u32)block_flags & (u32)Thread::FileBlocker::BlockFlags::Write) && can_write())
+ unblock_flags |= (u32)Thread::FileBlocker::BlockFlags::Write;
+ // TODO: Implement Thread::FileBlocker::BlockFlags::Exception
+
+ if ((u32)block_flags & (u32)Thread::FileBlocker::BlockFlags::SocketFlags) {
+ auto* sock = socket();
+ ASSERT(sock);
+ if (((u32)block_flags & (u32)Thread::FileBlocker::BlockFlags::Accept) && sock->can_accept())
+ unblock_flags |= (u32)Thread::FileBlocker::BlockFlags::Accept;
+ if (((u32)block_flags & (u32)Thread::FileBlocker::BlockFlags::Connect) && sock->setup_state() == Socket::SetupState::Completed)
+ unblock_flags |= (u32)Thread::FileBlocker::BlockFlags::Connect;
+ }
+ return (Thread::FileBlocker::BlockFlags)unblock_flags;
+}
+
 KResult FileDescription::stat(::stat& buffer)
 {
  LOCKER(m_lock);
@@ -113,6 +133,7 @@ off_t FileDescription::seek(off_t offset, int whence)
  // FIXME: Return -EINVAL if attempting to seek past the end of a seekable device.
 
  m_current_offset = new_offset;
+ evaluate_block_conditions();
  return m_current_offset;
 }
 
@@ -124,8 +145,11 @@ KResultOr<size_t> FileDescription::read(UserOrKernelBuffer& buffer, size_t count
  if (new_offset.has_overflow())
  return -EOVERFLOW;
  auto nread_or_error = m_file->read(*this, offset(), buffer, count);
- if (!nread_or_error.is_error() && m_file->is_seekable())
- m_current_offset += nread_or_error.value();
+ if (!nread_or_error.is_error()) {
+ if (m_file->is_seekable())
+ m_current_offset += nread_or_error.value();
+ evaluate_block_conditions();
+ }
  return nread_or_error;
 }
 
@@ -137,8 +161,11 @@ KResultOr<size_t> FileDescription::write(const UserOrKernelBuffer& data, size_t
  if (new_offset.has_overflow())
  return -EOVERFLOW;
  auto nwritten_or_error = m_file->write(*this, offset(), data, size);
- if (!nwritten_or_error.is_error() && m_file->is_seekable())
- m_current_offset += nwritten_or_error.value();
+ if (!nwritten_or_error.is_error()) {
+ if (m_file->is_seekable())
+ m_current_offset += nwritten_or_error.value();
+ evaluate_block_conditions();
+ }
  return nwritten_or_error;
 }
 
@@ -340,4 +367,9 @@ KResult FileDescription::chown(uid_t uid, gid_t gid)
  return m_file->chown(*this, uid, gid);
 }
 
+FileBlockCondition& FileDescription::block_condition()
+{
+ return m_file->block_condition();
+}
+
 }

Kernel/FileSystem/FileDescription.h

@@ -45,6 +45,8 @@ class FileDescription : public RefCounted<FileDescription> {
  static NonnullRefPtr<FileDescription> create(File&);
  ~FileDescription();
 
+ Thread::FileBlocker::BlockFlags should_unblock(Thread::FileBlocker::BlockFlags) const;
+
  bool is_readable() const { return m_readable; }
  bool is_writable() const { return m_writable; }
 
@@ -130,11 +132,18 @@ class FileDescription : public RefCounted<FileDescription> {
 
  KResult chown(uid_t, gid_t);
 
+ FileBlockCondition& block_condition();
+
 private:
  friend class VFS;
  explicit FileDescription(File&);
  FileDescription(FIFO&, FIFO::Direction);
 
+ void evaluate_block_conditions()
+ {
+ block_condition().unblock();
+ }
+
  RefPtr<Custody> m_custody;
  RefPtr<Inode> m_inode;
  NonnullRefPtr<File> m_file;

Kernel/FileSystem/InodeFile.cpp

@@ -47,8 +47,10 @@ InodeFile::~InodeFile()
 KResultOr<size_t> InodeFile::read(FileDescription& description, size_t offset, UserOrKernelBuffer& buffer, size_t count)
 {
  ssize_t nread = m_inode->read_bytes(offset, count, buffer, &description);
- if (nread > 0)
+ if (nread > 0) {
  Thread::current()->did_file_read(nread);
+ evaluate_block_conditions();
+ }
  if (nread < 0)
  return KResult(nread);
  return nread;
@@ -60,6 +62,7 @@ KResultOr<size_t> InodeFile::write(FileDescription& description, size_t offset,
  if (nwritten > 0) {
  m_inode->set_mtime(kgettimeofday().tv_sec);
  Thread::current()->did_file_write(nwritten);
+ evaluate_block_conditions();
  }
  if (nwritten < 0)
  return KResult(nwritten);

Kernel/FileSystem/InodeWatcher.cpp

@@ -78,6 +78,7 @@ KResultOr<size_t> InodeWatcher::read(FileDescription&, size_t, UserOrKernelBuffe
  });
  if (nwritten < 0)
  return KResult(nwritten);
+ evaluate_block_conditions();
  return bytes_to_write;
 }
 
@@ -97,18 +98,21 @@ void InodeWatcher::notify_inode_event(Badge<Inode>, Event::Type event_type)
 {
  LOCKER(m_lock);
  m_queue.enqueue({ event_type });
+ evaluate_block_conditions();
 }
 
 void InodeWatcher::notify_child_added(Badge<Inode>, const InodeIdentifier& child_id)
 {
  LOCKER(m_lock);
  m_queue.enqueue({ Event::Type::ChildAdded, child_id.index() });
+ evaluate_block_conditions();
 }
 
 void InodeWatcher::notify_child_removed(Badge<Inode>, const InodeIdentifier& child_id)
 {
  LOCKER(m_lock);
  m_queue.enqueue({ Event::Type::ChildRemoved, child_id.index() });
+ evaluate_block_conditions();
 }
 
 }