MDEV-29367 Refactor tpool::cache

Removed use std::vector's ba push_back(), pop_back() to  make it more
obvious that memory in the vectors won't be reallocated.

Also, "borrowed" elements can be debugged a little better now,
they are put into the start of the m_cache vector.

storage/innobase/os/os0file.cc

@@ -121,7 +121,7 @@ class io_slots
 
  size_t pending_io_count()
  {
- return (size_t)m_max_aio - m_cache.size();
+ return m_cache.pos();
  }
 
  tpool::task_group* get_task_group()

tpool/tpool_structs.h

@@ -40,79 +40,121 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111 - 1301 USA*/
 namespace tpool
 {
 
-enum cache_notification_mode
-{
- NOTIFY_ONE,
- NOTIFY_ALL
-};
-
 /**
  Generic "pointer" cache of a fixed size
  with fast put/get operations.
 
- Compared to STL containers, is faster/does not
- do allocations. However, put() operation will wait
- if there is no free items.
+ Compared to STL containers,e.g stack or queue
+ is faster/does not do allocations.
+
+ However, get() operation will wait if there is no free items.
+
+ We assume that put() will only put back the elements that
+ were retrieved previously with get().
 */
 template<typename T> class cache
 {
+ /** Protects updates of m_pos and m_cache members */
  std::mutex m_mtx;
+
+ /**
+ Notify waiting threads about "cache full" or "cache not empty" conditions
+ @see get() and wait()
+ */
  std::condition_variable m_cv;
- std::vector<T> m_base;
+
+ /** Cached items vector.Does not change after construction */
+ std::vector<T> m_base;
+
+ /**
+ Pointers to cached items. Protected by m_mtx. Does not grow after
+ construction. Elements in position [0,m_pos-1] are "borrowed",
+ elements in position [m_pos,capacity()-1] are "free"
+ */
  std::vector<T*> m_cache;
- cache_notification_mode m_notification_mode;
+
+ /** Number of threads waiting for "cache full" condition (s. wait())
+ Protected by m_mtx */
  int m_waiters;
 
+ /** Current cache size. Protected by m_mtx*/
+ size_t m_pos;
+
+private:
+
+ inline size_t capacity()
+ {
+ return m_base.size();
+ }
+
+ /**
+ @return true if cache is full (no items are borrowed)
+ */
  bool is_full()
  {
- return m_cache.size() == m_base.size();
+ return m_pos == 0;
+ }
+
+ /**
+ @return true if cache is empty (all items are borrowed)
+ */
+ bool is_empty()
+ {
+ return m_pos == capacity();
  }
 
 public:
- cache(size_t count, cache_notification_mode mode= tpool::cache_notification_mode::NOTIFY_ALL):
- m_mtx(), m_cv(), m_base(count),m_cache(count), m_notification_mode(mode),m_waiters()
+ /**
+ Constructor
+ @param size - maximum number of items in cache
+ */
+ cache(size_t size) : m_mtx(), m_cv(), m_base(size), m_cache(size),
+ m_waiters(), m_pos(0)
  {
- for(size_t i = 0 ; i < count; i++)
- m_cache[i]=&m_base[i];
+ for(size_t i= 0 ; i < size; i++)
+ m_cache[i]= &m_base[i];
  }
 
- T* get(bool blocking=true)
+ /**
+ Retrieve an item from cache. Waits for free item, if cache is
+ currently empty.
+ @return borrowed item
+ */
+ T* get()
  {
  std::unique_lock<std::mutex> lk(m_mtx);
- if (blocking)
- {
- while(m_cache.empty())
- m_cv.wait(lk);
- }
- else
- {
- if(m_cache.empty())
- return nullptr;
- }
- T* ret = m_cache.back();
- m_cache.pop_back();
- return ret;
+ while(is_empty())
+ m_cv.wait(lk);
+ assert(m_pos < capacity());
+ // return last element
+ return m_cache[m_pos++];
  }
 
-
+ /**
+ Put back an item to cache.
+ @param item - item to put back
+ */
  void put(T *ele)
  {
  std::unique_lock<std::mutex> lk(m_mtx);
- m_cache.push_back(ele);
- if (m_notification_mode == NOTIFY_ONE)
- m_cv.notify_one();
- else if(m_cache.size() == 1)
- m_cv.notify_all(); // Signal cache is not empty
- else if(m_waiters && is_full())
- m_cv.notify_all(); // Signal cache is full
+ assert(!is_full());
+ // put element to the logical end of the array
+ m_cache[--m_pos] = ele;
+
+ /* Notify waiters when the cache becomes
+ not empty, or when it becomes full */
+ if (m_pos == 1 || (m_waiters && is_full()))
+ m_cv.notify_all();
  }
 
+ /** Check if pointer represents cached element */
  bool contains(T* ele)
  {
- return ele >= &m_base[0] && ele <= &m_base[m_base.size() -1];
+ // No locking required, m_base does not change after construction.
+ return ele >= &m_base[0] && ele <= &m_base[capacity() - 1];
  }
 
- /* Wait until cache is full.*/
+ /** Wait until cache is full.*/
  void wait()
  {
  std::unique_lock<std::mutex> lk(m_mtx);
@@ -122,9 +164,13 @@ template<typename T> class cache
  m_waiters--;
  }
 
- TPOOL_SUPPRESS_TSAN size_t size()
+ /**
+ @return approximate number of "borrowed" items.
+ A "dirty" read, not used in any critical functionality.
+ */
+ TPOOL_SUPPRESS_TSAN size_t pos()
  {
- return m_cache.size();
+ return m_pos;
  }
 };