WIP MDEV-37482: Refactor btr_sea::hash_table

Let us prepare to push down the btr_sea::partition::latch
to the hash table itself. This is work in progress, only refactoring
the hash table, not moving the latch itself!

Author: dr-m

storage/innobase/btr/btr0sea.cc

@@ -63,6 +63,24 @@ struct ahi_node {
 #endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */
 };
 
+template<typename UnaryPred>
+inline ahi_node *btr_sea::hash_chain::find(UnaryPred u) const noexcept
+{
+  ahi_node *node= first;
+  while (node && !u(node))
+    node= node->next;
+  return node;
+}
+
+template<typename UnaryPred>
+inline ahi_node **btr_sea::hash_chain::search(UnaryPred u) noexcept
+{
+  ahi_node **prev= &first;
+  while (!u(*prev))
+    prev= &(*prev)->next;
+  return prev;
+}
+
 inline void btr_sea::partition::init() noexcept
 {
   latch.SRW_LOCK_INIT(btr_search_latch_key);
@@ -108,6 +126,16 @@ inline void btr_sea::partition::free() noexcept
   blocks_mutex.destroy();
 }
 
+inline void btr_sea::hash_table::create(ulint n) noexcept
+{
+  n_cells= ut_find_prime(n);
+  const size_t size= MY_ALIGN(pad(n_cells) * sizeof *array,
+                              CPU_LEVEL1_DCACHE_LINESIZE);
+  void *v= aligned_malloc(size, CPU_LEVEL1_DCACHE_LINESIZE);
+  memset_aligned<CPU_LEVEL1_DCACHE_LINESIZE>(v, 0, size);
+  array= static_cast<hash_chain*>(v);
+}
+
 inline void btr_sea::partition::alloc(ulint hash_size) noexcept
 {
   table.create(hash_size);
@@ -665,8 +693,8 @@ void btr_sea::partition::insert(uint32_t fold, const rec_t *rec) noexcept
 #endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */
   ut_ad(btr_search.enabled);
 
-  ahi_node **prev= table.cell_get(fold)->
-    search(&ahi_node::next, [fold](const ahi_node *node)
+  ahi_node **prev= table.cell_get(fold).
+    search([fold](const ahi_node *node)
     { return !node || node->fold == fold; });
   ahi_node *node= *prev;
 
@@ -761,8 +789,8 @@ buf_block_t *btr_sea::partition::cleanup_after_erase(ahi_node *erase) noexcept
   {
     /* Shrink the allocation by replacing the erased element with the top. */
     *erase= *top;
-    ahi_node **prev= table.cell_get(top->fold)->
-      search(&ahi_node::next, [top](const ahi_node *n) { return n == top; });
+    ahi_node **prev= table.cell_get(top->fold).
+      search([top](const ahi_node *n) { return n == top; });
     *prev= erase;
   }
 
@@ -805,12 +833,12 @@ static void ha_remove_all_nodes_to_page(btr_sea::partition &part,
                                         uint32_t fold, const page_t *page)
   noexcept
 {
-  hash_cell_t *cell= part.table.cell_get(fold);
+  btr_sea::hash_chain &cell= part.table.cell_get(fold);
   const uintptr_t page_size{srv_page_size};
 
 rewind:
   ahi_node **prev=
-    cell->search(&ahi_node::next, [page,page_size](const ahi_node *node)
+    cell.search([page,page_size](const ahi_node *node)
     { return !node || (uintptr_t(node->rec) ^ uintptr_t(page)) < page_size; });
 
   if (ahi_node *node= *prev)
@@ -824,7 +852,7 @@ static void ha_remove_all_nodes_to_page(btr_sea::partition &part,
   }
 
   /* Check that all nodes really got deleted */
-  ut_ad(!cell->find(&ahi_node::next, [page](const ahi_node* node)
+  ut_ad(!cell.find([page](const ahi_node* node)
   { return page_align(node->rec) == page; }));
 }
 
@@ -835,8 +863,8 @@ inline bool btr_sea::partition::erase(uint32_t fold, const rec_t *rec) noexcept
 #endif
   ut_ad(btr_search.enabled);
 
-  ahi_node **prev= table.cell_get(fold)->
-    search(&ahi_node::next, [rec](const ahi_node *node)
+  ahi_node **prev= table.cell_get(fold).
+    search([rec](const ahi_node *node)
     { return !node || node->rec == rec; });
 
   if (ahi_node *node= *prev)
@@ -862,7 +890,8 @@ updates the pointer to data if found.
 @param data      pointer to the data
 @param new_data  new pointer to the data
 @return whether the element was found */
-static bool ha_search_and_update_if_found(hash_table_t *table, uint32_t fold,
+static bool ha_search_and_update_if_found(btr_sea::hash_table *table,
+                                          uint32_t fold,
                                           const rec_t *data,
 #if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG
                                           /** block containing new_data */
@@ -875,9 +904,8 @@ static bool ha_search_and_update_if_found(hash_table_t *table, uint32_t fold,
 #endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */
   ut_ad(btr_search.enabled);
 
-  if (ahi_node *node= table->cell_get(fold)->
-      find(&ahi_node::next, [data](const ahi_node *node)
-      { return node->rec == data; }))
+  if (ahi_node *node= table->cell_get(fold).
+      find([data](const ahi_node *node){ return node->rec == data; }))
   {
 #if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG
     if (node->block != new_block)
@@ -1027,9 +1055,8 @@ btr_search_guess_on_hash(
     return false;
   }
 
-  const ahi_node *node= part.table.cell_get(fold)->
-    find(&ahi_node::next, [fold](const ahi_node* node)
-    { return node->fold == fold; });
+  const ahi_node *node= part.table.cell_get(fold).
+    find([fold](const ahi_node* node){ return node->fold == fold; });
 
   if (!node)
   {
@@ -1783,17 +1810,18 @@ void btr_search_update_hash_on_insert(btr_cur_t *cursor, bool reorg) noexcept
 # if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG
 __attribute__((nonnull))
 /** @return whether a range of the cells is valid */
-static bool ha_validate(const hash_table_t *table,
+static bool ha_validate(const btr_sea::hash_table *table,
                         ulint start_index, ulint end_index)
 {
   ut_a(start_index <= end_index);
   ut_a(end_index < table->n_cells);
 
   bool ok= true;
 
+  /* FIXME: skip latches */
   for (ulint i= start_index; i <= end_index; i++)
   {
-    for (auto node= static_cast<const ahi_node*>(table->array[i].node); node;
+    for (auto node= static_cast<const ahi_node*>(table->array[i].first); node;
          node= node->next)
     {
       if (table->calc_hash(node->fold) != i) {
@@ -1850,7 +1878,7 @@ static bool btr_search_hash_table_validate(THD *thd, ulint hash_table_id)
 	btr_sea::partition& part = btr_search.parts[hash_table_id];
 
 	cell_count = part.table.n_cells;
-
+	/* FIXME: skip latches */
 	for (i = 0; i < cell_count; i++) {
 		/* We release search latches every once in a while to
 		give other queries a chance to run. */
@@ -1882,7 +1910,7 @@ static bool btr_search_hash_table_validate(THD *thd, ulint hash_table_id)
 			}
 		}
 
-		node = static_cast<ahi_node*>(part.table.array[i].node);
+		node = static_cast<ahi_node*>(part.table.array[i].first);
 
 		for (; node != NULL; node = node->next) {
 			const buf_block_t*	block

storage/innobase/include/btr0sea.h

@@ -117,13 +117,94 @@ struct btr_sea
   @param resize whether buf_pool_t::resize() is the caller */
   ATTRIBUTE_COLD void enable(bool resize= false) noexcept;
 
+  /** Hash cell chain in hash_table */
+  struct hash_chain
+  {
+    /** pointer to the first block */
+    ahi_node *first;
+
+    /** Find an element.
+    @param u           unary predicate for searching the element
+    @return the first matching element
+    @retval nullptr if not found */
+    template<typename UnaryPred>
+    inline ahi_node *find(UnaryPred u) const noexcept;
+
+    /** Search for a pointer to an element.
+    @tparam UnaryPred  unary predicate
+    @param u           unary predicate for searching the element
+    @return pointer to the first matching element,
+    or to the last element in the chain */
+    template<typename UnaryPred>
+    inline ahi_node **search(UnaryPred u) noexcept;
+  };
+
+  /** Hash table with singly-linked overflow lists.
+  FIXME: deduplicate with buf_pool_t::page_hash_table */
+  struct hash_table
+  {
+    static_assert(CPU_LEVEL1_DCACHE_LINESIZE >= 64, "less than 64 bytes");
+    static_assert(!(CPU_LEVEL1_DCACHE_LINESIZE & 63),
+      "not a multiple of 64 bytes");
+
+    /** Number of array[] elements per page_hash_latch.
+    Must be one less than a power of 2. */
+    static constexpr size_t ELEMENTS_PER_LATCH= 64 / sizeof(void*) - 1;
+    static constexpr size_t EMPTY_SLOTS_PER_LATCH=
+      ((CPU_LEVEL1_DCACHE_LINESIZE / 64) - 1) * (64 / sizeof(void*));
+
+    /** number of payload elements in array[] */
+    Atomic_relaxed<ulint> n_cells;
+    /** the hash table, with pad(n_cells) elements, aligned to L1 cache size */
+    hash_chain *array;
+
+    /** Create the hash table.
+    @param n  the lower bound of n_cells */
+    inline void create(ulint n) noexcept;
+
+    /** Free the hash table. */
+    void free() noexcept { aligned_free(array); array= nullptr; }
+
+    /** @return the index of an array element */
+    ulint calc_hash(ulint fold) const noexcept
+    { return calc_hash(fold, n_cells); }
+    /** @return raw array index converted to padded index */
+    static ulint pad(ulint h) noexcept
+    {
+      ulint latches= h / ELEMENTS_PER_LATCH;
+      ulint empty_slots= latches * EMPTY_SLOTS_PER_LATCH;
+      return 1 + latches + empty_slots + h;
+    }
+  private:
+    /** @return the index of an array element */
+    static ulint calc_hash(ulint fold, ulint n_cells) noexcept
+    {
+      return pad(fold % n_cells);
+    }
+  public:
+    /** @return the latch covering a hash table chain */
+    static page_hash_latch &lock_get(hash_chain &chain) noexcept
+    {
+      static_assert(!((ELEMENTS_PER_LATCH + 1) & ELEMENTS_PER_LATCH),
+                    "must be one less than a power of 2");
+      const size_t addr= reinterpret_cast<size_t>(&chain);
+      ut_ad(addr & (ELEMENTS_PER_LATCH * sizeof chain));
+      return *reinterpret_cast<page_hash_latch*>
+        (addr & ~(ELEMENTS_PER_LATCH * sizeof chain));
+    }
+
+    /** Get a hash table slot. */
+    hash_chain &cell_get(ulint fold) const
+    { return array[calc_hash(fold, n_cells)]; }
+  };
+
   /** Partition of the hash table */
   struct partition
   {
     /** latch protecting table */
-    alignas(CPU_LEVEL1_DCACHE_LINESIZE) srw_spin_lock latch;
+    alignas(CPU_LEVEL1_DCACHE_LINESIZE) srw_spin_lock latch; /* FIXME: remove this */
     /** map of CRC-32C of rec prefix to rec_t* in buf_page_t::frame */
-    hash_table_t table;
+    hash_table table;
     /** latch protecting blocks, spare; may be acquired while holding latch */
     srw_mutex blocks_mutex;
     /** allocated blocks */

storage/innobase/srv/srv0srv.cc

@@ -746,7 +746,8 @@ srv_printf_innodb_monitor(
 			part.blocks_mutex.wr_lock();
 			fprintf(file, "Hash table size " ULINTPF
 				", node heap has " ULINTPF " buffer(s)\n",
-				part.table.n_cells, part.blocks.count + !!part.spare);
+				size_t{part.table.n_cells},
+                                part.blocks.count + !!part.spare);
 			part.blocks_mutex.wr_unlock();
 		}