MDEV-36089 New-style hint: [NO_]ROWID_FILTER

This commit introduces two optimizer hints: ROWID_FILTER and NO_ROWID_FILTER.
They allow explicit control over whether certain indexes are used to build
rowid filters.

Syntax:
  [NO_]ROWID_FILTER(tbl_name@query_block_name [index_name [, index_name] ...])

Hints can be applied at two levels:

Table-level (only table name specified): enables or disables rowid filters
  for the given table.

Index-level (table and index names specified): forces the server to use
  or ignore the listed indexes when building rowid filters.

Examples:
  SELECT /*+ rowid_filter(t1)*/ a FROM t1 WHERE a < 'e' AND b > 't';
  SELECT /*+ no_rowid_filter(t1)*/ a FROM t1 WHERE a < 'e' AND b > 't';
  SELECT /*+ rowid_filter(t1 key_a, key_b)*/ a FROM t1 WHERE a < 'e' AND b > 't';
  SELECT /*+ no_rowid_filter(t1 key_c)*/ a FROM t1 WHERE a < 'e' AND b > 't';

===== Interaction with other index hints =====
If [NO_]INDEX, [NO_]JOIN_INDEX hints are specified for the same table,
then only indexes whitelisted by INDEX() or JOIN_INDEX() can be considered
for rowid filters. ROWID_FILTER() hint cannot force the use of indexes that are
disabled by NO_INDEX() / NO_JOIN_INDEX() or omitted from INDEX() / JOIN_INDEX().
For example, if one index should be used for data access and another for
a rowid filter, the filter index must be mentioned in both hints:
  SELECT /*+ index(t1 key_a, key_b) rowid_filter(t1 key_b) a FROM t1 ....

Author: Olernov

mysql-test/main/opt_hint_rowid_filter.result

@@ -0,0 +1,180 @@
+--enable_prepare_warnings
+--disable_view_protocol # Since optimizer hints are not supported inside views
+
+CREATE TABLE t1 (a INT, b VARCHAR(10), c VARCHAR(100), d VARCHAR(200),
+  KEY key_a(a), KEY key_b(b), KEY key_c(c), KEY key_d(d));
+
+INSERT INTO t1 VALUES
+  (1,'w','z','F'), (1,'X','o','s'), (1,'q','c','a'), (5,'w','c','d'), (2,'j','m','k'),
+  (2,'Q','s','q'), (9,'e','J','B'), (2,'p','W','l'), (9,'o','F','Y'), (2,'g','S','M'),
+  (1,'Y','a','h'), (NULL,'Y','p','J'), (NULL,'s','x','M'), (NULL,'i','S','k'),
+  (1,'l','q','w'), (7,'r','e','_'), (4,'b','h','I'), (NULL,'E','c','z'),
+  (NULL,'M','a','j'), (3,'e','X','K'), (NULL,'p','r','R'), (9,'e','i','g'),
+  (3,'g','x','m'), (2,'h','y','p');
+
+ANALYZE TABLE t1;
+
+set optimizer_switch='rowid_filter=on';
+--echo # range|filter `key_d|key_b` is applied by default when there are no hints:
+EXPLAIN EXTENDED
+SELECT a FROM t1 WHERE c < 'e' AND b > 't' and d < 'e';
+
+--echo # Sample result set to compare against later
+SELECT a FROM t1 WHERE c < 'e' AND b > 't' and d < 'e';
+
+--echo # Disable rowid filter for t1
+EXPLAIN EXTENDED
+SELECT /*+ no_rowid_filter(t1)*/ a FROM t1 WHERE c < 'e' AND b > 't';
+
+--echo # Force using rowid filter made from `key_c` despite being less
+--echo # efficient then the one made from `key_b`
+EXPLAIN EXTENDED
+SELECT /*+ rowid_filter(t1 key_c)*/ a FROM t1 WHERE c < 'e' AND b > 't' and d < 'e';
+
+--echo # Allow the optimizer to choose the best rowid filter from key_a,key_b,key_c,key_d
+EXPLAIN EXTENDED
+SELECT /*+ rowid_filter(t1 key_a,key_b,key_c,key_d)*/ a FROM t1
+  WHERE c < 'e' AND b > 't' and d < 'e';
+
+--echo # Forbid `key_b` for rowid filtering, so `key_c` is used
+let $q= SELECT /*+ no_rowid_filter(t1 key_b)*/ a FROM t1
+  WHERE c < 'e' AND b > 't' and d < 'e';
+
+eval EXPLAIN EXTENDED $q;
+--echo # Validate the result set
+eval $q;
+
+--echo # Forbid both `key_b` and `key_c`, so no rowid filter can be applied
+EXPLAIN EXTENDED
+SELECT /*+ no_rowid_filter(t1 key_b, key_c)*/ a FROM t1
+  WHERE c < 'e' AND b > 't' and d < 'e';
+
+--echo # Disable rowid filter for indexes which would not be used anyway
+EXPLAIN EXTENDED
+SELECT /*+ no_rowid_filter(t1 key_a)*/ a FROM t1
+  WHERE c < 'e' AND b > 't' and d < 'e';
+
+EXPLAIN EXTENDED
+SELECT /*+ no_rowid_filter(t1 key_a,key_b)*/ a FROM t1
+  WHERE c < 'e' AND b > 't' and d < 'e';
+
+--echo # Test the hint with query blocks
+set optimizer_switch='derived_merge=off';
+
+EXPLAIN EXTENDED
+SELECT /*+ no_rowid_filter(t1@qb1)*/ * FROM (
+  SELECT /*+ qb_name(qb1)*/ a FROM t1 WHERE c < 'e' AND b > 't') dt;
+
+--echo # `key_b|key_c` access is forced
+EXPLAIN EXTENDED
+SELECT /*+ /*+ rowid_filter(t1@qb1 key_c)*/ * FROM (
+  SELECT /*+ qb_name(qb1)*/ a FROM t1 WHERE c < 'e' AND b > 't') dt;
+
+set optimizer_switch=default;
+
+--echo # Turn rowid filtering off and enable it selectively
+set optimizer_switch='rowid_filter=off';
+
+--echo # Make sure rowid filter is not used when there are no hints:
+EXPLAIN EXTENDED
+SELECT a FROM t1 WHERE c < 'e' AND b > 't' and d < 'e';
+
+--echo # Enable rowid filter for t1
+EXPLAIN EXTENDED
+SELECT /*+ rowid_filter(t1)*/ a FROM t1 WHERE c < 'e' AND b > 't' and d < 'e';
+
+--echo # Enable rowid filter only for some indexes of t1
+EXPLAIN EXTENDED
+SELECT /*+ rowid_filter(t1 key_a, key_c)*/ a FROM t1
+  WHERE c < 'e' AND b > 't' and d < 'e';
+
+EXPLAIN EXTENDED
+SELECT /*+ rowid_filter(t1 key_c)*/ a FROM t1
+  WHERE c < 'e' AND b > 't' and d < 'e';
+
+--echo # Enable rowid filter for the same index that is used to access data (impossible)
+EXPLAIN EXTENDED
+SELECT /*+ rowid_filter(t1 key_d)*/ a FROM t1 WHERE c < 'e' AND b > 't' and d < 'e';
+
+--echo # Conflicting hints
+EXPLAIN EXTENDED
+SELECT /*+ no_rowid_filter(t1) ROWID_FILTER(t1 key_a)*/a FROM t1
+  WHERE c < 'e' AND b > 't';
+
+EXPLAIN EXTENDED
+SELECT /*+ no_rowid_filter(t1 key_a, key_b) no_rowid_filter(t1 key_c, key_b)*/a FROM t1
+  WHERE c < 'e' AND b > 't' and d < 'e';
+
+EXPLAIN EXTENDED
+SELECT /*+ no_rowid_filter(t1 key_a,key_b) rowid_filter(t1 key_c)*/a FROM t1
+  WHERE c < 'e' AND b > 't' and d < 'e';
+
+# Warnings "Unresolved table/index name..." are generated during both prepare
+# and execution stages. So disable PS protocol to avoid duplication
+--disable_ps_protocol
+--echo # Wrong table name
+EXPLAIN EXTENDED
+SELECT /*+ no_rowid_filter(missing_table)*/a FROM t1
+  WHERE c < 'e' AND b > 't' and d < 'e';
+
+--echo # Some index names are wrong but key_c is applicable
+EXPLAIN EXTENDED
+SELECT /*+ rowid_filter(t1 wrong_key1, key_c, wrong_key2)*/ a FROM t1
+  WHERE c < 'e' AND b > 't' and d < 'e';
+--enable_ps_protocol
+
+--echo # Test cases mixing with index hints.
+--echo # Keys must be whitelisted in INDEX()/JOIN_INDEX() hint to be
+--echo # applicable as a rowid filter. ROWID_FILTER() hint cannot enable keys
+--echo # not enabled by INDEX()/JOIN_INDEX() hints or disabled by
+--echo # NO_INDEX()/NO_JOIN_INDEX() hints
+EXPLAIN EXTENDED
+SELECT /*+ index(t1 key_b,key_c) rowid_filter(t1 key_c)*/ a FROM t1
+  WHERE c < 'e' AND b > 't' and d < 'e';
+
+EXPLAIN EXTENDED
+SELECT /*+ join_index(t1 key_b,key_c) rowid_filter(t1 key_c)*/ a FROM t1
+  WHERE c < 'e' AND b > 't' and d < 'e';
+
+EXPLAIN EXTENDED
+SELECT /*+ index(t1 key_d,key_c) rowid_filter(t1 key_c)*/ a FROM t1
+  WHERE c < 'e' AND b > 't' and d < 'e';
+
+EXPLAIN EXTENDED
+SELECT /*+ no_index(t1) rowid_filter(t1 key_c)*/ a FROM t1
+  WHERE c < 'e' AND b > 't' and d < 'e';
+
+EXPLAIN EXTENDED
+SELECT /*+ no_index(t1 key_d) rowid_filter(t1 key_c)*/ a FROM t1
+  WHERE c < 'e' AND b > 't' and d < 'e';
+
+EXPLAIN EXTENDED
+SELECT /*+ no_index(t1 key_c) rowid_filter(t1 key_c)*/ a FROM t1
+  WHERE c < 'e' AND b > 't' and d < 'e';
+
+EXPLAIN EXTENDED
+SELECT /*+ index(t1) no_rowid_filter(t1 key_c)*/ a FROM t1
+  WHERE c < 'e' AND b > 't' and d < 'e';
+
+--echo # Test cases for old-style hints
+EXPLAIN EXTENDED
+SELECT a FROM t1 FORCE INDEX(key_d, key_b)
+  WHERE c < 'e' AND b > 't' and d < 'e';
+
+--echo # `key_c` is not listed in the hint so is not applicable as rowid filter
+--echo # (also see comment for new-style index hints above)
+EXPLAIN EXTENDED
+SELECT a FROM t1 FORCE INDEX(key_d)
+  WHERE c < 'e' AND b > 't' and d < 'e';
+
+--echo # `key_c` is blacklisted, so `key_b` is used for rowid filter
+EXPLAIN EXTENDED
+SELECT a FROM t1 IGNORE INDEX(key_c)
+  WHERE c < 'e' AND b > 't' and d < 'e';
+
+--echo # `key_b` is blacklisted, so `key_c` is used for rowid filter
+EXPLAIN EXTENDED
+SELECT a FROM t1 IGNORE INDEX(key_b)
+  WHERE c < 'e' AND b > 't' and d < 'e';
+
+DROP TABLE t1;

mysql-test/main/opt_hint_rowid_filter.test

@@ -1150,7 +1150,7 @@ int DsMrr_impl::dsmrr_init(handler *h_arg, RANGE_SEQ_IF *seq_funcs,
   buf_manager.redistribute_buffer_space= do_nothing;
 
   if (!hint_key_state(thd, table, h_arg->active_index,
-                      MRR_HINT_ENUM, OPTIMIZER_SWITCH_MRR) ||
+                MRR_HINT_ENUM, optimizer_flag(thd, OPTIMIZER_SWITCH_MRR)) ||
       mode & (HA_MRR_USE_DEFAULT_IMPL | HA_MRR_SORTED))
     goto use_default_impl;
 
@@ -1905,9 +1905,9 @@ bool DsMrr_impl::choose_mrr_impl(uint keyno, ha_rows rows, uint *flags,
   TABLE_SHARE *share= primary_file->get_table_share();
 
   const bool mrr_on= hint_key_state(thd, table, keyno, MRR_HINT_ENUM,
-                                    OPTIMIZER_SWITCH_MRR);
+                                    optimizer_flag(thd, OPTIMIZER_SWITCH_MRR));
   const bool force_dsmrr_by_hints=
-    hint_key_state(thd, table, keyno, MRR_HINT_ENUM, 0) ||
+    hint_key_state(thd, table, keyno, MRR_HINT_ENUM, false) ||
     hint_table_state(thd, table, BKA_HINT_ENUM, false);
 
   bool doing_cpk_scan= check_cpk_scan(thd, share, keyno, *flags);