CASSANDRA-18112 Support manual secondary index selection at the CQL level

CHANGES.txt

@@ -1,4 +1,5 @@
 5.1
+ * Support manual secondary index selection at the CQL level (CASSANDRA-18112)
  * When regulars CQL mutations run on Accord use the txn timestamp rather than server timestamp (CASSANDRA-20744)
  * When using BEGIN TRANSACTION if a complex mutation exists in the same statement as one that uses a reference, then the complex delete is dropped (CASSANDRA-20788)
  * Migrate all nodetool commands from airline to picocli (CASSANDRA-17445)

doc/cql3/CQL.textile

@@ -537,7 +537,7 @@ h3(#createIndexStmt). CREATE INDEX
 __Syntax:__
 
 bc(syntax).. 
-<create-index-stmt> ::= CREATE ( CUSTOM )? INDEX ( IF NOT EXISTS )? ( <indexname> )?
+<create-index-stmt> ::= CREATE ( CUSTOM )? INDEX ( IF NOT EXISTS )? ( <index-name> )?
                             ON <tablename> '(' <index-identifier> ')'
                             ( USING <string> ( WITH OPTIONS = <map-literal> )? )?
 
@@ -1138,6 +1138,7 @@ bc(syntax)..
                   ( PER PARTITION LIMIT <integer> )?
                   ( LIMIT <integer> )?
                   ( ALLOW FILTERING )?
+                  ( WITH <select-options> )?
 
 <select-clause> ::= DISTINCT? <selection-list>
 
@@ -1171,6 +1172,11 @@ bc(syntax)..
 <order-by> ::= <ordering> ( ',' <odering> )*
 <ordering> ::= <identifer> ( ASC | DESC )?
 <term-tuple> ::= '(' <term> (',' <term>)* ')'
+
+<select-options> ::= <select-option> ( AND <select-option> )*
+<select-option> ::= included_indexes  '=' <index-names>
+                  | excluded_indexes  '=' <index-names>
+<index-names> ::= '{' <identifier> ( ',' <identifier> )* '}'
 p. 
 __Sample:__
 

doc/modules/cassandra/examples/BNF/select_statement.bnf

@@ -6,6 +6,7 @@ select_statement::= SELECT [ JSON | DISTINCT ] ( select_clause | '*' )
 	[ PER PARTITION LIMIT (`integer` | `bind_marker`) ]  
 	[ LIMIT (`integer` | `bind_marker`) ]  
 	[ ALLOW FILTERING ]
+	[ WITH `select_options` ]
 select_clause::= `selector` [ AS `identifier` ] ( ',' `selector` [ AS `identifier` ] ) 
 selector::== `column_name` 
 	| `term`  
@@ -17,5 +18,9 @@ relation::= column_name operator term
 	'(' column_name ( ',' column_name )* ')' operator tuple_literal 
 	TOKEN '(' column_name# ( ',' column_name )* ')' operator term 
 operator::= '=' | '<' | '>' | '<=' | '>=' | '!=' | IN | NOT IN | CONTAINS | NOT CONTAINS | CONTAINS KEY | NOT CONTAINS KEY
-group_by_clause::= column_name ( ',' column_name )* 
+group_by_clause::= column_name ( ',' column_name )*
 ordering_clause::= column_name [ ASC | DESC ] ( ',' column_name [ ASC | DESC ] )*
+select_options::= `select_option` ( AND `select_option` )*
+select_option::= included_indexes  '=' `index_names`
+	           | excluded_indexes  '=' `index_names`
+index_names::= '{' index_name ( ',' index_name )* '}'

doc/modules/cassandra/examples/CQL/query_with_index_hints.cql

@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+CREATE INDEX birth_year_idx ON users (birth_year) USING 'sai';
+CREATE INDEX country_idx ON users (country) USING 'sai';
+
+SELECT * FROM users
+WHERE birth_year = 1981 AND country = 'FR'
+WITH included_indexes = {birth_year_idx} AND excluded_indexes = {country_idx};

doc/modules/cassandra/pages/developing/cql/dml.adoc

@@ -250,6 +250,28 @@ execute:
 include::cassandra:example$CQL/query_nofail_allow_filtering.cql[]
 ----
 
+[[index-hints]]
+=== Index hints
+
+`SELECT` statements allow to provide sets of included and excluded indexes:
+
+[source,cql]
+----
+include::example$CQL/query_with_index_hints.cql[]
+----
+The included indexes are indexes that should be used by the query.
+Queries will fail if it's not possible to use the included indexes.
+That might happen because the query doesn't have a restriction for those indexes,
+or because there is a restriction that can use the index,
+but it is not compatible with other restrictions,
+or because the underlying index implementation isn't able to use the index for whatever reason.
+
+The excluded indexes are indexes that should not be used by the query.
+Excluded indexes will never fail the query unless they reference a non-existent index,
+since it's always possible to exclude indexes that are not used by the query.
+
+The indexes mentioned in included or excluded sets must exist, otherwise the query will fail.
+
 [[insert-statement]]
 == INSERT
 

pylib/cqlshlib/cql3handling.py

@@ -759,6 +759,7 @@ def working_on_keyspace(ctxt):
                           ( "PER" "PARTITION" "LIMIT" perPartitionLimit=<wholenumber> )?
                           ( "LIMIT" limit=<wholenumber> )?
                           ( "ALLOW" "FILTERING" )?
+                          ( "WITH" <options> )?
                     ;
 <whereClause> ::= <relation> ( "AND" <relation> )*
                 ;
@@ -804,6 +805,14 @@ def working_on_keyspace(ctxt):
                             | <term>
                             ;
 
+<identifiers> ::= "{" <identifier> ( "," <identifier> )* "}"
+                  ;
+<options> ::= <option> ( "AND" <option> )*
+                  ;
+<option> ::= "included_indexes" "=" <identifiers>
+           | "excluded_indexes" "=" <identifiers>
+           ;
+
 <aggregateMathFunctions> ::= "COUNT" "(" star=( "*" | "1" ) ")"
              | "AVG" "(" [colname]=<cident> ")"
              | "MIN" "(" [colname]=<cident> ")"

pylib/cqlshlib/test/test_cqlsh_completion.py

@@ -1236,3 +1236,7 @@ def test_complete_in_show(self):
 
     def test_complete_in_tracing(self):
         self.trycompletions('TRACING ', choices=[';', '<enter>', 'OFF', 'ON'])
+
+    def test_complete_in_select_include_exclude_index(self):
+        self.trycompletions('SELECT * FROM system.peers WHERE native_port = 8100 WITH i', immediate='ncluded_indexes = { ',)
+        self.trycompletions('SELECT * FROM system.peers WHERE native_port = 8100 WITH e', immediate='xcluded_indexes = { ',)

src/antlr/Parser.g

@@ -307,6 +307,7 @@ selectStatement returns [SelectStatement.RawStatement expr]
         List<Selectable.Raw> groups = new ArrayList<>();
         boolean allowFiltering = false;
         boolean isJson = false;
+        SelectOptions options = new SelectOptions();
         stmtBegins();
     }
     : K_SELECT
@@ -319,6 +320,7 @@ selectStatement returns [SelectStatement.RawStatement expr]
       ( K_PER K_PARTITION K_LIMIT rows=intValue { perPartitionLimit = rows; } )?
       ( K_LIMIT rows=intValue { limit = rows; } )?
       ( K_ALLOW K_FILTERING  { allowFiltering = true; } )?
+      ( K_WITH properties[options] )?
       {
           SelectStatement.Parameters params = new SelectStatement.Parameters(orderings,
                                                                              groups,
@@ -327,7 +329,7 @@ selectStatement returns [SelectStatement.RawStatement expr]
                                                                              isJson,
                                                                              null);
           WhereClause where = wclause == null ? WhereClause.empty() : wclause.build();
-          $expr = new SelectStatement.RawStatement(cf, params, $sclause.selectors, where, limit, perPartitionLimit, stmtSrc());
+          $expr = new SelectStatement.RawStatement(cf, params, $sclause.selectors, where, limit, perPartitionLimit, stmtSrc(), options);
       }
     ;
 
@@ -345,7 +347,7 @@ letStatement returns [SelectStatement.RawStatement expr]
           SelectStatement.Parameters params = new SelectStatement.Parameters(Collections.emptyList(), Collections.emptyList(), false, false, false, $txnVar.text);
           WhereClause where = wclause == null ? WhereClause.empty() : wclause.build();
 
-          $expr = new SelectStatement.RawStatement(cf, params, assignments, where, limit, null, stmtSrc());
+          $expr = new SelectStatement.RawStatement(cf, params, assignments, where, limit, null, stmtSrc(), SelectOptions.EMPTY);
       }
     ;
 
@@ -1765,6 +1767,11 @@ indexName returns [QualifiedName name]
     : (ksName[name] '.')? idxName[name]
     ;
 
+indexNames returns [Set<QualifiedName> names]
+    @init { $names = new HashSet<QualifiedName>(); }
+    : '{' ( t1=indexName { names.add(t1); } ( ',' tn=indexName { names.add(tn); } )* )? '}'
+    ;
+
 columnFamilyName returns [QualifiedName name]
     @init { $name = new QualifiedName(); }
     : (ksName[name] '.')? cfName[name]
@@ -2053,9 +2060,15 @@ properties[PropertyDefinitions props]
     : property[props] (K_AND property[props])*
     ;
 
+indexProperty returns [String s]
+    : 'included_indexes' { s = "included_indexes"; }
+    | 'excluded_indexes' { s = "excluded_indexes"; }
+    ;
+
 property[PropertyDefinitions props]
     : k=noncol_ident '=' simple=propertyValue { try { $props.addProperty(k.toString(), simple); } catch (SyntaxException e) { addRecognitionError(e.getMessage()); } }
     | k=noncol_ident '=' map=fullMapLiteral { try { $props.addProperty(k.toString(), convertPropertyMap(map)); } catch (SyntaxException e) { addRecognitionError(e.getMessage()); } }
+    | s=indexProperty '=' names=indexNames { try { $props.addProperty(s, names); } catch (SyntaxException e) { addRecognitionError(e.getMessage()); } }
     ;
 
 propertyValue returns [String str]

src/java/org/apache/cassandra/cql3/CqlBuilder.java

@@ -19,8 +19,10 @@
 
 import java.util.Iterator;
 import java.util.Map;
-import java.util.Map.Entry;
+import java.util.Set;
 import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.function.Consumer;
 
 import org.apache.cassandra.cql3.functions.FunctionName;
 import org.apache.cassandra.db.marshal.AbstractType;
@@ -59,6 +61,11 @@ public CqlBuilder(int capacity)
         builder = new StringBuilder(capacity);
     }
 
+    public CqlBuilder append(Object o)
+    {
+        return append(String.valueOf(o));
+    }
+
     public CqlBuilder append(String str)
     {
         indentIfNeeded();
@@ -153,30 +160,47 @@ public CqlBuilder append(Map<String, String> map)
     }
 
     public CqlBuilder append(Map<String, String> map, boolean quoteValue)
+    {
+        return append(new TreeMap<>(map).entrySet(), e -> {
+            appendWithSingleQuotes(e.getKey());
+            builder.append(": ");
+            appendCollectionValue(e.getValue(), quoteValue);
+        });
+    }
+
+    public CqlBuilder append(Set<String> set, boolean quoteValue)
+    {
+        return append(new TreeSet<>(set),
+                      value -> appendCollectionValue(value, quoteValue));
+    }
+
+    private <T> CqlBuilder append(Iterable<T> iterable, Consumer<T> appender)
     {
         indentIfNeeded();
 
         builder.append('{');
 
-        Iterator<Entry<String, String>> iter = new TreeMap<>(map).entrySet()
-                                                                 .iterator();
-        while(iter.hasNext())
+        Iterator<T> iterator = iterable.iterator();
+        while (iterator.hasNext())
         {
-            Entry<String, String> e = iter.next();
-            appendWithSingleQuotes(e.getKey());
-            builder.append(": ");
-            if (quoteValue)
-                appendWithSingleQuotes(e.getValue());
-            else
-                builder.append(e.getValue());
+            appender.accept(iterator.next());
 
-            if (iter.hasNext())
+            if (iterator.hasNext())
                 builder.append(", ");
         }
+
         builder.append('}');
         return this;
     }
 
+    private void appendCollectionValue(String value, boolean quote)
+    {
+        if (quote)
+            appendWithSingleQuotes(value);
+        else
+            builder.append(value);
+    }
+
     public <T> CqlBuilder appendWithSeparators(Iterable<T> iterable, Appender<T> appender, String separator)
     {
         return appendWithSeparators(iterable.iterator(), appender, separator);
@@ -223,4 +247,49 @@ public String toString()
     {
         return builder.toString();
     }
+
+    /**
+     * Builds a `WITH option1 = ... AND option2 = ... AND option3 = ... clause
+     *
+     * @param builder a consumer to receive a builder for the options to append
+     */
+    public CqlBuilder appendOptions(Consumer<OptionsBuilder> builder)
+    {
+        builder.accept(new OptionsBuilder(this));
+        return this;
+    }
+
+    public static class OptionsBuilder
+    {
+        private final CqlBuilder builder;
+        private boolean empty = true;
+
+        OptionsBuilder(CqlBuilder builder)
+        {
+            this.builder = builder;
+        }
+
+        public OptionsBuilder append(String name, Map<String, String> options)
+        {
+            return options.isEmpty() ? this : append(name, () -> builder.append(options, false));
+        }
+
+        public OptionsBuilder append(String name, Set<String> options)
+        {
+            return options.isEmpty() ? this : append(name, () -> builder.append(options, false));
+        }
+
+        public OptionsBuilder append(String name, String options)
+        {
+            return options.isEmpty() ? this : append(name, () -> builder.append(options));
+        }
+
+        private OptionsBuilder append(String name, Runnable appender)
+        {
+            builder.append((empty ? " WITH " : " AND ") + name + " = ");
+            empty = false;
+            appender.run();
+            return this;
+        }
+    }
 }

src/java/org/apache/cassandra/cql3/QueryProcessor.java

@@ -589,19 +589,29 @@ public static UntypedResultSet executeInternalWithNowInSec(String query, long no
     public static UntypedResultSet execute(String query, ConsistencyLevel cl, QueryState state, Object... values)
     throws RequestExecutionException
     {
-        try
-        {
-            Prepared prepared = prepareInternal(query);
-            ResultMessage result = prepared.statement.execute(state, makeInternalOptionsWithNowInSec(prepared.statement, state.getNowInSeconds(), values, cl), Dispatcher.RequestTime.forImmediateExecution());
-            if (result instanceof ResultMessage.Rows)
-                return UntypedResultSet.create(((ResultMessage.Rows)result).result);
-            else
-                return null;
-        }
-        catch (RequestValidationException e)
-        {
-            throw new RuntimeException("Error validating " + query, e);
-        }
+        Prepared prepared = prepareInternal(query);
+        ResultMessage result = prepared.statement.execute(state, makeInternalOptionsWithNowInSec(prepared.statement, state.getNowInSeconds(), values, cl), Dispatcher.RequestTime.forImmediateExecution());
+        if (result instanceof ResultMessage.Rows)
+            return UntypedResultSet.create(((ResultMessage.Rows)result).result);
+        else
+            return null;
+    }
+
+    /**
+     * Same than {@link #execute(String, ConsistencyLevel, Object...)}, but to use for queries we know are only executed
+     * once so that the created statement object is not cached.
+     */
+    @VisibleForTesting
+    static UntypedResultSet executeOnce(String query, ConsistencyLevel cl, Object... values)
+    {
+        QueryState queryState = internalQueryState();
+        CQLStatement statement = parseStatement(query, queryState.getClientState());
+        statement.validate(queryState.getClientState());
+        ResultMessage result = statement.execute(queryState, makeInternalOptionsWithNowInSec(statement, queryState.getNowInSeconds(), values, cl), Dispatcher.RequestTime.forImmediateExecution());
+        if (result instanceof ResultMessage.Rows)
+            return UntypedResultSet.create(((ResultMessage.Rows)result).result);
+        else
+            return null;
     }
 
     public static UntypedResultSet executeInternalWithPaging(String query, int pageSize, Object... values)