Droid/redis performance optimizations

all protos/Aggregation.proto

@@ -0,0 +1,15 @@
+syntax = "proto3";
+package feast.core;
+
+option go_package = "github.com/feast-dev/feast/go/protos/feast/core";
+option java_outer_classname = "AggregationProto";
+option java_package = "feast.proto.core";
+
+import "google/protobuf/duration.proto";
+
+message Aggregation {
+    string column = 1;
+    string function = 2;
+    google.protobuf.Duration time_window = 3;
+    google.protobuf.Duration slide_interval = 4;
+}

all protos/Connector.proto

@@ -0,0 +1,34 @@
+syntax = "proto3";
+
+package grpc.connector;
+
+import "google/protobuf/timestamp.proto";
+import "feast/types/Value.proto";
+import "feast/types/EntityKey.proto";
+import "feast/serving/ServingService.proto";
+
+option go_package = "github.com/feast-dev/feast/go/protos/feast/serving";
+
+message ConnectorFeature {
+	feast.serving.FeatureReferenceV2 reference = 1;
+	google.protobuf.Timestamp timestamp = 2;
+	feast.types.Value value = 3;
+}
+
+message ConnectorFeatureList {
+	repeated ConnectorFeature  featureList = 1; 
+}
+
+service OnlineStore {
+    rpc OnlineRead(OnlineReadRequest) returns (OnlineReadResponse);
+}
+
+message OnlineReadRequest {
+    repeated feast.types.EntityKey entityKeys = 1;
+    string view = 2;
+    repeated string features = 3;
+}
+
+message OnlineReadResponse {
+    repeated ConnectorFeatureList results = 1;
+}

all protos/DataFormat.proto

@@ -0,0 +1,61 @@
+//
+// Copyright 2020 The Feast Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+
+syntax = "proto3";
+package feast.core;
+
+option go_package = "github.com/feast-dev/feast/go/protos/feast/core";
+option java_outer_classname = "DataFormatProto";
+option java_package = "feast.proto.core";
+
+// Defines the file format encoding the features/entity data in files
+message FileFormat {
+  // Defines options for the Parquet data format
+  message ParquetFormat {}
+
+  oneof format {
+    ParquetFormat parquet_format = 1;
+  }
+}
+
+// Defines the data format encoding features/entity data in data streams
+message StreamFormat {
+  // Defines options for the protobuf data format
+  message ProtoFormat {
+    // Classpath to the generated Java Protobuf class that can be used to decode
+    // Feature data from the obtained stream message
+    string class_path = 1;
+  }
+
+  // Defines options for the avro data format
+  message AvroFormat {
+    // Optional if used in a File DataSource as schema is embedded in avro file.
+    // Specifies the schema of the Avro message as JSON string.
+    string schema_json = 1;
+  }
+
+  message JsonFormat {
+    string schema_json = 1;
+  }
+
+  // Specifies the data format and format specific options
+  oneof format {
+    AvroFormat avro_format = 1;
+    ProtoFormat proto_format = 2;
+    JsonFormat json_format = 3;
+  }
+}

all protos/DataSource.proto

@@ -0,0 +1,271 @@
+//
+// Copyright 2020 The Feast Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+
+syntax = "proto3";
+package feast.core;
+
+option go_package = "github.com/feast-dev/feast/go/protos/feast/core";
+option java_outer_classname = "DataSourceProto";
+option java_package = "feast.proto.core";
+
+import "google/protobuf/duration.proto";
+import "google/protobuf/timestamp.proto";
+import "feast/core/DataFormat.proto";
+import "feast/types/Value.proto";
+import "feast/core/Feature.proto";
+
+// Defines a Data Source that can be used source Feature data
+// Next available id: 28
+message DataSource {
+  // Field indexes should *not* be reused. Not sure if fields 6-10 were used previously or not,
+  // but they are going to be reserved for backwards compatibility.
+  reserved 6 to 10;
+
+  // Type of Data Source.
+  // Next available id: 12
+  enum SourceType {
+    INVALID = 0;
+    BATCH_FILE = 1;
+    BATCH_SNOWFLAKE = 8;
+    BATCH_BIGQUERY = 2;
+    BATCH_REDSHIFT = 5;
+    STREAM_KAFKA = 3;
+    STREAM_KINESIS = 4;
+    CUSTOM_SOURCE = 6;
+    REQUEST_SOURCE = 7;
+    PUSH_SOURCE = 9;
+    BATCH_TRINO = 10;
+    BATCH_SPARK = 11;
+    BATCH_ATHENA = 12;
+  }
+
+  // Unique name of data source within the project
+  string name = 20;
+
+  // Name of Feast project that this data source belongs to.
+  string project = 21;
+
+  string description = 23;
+
+  map<string, string> tags = 24;
+
+  string owner = 25;
+
+  SourceType type = 1;
+
+  // Defines mapping between fields in the sourced data
+  // and fields in parent FeatureTable.
+  map<string, string> field_mapping = 2;
+
+  // Must specify event timestamp column name
+  string timestamp_field = 3;
+
+  // (Optional) Specify partition column
+  // useful for file sources
+  string date_partition_column = 4;
+
+  // Must specify creation timestamp column name
+  string created_timestamp_column = 5;
+
+  // This is an internal field that is represents the python class for the data source object a proto object represents.
+  // This should be set by feast, and not by users.
+  // The field is used primarily by custom data sources and is mandatory for them to set. Feast may set it for
+  // first party sources as well.
+  string data_source_class_type = 17;
+
+  // Optional batch source for streaming sources for historical features and materialization.
+  DataSource batch_source = 26;
+
+  SourceMeta meta = 50;
+
+  message SourceMeta {
+    google.protobuf.Timestamp earliestEventTimestamp = 1;
+    google.protobuf.Timestamp latestEventTimestamp = 2;
+  }
+
+  // Defines options for DataSource that sources features from a file
+  message FileOptions {
+    FileFormat file_format = 1;
+
+    // Target URL of file to retrieve and source features from.
+    // s3://path/to/file for AWS S3 storage
+    // gs://path/to/file for GCP GCS storage
+    // file:///path/to/file for local storage
+    string uri  = 2;
+
+    // override AWS S3 storage endpoint with custom S3 endpoint
+    string s3_endpoint_override  = 3;
+  }
+
+  // Defines options for DataSource that sources features from a BigQuery Query
+  message BigQueryOptions {
+    // Full table reference in the form of [project:dataset.table]
+    string table = 1;
+
+    // SQL query that returns a table containing feature data. Must contain an event_timestamp column, and respective
+    // entity columns
+    string query = 2;
+  }
+
+  // Defines options for DataSource that sources features from a Trino Query
+  message TrinoOptions {
+    // Full table reference in the form of [project:dataset.table]
+    string table = 1;
+
+    // SQL query that returns a table containing feature data. Must contain an event_timestamp column, and respective
+    // entity columns
+    string query = 2;
+  }
+
+  // Defines options for DataSource that sources features from Kafka messages.
+  // Each message should be a Protobuf that can be decoded with the generated
+  // Java Protobuf class at the given class path
+  message KafkaOptions {
+    // Comma separated list of Kafka bootstrap servers. Used for feature tables without a defined source host[:port]]
+    string kafka_bootstrap_servers = 1;
+
+    // Kafka topic to collect feature data from.
+    string topic = 2;
+
+    // Defines the stream data format encoding feature/entity data in Kafka messages.
+    StreamFormat message_format = 3;
+
+    // Watermark delay threshold for stream data
+    google.protobuf.Duration watermark_delay_threshold = 4;
+  }
+
+  // Defines options for DataSource that sources features from Kinesis records.
+  // Each record should be a Protobuf that can be decoded with the generated
+  // Java Protobuf class at the given class path
+  message KinesisOptions {
+    // AWS region of the Kinesis stream
+    string region = 1;
+
+    // Name of the Kinesis stream to obtain feature data from.
+    string stream_name = 2;
+
+    // Defines the data format encoding the feature/entity data in Kinesis records.
+    // Kinesis Data Sources support Avro and Proto as data formats.
+    StreamFormat record_format = 3;
+  }
+
+  // Defines options for DataSource that sources features from a Redshift Query
+  message RedshiftOptions {
+    // Redshift table name
+    string table = 1;
+
+    // SQL query that returns a table containing feature data. Must contain an event_timestamp column, and respective
+    // entity columns
+    string query = 2;
+
+    // Redshift schema name
+    string schema = 3;
+
+    // Redshift database name
+    string database = 4;
+  }
+
+  // Defines options for DataSource that sources features from a Athena Query
+  message AthenaOptions {
+    // Athena table name
+    string table = 1;
+
+    // SQL query that returns a table containing feature data. Must contain an event_timestamp column, and respective
+    // entity columns
+    string query = 2;
+
+    // Athena database name
+    string database = 3;
+
+    // Athena schema name
+    string data_source = 4;
+  }
+
+  // Defines options for DataSource that sources features from a Snowflake Query
+  message SnowflakeOptions {
+    // Snowflake table name
+    string table = 1;
+
+    // SQL query that returns a table containing feature data. Must contain an event_timestamp column, and respective
+    // entity columns
+    string query = 2;
+
+    // Snowflake schema name
+    string schema = 3;
+
+    // Snowflake schema name
+    string database = 4;
+
+    // Snowflake warehouse name
+    string warehouse = 5;
+  }
+
+  // Defines options for DataSource that sources features from a spark table/query
+  message SparkOptions {
+    // Table name
+    string table = 1;
+
+    // Spark SQl query that returns the table, this is an alternative to `table`
+    string query = 2;
+
+    // Path from which spark can read the table, this is an alternative to `table`
+    string path = 3;
+
+    // Format of files at `path` (e.g. parquet, avro, etc)
+    string file_format = 4;
+  }
+
+  // Defines configuration for custom third-party data sources.
+  message CustomSourceOptions {
+    // Serialized configuration information for the data source. The implementer of the custom data source is
+    // responsible for serializing and deserializing data from bytes
+    bytes configuration = 1;
+  }
+
+  // Defines options for DataSource that sources features from request data
+  message RequestDataOptions {
+    reserved 1;
+    // Mapping of feature name to type
+    map<string, feast.types.ValueType.Enum> deprecated_schema = 2;
+
+    repeated FeatureSpecV2 schema = 3;
+
+  }
+
+  // Defines options for DataSource that supports pushing data to it. This allows data to be pushed to
+  // the online store on-demand, such as by stream consumers.
+  message PushOptions {
+    reserved 1;
+  }
+
+
+  // DataSource options.
+  oneof options {
+    FileOptions file_options = 11;
+    BigQueryOptions bigquery_options = 12;
+    KafkaOptions kafka_options = 13;
+    KinesisOptions kinesis_options = 14;
+    RedshiftOptions redshift_options = 15;
+    RequestDataOptions request_data_options = 18;
+    CustomSourceOptions custom_options = 16;
+    SnowflakeOptions snowflake_options = 19;
+    PushOptions push_options = 22;
+    SparkOptions spark_options = 27;
+    TrinoOptions trino_options = 30;
+    AthenaOptions athena_options = 35;
+  }
+}