Merge pull request #179 from poissoncorp/RDBC-692

RDBC-692 Counters - #3

Author: ml054

ravendb/documents/indexes/counters.py

@@ -0,0 +1,193 @@
+from typing import Dict, Set, Optional, Callable, Union
+
+from ravendb import constants
+from ravendb.documents.conventions import DocumentConventions
+from ravendb.documents.indexes.definitions import (
+    FieldStorage,
+    FieldIndexing,
+    FieldTermVector,
+    IndexDefinition,
+    IndexSourceType,
+    IndexFieldOptions,
+    IndexType,
+)
+from ravendb.documents.indexes.index_creation import AbstractIndexCreationTaskBase, AbstractIndexDefinitionBuilder
+from ravendb.documents.indexes.spatial.configuration import SpatialOptions, SpatialOptionsFactory
+
+
+class CountersIndexDefinition(IndexDefinition):
+    @property
+    def source_type(self) -> IndexSourceType:
+        return IndexSourceType.COUNTERS
+
+
+class AbstractGenericCountersIndexCreationTask(AbstractIndexCreationTaskBase[CountersIndexDefinition]):
+    def __init__(self):
+        super(AbstractGenericCountersIndexCreationTask, self).__init__()
+        self._reduce: Optional[str] = None
+
+        self._stores_strings: Dict[str, FieldStorage] = {}
+        self._indexes_strings: Dict[str, FieldIndexing] = {}
+        self._analyzers_strings: Dict[str, str] = {}
+        self._index_suggestions: Set[str] = set()
+        self._term_vectors_strings: Dict[str, FieldTermVector] = {}
+        self._spatial_options_strings: Dict[str, SpatialOptions] = {}
+
+        self._output_reduce_to_collection: Optional[str] = None
+        self._pattern_for_output_reduce_to_collection_references: Optional[str] = None
+        self._pattern_references_collection_name: Optional[str] = None
+
+    @property
+    def is_map_reduce(self) -> bool:
+        return self._reduce is not None
+
+    def _index(self, field: str, indexing: FieldIndexing) -> None:
+        self._indexes_strings[field] = indexing
+
+    def _spatial(self, field: str, indexing: Callable[[SpatialOptionsFactory], SpatialOptions]) -> None:
+        self._spatial_options_strings[field] = indexing(SpatialOptionsFactory())
+
+    def _store_all_fields(self, storage: FieldStorage) -> None:
+        self._stores_strings[constants.Documents.Indexing.Fields.ALL_FIELDS] = storage
+
+    def _store(self, field: str, storage: FieldStorage) -> None:
+        self._stores_strings[field] = storage
+
+    def _analyze(self, field: str, analyzer: str) -> None:
+        self._analyzers_strings[field] = analyzer
+
+    def _term_vector(self, field: str, term_vector: FieldTermVector) -> None:
+        self._term_vectors_strings[field] = term_vector
+
+    def _suggestion(self, field: str) -> None:
+        self._index_suggestions.add(field)
+
+
+class CountersIndexDefinitionBuilder(AbstractIndexDefinitionBuilder[CountersIndexDefinition]):
+    def __init__(self, index_name: Optional[str] = None):
+        super(CountersIndexDefinitionBuilder, self).__init__(index_name)
+        self.map: Optional[str] = None
+
+    def _new_index_definition(self) -> CountersIndexDefinition:
+        return CountersIndexDefinition()
+
+    def to_index_definition(
+        self, conventions: DocumentConventions, validate_map: bool = True
+    ) -> CountersIndexDefinition:
+        if self.map is None and validate_map:
+            raise RuntimeError(
+                f"Map is required to generate an index, you "
+                f"cannot create an index without a valid map property (in index {self._index_name})."
+            )
+
+        return super(CountersIndexDefinitionBuilder, self).to_index_definition(conventions, validate_map)
+
+    def _to_index_definition(self, index_definition: CountersIndexDefinition, conventions: DocumentConventions) -> None:
+        if self.map is None:
+            return
+
+        index_definition.maps.add(self.map)
+
+
+class AbstractCountersIndexCreationTask(AbstractGenericCountersIndexCreationTask):
+    def __init__(self):
+        super(AbstractCountersIndexCreationTask, self).__init__()
+        self.map: Optional[str] = None
+
+    def create_index_definition(self) -> CountersIndexDefinition:
+        if self.conventions is None:
+            self.conventions = DocumentConventions()
+
+        index_definition_builder = CountersIndexDefinitionBuilder(self.index_name)
+        index_definition_builder.indexes_strings = self._indexes_strings
+        index_definition_builder.analyzers_strings = self._analyzers_strings
+        index_definition_builder.map = self.map
+        index_definition_builder.reduce = self._reduce
+        index_definition_builder.stores_strings = self._stores_strings
+        index_definition_builder.suggestions_options = self._index_suggestions
+        index_definition_builder.term_vectors_strings = self._term_vectors_strings
+        index_definition_builder.spatial_indexes_strings = self._spatial_options_strings
+        index_definition_builder.output_reduce_to_collection = self._output_reduce_to_collection
+        index_definition_builder.pattern_for_output_reduce_to_collection_references = (
+            self._pattern_for_output_reduce_to_collection_references
+        )
+        index_definition_builder.pattern_references_collection_name = self._pattern_references_collection_name
+        index_definition_builder.additional_sources = self.additional_sources
+        index_definition_builder.additional_assemblies = self.additional_assemblies
+        index_definition_builder.configuration = self.configuration
+        index_definition_builder.lock_mode = self.lock_mode
+        index_definition_builder.priority = self.priority
+        index_definition_builder.state = self.state
+        index_definition_builder.deployment_mode = self.deployment_mode
+
+        return index_definition_builder.to_index_definition(self.conventions)
+
+
+class AbstractJavaScriptCountersIndexCreationTask(AbstractIndexCreationTaskBase[CountersIndexDefinition]):
+    def __init__(self):
+        super(AbstractJavaScriptCountersIndexCreationTask, self).__init__()
+        self._definition = CountersIndexDefinition()
+
+    @property
+    def maps(self) -> Set[str]:
+        return self._definition.maps
+
+    @maps.setter
+    def maps(self, value: Set[str]):
+        self._definition.maps = value
+
+    @property
+    def fields(self) -> Dict[str, IndexFieldOptions]:
+        return self._definition.fields
+
+    @fields.setter
+    def fields(self, value: Dict[str, IndexFieldOptions]):
+        self._definition.fields = value
+
+    @property
+    def reduce(self) -> str:
+        return self._definition.reduce
+
+    @reduce.setter
+    def reduce(self, value: Dict[str, IndexFieldOptions]):
+        self._definition.reduce = value
+
+    @property
+    def is_map_reduce(self) -> bool:
+        return self.reduce is not None
+
+    @property
+    def output_reduce_to_collection(self) -> str:
+        return self._definition.output_reduce_to_collection
+
+    @output_reduce_to_collection.setter
+    def output_reduce_to_collection(self, value: Dict[str, IndexFieldOptions]):
+        self._definition.output_reduce_to_collection = value
+
+    @property
+    def pattern_references_collection_name(self) -> str:
+        return self._definition.pattern_references_collection_name
+
+    @pattern_references_collection_name.setter
+    def pattern_references_collection_name(self, value: Dict[str, IndexFieldOptions]):
+        self._definition.pattern_references_collection_name = value
+
+    @property
+    def pattern_for_output_reduce_to_collection_references(self) -> str:
+        return self._definition.pattern_for_output_reduce_to_collection_references
+
+    @pattern_for_output_reduce_to_collection_references.setter
+    def pattern_for_output_reduce_to_collection_references(self, value: Dict[str, IndexFieldOptions]):
+        self._definition.pattern_for_output_reduce_to_collection_references = value
+
+    def create_index_definition(self) -> CountersIndexDefinition:
+        self._definition.name = self.index_name
+        self._definition.type = IndexType.JAVA_SCRIPT_MAP_REDUCE if self.is_map_reduce else IndexType.JAVA_SCRIPT_MAP
+        self._definition.additional_sources = self.additional_sources or {}
+        self._definition.additional_assemblies = self.additional_assemblies or set()
+        self._definition.configuration = self.configuration
+        self._definition.lock_mode = self.lock_mode
+        self._definition.priority = self.priority
+        self._definition.state = self.state
+        self._definition.deployment_mode = self.deployment_mode
+        return self._definition

ravendb/documents/session/document_session.py

@@ -346,8 +346,8 @@ def __load_internal_stream(self, keys: List[str], operation: LoadOperation, stre
 
     def load_starting_with(
         self,
-        object_type: Type[_T],
         id_prefix: str,
+        object_type: Optional[Type[_T]] = None,
         matches: Optional[str] = None,
         start: Optional[int] = None,
         page_size: Optional[int] = None,
@@ -727,17 +727,16 @@ def __load_starting_with_internal(
 
         def load_starting_with(
             self,
-            object_type: type,
             id_prefix: str,
-            matches: str,
-            start: int,
-            page_size: int,
-            exclude: str,
-            start_after: str,
-        ) -> object:
-            load_starting_with_operation = LoadStartingWithOperation(self._session)
-            self.__load_starting_with_internal(
-                id_prefix, load_starting_with_operation, None, matches, start, page_size, exclude, start_after
+            object_type: Optional[Type[_T]] = None,
+            matches: Optional[str] = None,
+            start: Optional[int] = None,
+            page_size: Optional[int] = None,
+            exclude: Optional[str] = None,
+            start_after: Optional[str] = None,
+        ) -> List[_T]:
+            return self._session.load_starting_with(
+                id_prefix, object_type, matches, start, page_size, exclude, start_after
             )
 
         def load_starting_with_into_stream(

ravendb/documents/session/operations/lazy.py

@@ -210,8 +210,8 @@ def load(
 
     def load_starting_with(
         self,
-        object_type: Type[_T],
         id_prefix: str,
+        object_type: Optional[Type[_T]],
         matches: str = None,
         start: int = 0,
         page_size: int = 25,

ravendb/tests/counters_tests/test_query_on_counters.py

@@ -326,6 +326,60 @@ def test_session_query_include_counter_and_document(self):
 
             self.assertEqual(1, session.advanced.number_of_requests)
 
+    def test_session_query_include_all_counters(self):
+        with self.store.open_session() as session:
+            session.store(Order(company="companies/1-A"), "orders/1-A")
+            session.store(Order(company="companies/2-A"), "orders/2-A")
+            session.store(Order(company="companies/3-A"), "orders/3-A")
+
+            session.counters_for("orders/1-A").increment("Downloads", 100)
+            session.counters_for("orders/2-A").increment("Downloads", 200)
+            session.counters_for("orders/3-A").increment("Downloads", 300)
+
+            session.counters_for("orders/1-A").increment("Likes", 1000)
+            session.counters_for("orders/2-A").increment("Likes", 2000)
+
+            session.counters_for("orders/1-A").increment("Votes", 10000)
+            session.counters_for("orders/3-A").increment("Cats", 5)
+
+            session.save_changes()
+
+        with self.store.open_session() as session:
+            query = session.query(object_type=Order).include(lambda i: i.include_all_counters())
+
+            self.assertEqual("from 'Orders' include counters()", query._to_string())
+
+            query_result = list(query)
+            self.assertEqual(1, session.advanced.number_of_requests)
+
+            # included counters should be in cache
+            order = query_result[0]
+            self.assertEqual("orders/1-A", order.Id)
+
+            dic = session.counters_for_entity(order).get_all()
+            self.assertEqual(3, len(dic))
+            self.assertIn(("downloads", 100), dic.items())
+            self.assertIn(("likes", 1000), dic.items())
+            self.assertIn(("votes", 10000), dic.items())
+
+            order = query_result[1]
+            self.assertEqual("orders/2-A", order.Id)
+
+            dic = session.counters_for_entity(order).get_all()
+            self.assertEqual(2, len(dic))
+            self.assertIn(("downloads", 200), dic.items())
+            self.assertIn(("likes", 2000), dic.items())
+
+            order = query_result[2]
+            self.assertEqual("orders/3-A", order.Id)
+
+            dic = session.counters_for_entity(order).get_all()
+            self.assertEqual(2, len(dic))
+            self.assertIn(("downloads", 300), dic.items())
+            self.assertIn(("cats", 5), dic.items())
+
+            self.assertEqual(1, session.advanced.number_of_requests)
+
     def test_session_query_include_all_counters_of_document_and_of_related_document(self):
         with self.store.open_session() as session:
             session.store(Order(employee="employees/1-A"), "orders/1-A")
@@ -413,36 +467,6 @@ def test_counters_should_be_cached_on_all_docs_collection(self):
             counter_value = session.counters_for("orders/1-A").get("downloads")
             self.assertEqual(500, counter_value)
 
-    def test_counters_should_be_cached_on_collection(self):
-        with self.store.open_session() as session:
-            session.store(Order(company="companies/1-A"), "orders/1-A")
-            session.counters_for("orders/1-A").increment("downloads", 100)
-            session.save_changes()
-
-        with self.store.open_session() as session:
-            list(session.query(object_type=Order).include(lambda i: i.include_counters("downloads")))
-            counter_value = session.counters_for("orders/1-A").get("downloads")
-            self.assertEqual(100, counter_value)
-
-            session.counters_for("orders/1-A").increment("downloads", 200)
-            session.save_changes()
-
-            list(session.query(object_type=Order).include(lambda i: i.include_counters("downloads")))
-
-            counter_value = session.counters_for("orders/1-A").get("downloads")
-            self.assertEqual(300, counter_value)
-
-            session.counters_for("orders/1-A").increment("downloads", 200)
-            session.save_changes()
-
-            list(session.query(object_type=Order).include(lambda i: i.include_counters("downloads")))
-
-            counter_value = session.counters_for("orders/1-A").get("downloads")
-            self.assertEqual(500, counter_value)
-
-        with self.store.open_session() as session:
-            session.load("orders/1-A", Order)
-
     def test_counters_caching_should_handle_deletion__include_counters(self):
         self._counters_caching_should_handle_deletion(
             lambda session: list(
@@ -655,3 +679,35 @@ def test_raw_query_js_projection_with_counter_raw_values(self):
             self.assertEqual("Pigpen", query[2].name)
             self.assertEqual(500, query[2].likes.values().__iter__().__next__())
             self.assertIsNone(query[2].downloads)
+
+    def test_counters_should_be_cached_on_collection(self):
+        with self.store.open_session() as session:
+            session.store(Order(company="companies/1-A"), "orders/1-A")
+            session.counters_for("orders/1-A").increment("downloads", 100)
+            session.save_changes()
+
+        with self.store.open_session() as session:
+            list(session.query(object_type=Order).include(lambda i: i.include_counter("downloads")))
+            self.assertEqual(100, session.counters_for("orders/1-A").get("downloads"))
+            session.counters_for("orders/1-A").increment("downloads", 200)
+            session.save_changes()
+
+            list(session.query(object_type=Order).include(lambda i: i.include_counters("downloads")))
+            self.assertEqual(300, session.counters_for("orders/1-A").get("downloads"))
+            session.counters_for("orders/1-A").increment("downloads", 200)
+            session.save_changes()
+
+            list(session.query(object_type=Order).include(lambda i: i.include_counters("downloads")))
+            self.assertEqual(500, session.counters_for("orders/1-A").get("downloads"))
+
+        with self.store.open_session() as session:
+            session.load("orders/1-A", Order, lambda i: i.include_counter("downloads"))
+            counter_value = session.counters_for("orders/1-A").get("downloads")
+            self.assertEqual(500, counter_value)
+
+            session.counters_for("orders/1-A").increment("downloads", 200)
+            session.save_changes()
+
+            session.load("order/1-A", Order, lambda i: i.include_counter("downloads"))
+            counter_value = session.counters_for("orders/1-A").get("downloads")
+            self.assertEqual(700, counter_value)