From 033e1fa69e833edf05e7d4b1e2d74331b24a8254 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Sun, 24 Dec 2023 11:56:41 +0100 Subject: [PATCH 01/26] init commit rework --- cognition_objects/project.py | 15 +++++++++++++ cognition_objects/strategy_step.py | 15 +++++++++++++ enums.py | 34 +++++++++++++++++++++++++++++- models.py | 11 ++++++++-- 4 files changed, 72 insertions(+), 3 deletions(-) diff --git a/cognition_objects/project.py b/cognition_objects/project.py index 6d2d398a..e9964b5a 100644 --- a/cognition_objects/project.py +++ b/cognition_objects/project.py @@ -61,6 +61,15 @@ def routing( record_dict['routing'] = 'Low-code strategy' return record_dict, scope_dict +""" + + execute_query_enrichment_if_source_code = """from typing import Dict, Any, Tuple + +def check_execute( + record_dict: Dict[str, Any], scope_dict: Dict[str, Any] +) -> bool: + return True + """ project: CognitionProject = CognitionProject( @@ -75,6 +84,7 @@ def routing( refinery_relevance_project_id=refinery_relevances_project_id, operator_routing_source_code=operator_routing_source_code, refinery_synchronization_interval_option=enums.RefinerySynchronizationIntervalOption.NEVER.value, + execute_query_enrichment_if_source_code=execute_query_enrichment_if_source_code, ) general.add(project, with_commit) return project @@ -86,6 +96,7 @@ def update( description: Optional[str] = None, operator_routing_source_code: Optional[str] = None, refinery_synchronization_interval_option: Optional[str] = None, + execute_query_enrichment_if_source_code: Optional[str] = None, with_commit: bool = True, ) -> CognitionProject: project: CognitionProject = get(project_id) @@ -99,6 +110,10 @@ def update( project.refinery_synchronization_interval_option = ( refinery_synchronization_interval_option ) + if execute_query_enrichment_if_source_code is not None: + project.execute_query_enrichment_if_source_code = ( + execute_query_enrichment_if_source_code + ) general.flush_or_commit(with_commit) return project diff --git a/cognition_objects/strategy_step.py b/cognition_objects/strategy_step.py index 92ee0846..f729735f 100644 --- a/cognition_objects/strategy_step.py +++ b/cognition_objects/strategy_step.py @@ -54,9 +54,19 @@ def create( step_type: str, position: int, config: Dict, + progress_text: str, with_commit: bool = True, created_at: Optional[datetime] = None, ) -> CognitionStrategyStep: + + execute_if_source_code = """from typing import Dict, Any, Tuple + +def check_execute( + record_dict: Dict[str, Any], scope_dict: Dict[str, Any] +) -> bool: + return True +""" + strategy: CognitionStrategyStep = CognitionStrategyStep( project_id=project_id, strategy_id=strategy_id, @@ -67,6 +77,8 @@ def create( step_type=step_type, position=position, config=config, + progress_text=progress_text, + execute_if_source_code=execute_if_source_code, ) general.add(strategy, with_commit) @@ -80,6 +92,7 @@ def update( description: Optional[str] = None, position: Optional[int] = None, config: Optional[Dict] = None, + progress_text: Optional[str] = None, with_commit: bool = True, ) -> CognitionStrategyStep: strategy_step: CognitionStrategyStep = get(project_id, strategy_step_id) @@ -93,6 +106,8 @@ def update( if config is not None: strategy_step.config = config flag_modified(strategy_step, "config") + if progress_text is not None: + strategy_step.progress_text = progress_text general.flush_or_commit(with_commit) return strategy_step diff --git a/enums.py b/enums.py index 8f67194e..87ab5fd3 100644 --- a/enums.py +++ b/enums.py @@ -488,20 +488,41 @@ class StrategyStepType(Enum): NONE = "NONE" PYTHON = "PYTHON" LLM = "LLM" + SELECTION = "SELECTION" + QUERY_REPHRASING = "QUERY_REPHRASING" + WEBSEARCH = "WEBSEARCH" def get_description(self): return STEP_DESCRIPTIONS.get(self, "No description available") + + def get_progress_text(self): + return STEP_PROGRESS_TEXTS.get(self, "No progress text available") STEP_DESCRIPTIONS = { StrategyStepType.RETRIEVAL: "Fetch facts from a DB", StrategyStepType.RELEVANCE: "Classify retrieved facts", - StrategyStepType.NONE: "Dummy step", StrategyStepType.PYTHON: "Custom python function", StrategyStepType.LLM: "Run a LLM", + StrategyStepType.NONE: "Dummy step", + StrategyStepType.SELECTION: "Select data", + StrategyStepType.QUERY_REPHRASING: "Rephrase query", + StrategyStepType.WEBSEARCH: "Search the web", +} + +STEP_PROGRESS_TEXTS = { + StrategyStepType.RETRIEVAL: "Retrieving facts", + StrategyStepType.RELEVANCE: "Classifying facts", + StrategyStepType.NONE: "Dummy step", + StrategyStepType.PYTHON: "Running custom python function", + StrategyStepType.LLM: "Running LLM", + StrategyStepType.SELECTION: "Selecting data", + StrategyStepType.QUERY_REPHRASING: "Rephrasing query", + StrategyStepType.WEBSEARCH: "Searching the web", } + class PipelineStep(Enum): INCOMING_QUESTION = "INCOMING_QUESTION" INCOMING_QUESTION_TRY = "INCOMING_QUESTION_TRY" @@ -560,3 +581,14 @@ def try_parse_enum_value(string: str, enumType: Enum, raise_me: bool = True) -> raise ValueError(f"Invalid value {string} for enum {enumType}") return return parsed + +class EmitType(Enum): + ANSWER = "ANSWER" + RETRIEVAL_RESULTS = "RETRIEVAL_RESULTS" + FOLLOW_UPS = "FOLLOW_UPS" + SELECTION = "SELECTION" + QUERY_REPHRASING = "QUERY_REPHRASING" + +class CognitionLLMStepUsageType(Enum): + BASE = "BASE" + QUERY_REPHRASING = "QUERY_REPHRASING" \ No newline at end of file diff --git a/models.py b/models.py index c4935711..0faecb6c 100644 --- a/models.py +++ b/models.py @@ -1046,6 +1046,8 @@ class CognitionProject(Base): operator_routing_source_code = Column(String) wizard_running = Column(Boolean, default=False) refinery_synchronization_interval_option = Column(String) + interface_type = Column(String) + execute_query_enrichment_if_source_code = Column(String) class CognitionStrategy(Base): @@ -1093,6 +1095,9 @@ class CognitionStrategyStep(Base): step_type = Column(String) position = Column(Integer) config = Column(JSON) + progress_text = Column(String) + enable_emissions = Column(Boolean, default=True) + execute_if_source_code = Column(String) class CognitionConversation(Base): @@ -1140,10 +1145,11 @@ class CognitionMessage(Base): created_at = Column(DateTime, default=sql.func.now()) question = Column(String) facts = Column(ARRAY(JSON)) + selection_widget = Column(ARRAY(JSON)) answer = Column(String) - # None = not yet answered, True = positive, false = negative - positive_feedback = Column(Boolean) + feedback_value = Column(String) + feedback_category = Column(String) feedback_message = Column(String) scope_dict_diff_previous_conversation = Column(JSON) @@ -1184,6 +1190,7 @@ class CognitionPipelineLogs(Base): record_dict_diff_previous_message = Column(JSON) content = Column(ARRAY(String)) time_elapsed = Column(Float) + skipped_step = Column(Boolean, default=False) class CognitionEnvironmentVariable(Base): From 2d341bf7015fe1b280d59eea910cd7eb8f77af18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Sun, 24 Dec 2023 13:50:28 +0100 Subject: [PATCH 02/26] update message --- cognition_objects/message.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/cognition_objects/message.py b/cognition_objects/message.py index da306188..d26d5373 100644 --- a/cognition_objects/message.py +++ b/cognition_objects/message.py @@ -1,4 +1,4 @@ -from typing import List, Optional +from typing import Any, Dict, List, Optional from datetime import datetime from ..business_objects import general from ..session import session @@ -91,6 +91,35 @@ def create( return message +def update( + project_id: str, + message_id: str, + answer: Optional[str] = None, + facts: Optional[List[Dict[str, Any]]] = None, + selection_widget: Optional[List[Dict[str, Any]]] = None, + feedback_value: Optional[str] = None, + feedback_category: Optional[str] = None, + feedback_message: Optional[str] = None, + with_commit: bool = True, +) -> CognitionMessage: + message = get(project_id, message_id) + if answer is not None: + message.answer = answer + if facts is not None: + message.facts = facts + if selection_widget is not None: + message.selection_widget = selection_widget + if feedback_value is not None: + message.feedback_value = feedback_value + if feedback_category is not None: + message.feedback_category = feedback_category + if feedback_message is not None: + message.feedback_message = feedback_message + + general.flush_or_commit(with_commit) + + return message + def delete(project_id: str, message_id: str, with_commit: bool = True) -> None: session.query(CognitionMessage).filter( From 92325c1df1ce3ba735c0009a8ffc4d718f042f0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Mon, 25 Dec 2023 01:12:37 +0100 Subject: [PATCH 03/26] change order --- cognition_objects/conversation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cognition_objects/conversation.py b/cognition_objects/conversation.py index d1a7f491..d89c81f9 100644 --- a/cognition_objects/conversation.py +++ b/cognition_objects/conversation.py @@ -42,7 +42,7 @@ def get_all_paginated_by_project_id( paginated_result = ( session.query(CognitionConversation) .filter(CognitionConversation.project_id == project_id) - .order_by(CognitionConversation.created_at.asc()) + .order_by(CognitionConversation.created_at.desc()) .limit(limit) .offset((page - 1) * limit) .all() From 6848097ec179fc8486928b936bb97345e416dce7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Mon, 25 Dec 2023 01:45:17 +0100 Subject: [PATCH 04/26] custom colors --- cognition_objects/project.py | 6 ++++++ models.py | 3 +++ 2 files changed, 9 insertions(+) diff --git a/cognition_objects/project.py b/cognition_objects/project.py index e9964b5a..c3664c98 100644 --- a/cognition_objects/project.py +++ b/cognition_objects/project.py @@ -94,6 +94,8 @@ def update( project_id: str, name: Optional[str] = None, description: Optional[str] = None, + customer_color_primary: Optional[str] = None, + customer_color_secondary: Optional[str] = None, operator_routing_source_code: Optional[str] = None, refinery_synchronization_interval_option: Optional[str] = None, execute_query_enrichment_if_source_code: Optional[str] = None, @@ -104,6 +106,10 @@ def update( project.name = name if description is not None: project.description = description + if customer_color_primary is not None: + project.customer_color_primary = customer_color_primary + if customer_color_secondary is not None: + project.customer_color_secondary = customer_color_secondary if operator_routing_source_code is not None: project.operator_routing_source_code = operator_routing_source_code if refinery_synchronization_interval_option is not None: diff --git a/models.py b/models.py index 0faecb6c..deff57ae 100644 --- a/models.py +++ b/models.py @@ -1049,6 +1049,9 @@ class CognitionProject(Base): interface_type = Column(String) execute_query_enrichment_if_source_code = Column(String) + customer_logo_uri = Column(String) + customer_color_primary = Column(String, default="#4ade80") + customer_color_secondary = Column(String, default="#4ade80") class CognitionStrategy(Base): __tablename__ = Tablenames.STRATEGY.value From 1ba776027ececf34c6174c9c51c71b1cbea910b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Mon, 25 Dec 2023 11:54:33 +0100 Subject: [PATCH 05/26] remove column for uri --- models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/models.py b/models.py index deff57ae..abfe8e3a 100644 --- a/models.py +++ b/models.py @@ -1049,7 +1049,6 @@ class CognitionProject(Base): interface_type = Column(String) execute_query_enrichment_if_source_code = Column(String) - customer_logo_uri = Column(String) customer_color_primary = Column(String, default="#4ade80") customer_color_secondary = Column(String, default="#4ade80") From 0021319a4c49b47424dada8dda31d93f89e34de6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Mon, 25 Dec 2023 23:55:54 +0100 Subject: [PATCH 06/26] custom colors --- cognition_objects/project.py | 5 +++++ models.py | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/cognition_objects/project.py b/cognition_objects/project.py index c3664c98..2cc25f2f 100644 --- a/cognition_objects/project.py +++ b/cognition_objects/project.py @@ -95,6 +95,7 @@ def update( name: Optional[str] = None, description: Optional[str] = None, customer_color_primary: Optional[str] = None, + customer_color_primary_only_accent: Optional[bool] = None, customer_color_secondary: Optional[str] = None, operator_routing_source_code: Optional[str] = None, refinery_synchronization_interval_option: Optional[str] = None, @@ -108,6 +109,10 @@ def update( project.description = description if customer_color_primary is not None: project.customer_color_primary = customer_color_primary + if customer_color_primary_only_accent is not None: + project.customer_color_primary_only_accent = ( + customer_color_primary_only_accent + ) if customer_color_secondary is not None: project.customer_color_secondary = customer_color_secondary if operator_routing_source_code is not None: diff --git a/models.py b/models.py index abfe8e3a..ad276163 100644 --- a/models.py +++ b/models.py @@ -1049,8 +1049,9 @@ class CognitionProject(Base): interface_type = Column(String) execute_query_enrichment_if_source_code = Column(String) - customer_color_primary = Column(String, default="#4ade80") - customer_color_secondary = Column(String, default="#4ade80") + customer_color_primary = Column(String, default="#18181b") + customer_color_primary_only_accent = Column(Boolean, default=False) + customer_color_secondary = Column(String, default="#9333ea") class CognitionStrategy(Base): __tablename__ = Tablenames.STRATEGY.value From 1c258e5953991fc81208b5e2678282bc74a76153 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Tue, 26 Dec 2023 16:32:10 +0100 Subject: [PATCH 07/26] i18n --- business_objects/user.py | 7 +++++++ models.py | 1 + 2 files changed, 8 insertions(+) diff --git a/business_objects/user.py b/business_objects/user.py index b44a74d6..609c731f 100644 --- a/business_objects/user.py +++ b/business_objects/user.py @@ -100,6 +100,13 @@ def update_organization( user.organization_id = organization_id general.flush_or_commit(with_commit) +def update_language_display( + user_id: str, language_display: str, with_commit: bool = False +) -> None: + user = get(user_id) + user.language_display = language_display + general.flush_or_commit(with_commit) + def __create_migration_user() -> str: organization_item = organization.get_by_name("migration") diff --git a/models.py b/models.py index ad276163..86c5ae5c 100644 --- a/models.py +++ b/models.py @@ -157,6 +157,7 @@ class User(Base): index=True, ) role = Column(String, default=UserRoles.ENGINEER.value) # enum UserRoles + language_display = Column(String, default="en") notifications = parent_to_child_relationship( Tablenames.USER, Tablenames.NOTIFICATION, From 3f72e98af4f8eae44d139ca2d3b78b86b8b5de90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Wed, 27 Dec 2023 02:28:01 +0100 Subject: [PATCH 08/26] emit and skips --- cognition_objects/pipeline_log.py | 2 ++ cognition_objects/strategy_step.py | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/cognition_objects/pipeline_log.py b/cognition_objects/pipeline_log.py index 787cff0f..2acfd504 100644 --- a/cognition_objects/pipeline_log.py +++ b/cognition_objects/pipeline_log.py @@ -64,6 +64,7 @@ def create( time_elapsed: float, record_dict_diff_previous: Dict[str, Any], scope_dict_diff_previous: Dict[str, Any], + skipped_step: Optional[bool] = None, with_commit: bool = True, created_at: Optional[datetime] = None, ) -> CognitionPipelineLogs: @@ -80,6 +81,7 @@ def create( time_elapsed=time_elapsed, record_dict_diff_previous_message=record_dict_diff_previous, scope_dict_diff_previous_message=scope_dict_diff_previous, + skipped_step=skipped_step, ) general.add(log, with_commit) diff --git a/cognition_objects/strategy_step.py b/cognition_objects/strategy_step.py index f729735f..7935c9e3 100644 --- a/cognition_objects/strategy_step.py +++ b/cognition_objects/strategy_step.py @@ -58,7 +58,6 @@ def create( with_commit: bool = True, created_at: Optional[datetime] = None, ) -> CognitionStrategyStep: - execute_if_source_code = """from typing import Dict, Any, Tuple def check_execute( @@ -93,6 +92,8 @@ def update( position: Optional[int] = None, config: Optional[Dict] = None, progress_text: Optional[str] = None, + enable_emissions: Optional[bool] = None, + execute_if_source_code: Optional[str] = None, with_commit: bool = True, ) -> CognitionStrategyStep: strategy_step: CognitionStrategyStep = get(project_id, strategy_step_id) @@ -108,6 +109,10 @@ def update( flag_modified(strategy_step, "config") if progress_text is not None: strategy_step.progress_text = progress_text + if enable_emissions is not None: + strategy_step.enable_emissions = enable_emissions + if execute_if_source_code is not None: + strategy_step.execute_if_source_code = execute_if_source_code general.flush_or_commit(with_commit) return strategy_step From 870a27715f381750574e69d5c33a0d253fb59bd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Wed, 27 Dec 2023 12:33:44 +0100 Subject: [PATCH 09/26] feedback responses --- cognition_objects/conversation.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cognition_objects/conversation.py b/cognition_objects/conversation.py index d89c81f9..f4bbce52 100644 --- a/cognition_objects/conversation.py +++ b/cognition_objects/conversation.py @@ -104,6 +104,9 @@ def update_message( conversation_id: str, message_id: str, answer: Optional[str] = None, + feedback_value: Optional[str] = None, + feedback_category: Optional[str] = None, + feedback_message: Optional[str] = None, strategy_id: Optional[str] = None, scope_dict_diff_previous_conversation: Optional[Dict[str, Any]] = None, with_commit: bool = True, @@ -113,6 +116,12 @@ def update_message( message_entity.strategy_id = strategy_id if answer is not None: message_entity.answer = answer + if feedback_value is not None: + message_entity.feedback_value = feedback_value + if feedback_category is not None: + message_entity.feedback_category = feedback_category + if feedback_message is not None: + message_entity.feedback_message = feedback_message if scope_dict_diff_previous_conversation is not None: message_entity.scope_dict_diff_previous_conversation = ( scope_dict_diff_previous_conversation From ab825cf596e9213cc78b9a1b4ff55a97a07d32c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Wed, 27 Dec 2023 12:53:18 +0100 Subject: [PATCH 10/26] strategy complexities --- enums.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/enums.py b/enums.py index 87ab5fd3..9cf58267 100644 --- a/enums.py +++ b/enums.py @@ -494,7 +494,7 @@ class StrategyStepType(Enum): def get_description(self): return STEP_DESCRIPTIONS.get(self, "No description available") - + def get_progress_text(self): return STEP_PROGRESS_TEXTS.get(self, "No progress text available") @@ -522,7 +522,6 @@ def get_progress_text(self): } - class PipelineStep(Enum): INCOMING_QUESTION = "INCOMING_QUESTION" INCOMING_QUESTION_TRY = "INCOMING_QUESTION_TRY" @@ -582,6 +581,7 @@ def try_parse_enum_value(string: str, enumType: Enum, raise_me: bool = True) -> return return parsed + class EmitType(Enum): ANSWER = "ANSWER" RETRIEVAL_RESULTS = "RETRIEVAL_RESULTS" @@ -589,6 +589,13 @@ class EmitType(Enum): SELECTION = "SELECTION" QUERY_REPHRASING = "QUERY_REPHRASING" + class CognitionLLMStepUsageType(Enum): BASE = "BASE" - QUERY_REPHRASING = "QUERY_REPHRASING" \ No newline at end of file + QUERY_REPHRASING = "QUERY_REPHRASING" + + +class StrategyComplexity(Enum): + SIMPLE = "SIMPLE" + MEDIUM = "MEDIUM" + COMPLEX = "COMPLEX" From 9200d02213f9569105a33130431621a4c367e00f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Wed, 27 Dec 2023 17:03:41 +0100 Subject: [PATCH 11/26] consumption statistics --- cognition_objects/consumption_log.py | 90 ++++++++++++++++++++++++++++ enums.py | 13 ++++ models.py | 35 +++++++++++ 3 files changed, 138 insertions(+) create mode 100644 cognition_objects/consumption_log.py diff --git a/cognition_objects/consumption_log.py b/cognition_objects/consumption_log.py new file mode 100644 index 00000000..db72b559 --- /dev/null +++ b/cognition_objects/consumption_log.py @@ -0,0 +1,90 @@ +from typing import Dict, List, Optional, Tuple, Any + +from datetime import datetime + +from submodules.model import enums + +from ..cognition_objects import message +from ..business_objects import general +from ..session import session +from ..models import CognitionConsumptionLog + + +def get(project_id: str, log_id: str) -> CognitionConsumptionLog: + return ( + session.query(CognitionConsumptionLog) + .filter( + CognitionConsumptionLog.project_id == project_id, + CognitionConsumptionLog.id == log_id, + ) + .first() + ) + + +def get_all_by_project_id( + project_id: str, +) -> List[CognitionConsumptionLog]: + return ( + session.query(CognitionConsumptionLog) + .filter( + CognitionConsumptionLog.project_id == project_id, + ) + .order_by(CognitionConsumptionLog.created_at.asc()) + .all() + ) + + +def get_all_by_project_id_for_year( + project_id: str, + year: int, +) -> List[CognitionConsumptionLog]: + return ( + session.query(CognitionConsumptionLog) + .filter( + CognitionConsumptionLog.project_id == project_id, + CognitionConsumptionLog.created_at >= datetime(year, 1, 1), + CognitionConsumptionLog.created_at < datetime(year + 1, 1, 1), + ) + .order_by(CognitionConsumptionLog.created_at.asc()) + .all() + ) + + +def create( + project_id: str, + user_id: str, + conversation_id: str, + message_id: str, + state: str, + with_commit: bool = True, + created_at: Optional[datetime] = None, +) -> CognitionConsumptionLog: + conversation: CognitionConsumptionLog = CognitionConsumptionLog( + project_id=project_id, + created_by=user_id, + conversation_id=conversation_id, + message_id=message_id, + created_at=created_at, + state=state, + ) + general.add(conversation, with_commit) + return conversation + + +def update( + project_id: str, + log_id: str, + strategy_id: Optional[str] = None, + complexity: Optional[str] = None, + state: Optional[str] = None, + with_commit: bool = True, +) -> CognitionConsumptionLog: + log = get(project_id=project_id, log_id=log_id) + if strategy_id is not None: + log.strategy_id = strategy_id + if complexity is not None: + log.complexity = complexity + if state is not None: + log.state = state + general.flush_or_commit(with_commit) + return log diff --git a/enums.py b/enums.py index 9cf58267..701b1fa5 100644 --- a/enums.py +++ b/enums.py @@ -130,6 +130,7 @@ class Tablenames(Enum): LLM_STEP = "llm_step" MARKDOWN_LLM_LOGS = "markdown_llm_logs" MARKDOWN_DATASET = "markdown_dataset" + CONSUMPTION_LOG = "consumption_log" def snake_case_to_pascal_case(self): # the type name of a table is needed to create backrefs @@ -599,3 +600,15 @@ class StrategyComplexity(Enum): SIMPLE = "SIMPLE" MEDIUM = "MEDIUM" COMPLEX = "COMPLEX" + + +class StrategyComplexityPrice(Enum): + SIMPLE = 0.25 + MEDIUM = 0.5 + COMPLEX = 0.75 + + +class ConsumptionLogState(Enum): + CREATED = "CREATED" + FINISHED = "FINISHED" + FAILED = "FAILED" diff --git a/models.py b/models.py index 86c5ae5c..19a702c3 100644 --- a/models.py +++ b/models.py @@ -1054,6 +1054,7 @@ class CognitionProject(Base): customer_color_primary_only_accent = Column(Boolean, default=False) customer_color_secondary = Column(String, default="#9333ea") + class CognitionStrategy(Base): __tablename__ = Tablenames.STRATEGY.value __table_args__ = {"schema": "cognition"} @@ -1355,3 +1356,37 @@ class CognitionRefinerySynchronizationTask(Base): state = Column(String) # e.g. CREATED, FINISHED, FAILED logs = Column(ARRAY(String)) num_records_created = Column(Integer) + + +class CognitionConsumptionLog(Base): + __tablename__ = Tablenames.CONSUMPTION_LOG.value + __table_args__ = {"schema": "cognition"} + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + project_id = Column( + UUID(as_uuid=True), + ForeignKey(f"cognition.{Tablenames.PROJECT.value}.id", ondelete="CASCADE"), + index=True, + ) + strategy_id = Column( + UUID(as_uuid=True), + ForeignKey(f"cognition.{Tablenames.STRATEGY.value}.id", ondelete="CASCADE"), + index=True, + ) + conversation_id = Column( + UUID(as_uuid=True), + ForeignKey(f"cognition.{Tablenames.CONVERSATION.value}.id", ondelete="CASCADE"), + index=True, + ) + message_id = Column( + UUID(as_uuid=True), + ForeignKey(f"cognition.{Tablenames.MESSAGE.value}.id", ondelete="CASCADE"), + index=True, + ) + created_by = Column( + UUID(as_uuid=True), + ForeignKey(f"{Tablenames.USER.value}.id", ondelete="SET NULL"), + index=True, + ) + created_at = Column(DateTime, default=sql.func.now()) + complexity = Column(String) + state = Column(String) From 3a5cd18bdbd5eb8861c84ad6b5ebbee43b346fc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Wed, 27 Dec 2023 22:37:20 +0100 Subject: [PATCH 12/26] adds truncate context node --- enums.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/enums.py b/enums.py index 701b1fa5..a1011ed8 100644 --- a/enums.py +++ b/enums.py @@ -492,6 +492,7 @@ class StrategyStepType(Enum): SELECTION = "SELECTION" QUERY_REPHRASING = "QUERY_REPHRASING" WEBSEARCH = "WEBSEARCH" + TRUNCATE_CONTEXT = "TRUNCATE_CONTEXT" def get_description(self): return STEP_DESCRIPTIONS.get(self, "No description available") @@ -509,6 +510,7 @@ def get_progress_text(self): StrategyStepType.SELECTION: "Select data", StrategyStepType.QUERY_REPHRASING: "Rephrase query", StrategyStepType.WEBSEARCH: "Search the web", + StrategyStepType.TRUNCATE_CONTEXT: "Truncate context", } STEP_PROGRESS_TEXTS = { @@ -520,6 +522,7 @@ def get_progress_text(self): StrategyStepType.SELECTION: "Selecting data", StrategyStepType.QUERY_REPHRASING: "Rephrasing query", StrategyStepType.WEBSEARCH: "Searching the web", + StrategyStepType.TRUNCATE_CONTEXT: "Truncating context", } From ab5cf1ce542f3c1c43edc1275d444b033228611e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Thu, 28 Dec 2023 00:12:38 +0100 Subject: [PATCH 13/26] compute header --- cognition_objects/message.py | 7 ++++++- enums.py | 3 +++ models.py | 1 + 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/cognition_objects/message.py b/cognition_objects/message.py index d26d5373..d0ecc5a3 100644 --- a/cognition_objects/message.py +++ b/cognition_objects/message.py @@ -85,12 +85,14 @@ def create( created_at=created_at, question=question, facts=[], + header=question, ) general.add(message, with_commit) return message + def update( project_id: str, message_id: str, @@ -100,6 +102,7 @@ def update( feedback_value: Optional[str] = None, feedback_category: Optional[str] = None, feedback_message: Optional[str] = None, + header: Optional[str] = None, with_commit: bool = True, ) -> CognitionMessage: message = get(project_id, message_id) @@ -115,7 +118,9 @@ def update( message.feedback_category = feedback_category if feedback_message is not None: message.feedback_message = feedback_message - + if header is not None: + message.header = header + general.flush_or_commit(with_commit) return message diff --git a/enums.py b/enums.py index a1011ed8..34f5e406 100644 --- a/enums.py +++ b/enums.py @@ -493,6 +493,7 @@ class StrategyStepType(Enum): QUERY_REPHRASING = "QUERY_REPHRASING" WEBSEARCH = "WEBSEARCH" TRUNCATE_CONTEXT = "TRUNCATE_CONTEXT" + HEADER = "HEADER" def get_description(self): return STEP_DESCRIPTIONS.get(self, "No description available") @@ -511,6 +512,7 @@ def get_progress_text(self): StrategyStepType.QUERY_REPHRASING: "Rephrase query", StrategyStepType.WEBSEARCH: "Search the web", StrategyStepType.TRUNCATE_CONTEXT: "Truncate context", + StrategyStepType.HEADER: "Writing header", } STEP_PROGRESS_TEXTS = { @@ -523,6 +525,7 @@ def get_progress_text(self): StrategyStepType.QUERY_REPHRASING: "Rephrasing query", StrategyStepType.WEBSEARCH: "Searching the web", StrategyStepType.TRUNCATE_CONTEXT: "Truncating context", + StrategyStepType.HEADER: "Headline generation", } diff --git a/models.py b/models.py index 19a702c3..d8a1292e 100644 --- a/models.py +++ b/models.py @@ -1152,6 +1152,7 @@ class CognitionMessage(Base): facts = Column(ARRAY(JSON)) selection_widget = Column(ARRAY(JSON)) answer = Column(String) + header = Column(String) feedback_value = Column(String) feedback_category = Column(String) From 954f4879cf2918afd361f62e4acf2c8ceb9acc03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Thu, 28 Dec 2023 12:32:44 +0100 Subject: [PATCH 14/26] adds when to use to enums --- enums.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/enums.py b/enums.py index 34f5e406..6745d577 100644 --- a/enums.py +++ b/enums.py @@ -497,6 +497,9 @@ class StrategyStepType(Enum): def get_description(self): return STEP_DESCRIPTIONS.get(self, "No description available") + + def get_when_to_use(self): + return STEP_WHEN_TO_USE.get(self, "No description available") def get_progress_text(self): return STEP_PROGRESS_TEXTS.get(self, "No progress text available") @@ -515,6 +518,19 @@ def get_progress_text(self): StrategyStepType.HEADER: "Writing header", } +STEP_WHEN_TO_USE = { + StrategyStepType.RETRIEVAL: "When you want to retrieve facts from a database", + StrategyStepType.RELEVANCE: "When you want to classify retrieved facts", + StrategyStepType.PYTHON: "When you want to run a custom python function", + StrategyStepType.LLM: "When you want to run a LLM", + StrategyStepType.NONE: "Dummy step", + StrategyStepType.SELECTION: "When you want to select data", + StrategyStepType.QUERY_REPHRASING: "When you want to rephrase a query", + StrategyStepType.WEBSEARCH: "When you want to search the web", + StrategyStepType.TRUNCATE_CONTEXT: "When you want to truncate context", + StrategyStepType.HEADER: "When you want to write a header", +} + STEP_PROGRESS_TEXTS = { StrategyStepType.RETRIEVAL: "Retrieving facts", StrategyStepType.RELEVANCE: "Classifying facts", From 16da3cd68b3f5948d392943a07b277028d597d0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Thu, 28 Dec 2023 18:45:13 +0100 Subject: [PATCH 15/26] header in conversation, not message --- cognition_objects/conversation.py | 3 +++ cognition_objects/message.py | 4 ---- models.py | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/cognition_objects/conversation.py b/cognition_objects/conversation.py index f4bbce52..086db514 100644 --- a/cognition_objects/conversation.py +++ b/cognition_objects/conversation.py @@ -90,11 +90,14 @@ def update( project_id: str, conversation_id: str, scope_dict: Optional[Dict[str, Any]] = None, + header: Optional[str] = None, with_commit: bool = True, ) -> CognitionConversation: conversation_entity = get(project_id, conversation_id) if scope_dict is not None: conversation_entity.scope_dict = scope_dict + if header is not None: + conversation_entity.header = header general.flush_or_commit(with_commit) return conversation_entity diff --git a/cognition_objects/message.py b/cognition_objects/message.py index d0ecc5a3..5bde633f 100644 --- a/cognition_objects/message.py +++ b/cognition_objects/message.py @@ -85,7 +85,6 @@ def create( created_at=created_at, question=question, facts=[], - header=question, ) general.add(message, with_commit) @@ -102,7 +101,6 @@ def update( feedback_value: Optional[str] = None, feedback_category: Optional[str] = None, feedback_message: Optional[str] = None, - header: Optional[str] = None, with_commit: bool = True, ) -> CognitionMessage: message = get(project_id, message_id) @@ -118,8 +116,6 @@ def update( message.feedback_category = feedback_category if feedback_message is not None: message.feedback_message = feedback_message - if header is not None: - message.header = header general.flush_or_commit(with_commit) diff --git a/models.py b/models.py index d8a1292e..4713d57b 100644 --- a/models.py +++ b/models.py @@ -1121,6 +1121,7 @@ class CognitionConversation(Base): ) created_at = Column(DateTime, default=sql.func.now()) scope_dict = Column(JSON) + header = Column(String) class CognitionMessage(Base): @@ -1152,7 +1153,6 @@ class CognitionMessage(Base): facts = Column(ARRAY(JSON)) selection_widget = Column(ARRAY(JSON)) answer = Column(String) - header = Column(String) feedback_value = Column(String) feedback_category = Column(String) From 0ac1b672dbbc24dc20321f91d0236cbbe86e0d22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Thu, 28 Dec 2023 20:04:23 +0100 Subject: [PATCH 16/26] error handling --- cognition_objects/conversation.py | 14 ++++++++++++++ models.py | 1 + 2 files changed, 15 insertions(+) diff --git a/cognition_objects/conversation.py b/cognition_objects/conversation.py index 086db514..6cb4e413 100644 --- a/cognition_objects/conversation.py +++ b/cognition_objects/conversation.py @@ -91,6 +91,7 @@ def update( conversation_id: str, scope_dict: Optional[Dict[str, Any]] = None, header: Optional[str] = None, + error: Optional[str] = None, with_commit: bool = True, ) -> CognitionConversation: conversation_entity = get(project_id, conversation_id) @@ -98,6 +99,19 @@ def update( conversation_entity.scope_dict = scope_dict if header is not None: conversation_entity.header = header + if error is not None: + conversation_entity.error = error + general.flush_or_commit(with_commit) + return conversation_entity + + +def clear_error( + project_id: str, + conversation_id: str, + with_commit: bool = True, +) -> CognitionConversation: + conversation_entity = get(project_id, conversation_id) + conversation_entity.error = None general.flush_or_commit(with_commit) return conversation_entity diff --git a/models.py b/models.py index 4713d57b..d3b9e235 100644 --- a/models.py +++ b/models.py @@ -1122,6 +1122,7 @@ class CognitionConversation(Base): created_at = Column(DateTime, default=sql.func.now()) scope_dict = Column(JSON) header = Column(String) + error = Column(String) class CognitionMessage(Base): From c529398ebad6d9d43670ef1c82337c9df5128052 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Fri, 29 Dec 2023 01:26:46 +0100 Subject: [PATCH 17/26] adds team --- business_objects/user.py | 1 + cognition_objects/team.py | 71 ++++++++++++++++++++++++++++++++ cognition_objects/team_member.py | 57 +++++++++++++++++++++++++ enums.py | 4 +- models.py | 51 +++++++++++++++++++++++ 5 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 cognition_objects/team.py create mode 100644 cognition_objects/team_member.py diff --git a/business_objects/user.py b/business_objects/user.py index 609c731f..ab83c035 100644 --- a/business_objects/user.py +++ b/business_objects/user.py @@ -100,6 +100,7 @@ def update_organization( user.organization_id = organization_id general.flush_or_commit(with_commit) + def update_language_display( user_id: str, language_display: str, with_commit: bool = False ) -> None: diff --git a/cognition_objects/team.py b/cognition_objects/team.py new file mode 100644 index 00000000..e1f19978 --- /dev/null +++ b/cognition_objects/team.py @@ -0,0 +1,71 @@ +from typing import List, Optional, Dict +from datetime import datetime +from ..business_objects import general +from ..session import session +from ..models import ( + CognitionTeam, +) +from ..util import prevent_sql_injection +from sqlalchemy import or_ + + +def get(team_id: str) -> CognitionTeam: + return ( + session.query(CognitionTeam) + .filter( + CognitionTeam.id == team_id, + ) + .first() + ) + + +def get_all_by_org_id(org_id: str) -> List[CognitionTeam]: + # Note, atm this doesn't mean all but all on org level + return ( + session.query(CognitionTeam) + .filter(CognitionTeam.organization_id == org_id) + .order_by(CognitionTeam.created_at.asc()) + .all() + ) + + +def create( + organization_id: str, + user_id: str, + name: str, + description: str, + with_commit: bool = True, + created_at: Optional[datetime] = None, +) -> CognitionTeam: + team: CognitionTeam = CognitionTeam( + created_by=user_id, + organization_id=organization_id, + name=name, + description=description, + created_at=created_at, + ) + general.add(team, with_commit) + return team + + +def update( + team_id: str, + name: Optional[str] = None, + description: Optional[str] = None, + with_commit: bool = True, +) -> CognitionTeam: + team: CognitionTeam = get(team_id) + + if name is not None: + team.name = name + if description is not None: + team.description = description + general.flush_or_commit(with_commit) + return team + + +def delete(team_id: str, with_commit: bool = True) -> None: + session.query(CognitionTeam).filter( + CognitionTeam.id == team_id, + ).delete() + general.flush_or_commit(with_commit) diff --git a/cognition_objects/team_member.py b/cognition_objects/team_member.py new file mode 100644 index 00000000..78000426 --- /dev/null +++ b/cognition_objects/team_member.py @@ -0,0 +1,57 @@ +from typing import List, Optional, Dict +from datetime import datetime +from ..business_objects import general +from ..session import session +from ..models import ( + CognitionTeamMember, +) + + +def get(team_member_id: str) -> CognitionTeamMember: + return ( + session.query(CognitionTeamMember) + .filter( + CognitionTeamMember.id == team_member_id, + ) + .first() + ) + + +def get_all_by_team_id(team_id: str) -> List[CognitionTeamMember]: + return ( + session.query(CognitionTeamMember) + .filter(CognitionTeamMember.team_id == team_id) + .order_by(CognitionTeamMember.created_at.asc()) + .all() + ) + + +def get_all_by_user_id(user_id: str) -> List[CognitionTeamMember]: + return ( + session.query(CognitionTeamMember) + .filter(CognitionTeamMember.user_id == user_id) + .order_by(CognitionTeamMember.created_at.asc()) + .all() + ) + + +def create( + team_id: str, + user_id: str, + with_commit: bool = True, + created_at: Optional[datetime] = None, +) -> CognitionTeamMember: + team: CognitionTeamMember = CognitionTeamMember( + created_by=user_id, + team_id=team_id, + created_at=created_at, + ) + general.add(team, with_commit) + return team + + +def delete(team_member_id: str, with_commit: bool = True) -> None: + session.query(CognitionTeamMember).filter( + CognitionTeamMember.id == team_member_id, + ).delete() + general.flush_or_commit(with_commit) diff --git a/enums.py b/enums.py index 6745d577..1ca13679 100644 --- a/enums.py +++ b/enums.py @@ -131,6 +131,8 @@ class Tablenames(Enum): MARKDOWN_LLM_LOGS = "markdown_llm_logs" MARKDOWN_DATASET = "markdown_dataset" CONSUMPTION_LOG = "consumption_log" + TEAM = "team" + TEAM_MEMBER = "team_member" def snake_case_to_pascal_case(self): # the type name of a table is needed to create backrefs @@ -497,7 +499,7 @@ class StrategyStepType(Enum): def get_description(self): return STEP_DESCRIPTIONS.get(self, "No description available") - + def get_when_to_use(self): return STEP_WHEN_TO_USE.get(self, "No description available") diff --git a/models.py b/models.py index d3b9e235..fc79a6c1 100644 --- a/models.py +++ b/models.py @@ -1011,6 +1011,27 @@ class TaskQueue(Base): # --- COGNITION TABLES +class CognitionTeam(Base): + __tablename__ = Tablenames.TEAM.value + __table_args__ = {"schema": "cognition"} + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + + organization_id = Column( + UUID(as_uuid=True), + ForeignKey(f"{Tablenames.ORGANIZATION.value}.id", ondelete="CASCADE"), + index=True, + ) + created_by = Column( + UUID(as_uuid=True), + ForeignKey(f"{Tablenames.USER.value}.id", ondelete="SET NULL"), + index=True, + ) + + created_at = Column(DateTime, default=sql.func.now()) + name = Column(String) + description = Column(String) + + class CognitionProject(Base): __tablename__ = Tablenames.PROJECT.value __table_args__ = {"schema": "cognition"} @@ -1020,6 +1041,11 @@ class CognitionProject(Base): ForeignKey(f"{Tablenames.ORGANIZATION.value}.id", ondelete="CASCADE"), index=True, ) + team_id = Column( + UUID(as_uuid=True), + ForeignKey(f"{Tablenames.TEAM.value}.id", ondelete="CASCADE"), + index=True, + ) refinery_references_project_id = Column( UUID(as_uuid=True), ForeignKey(f"{Tablenames.PROJECT.value}.id", ondelete="SET NULL"), @@ -1392,3 +1418,28 @@ class CognitionConsumptionLog(Base): created_at = Column(DateTime, default=sql.func.now()) complexity = Column(String) state = Column(String) + + +class CognitionTeamMember(Base): + __tablename__ = Tablenames.TEAM_MEMBER.value + __table_args__ = {"schema": "cognition"} + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + + team_id = Column( + UUID(as_uuid=True), + ForeignKey(f"cognition.{Tablenames.TEAM.value}.id", ondelete="CASCADE"), + index=True, + ) + + user_id = Column( + UUID(as_uuid=True), + ForeignKey(f"{Tablenames.USER.value}.id", ondelete="CASCADE"), + index=True, + ) + + created_at = Column(DateTime, default=sql.func.now()) + created_by = Column( + UUID(as_uuid=True), + ForeignKey(f"{Tablenames.USER.value}.id", ondelete="SET NULL"), + index=True, + ) From f12485aba2010e61de64660f7a75aa06f9a2932a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Fri, 29 Dec 2023 02:11:18 +0100 Subject: [PATCH 18/26] adds team --- cognition_objects/team_member.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/cognition_objects/team_member.py b/cognition_objects/team_member.py index 78000426..b451eb96 100644 --- a/cognition_objects/team_member.py +++ b/cognition_objects/team_member.py @@ -38,16 +38,18 @@ def get_all_by_user_id(user_id: str) -> List[CognitionTeamMember]: def create( team_id: str, user_id: str, + created_by: str, with_commit: bool = True, created_at: Optional[datetime] = None, ) -> CognitionTeamMember: - team: CognitionTeamMember = CognitionTeamMember( + team_member: CognitionTeamMember = CognitionTeamMember( created_by=user_id, team_id=team_id, + user_id=user_id, created_at=created_at, ) - general.add(team, with_commit) - return team + general.add(team_member, with_commit) + return team_member def delete(team_member_id: str, with_commit: bool = True) -> None: @@ -55,3 +57,13 @@ def delete(team_member_id: str, with_commit: bool = True) -> None: CognitionTeamMember.id == team_member_id, ).delete() general.flush_or_commit(with_commit) + + +def delete_by_team_and_user_id( + team_id: str, user_id: str, with_commit: bool = True +) -> None: + session.query(CognitionTeamMember).filter( + CognitionTeamMember.team_id == team_id, + CognitionTeamMember.user_id == user_id, + ).delete() + general.flush_or_commit(with_commit) From 07591d45e7690834219b51d77ed0ae438463575d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Fri, 29 Dec 2023 12:19:39 +0100 Subject: [PATCH 19/26] adds team --- cognition_objects/project.py | 4 +- cognition_objects/team_member.py | 11 ++++ cognition_objects/team_project_access.py | 69 ++++++++++++++++++++++++ enums.py | 1 + models.py | 30 +++++++++-- 5 files changed, 107 insertions(+), 8 deletions(-) create mode 100644 cognition_objects/team_project_access.py diff --git a/cognition_objects/project.py b/cognition_objects/project.py index 2cc25f2f..e76866cf 100644 --- a/cognition_objects/project.py +++ b/cognition_objects/project.py @@ -110,9 +110,7 @@ def update( if customer_color_primary is not None: project.customer_color_primary = customer_color_primary if customer_color_primary_only_accent is not None: - project.customer_color_primary_only_accent = ( - customer_color_primary_only_accent - ) + project.customer_color_primary_only_accent = customer_color_primary_only_accent if customer_color_secondary is not None: project.customer_color_secondary = customer_color_secondary if operator_routing_source_code is not None: diff --git a/cognition_objects/team_member.py b/cognition_objects/team_member.py index b451eb96..1e8631e8 100644 --- a/cognition_objects/team_member.py +++ b/cognition_objects/team_member.py @@ -35,6 +35,17 @@ def get_all_by_user_id(user_id: str) -> List[CognitionTeamMember]: ) +def get_by_team_and_user_id(team_id: str, user_id: str) -> CognitionTeamMember: + return ( + session.query(CognitionTeamMember) + .filter( + CognitionTeamMember.team_id == team_id, + CognitionTeamMember.user_id == user_id, + ) + .first() + ) + + def create( team_id: str, user_id: str, diff --git a/cognition_objects/team_project_access.py b/cognition_objects/team_project_access.py new file mode 100644 index 00000000..6db65ca9 --- /dev/null +++ b/cognition_objects/team_project_access.py @@ -0,0 +1,69 @@ +from typing import List, Optional, Dict +from datetime import datetime +from ..business_objects import general +from ..session import session +from ..models import ( + CognitionTeamProjectAccess, +) + + +def get(team_project_access_id: str) -> CognitionTeamProjectAccess: + return ( + session.query(CognitionTeamProjectAccess) + .filter( + CognitionTeamProjectAccess.id == team_project_access_id, + ) + .first() + ) + + +def get_all_by_team_id(team_id: str) -> List[CognitionTeamProjectAccess]: + return ( + session.query(CognitionTeamProjectAccess) + .filter(CognitionTeamProjectAccess.team_id == team_id) + .order_by(CognitionTeamProjectAccess.created_at.asc()) + .all() + ) + + +def get_all_by_project_id(project_id: str) -> List[CognitionTeamProjectAccess]: + return ( + session.query(CognitionTeamProjectAccess) + .filter(CognitionTeamProjectAccess.project_id == project_id) + .order_by(CognitionTeamProjectAccess.created_at.asc()) + .all() + ) + + +def create( + user_id: str, + team_id: str, + project_id: str, + with_commit: bool = True, + created_at: Optional[datetime] = None, +) -> CognitionTeamProjectAccess: + team_project_access: CognitionTeamProjectAccess = CognitionTeamProjectAccess( + created_by=user_id, + team_id=team_id, + project_id=project_id, + created_at=created_at, + ) + general.add(team_project_access, with_commit) + return team_project_access + + +def delete(team_project_access_id: str, with_commit: bool = True) -> None: + session.query(CognitionTeamProjectAccess).filter( + CognitionTeamProjectAccess.id == team_project_access_id, + ).delete() + general.flush_or_commit(with_commit) + + +def delete_by_team_and_project_id( + team_id: str, project_id: str, with_commit: bool = True +) -> None: + session.query(CognitionTeamProjectAccess).filter( + CognitionTeamProjectAccess.team_id == team_id, + CognitionTeamProjectAccess.project_id == project_id, + ).delete() + general.flush_or_commit(with_commit) diff --git a/enums.py b/enums.py index 1ca13679..79c11a40 100644 --- a/enums.py +++ b/enums.py @@ -133,6 +133,7 @@ class Tablenames(Enum): CONSUMPTION_LOG = "consumption_log" TEAM = "team" TEAM_MEMBER = "team_member" + TEAM_PROJECT_ACCESS = "team_project_access" def snake_case_to_pascal_case(self): # the type name of a table is needed to create backrefs diff --git a/models.py b/models.py index fc79a6c1..5d30b9bd 100644 --- a/models.py +++ b/models.py @@ -1041,11 +1041,6 @@ class CognitionProject(Base): ForeignKey(f"{Tablenames.ORGANIZATION.value}.id", ondelete="CASCADE"), index=True, ) - team_id = Column( - UUID(as_uuid=True), - ForeignKey(f"{Tablenames.TEAM.value}.id", ondelete="CASCADE"), - index=True, - ) refinery_references_project_id = Column( UUID(as_uuid=True), ForeignKey(f"{Tablenames.PROJECT.value}.id", ondelete="SET NULL"), @@ -1443,3 +1438,28 @@ class CognitionTeamMember(Base): ForeignKey(f"{Tablenames.USER.value}.id", ondelete="SET NULL"), index=True, ) + + +class CognitionTeamProjectAccess(Base): + __tablename__ = Tablenames.TEAM_PROJECT_ACCESS.value + __table_args__ = {"schema": "cognition"} + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + + team_id = Column( + UUID(as_uuid=True), + ForeignKey(f"cognition.{Tablenames.TEAM.value}.id", ondelete="CASCADE"), + index=True, + ) + + project_id = Column( + UUID(as_uuid=True), + ForeignKey(f"cognition.{Tablenames.PROJECT.value}.id", ondelete="CASCADE"), + index=True, + ) + + created_at = Column(DateTime, default=sql.func.now()) + created_by = Column( + UUID(as_uuid=True), + ForeignKey(f"{Tablenames.USER.value}.id", ondelete="SET NULL"), + index=True, + ) From 488efe60720b3facc3d72eb8b6531d7730b1830a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Fri, 29 Dec 2023 19:03:03 +0100 Subject: [PATCH 20/26] markdown dataset sync refinery --- business_objects/project.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/business_objects/project.py b/business_objects/project.py index 49e1b1f4..b19fb6b9 100644 --- a/business_objects/project.py +++ b/business_objects/project.py @@ -212,7 +212,9 @@ def get_zero_shot_project_config(project_id: str, payload_id: str) -> Any: def get_or_create_queue_project( - org_id: str, user_id: str, with_commit: bool = False + org_id: str, + user_id: str, + with_commit: bool = False, ) -> Project: ## user_id is a "last used by" indicator @@ -244,12 +246,37 @@ def get_or_create_queue_project( return prj +def create_dataset_project( + org_id: str, + user_id: str, + name: str, + description: str, + tokenizer: str, + with_commit: bool = False, +) -> Project: + ## user_id is a "last used by" indicator + + prj = create( + organization_id=org_id, + name=name, + description=description, + created_by=user_id, + tokenizer=tokenizer, + tokenizer_blank=tokenizer[:2], + with_commit=with_commit, + ) + + return prj + + def create( organization_id: str, name: str, description: str, created_by: str, created_at: Optional[str] = None, + tokenizer: Optional[str] = None, + tokenizer_blank: Optional[str] = None, with_commit: bool = False, status: enums.ProjectStatus = enums.ProjectStatus.INIT_UPLOAD, ) -> Project: @@ -260,6 +287,8 @@ def create( created_by=created_by, created_at=created_at, status=status.value, + tokenizer=tokenizer, + tokenizer_blank=tokenizer_blank, ) general.add(project, with_commit) return project From b3f6d20122dc01cd0293dcf8c34430ae4df1580b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Sat, 30 Dec 2023 13:21:51 +0100 Subject: [PATCH 21/26] initial synopsis --- cognition_objects/project.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cognition_objects/project.py b/cognition_objects/project.py index e76866cf..efb9215e 100644 --- a/cognition_objects/project.py +++ b/cognition_objects/project.py @@ -44,6 +44,7 @@ def create( color: str, org_id: str, user_id: str, + interface_type: str, refinery_references_project_id: str, refinery_queries_project_id: str, refinery_relevances_project_id: str, @@ -79,6 +80,7 @@ def check_execute( organization_id=org_id, created_by=user_id, created_at=created_at, + interface_type=interface_type, refinery_references_project_id=refinery_references_project_id, refinery_question_project_id=refinery_queries_project_id, refinery_relevance_project_id=refinery_relevances_project_id, From 414a9ded6daf2799340d357bd1f760325e2e2695 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Sun, 31 Dec 2023 00:12:01 +0100 Subject: [PATCH 22/26] early prototype synopsis --- cognition_objects/conversation.py | 15 ++++++ cognition_objects/markdown_dataset.py | 16 +++--- cognition_objects/spreadsheet.py | 74 +++++++++++++++++++++++++++ cognition_objects/spreadsheet_row.py | 54 +++++++++++++++++++ enums.py | 2 + models.py | 51 ++++++++++++++++++ 6 files changed, 206 insertions(+), 6 deletions(-) create mode 100644 cognition_objects/spreadsheet.py create mode 100644 cognition_objects/spreadsheet_row.py diff --git a/cognition_objects/conversation.py b/cognition_objects/conversation.py index 6cb4e413..7ea0aa53 100644 --- a/cognition_objects/conversation.py +++ b/cognition_objects/conversation.py @@ -19,6 +19,19 @@ def get(project_id: str, conversation_id: str) -> CognitionConversation: ) +def get_all_by_spreadsheet_row_id( + spreadsheet_row_id: str, +) -> List[CognitionConversation]: + return ( + session.query(CognitionConversation) + .filter( + CognitionConversation.synopsis_spreadsheet_row_id == spreadsheet_row_id, + ) + .order_by(CognitionConversation.created_at.asc()) + .all() + ) + + def get_all_paginated_by_project_id( project_id: str, page: int, limit: int ) -> Tuple[int, int, List[CognitionConversation]]: @@ -56,6 +69,7 @@ def create( project_id: str, user_id: str, with_commit: bool = True, + spreadsheet_row_id: Optional[str] = None, created_at: Optional[datetime] = None, ) -> CognitionConversation: conversation: CognitionConversation = CognitionConversation( @@ -63,6 +77,7 @@ def create( created_by=user_id, created_at=created_at, scope_dict={}, + synopsis_spreadsheet_row_id=spreadsheet_row_id, ) general.add(conversation, with_commit) return conversation diff --git a/cognition_objects/markdown_dataset.py b/cognition_objects/markdown_dataset.py index 64468490..00e5ab9b 100644 --- a/cognition_objects/markdown_dataset.py +++ b/cognition_objects/markdown_dataset.py @@ -52,17 +52,21 @@ def get_enriched(org_id: str, id: str) -> Dict[str, Any]: def get_all_paginated_for_category_origin( org_id: str, - category_origin: str, page: int, limit: int, + category_origin: Optional[str] = None, ) -> Tuple[int, int, List[CognitionMarkdownDataset]]: - total_count = ( - session.query(CognitionMarkdownDataset.id) - .filter(CognitionMarkdownDataset.organization_id == org_id) - .filter(CognitionMarkdownDataset.category_origin == category_origin) - .count() + query = session.query(CognitionMarkdownDataset.id).filter( + CognitionMarkdownDataset.organization_id == org_id ) + if category_origin is not None: + query = query.filter( + CognitionMarkdownDataset.category_origin == category_origin + ) + + total_count = query.count() + num_pages = int(total_count / limit) if total_count % limit > 0: num_pages += 1 diff --git a/cognition_objects/spreadsheet.py b/cognition_objects/spreadsheet.py new file mode 100644 index 00000000..10464439 --- /dev/null +++ b/cognition_objects/spreadsheet.py @@ -0,0 +1,74 @@ +from typing import Dict, List, Optional, Tuple, Any + +from datetime import datetime + +from ..business_objects import general +from ..session import session +from ..models import CognitionSynopsisSpreadsheet + + +def get(spreadsheet_id: str) -> CognitionSynopsisSpreadsheet: + return ( + session.query(CognitionSynopsisSpreadsheet) + .filter( + CognitionSynopsisSpreadsheet.id == spreadsheet_id, + ) + .first() + ) + + +def get_all_by_project_id( + project_id: str, +) -> List[CognitionSynopsisSpreadsheet]: + return ( + session.query(CognitionSynopsisSpreadsheet) + .filter( + CognitionSynopsisSpreadsheet.project_id == project_id, + ) + .order_by(CognitionSynopsisSpreadsheet.created_at.asc()) + .all() + ) + + +def create( + project_id: str, + user_id: str, + dataset_id: str, + name: str, + filter_name: str, + with_commit: bool = True, + created_at: Optional[datetime] = None, +) -> CognitionSynopsisSpreadsheet: + spreadsheet = CognitionSynopsisSpreadsheet( + project_id=project_id, + dataset_id=dataset_id, + name=name, + filter_attribute_name=filter_name, + created_at=created_at, + created_by=user_id, + ) + general.add(spreadsheet, with_commit) + return spreadsheet + + +def update( + spreadsheet_id: str, + name: Optional[str] = None, + filter_name: Optional[str] = None, + with_commit: bool = True, +) -> CognitionSynopsisSpreadsheet: + spreadsheet = get(spreadsheet_id) + if name is not None: + spreadsheet.name = name + if filter_name is not None: + spreadsheet.filter_name = filter_name + + general.flush_or_commit(with_commit) + return spreadsheet + + +def delete(spreadsheet_id: str, with_commit: bool = True) -> None: + session.query(CognitionSynopsisSpreadsheet).filter( + CognitionSynopsisSpreadsheet.id == spreadsheet_id, + ).delete() + general.flush_or_commit(with_commit) diff --git a/cognition_objects/spreadsheet_row.py b/cognition_objects/spreadsheet_row.py new file mode 100644 index 00000000..3480fa97 --- /dev/null +++ b/cognition_objects/spreadsheet_row.py @@ -0,0 +1,54 @@ +from typing import Dict, List, Optional, Tuple, Any + +from datetime import datetime + +from ..business_objects import general +from ..session import session +from ..models import CognitionSynopsisSpreadsheetRow + + +def get(spreadsheet_row_id: str) -> CognitionSynopsisSpreadsheetRow: + return ( + session.query(CognitionSynopsisSpreadsheetRow) + .filter( + CognitionSynopsisSpreadsheetRow.id == spreadsheet_row_id, + ) + .first() + ) + + +def get_all_by_spreadsheet_id( + spreadsheet_id: str, +) -> List[CognitionSynopsisSpreadsheetRow]: + return ( + session.query(CognitionSynopsisSpreadsheetRow) + .filter( + CognitionSynopsisSpreadsheetRow.spreadsheet_id == spreadsheet_id, + ) + .order_by(CognitionSynopsisSpreadsheetRow.created_at.asc()) + .all() + ) + + +def create( + spreadsheet_id: str, + project_id: str, + user_id: str, + with_commit: bool = True, + created_at: Optional[datetime] = None, +) -> CognitionSynopsisSpreadsheetRow: + spreadsheet_row = CognitionSynopsisSpreadsheetRow( + project_id=project_id, + spreadsheet_id=spreadsheet_id, + created_at=created_at, + created_by=user_id, + ) + general.add(spreadsheet_row, with_commit) + return spreadsheet_row + + +def delete(spreadsheet_row_id: str, with_commit: bool = True) -> None: + session.query(CognitionSynopsisSpreadsheetRow).filter( + CognitionSynopsisSpreadsheetRow.id == spreadsheet_row_id, + ).delete() + general.flush_or_commit(with_commit) diff --git a/enums.py b/enums.py index 79c11a40..5c383982 100644 --- a/enums.py +++ b/enums.py @@ -134,6 +134,8 @@ class Tablenames(Enum): TEAM = "team" TEAM_MEMBER = "team_member" TEAM_PROJECT_ACCESS = "team_project_access" + SYNOPSIS_SPREADSHEET = "synopsis_spreadsheet" + SYNOPSIS_SPREADSHEET_ROW = "synopsis_spreadsheet_row" def snake_case_to_pascal_case(self): # the type name of a table is needed to create backrefs diff --git a/models.py b/models.py index 5d30b9bd..cad929cc 100644 --- a/models.py +++ b/models.py @@ -1135,6 +1135,7 @@ class CognitionConversation(Base): ForeignKey(f"cognition.{Tablenames.PROJECT.value}.id", ondelete="CASCADE"), index=True, ) + synopsis_spreadsheet_row_id = String() created_by = Column( UUID(as_uuid=True), ForeignKey(f"{Tablenames.USER.value}.id", ondelete="SET NULL"), @@ -1463,3 +1464,53 @@ class CognitionTeamProjectAccess(Base): ForeignKey(f"{Tablenames.USER.value}.id", ondelete="SET NULL"), index=True, ) + + +class CognitionSynopsisSpreadsheet(Base): + __tablename__ = Tablenames.SYNOPSIS_SPREADSHEET.value + __table_args__ = {"schema": "cognition"} + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + dataset_id = Column( + UUID(as_uuid=True), + ForeignKey( + f"cognition.{Tablenames.MARKDOWN_DATASET.value}.id", ondelete="CASCADE" + ), + index=True, + ) + project_id = Column( + UUID(as_uuid=True), + ForeignKey(f"cognition.{Tablenames.PROJECT.value}.id", ondelete="CASCADE"), + index=True, + ) + name = Column(String) + filter_attribute_name = Column(String) + created_at = Column(DateTime, default=sql.func.now()) + created_by = Column( + UUID(as_uuid=True), + ForeignKey(f"{Tablenames.USER.value}.id", ondelete="SET NULL"), + index=True, + ) + + +class CognitionSynopsisSpreadsheetRow(Base): + __tablename__ = Tablenames.SYNOPSIS_SPREADSHEET_ROW.value + __table_args__ = {"schema": "cognition"} + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + project_id = Column( + UUID(as_uuid=True), + ForeignKey(f"cognition.{Tablenames.PROJECT.value}.id", ondelete="CASCADE"), + index=True, + ) + spreadsheet_id = Column( + UUID(as_uuid=True), + ForeignKey( + f"cognition.{Tablenames.SYNOPSIS_SPREADSHEET.value}.id", ondelete="CASCADE" + ), + index=True, + ) + created_at = Column(DateTime, default=sql.func.now()) + created_by = Column( + UUID(as_uuid=True), + ForeignKey(f"{Tablenames.USER.value}.id", ondelete="SET NULL"), + index=True, + ) From cb82aa3b1526eff0c7a27e5c8fa8c1f499653469 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Sun, 31 Dec 2023 11:12:44 +0100 Subject: [PATCH 23/26] early prototype synopsis --- cognition_objects/conversation.py | 2 ++ models.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/cognition_objects/conversation.py b/cognition_objects/conversation.py index 7ea0aa53..878f428d 100644 --- a/cognition_objects/conversation.py +++ b/cognition_objects/conversation.py @@ -69,6 +69,7 @@ def create( project_id: str, user_id: str, with_commit: bool = True, + synopsis_column: Optional[str] = None, spreadsheet_row_id: Optional[str] = None, created_at: Optional[datetime] = None, ) -> CognitionConversation: @@ -77,6 +78,7 @@ def create( created_by=user_id, created_at=created_at, scope_dict={}, + synopsis_column=synopsis_column, synopsis_spreadsheet_row_id=spreadsheet_row_id, ) general.add(conversation, with_commit) diff --git a/models.py b/models.py index cad929cc..89b10ec2 100644 --- a/models.py +++ b/models.py @@ -1135,7 +1135,6 @@ class CognitionConversation(Base): ForeignKey(f"cognition.{Tablenames.PROJECT.value}.id", ondelete="CASCADE"), index=True, ) - synopsis_spreadsheet_row_id = String() created_by = Column( UUID(as_uuid=True), ForeignKey(f"{Tablenames.USER.value}.id", ondelete="SET NULL"), @@ -1146,6 +1145,10 @@ class CognitionConversation(Base): header = Column(String) error = Column(String) + # synopsis-specific + synopsis_spreadsheet_row_id = Column(String) + synopsis_column = Column(String) + class CognitionMessage(Base): __tablename__ = Tablenames.MESSAGE.value From dae5acfe58721bfcd90fafa83e613c6238c3f693 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Sun, 31 Dec 2023 12:17:36 +0100 Subject: [PATCH 24/26] allow deleting questions --- cognition_objects/spreadsheet.py | 3 ++- cognition_objects/spreadsheet_row.py | 9 ++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/cognition_objects/spreadsheet.py b/cognition_objects/spreadsheet.py index 10464439..f71a4013 100644 --- a/cognition_objects/spreadsheet.py +++ b/cognition_objects/spreadsheet.py @@ -67,8 +67,9 @@ def update( return spreadsheet -def delete(spreadsheet_id: str, with_commit: bool = True) -> None: +def delete(project_id: str, spreadsheet_id: str, with_commit: bool = True) -> None: session.query(CognitionSynopsisSpreadsheet).filter( CognitionSynopsisSpreadsheet.id == spreadsheet_id, + CognitionSynopsisSpreadsheet.project_id == project_id, ).delete() general.flush_or_commit(with_commit) diff --git a/cognition_objects/spreadsheet_row.py b/cognition_objects/spreadsheet_row.py index 3480fa97..f868ca86 100644 --- a/cognition_objects/spreadsheet_row.py +++ b/cognition_objects/spreadsheet_row.py @@ -47,8 +47,15 @@ def create( return spreadsheet_row -def delete(spreadsheet_row_id: str, with_commit: bool = True) -> None: +def delete( + project_id: str, + spreadsheet_id: str, + spreadsheet_row_id: str, + with_commit: bool = True, +) -> None: session.query(CognitionSynopsisSpreadsheetRow).filter( CognitionSynopsisSpreadsheetRow.id == spreadsheet_row_id, + CognitionSynopsisSpreadsheetRow.spreadsheet_id == spreadsheet_id, + CognitionSynopsisSpreadsheetRow.project_id == project_id, ).delete() general.flush_or_commit(with_commit) From bced3406eb4aed7902a85a8a29f7b5552b24f664 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Sun, 31 Dec 2023 14:07:11 +0100 Subject: [PATCH 25/26] adds project id to get --- cognition_objects/spreadsheet.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cognition_objects/spreadsheet.py b/cognition_objects/spreadsheet.py index f71a4013..cf4befc4 100644 --- a/cognition_objects/spreadsheet.py +++ b/cognition_objects/spreadsheet.py @@ -7,11 +7,12 @@ from ..models import CognitionSynopsisSpreadsheet -def get(spreadsheet_id: str) -> CognitionSynopsisSpreadsheet: +def get(project_id: str, spreadsheet_id: str) -> CognitionSynopsisSpreadsheet: return ( session.query(CognitionSynopsisSpreadsheet) .filter( CognitionSynopsisSpreadsheet.id == spreadsheet_id, + CognitionSynopsisSpreadsheet.project_id == project_id, ) .first() ) @@ -52,12 +53,13 @@ def create( def update( + project_id: str, spreadsheet_id: str, name: Optional[str] = None, filter_name: Optional[str] = None, with_commit: bool = True, ) -> CognitionSynopsisSpreadsheet: - spreadsheet = get(spreadsheet_id) + spreadsheet = get(project_id, spreadsheet_id) if name is not None: spreadsheet.name = name if filter_name is not None: From 750d7337d63410e40a01e56cedfe71b3c6587b47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Sun, 31 Dec 2023 17:00:18 +0100 Subject: [PATCH 26/26] allow task definition in synopsis --- cognition_objects/spreadsheet.py | 8 ++++++-- models.py | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/cognition_objects/spreadsheet.py b/cognition_objects/spreadsheet.py index cf4befc4..ab6a4d27 100644 --- a/cognition_objects/spreadsheet.py +++ b/cognition_objects/spreadsheet.py @@ -34,9 +34,11 @@ def get_all_by_project_id( def create( project_id: str, user_id: str, - dataset_id: str, name: str, - filter_name: str, + synopsis_type: str, + dataset_id: Optional[str] = None, + filter_name: Optional[str] = None, + task_scope_dict: Optional[Dict[str, Any]] = None, with_commit: bool = True, created_at: Optional[datetime] = None, ) -> CognitionSynopsisSpreadsheet: @@ -45,6 +47,8 @@ def create( dataset_id=dataset_id, name=name, filter_attribute_name=filter_name, + synopsis_type=synopsis_type, + task_scope_dict=task_scope_dict, created_at=created_at, created_by=user_id, ) diff --git a/models.py b/models.py index 89b10ec2..41aad4d6 100644 --- a/models.py +++ b/models.py @@ -1486,6 +1486,8 @@ class CognitionSynopsisSpreadsheet(Base): index=True, ) name = Column(String) + synopsis_type = Column(String) + task_scope_dict = Column(JSON) filter_attribute_name = Column(String) created_at = Column(DateTime, default=sql.func.now()) created_by = Column(