Add knowledge tests

config.yml

@@ -3,10 +3,10 @@ language: en
 pipeline:
 - name: LLMBasedRouter
   calm_entry:
-    sticky: "handles everything else, including finance and pizza orders"
+    sticky: "handles chitchat, everything else, including finance and pizza orders"
   nlu_entry:
-    sticky: "handles everything around health advice and hotel or restaurant bookings"
-    non_sticky: "handles chitchat"
+    sticky: "handles nothing"
+    non_sticky: "handles nothing either"
 - name: WhitespaceTokenizer
 - name: CountVectorsFeaturizer
 - name: CountVectorsFeaturizer
@@ -20,11 +20,6 @@ pipeline:
   random_state: 42
   ranking_length: 10
 - name: CRFEntityExtractor
-- name: DucklingEntityExtractor
-  url: "http://localhost:8000"
-  dimensions: [ "time", "number" ]
-  timezone: "Europe/Berlin"
-  timeout: 3
 - name: NLUCommandAdapter
 - name: SingleStepLLMCommandGenerator
   llm:
@@ -41,8 +36,8 @@ policies:
 - name: EnterpriseSearchPolicy
   #use_generative_llm: false
   #prompt: rag.jinja2
-  vector_store:
-    type: "addons.qdrant.Qdrant_Store"
+  # vector_store:
+  #   type: "addons.qdrant.Qdrant_Store"
   embeddings:
     type: "huggingface"
     model_name: "BAAI/bge-small-en-v1.5"

e2e_tests/knowledge/business-hours.yml

@@ -0,0 +1,8 @@
+test_cases:
+  - test_case: 3acd4e72521c43329accd5ee31b9c62b
+    steps:
+    - user: what are your business hours?
+    # it mentions "provided documents" but it shouldn't!
+    - utter: I'm sorry, but the provided documents do not contain information about the business hours of the FinX service. If you need assistance with a different topic, feel free to ask.
+    # - condition: utter matches not "(?i).*provided documents.*"
+    - utter: utter_can_do_something_else

e2e_tests/knowledge/change-password.yml

@@ -0,0 +1,8 @@
+test_cases:
+  - test_case: 8d54444606c245e0902425f9804c7438
+    steps:
+    - user: how do I change my password?
+    # knowledge answer
+    - utter: To change your password in FinX, go to your account settings within the app. Look for the option to change your password and follow the prompts to update it securely.
+    # - condition: utter matches "(?i).*password in FinX.*"
+    - utter: utter_can_do_something_else

e2e_tests/knowledge/contact-support.yml

@@ -0,0 +1,7 @@
+test_cases:
+  - test_case: 05f1fe4cb4ed4a4a86566e999b5b4a0f
+    steps:
+    - user: how do i contact customer support?
+    # fail, no self awareness 😆
+    - utter: You can contact FinX customer support through the chatbot within the FinX app. They have a dedicated team available to assist you with any questions or concerns.
+    - utter: utter_can_do_something_else

e2e_tests/knowledge/defective-item.yml

@@ -0,0 +1,7 @@
+test_cases:
+  - test_case: 1157a7effdec4883a6ea2a6471ea5b1a
+    steps:
+    - user: can i return a defective item
+    # terrible rag response
+    - utter: If you have received a defective item, please contact our support team immediately for assistance. We will do our best to help you resolve the situation promptly.
+    - utter: utter_can_do_something_else

e2e_tests/knowledge/exchange-item.yml

@@ -0,0 +1,6 @@
+test_cases:
+  - test_case: 7113513b98e94e9b9d1ed748832b7007
+    steps:
+    - user: can i exchange an item for a different size?
+    # fail, wrong command
+    - utter: utter_human_handoff_not_available

e2e_tests/knowledge/free-shipping.yml

@@ -0,0 +1,8 @@
+test_cases:
+  - test_case: 9baadf90d9704c25846cf959ad94b6b7
+    steps:
+    - user: do you offer free shipping
+    # fail, it mentions "provided documents" but it shouldn't!
+    - utter: Based on the information provided, FinX offers finPAY Cards with exclusive discounts and secure payments, but there is no mention of free shipping. For more details on any potential shipping fees, it is recommended to contact FinX customer support through the chatbot within the FinX app.
+    # - condition: utter matches not "(?i).*provided documents.*"
+    - utter: utter_can_do_something_else

e2e_tests/knowledge/gift-wrapping.yml

@@ -0,0 +1,8 @@
+test_cases:
+  - test_case: f028065cd38c4f2898a23cafddfa79f1
+    steps:
+    - user: do you offer gift wrapping services?
+    # it mentions "provided documents" but it shouldn't!
+    - utter: I'm sorry, but based on the provided documents and recent conversation, there is no information indicating that gift wrapping services are offered by FinX. If you have any other questions or need assistance with something else, feel free to ask!
+    # - condition: utter matches not "(?i).*provided documents.*"
+    - utter: utter_can_do_something_else

e2e_tests/knowledge/latest-iphone.yml

@@ -0,0 +1,8 @@
+test_cases:
+  - test_case: 1cc528b2755149e1ace525a1aefb7668
+    steps:
+    - user: what are the specifications of the latest iphone?
+    # it mentions "provided documents" but it shouldn't!
+    - utter: I do not have information on the specifications of the latest iPhone based on the provided documents and conversation context.
+    # - condition: utter matches not "(?i).*provided documents.*"
+    - utter: utter_can_do_something_else

e2e_tests/knowledge/nearest-branch.yml

@@ -0,0 +1,8 @@
+test_cases:
+  - test_case: 6250e74065584c15953d8b7baadd3d51
+    steps:
+    - user: where is your nearest branch location?
+    # it mentions "provided documents" but it shouldn't!
+    - utter: I'm sorry, but based on the provided documents and context, there is no information available about the nearest branch location. FinX operates primarily through its app and online services, offering convenience and accessibility without physical branch locations.
+    # - condition: utter matches not "(?i).*provided documents.*"
+    - utter: utter_can_do_something_else

e2e_tests/knowledge/ongoing-promotions.yml

@@ -0,0 +1,8 @@
+test_cases:
+  - test_case: aa1cf79e5bf3462296d75c6f1ceaf4b1
+    steps:
+    - user: do you have ongoing promotions or sales
+    # knowledge answer
+    - utter: Yes, we offer exclusive discounts and weekly offers with the finPAY Card. Enjoy savings and benefits at various merchants by using your card.
+    # - condition: utter matches "(?i).*download the app.*"
+    - utter: utter_can_do_something_else

e2e_tests/knowledge/open-bank-account.yml

@@ -0,0 +1,8 @@
+test_cases:
+  - test_case: 6a797d4fb40441fd989276050c83548d
+    steps:
+    - user: how do I open a bank account
+    # this is a knowledge answer
+    - utter: To open a bank account with FinX, download the app and follow the steps outlined for account verification. Once verified, you can enjoy higher transaction limits and all the benefits FinX has to offer.
+    # - condition: utter matches "(?i).*download the app.*"
+    - utter: utter_can_do_something_else

e2e_tests/knowledge/open-bank-account1.yml

@@ -0,0 +1,10 @@
+test_cases:
+  - test_case: 8ccdd391afbd423eb2507c3ff71911c2
+    steps:
+    - user: hey
+    - utter: utter_greet
+    - user: i'd like to open a bank account, tell me about it
+    # knowledge answer
+    - utter: To open a bank account with FinX, download the app and follow the account verification process to unlock higher transaction limits and enjoy secure payments with finPAY Cards. Your privacy and security are a top priority, and all data provided is handled with the utmost care according to strict privacy policies.
+    # - condition: utter matches "(?i).*download the app.*"
+    - utter: utter_can_do_something_else

e2e_tests/knowledge/open-bank-account2.yml

@@ -0,0 +1,13 @@
+test_cases:
+  - test_case: 0d71fbdc05e04a98bb60f7149aa88eb8
+    steps:
+    - user: hi, is this a bank app
+    - utter: utter_greet
+    - user: do you have accounts
+    - utter: Yes, this is a payment app called FinX that allows hassle-free transactions, secure payments, and easy management of finances. You can verify your account to unlock higher transaction limits and enjoy more flexibility with your funds.
+    - utter: utter_can_do_something_else
+    - user: can i open one?
+    # knowledge answer
+    - utter: To open a bank account with FinX, download the app and follow the account verification process to unlock higher transaction limits and enjoy secure payments with finPAY Cards. Your privacy and security are a top priority, and all data provided is handled with the utmost care according to strict privacy policies.
+    # - condition: utter matches "(?i).*download the app.*"
+    - utter: utter_can_do_something_else

e2e_tests/knowledge/open-bank-account3.yml

@@ -0,0 +1,8 @@
+test_cases:
+  - test_case: 987fb609e48a410ea30e48ad577a2130
+    steps:
+    - user: open account
+    # knowledge answer
+    - utter: To open a bank account with FinX, download the app and follow the account verification process to unlock higher transaction limits and enjoy secure payments with finPAY Cards. Your privacy and security are a top priority, and all data provided is handled with the utmost care according to strict privacy policies.
+    # - condition: utter matches "(?i).*download the app.*"
+    - utter: utter_can_do_something_else

e2e_tests/knowledge/order-status.yml

@@ -0,0 +1,6 @@
+test_cases:
+  - test_case: c6eac4cb2a554d889259cea01e0bda02
+    steps:
+    - user: can i check my order status online?
+    # this should have been a knowledge answer!
+    - utter: I'm sorry about that. Let's correct your order. What would you like to change?

e2e_tests/knowledge/payment-options.yml

@@ -0,0 +1,8 @@
+test_cases:
+  - test_case: b0126f2a6f7f46cfb6a0497eb8982507
+    steps:
+    - user: what payment methods do you accept?
+    # knowledge answer
+    - utter: We accept finPAY Cards, which offer secure transactions and exclusive discounts at participating merchants. You can easily request your finPAY Card through your FinX account without any additional costs.
+    # - condition: utter matches "(?i).*finPAY.*"
+    - utter: utter_can_do_something_else

e2e_tests/knowledge/recurring-payment.yml

@@ -0,0 +1,6 @@
+test_cases:
+  - test_case: da112465e5a24549ac2d676dbcddc350
+    steps:
+    - user: how do i set up recurring payment
+    # pass, shouldn't have used knowledge
+    - utter: utter_ask_recurrent_payment_type

e2e_tests/knowledge/refund-request.yml

@@ -0,0 +1,7 @@
+test_cases:
+  - test_case: f6af3ceecf93475c82ffaee74811d05c
+    steps:
+    - user: whats the status on my refund request??
+    # terrible response
+    - utter: If you have a refund request, please contact our support team immediately for assistance. They will be able to help you with the status of your refund and provide further guidance.
+    - utter: utter_can_do_something_else

e2e_tests/knowledge/return-policy.yml

@@ -0,0 +1,8 @@
+test_cases:
+  - test_case: 305b4fec6b7741149c48896e1c5f5f2a
+    steps:
+    - user: what is your return policy?
+    # it mentions "provided documents" but it shouldn't!
+    - utter: I do not know the return policy as it is not mentioned in the provided documents or the context of our conversation.
+    # - condition: utter matches not "(?i).*provided documents.*"
+    - utter: utter_can_do_something_else

e2e_tests/knowledge/student-discounts.yml

@@ -0,0 +1,6 @@
+test_cases:
+  - test_case: 49e221b4c7ed4a969c1c590f112ad628
+    steps:
+    - user: do you offer discounts for students?
+    - utter: Yes, we offer exclusive discounts and weekly offers with our finPAY Card. By becoming a member of the FinX community and ordering your finPAY Card, you can enjoy a range of benefits including discounts at various participating merchants.
+    - utter: utter_can_do_something_else

e2e_tests/knowledge/support-policy.yml

@@ -0,0 +1,10 @@
+test_cases:
+  - test_case: ff7b6b16576d49f58f08759ce40ae02d
+    steps:
+    - user: what is your refund policy?
+    # this test is completely messed up!
+    # wrong answer too IMO
+    - utter: If you are inquiring about our refund policy, please contact our support team for assistance. They will be able to provide you with the necessary information and guidance regarding refunds.
+    - utter: utter_can_do_something_else
+    - user: i bought something online
+    - utter: utter_greet

e2e_tests/knowledge/technical-support.yml

@@ -0,0 +1,8 @@
+test_cases:
+  - test_case: 46851cd0958d4f89b002056475700506
+    steps:
+    - user: do you provide technical support for your products?
+    # knowledge answer
+    - utter: Yes, we provide customer support for our products. You can reach out to our dedicated customer support team through the chatbot within the FinX app for assistance with any technical issues or questions you may have.
+    # - condition: utter matches "(?i).*password in FinX.*"
+    - utter: utter_can_do_something_else

e2e_tests/knowledge/terms-of-service.yml

@@ -0,0 +1,8 @@
+test_cases:
+  - test_case: 67cbc5caaaf043dcaab1d6a4337b7af5
+    steps:
+    - user: Are there any upcoming changes to your terms of service?
+    # it mentions "provided documents" but it shouldn't!
+    - utter: Based on the provided documents, there is no mention of upcoming changes to the terms of service for FinX. The focus is on security, convenience, and the services offered within the app.
+    # - condition: utter matches not "(?i).*provided documents.*"
+    - utter: utter_can_do_something_else

e2e_tests/knowledge/track-order.yml

@@ -0,0 +1,8 @@
+test_cases:
+  - test_case: b67f0a2e14bb4c7495753c92c76827fa
+    steps:
+    - user: how do i track my order
+    # knowledge answer
+    - utter: To track your order, you can log in to your FinX account and navigate to the "Cards" section. From there, you can follow the prompts to track the status of your finPAY Card. If you encounter any issues, you can reach out to FinX customer support for assistance through the chatbot within the app.
+    # - condition: utter matches "(?i).*track the status.*"
+    - utter: utter_can_do_something_else

e2e_tests/knowledge/tshirt-size.yml

@@ -0,0 +1,8 @@
+test_cases:
+  - test_case: 0901702cb1354dd8878d682cf83d56f8
+    steps:
+    - user: what are the sizes of your t shirts
+    # knowledge answer
+    - utter: I do not know the sizes of t-shirts available.
+    # - condition: utter matches "(?i).*do not know.*"
+    - utter: utter_can_do_something_else

e2e_tests/knowledge/update-account.yml

@@ -0,0 +1,7 @@
+test_cases:
+  - test_case: dd2784e82ea2428c876391b3971d4acf
+    steps:
+    - user: how can I update my account information?
+    # clarification triggered, Clarify(verify_account, replace_card)
+    # i think this should be a knowledge answer
+    - utter: utter_clarification_options_rasa

e2e_tests/knowledge/update-info.yml

@@ -0,0 +1,7 @@
+test_cases:
+  - test_case: 32f11189a8a44cbf9581edc3f559b730
+    steps:
+    - user: how do i update personal info
+    # knowledge answer but wrong
+    - utter: To update your personal information, you can verify your account with FinX. Follow the account verification process within the app to ensure your data is secure and to potentially access higher transaction limits.
+    - utter: utter_can_do_something_else

endpoints.yml

@@ -58,12 +58,12 @@ action_endpoint:
 nlg:
   type: rephrase
 
-#tracing:
-#  type: jaeger
-#  host: localhost
-#  port: 6831
-#  service_name: rasa
-#  sync_export: ~
+tracing:
+ type: jaeger
+ host: localhost
+ port: 6831
+ service_name: rasa
+ sync_export: ~
 
 vector_store:
   type: qdrant

evals/commands.csv

@@ -0,0 +1,51 @@
+Question,Commands
+How do I open a bank account?,SearchAndReply
+What's the weather like today?,ChitChat
+Can you tell me a joke?,ChitChat
+Book a restaurant,StartFlow
+Transfer money,StartFlow
+How do I change my password?,SearchAndReply
+What are your business hours?,SearchAndReply
+What is your return policy?,SearchAndReply
+Do you offer discounts for students?,SearchAndReply
+Where is the nearest branch location?,SearchAndReply
+Can I check my order status online?,SearchAndReply
+What payment methods do you accept?,SearchAndReply
+How do I track my order?,SearchAndReply
+What are the specifications of the latest iPhone model?,SearchAndReply
+Do you provide technical support for your products?,SearchAndReply
+What is your refund policy for digital purchases?,SearchAndReply
+How can I contact customer support?,SearchAndReply
+Do you offer free shipping on all orders?,SearchAndReply
+What is the status of my refund request?,SearchAndReply
+Can I return a defective item?,SearchAndReply
+Do you have any ongoing promotions or sales?,SearchAndReply
+How can I update my account information?,SearchAndReply
+What are the dimensions of your large-sized t-shirts?,SearchAndReply
+Do you offer gift wrapping services?,SearchAndReply
+Can I exchange an item for a different size?,SearchAndReply
+I want to transfer money to a friend.,StartFlow
+Book a table for two at a restaurant.,StartFlow
+Can I check my transaction history?,StartFlow
+How do I add a contact?,StartFlow
+I need to replace my lost credit card.,"StartFlow, SetSlot"
+What are the steps to set up a recurring payment?,SearchAndReply
+I'd like to check my account balance.,StartFlow
+Do you offer any health advice or recommendations?,StartFlow
+Can I get a list of my recent purchases?,StartFlow
+How can I update my personal information?,SearchAndReply
+"I want to make a reservation, but I'm not sure about the date.","StartFlow, SetSlot"
+Can I transfer money to myself?,"StartFlow, SetSlot"
+"I need help with my account, but I don't know where to start.",Clarify
+Can we move on to the next question?,SkipQuestion
+I want to cancel the current transaction.,CancelFlow
+Let's skip this step and come back to it later.,SkipQuestion
+I'm getting frustrated. Can I speak to a human?,HumanHandoff
+This is taking too long. Can I talk to a real person?,HumanHandoff
+I need more complex assistance than what the chatbot can provide.,HumanHandoff
+"I want to book a vacation package, including flights and accommodation.",Clarify
+Can I transfer money between different types of accounts?,StartFlow
+I need to update my contact information and change my password.,"StartFlow, StartFlow"
+Are there any upcoming changes to your terms of service?,SearchAndReply
+How do I provide feedback or report a bug?,ChitChat
+Can I get a summary of my recent activities with your service?,StartFlow