Apply temperature compatibility fix from PR #939 and format code

- Change temperature default from 1e-8 to 0.01 for better LLM compatibility
- Fix issues with Mistral7b and Mixtral8x22 hosted via Hugging Face TGI
- Apply code formatting fixes

Author: anistark

examples/ragas_examples/_version.py

@@ -0,0 +1,34 @@
+# file generated by setuptools-scm
+# don't change, don't track in version control
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple
+    from typing import Union
+
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
+else:
+    VERSION_TUPLE = object
+    COMMIT_ID = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
+
+__version__ = version = "0.3.3.dev51+g8e7ead097"
+__version_tuple__ = version_tuple = (0, 3, 3, "dev51", "g8e7ead097")
+
+__commit_id__ = commit_id = "g8e7ead097"

src/ragas/callbacks.py

@@ -135,7 +135,6 @@ def parse_run_traces(
     traces: t.Dict[str, ChainRun],
     parent_run_id: t.Optional[str] = None,
 ) -> t.List[t.Dict[str, t.Any]]:
-
     root_traces = [
         chain_trace
         for chain_trace in traces.values()

src/ragas/embeddings/base.py

@@ -96,7 +96,8 @@ def __get_pydantic_core_schema__(
         Define how Pydantic generates a schema for BaseRagasEmbeddings.
         """
         return core_schema.no_info_after_validator_function(
-            cls, core_schema.is_instance_schema(cls)  # The validator function
+            cls,
+            core_schema.is_instance_schema(cls),  # The validator function
         )
 
 
@@ -269,9 +270,9 @@ def embed_documents(self, texts: List[str]) -> List[List[float]]:
         from sentence_transformers.SentenceTransformer import SentenceTransformer
         from torch import Tensor
 
-        assert isinstance(
-            self.model, SentenceTransformer
-        ), "Model is not of the type Bi-encoder"
+        assert isinstance(self.model, SentenceTransformer), (
+            "Model is not of the type Bi-encoder"
+        )
         embeddings = self.model.encode(
             texts, normalize_embeddings=True, **self.encode_kwargs
         )
@@ -286,9 +287,9 @@ def predict(self, texts: List[List[str]]) -> List[List[float]]:
         from sentence_transformers.cross_encoder import CrossEncoder
         from torch import Tensor
 
-        assert isinstance(
-            self.model, CrossEncoder
-        ), "Model is not of the type CrossEncoder"
+        assert isinstance(self.model, CrossEncoder), (
+            "Model is not of the type CrossEncoder"
+        )
 
         predictions = self.model.predict(texts, **self.encode_kwargs)
 

src/ragas/integrations/langchain.py

@@ -48,14 +48,14 @@ def __init__(self, metric: Metric, **kwargs: t.Any):
             t.cast(MetricWithLLM, self.metric).llm = LangchainLLMWrapper(llm)
         if isinstance(self.metric, MetricWithEmbeddings):
             embeddings = get_or_init(kwargs, "embeddings", OpenAIEmbeddings)
-            t.cast(MetricWithEmbeddings, self.metric).embeddings = (
-                LangchainEmbeddingsWrapper(embeddings)
-            )
+            t.cast(
+                MetricWithEmbeddings, self.metric
+            ).embeddings = LangchainEmbeddingsWrapper(embeddings)
         self.metric.init(run_config)
 
-        assert isinstance(
-            self.metric, SingleTurnMetric
-        ), "Metric must be SingleTurnMetric"
+        assert isinstance(self.metric, SingleTurnMetric), (
+            "Metric must be SingleTurnMetric"
+        )
 
     @property
     def input_keys(self) -> list[str]:
@@ -87,9 +87,9 @@ def _call(
         _run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager()
         callbacks = _run_manager.get_child()
 
-        assert isinstance(
-            self.metric, SingleTurnMetric
-        ), "Metric must be SingleTurnMetric"
+        assert isinstance(self.metric, SingleTurnMetric), (
+            "Metric must be SingleTurnMetric"
+        )
         score = self.metric.single_turn_score(
             inputs,
             callbacks=callbacks,
@@ -119,9 +119,9 @@ async def _acall(
         _run_manager = run_manager or AsyncCallbackManagerForChainRun.get_noop_manager()
         # TODO: currently AsyncCallbacks are not supported in ragas
         _run_manager.get_child()
-        assert isinstance(
-            self.metric, SingleTurnMetric
-        ), "Metric must be SingleTurnMetric"
+        assert isinstance(self.metric, SingleTurnMetric), (
+            "Metric must be SingleTurnMetric"
+        )
         score = await self.metric.single_turn_ascore(
             inputs,
             callbacks=[],
@@ -160,9 +160,9 @@ def _validate_langsmith_eval(self, run: Run, example: t.Optional[Example]) -> No
                 "Expected 'question' and 'ground_truth' in example."
                 f"Got: {[k for k in example.inputs.keys()]}"
             )
-        assert (
-            run.outputs is not None
-        ), "the current run has no outputs. The chain should output 'answer' and 'contexts' keys."
+        assert run.outputs is not None, (
+            "the current run has no outputs. The chain should output 'answer' and 'contexts' keys."
+        )
         output_keys = get_required_columns_v1(self.metric)
         output_keys = [
             key for key in output_keys if key not in ["question", "ground_truth"]

src/ragas/integrations/langgraph.py

@@ -46,7 +46,6 @@ def _validate_string_content(message, message_type: str) -> str:
         return message.content
 
     def _extract_metadata(message) -> dict:
-
         return {k: v for k, v in message.__dict__.items() if k != "content"}
 
     if metadata:

src/ragas/llms/base.py

@@ -71,8 +71,7 @@ def generate_text(
         temperature: float = 0.01,
         stop: t.Optional[t.List[str]] = None,
         callbacks: Callbacks = None,
-    ) -> LLMResult:
-        ...
+    ) -> LLMResult: ...
 
     @abstractmethod
     async def agenerate_text(
@@ -82,8 +81,7 @@ async def agenerate_text(
         temperature: t.Optional[float] = 0.01,
         stop: t.Optional[t.List[str]] = None,
         callbacks: Callbacks = None,
-    ) -> LLMResult:
-        ...
+    ) -> LLMResult: ...
 
     async def generate(
         self,

src/ragas/losses.py

@@ -22,7 +22,8 @@ def __get_pydantic_core_schema__(
         Define how Pydantic generates a schema for BaseRagasEmbeddings.
         """
         return core_schema.no_info_after_validator_function(
-            cls, core_schema.is_instance_schema(cls)  # The validator function
+            cls,
+            core_schema.is_instance_schema(cls),  # The validator function
         )
 
 
@@ -34,7 +35,6 @@ class MSELoss(Loss):
     reduction: t.Literal["mean", "sum"] = "mean"
 
     def __call__(self, predicted: t.List[float], actual: t.List[float]) -> float:
-
         errors = [(p - a) ** 2 for p, a in zip(predicted, actual)]
         if self.reduction == "mean":
             return sum(errors) / len(errors)

src/ragas/metrics/_answer_relevance.py

@@ -95,9 +95,9 @@ class ResponseRelevancy(MetricWithLLM, MetricWithEmbeddings, SingleTurnMetric):
     strictness: int = 3
 
     def calculate_similarity(self, question: str, generated_questions: list[str]):
-        assert (
-            self.embeddings is not None
-        ), f"Error: '{self.name}' requires embeddings to be set."
+        assert self.embeddings is not None, (
+            f"Error: '{self.name}' requires embeddings to be set."
+        )
         question_vec = np.asarray(self.embeddings.embed_query(question)).reshape(1, -1)
         gen_question_vec = np.asarray(
             self.embeddings.embed_documents(generated_questions)

src/ragas/metrics/_answer_similarity.py

@@ -65,9 +65,9 @@ async def _single_turn_ascore(
         return await self._ascore(row, callbacks)
 
     async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
-        assert (
-            self.embeddings is not None
-        ), f"Error: '{self.name}' requires embeddings to be set."
+        assert self.embeddings is not None, (
+            f"Error: '{self.name}' requires embeddings to be set."
+        )
 
         ground_truth = t.cast(str, row["reference"])
         answer = t.cast(str, row["response"])

src/ragas/metrics/_bleu_score.py

@@ -32,7 +32,6 @@ def init(self, run_config: RunConfig):
     async def _single_turn_ascore(
         self, sample: SingleTurnSample, callbacks: Callbacks
     ) -> float:
-
         reference, response = sample.reference, sample.response
         assert isinstance(reference, str), "BleuScore expects a valid reference string"
         assert isinstance(response, str), "BleuScore expects a valid response string"