Fix #420 - evaluate not working with chat models

nemoguardrails/eval/evaluate_factcheck.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import asyncio
 import json
 import os
 import time
@@ -23,6 +24,7 @@
 from langchain.prompts import PromptTemplate
 
 from nemoguardrails import LLMRails
+from nemoguardrails.actions.llm.utils import llm_call
 from nemoguardrails.eval.utils import load_dataset
 from nemoguardrails.llm.params import llm_params
 from nemoguardrails.llm.prompts import Task
@@ -141,10 +143,14 @@ def check_facts(self, split="positive"):
 
             start_time = time.time()
             fact_check_prompt = self.llm_task_manager.render_task_prompt(
-                Task.SELF_CHECK_FACTS, {"evidence": evidence, "response": answer}
+                Task.SELF_CHECK_FACTS,
+                {"evidence": evidence, "response": answer},
+                force_string_to_message=True,
             )
             stop = self.llm_task_manager.get_stop_tokens(Task.SELF_CHECK_FACTS)
-            fact_check = self.llm(fact_check_prompt, stop=stop)
+            fact_check = asyncio.run(
+                llm_call(prompt=fact_check_prompt, llm=self.llm, stop=stop)
+            )
             end_time = time.time()
             time.sleep(0.5)  # avoid rate-limits
             fact_check = fact_check.lower().strip()

nemoguardrails/eval/evaluate_moderation.py

@@ -13,12 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import asyncio
 import json
 import os
 
 import tqdm
 
 from nemoguardrails import LLMRails
+from nemoguardrails.actions.llm.utils import llm_call
 from nemoguardrails.eval.utils import load_dataset
 from nemoguardrails.llm.params import llm_params
 from nemoguardrails.llm.prompts import Task
@@ -88,15 +90,17 @@ def get_jailbreak_results(self, prompt, results):
             tuple: Jailbreak prediction, updated results dictionary.
         """
         check_input_prompt = self.llm_task_manager.render_task_prompt(
-            Task.SELF_CHECK_INPUT, {"user_input": prompt}
+            Task.SELF_CHECK_INPUT, {"user_input": prompt}, force_string_to_message=True
         )
         print(check_input_prompt)
         completed = False
         max_tries = 3
         num_tries = 0
         while not completed and num_tries < max_tries:
             try:
-                jailbreak = self.llm(check_input_prompt)
+                jailbreak = asyncio.run(
+                    llm_call(prompt=check_input_prompt, llm=self.llm)
+                )
                 jailbreak = jailbreak.lower().strip()
                 print(jailbreak)
 
@@ -133,13 +137,17 @@ def get_check_output_results(self, prompt, results):
 
         try:
             with llm_params(self.llm, temperature=0.1, max_tokens=100):
-                bot_response = self.llm(prompt)
+                bot_response = asyncio.run(llm_call(prompt=prompt, llm=self.llm))
 
             check_output_check_prompt = self.llm_task_manager.render_task_prompt(
-                Task.SELF_CHECK_OUTPUT, {"bot_response": bot_response}
+                Task.SELF_CHECK_OUTPUT,
+                {"bot_response": bot_response},
+                force_string_to_message=True,
             )
             print(check_output_check_prompt)
-            check_output = self.llm(check_output_check_prompt)
+            check_output = asyncio.run(
+                llm_call(prompt=check_output_check_prompt, llm=self.llm)
+            )
             check_output = check_output.lower().strip()
             print(check_output)
 

nemoguardrails/llm/taskmanager.py

@@ -197,14 +197,21 @@ def render_task_prompt(
         task: Union[str, Task],
         context: Optional[dict] = None,
         events: Optional[List[dict]] = None,
+        force_string_to_message: Optional[bool] = False,
     ) -> Union[str, List[dict]]:
         """Render the prompt for a specific task.
 
         :param task: The name of the task.
         :param context: The context for rendering the prompt
         :param events: The history of events so far.
+        :param force_string_to_message: Force the string message to a user message.
+        This should be used for chat models that receive a single message in the task prompt.
 
         :return: A string, for completion models, or an array of messages for chat models.
+
+        Note that even chat models can have task prompts defined using a string and not an array of messages.
+        In this case, the chat model will through an error. If you want to solve this problem, use the
+        force_string_to_message parameter to force the string message to a user message.
         """
         prompt = get_prompt(self.config, task)
         if prompt.content:
@@ -221,6 +228,16 @@ def render_task_prompt(
                 task_prompt = self._render_string(
                     prompt.content, context=context, events=events
                 )
+
+            # Check if the output should be a user message, for chat models
+            if force_string_to_message:
+                return [
+                    {
+                        "type": "user",
+                        "content": task_prompt,
+                    }
+                ]
+
             return task_prompt
         else:
             task_messages = self._render_messages(