Merge remote-tracking branch 'upstream/v2' into PR/feat-1-tool

Author: wangliang181230

.github/workflows/build-and-push.yml

@@ -7,7 +7,7 @@ on:
     inputs:
       dockerImageTag:
         description: 'Image Tag'
-        default: 'v2.9.0-dev'
+        default: 'v2.10.0-dev'
         required: true
       dockerImageTagWithLatest:
         description: '是否发布latest tag（正式发版时选择，测试版本切勿选择）'

apps/application/chat_pipeline/step/chat_step/impl/base_chat_step.py

@@ -279,11 +279,10 @@ def _handle_mcp_request(self, mcp_source, mcp_servers, mcp_tool_ids, tool_ids,
                     continue
                 executor = ToolExecutor()
                 if tool.init_params is not None:
-                    params = json.loads(rsa_long_decrypt(tool.init_params))
                     tool_init_params = json.loads(rsa_long_decrypt(tool.init_params))
                 else:
-                    params = {}
-                tool_config = executor.get_tool_mcp_config(tool, params)
+                    tool_init_params = {i["field"]: i.get('default_value') for i in tool.init_field_list}
+                tool_config = executor.get_tool_mcp_config(tool, tool_init_params)
 
                 mcp_servers_config[str(tool.id)] = tool_config
 

apps/application/flow/step_node/ai_chat_step_node/i_chat_node.py

@@ -49,6 +49,10 @@ class ChatNodeSerializer(serializers.Serializer):
                                            label=_("Skill IDs"), )
     mcp_output_enable = serializers.BooleanField(required=False, default=True, label=_("Whether to enable MCP output"))
 
+    video_list = serializers.ListField(required=False, label=_("video"))
+
+    image_list = serializers.ListField(required=False, label=_("picture"))
+
 
 class IChatNode(INode):
     type = 'ai-chat-node'

apps/application/flow/step_node/ai_chat_step_node/impl/base_chat_node.py

@@ -6,10 +6,12 @@
     @date：2024/6/4 14:30
     @desc:
 """
+import base64
 import json
 import re
 import time
 from functools import reduce
+from imghdr import what
 from typing import List, Dict
 
 from django.db.models import QuerySet
@@ -25,6 +27,7 @@
 from common.utils.rsa_util import rsa_long_decrypt
 from common.utils.shared_resource_auth import filter_authorized_ids
 from common.utils.tool_code import ToolExecutor
+from knowledge.models import File
 from models_provider.models import Model
 from models_provider.tools import get_model_credential, get_model_instance_by_model_workspace_id
 from tools.models import Tool, ToolType
@@ -196,11 +199,11 @@ def execute(self, model_id, system, prompt, dialogue_number, history_chat_record
                                                    self.runtime_node_id)
         self.context['history_message'] = [{'content': message.content, 'role': message.type} for message in
                                            (history_message if history_message is not None else [])]
-        question = self.generate_prompt_question(prompt)
+        question = self.generate_prompt_question(prompt, chat_model)
         self.context['question'] = question.content
         system = self.workflow_manage.generate_prompt(system)
         self.context['system'] = system
-        message_list = self.generate_message_list(prompt, history_message)
+        message_list = self.generate_message_list(question, history_message)
         self.context['message_list'] = message_list
 
         # 过滤tool_id
@@ -274,11 +277,10 @@ def _handle_mcp_request(self, mcp_source, mcp_servers, mcp_tool_id, mcp_tool_ids
                     continue
                 executor = ToolExecutor()
                 if tool.init_params is not None:
-                    params = json.loads(rsa_long_decrypt(tool.init_params))
                     tool_init_params = json.loads(rsa_long_decrypt(tool.init_params))
                 else:
-                    params = {}
-                tool_config = executor.get_tool_mcp_config(tool, params)
+                    tool_init_params = {i["field"]: i.get('default_value') for i in tool.init_field_list}
+                tool_config = executor.get_tool_mcp_config(tool, tool_init_params)
 
                 mcp_servers_config[str(tool.id)] = tool_config
 
@@ -383,14 +385,76 @@ def get_history_message(history_chat_record, dialogue_number, dialogue_type, run
             range(start_index if start_index > 0 else 0, len(history_chat_record))], [])
         for message in history_message:
             if isinstance(message.content, str):
-                message.content = re.sub('<form_rander>[\d\D]*?<\/form_rander>', '', message.content)
+                message.content = re.sub(r'<form_rander>.*?<\/form_rander>', '', message.content, flags=re.DOTALL)
         return history_message
 
-    def generate_prompt_question(self, prompt):
-        return HumanMessage(self.workflow_manage.generate_prompt(prompt))
+    def generate_prompt_question(self, prompt, model):
+        image = self.get_image()
+        video = self.get_video()
+        videos = []
+        images = []
+        if image:
+            images = self._process_images(image)
+        if video:
+            videos = self._process_videos(video, model)
+        return HumanMessage(
+            content=[*videos, *images, {'type': 'text', 'text': self.workflow_manage.generate_prompt(prompt)}])
+
+    def get_image(self):
+        if 'image_list' in self.node_params_serializer.data:
+            image = self.workflow_manage.get_reference_field(self.node_params_serializer.data.get('image_list')[0],
+                                                             self.node_params_serializer.data.get('image_list')[1:])
+            return image
+        return None
+
+    def get_video(self):
+        if 'video_list' in self.node_params_serializer.data:
+            video = self.workflow_manage.get_reference_field(self.node_params_serializer.data.get('video_list')[0],
+                                                             self.node_params_serializer.data.get('video_list')[1:])
+            return video
+        return None
 
-    def generate_message_list(self, prompt: str, history_message):
-        return [*history_message, HumanMessage(self.workflow_manage.generate_prompt(prompt))]
+    def _process_videos(self, image, video_model):
+        videos = []
+        if isinstance(image, str) and image.startswith('http'):
+            videos.append({'type': 'video_url', 'video_url': {'url': image}})
+        elif image is not None and len(image) > 0:
+            for img in image:
+                if 'file_id' in img:
+                    file_id = img['file_id']
+                    file = QuerySet(File).filter(id=file_id).first()
+                    url = video_model.upload_file_and_get_url(file.get_bytes(), file.file_name)
+                    videos.append(
+                        {'type': 'video_url', 'video_url': {'url': url}})
+                elif 'url' in img and img['url'].startswith('http'):
+                    videos.append(
+                        {'type': 'video_url', 'video_url': {'url': img['url']}})
+        return videos
+
+    def _process_images(self, image):
+        """
+        处理图像数据，转换为模型可识别的格式
+        """
+        images = []
+        if isinstance(image, str) and image.startswith('http'):
+            images.append({'type': 'image_url', 'image_url': {'url': image}})
+        elif image is not None and len(image) > 0:
+            for img in image:
+                if 'file_id' in img:
+                    file_id = img['file_id']
+                    file = QuerySet(File).filter(id=file_id).first()
+                    image_bytes = file.get_bytes()
+                    base64_image = base64.b64encode(image_bytes).decode("utf-8")
+                    image_format = what(None, image_bytes)
+                    images.append(
+                        {'type': 'image_url', 'image_url': {'url': f'data:image/{image_format};base64,{base64_image}'}})
+                elif 'url' in img and img['url'].startswith('http'):
+                    images.append(
+                        {'type': 'image_url', 'image_url': {'url': img["url"]}})
+        return images
+
+    def generate_message_list(self, question, history_message):
+        return [*history_message, question]
 
     @staticmethod
     def reset_message_list(message_list: List[BaseMessage], answer_text):

apps/application/flow/step_node/application_node/impl/base_application_node.py

@@ -132,15 +132,14 @@ def reset_application_node_dict(application_node_dict, runtime_node_id, node_dat
             application_node = application_node_dict[key]
             if application_node.get('runtime_node_id') == runtime_node_id:
                 content: str = application_node.get('content')
-                match = re.search('<form_rander>.*?</form_rander>', content)
+                match = re.search(r'<form_rander>.*?<\/form_rander>', content, flags=re.DOTALL)
                 if match:
                     form_setting_str = match.group().replace('<form_rander>', '').replace('</form_rander>', '')
                     form_setting = json.loads(form_setting_str)
                     form_setting['is_submit'] = True
                     form_setting['form_data'] = node_data
                     value = f'<form_rander>{json.dumps(form_setting)}</form_rander>'
-                    res = re.sub('<form_rander>.*?</form_rander>',
-                                 '${value}', content)
+                    res = re.sub(r'<form_rander>.*?<\/form_rander>', '${value}', content, flags=re.DOTALL)
                     application_node['content'] = res.replace('${value}', value)
     except Exception as e:
         maxkb_logger.warning(f'reset_application_node_dict error: {e}', exc_info=True)

apps/application/flow/step_node/intent_node/impl/base_intent_node.py

@@ -133,7 +133,7 @@ def get_history_message(history_chat_record, dialogue_number):
 
         for message in history_message:
             if isinstance(message.content, str):
-                message.content = re.sub('<form_rander>[\d\D]*?<\/form_rander>', '', message.content)
+                message.content = re.sub(r'<form_rander>.*?<\/form_rander>', '', message.content, flags=re.DOTALL)
         return history_message
 
     def build_system_prompt(self) -> str:

apps/application/flow/step_node/loop_node/impl/base_loop_node.py

@@ -216,10 +216,10 @@ def loop(workflow_manage_new_instance, node: INode, generate_loop):
 def get_tokens(loop_node_data):
     message_tokens = 0
     answer_tokens = 0
-    for details in loop_node_data:
-        message_tokens += sum([row.get('message_tokens') for row in details.values() if
+    for details in (loop_node_data or {}):
+        message_tokens += sum([row.get('message_tokens') or 0 for row in details.values() if
                                'message_tokens' in row and row.get('message_tokens') is not None])
-        answer_tokens += sum([row.get('answer_tokens') for row in details.values() if
+        answer_tokens += sum([row.get('answer_tokens') or 0 for row in details.values() if
                               'answer_tokens' in row and row.get('answer_tokens') is not None])
     return {'message_tokens': message_tokens, 'answer_tokens': answer_tokens}
 

apps/application/flow/step_node/parameter_extraction_node/impl/base_parameter_extraction_node.py

@@ -62,24 +62,24 @@ def generate_content(input_variable, variable_list):
     return value
 
 
-def json_loads(response, expected_fields):
+def json_loads(response, variable_list):
     if not response or not isinstance(response, str):
-        return {field: None for field in expected_fields}
+        return generate_example(variable_list)
 
     cleaned = response.strip()
 
     extraction_strategies = [
         lambda: json.loads(cleaned),
         lambda: json.loads(re.search(r'```(?:json)?\s*(\{.*?\})\s*```', cleaned, re.DOTALL).group(1)),
-        lambda: json.loads(re.search(r'(\{[\s\S]*\})', cleaned).group(1)),
+        lambda: json.loads(re.search(r'(\{.*\})', cleaned, flags=re.DOTALL).group(1)),
     ]
     for strategy in extraction_strategies:
         try:
             result = strategy()
             return result
         except:
             continue
-    return generate_example(expected_fields)
+    return generate_example(variable_list)
 
 
 class BaseParameterExtractionNode(IParameterExtractionNode):

apps/application/flow/step_node/question_node/impl/base_question_node.py

@@ -130,7 +130,7 @@ def get_history_message(history_chat_record, dialogue_number):
             range(start_index if start_index > 0 else 0, len(history_chat_record))], [])
         for message in history_message:
             if isinstance(message.content, str):
-                message.content = re.sub('<form_rander>[\d\D]*?<\/form_rander>', '', message.content)
+                message.content = re.sub(r'<form_rander>.*?<\/form_rander>', '', message.content, flags=re.DOTALL)
         return history_message
 
     def generate_prompt_question(self, prompt):

apps/application/flow/step_node/tool_lib_node/impl/base_tool_lib_node.py

@@ -58,26 +58,24 @@ def get_field_value(debug_field_list, name, is_required):
 
 
 def valid_reference_value(_type, value, name):
-    try:
-        if _type == 'int':
-            instance_type = int | float
-        elif _type == 'boolean':
-            instance_type = bool
-        elif _type == 'float':
-            instance_type = float | int
-        elif _type == 'dict':
-            value = json.loads(value) if isinstance(value, str) else value
-            instance_type = dict
-        elif _type == 'array':
-            value = json.loads(value) if isinstance(value, str) else value
-            instance_type = list
-        elif _type == 'string':
-            instance_type = str
-        else:
-            raise Exception(_(
-                'Field: {name} Type: {_type} Value: {value} Unsupported types'
-            ).format(name=name, _type=_type, value=value))
-    except:
+    if _type == 'int':
+        instance_type = int | float
+    elif _type == 'boolean':
+        instance_type = bool
+    elif _type == 'float':
+        instance_type = float | int
+    elif _type == 'dict':
+        value = json.loads(value) if isinstance(value, str) else value
+        instance_type = dict
+    elif _type == 'array':
+        value = json.loads(value) if isinstance(value, str) else value
+        instance_type = list
+    elif _type == 'string':
+        instance_type = str
+    else:
+        maxkb_logger.error(_(
+            'Field: {name} Type: {_type} Value: {value} Unsupported this type'
+        ).format(name=name, _type=_type, value=value))
         return value
     if not isinstance(value, instance_type):
         raise Exception(_(