diff --git a/api/src/main/models/base.py b/api/src/main/models/base.py
index 61baa8d..358c990 100644
--- a/api/src/main/models/base.py
+++ b/api/src/main/models/base.py
@@ -7,6 +7,8 @@
from ..backend import BackendType
from ..utils import FunctionCalling, FunctionCallResult
+import json
+
@dataclass
class Tags:
@@ -60,12 +62,13 @@ def clean_up(self):
self.__class__.__instance = None
def parse_tool_calling(
- self,
- outputs,
- chat_history: ChatHistory,
- tools: List[Dict[str, str]],
- stream: bool = True,
- print_output: bool = False
+ self,
+ outputs,
+ chat_history: ChatHistory,
+ tools: List[Dict[str, str]],
+ stream: bool = True,
+ print_output: bool = False,
+ tool_call_caches: Optional[dict[str, str]] = None
) -> Union[Generator[str, None, None], str]:
""" Parse tool calling from the model's output """
result_obj = FunctionCallResult()
@@ -78,12 +81,13 @@ def parse_tool_calling(
if self.special_tags.TOOLCALL in word: # Start of a tool call
started = True
if buffer:
- buffer = 0
+ buffer = ""
elif self.special_tags.TOOLCALL_END in word: # End of a tool call
if buffer:
result_obj.stage(
buffer,
- (self.special_tags.TOOLCALL, self.special_tags.TOOLCALL_END)
+ (self.special_tags.TOOLCALL, self.special_tags.TOOLCALL_END),
+ tool_call_caches
)
state = result_obj.state
if state is not None:
@@ -119,8 +123,10 @@ def parse_tool_calling(
final_result = result_obj.finalize(
chat_history,
(self.special_tags.TOOLCALL, self.special_tags.TOOLCALL_END),
- print_output=print_output
+ print_output=print_output,
+ tool_call_caches=tool_call_caches
)
+
if queued > 0 and stat % 2 == 0:
print(f"\r{spinner[(stat//2) % len(spinner)]} Waiting for tool calls to finish...", end="", flush=True)
if final_result is False:
@@ -136,21 +142,22 @@ def parse_tool_calling(
return outputs + final_result
def chat(
- self,
- chat_history: ChatHistory,
- user_prompt: str,
- system_prompt: str = "",
- tools: Optional[List[Dict[str, str]]] = None,
- temperature: float = 0.2,
- top_p: float = 0.95,
- top_k: int = 40,
- min_p: float = 0.05,
- typical_p: float = 1.0,
- stream: bool = True,
- max_new_tokens: int = 1024,
- repeat_penalty: float = 1.0,
- print_output: bool = False,
- **kwargs
+ self,
+ chat_history: ChatHistory,
+ user_prompt: str,
+ system_prompt: str = "",
+ tools: Optional[List[Dict[str, str]]] = None,
+ temperature: float = 0.2,
+ top_p: float = 0.95,
+ top_k: int = 40,
+ min_p: float = 0.05,
+ typical_p: float = 1.0,
+ stream: bool = True,
+ max_new_tokens: int = 1024,
+ repeat_penalty: float = 1.0,
+ print_output: bool = False,
+ tool_call_caches: Optional[dict[str, str]] = None,
+ **kwargs
) -> Union[Generator[str, None, None], str]:
""" Process a chat request
@@ -229,7 +236,8 @@ def adaptive_special_tag_buffering(outs, wait_tokens_for=6):
chat_history=chat_history,
tools=tools,
stream=stream,
- print_output=print_output
+ print_output=print_output,
+ tool_call_caches= tool_call_caches
)
if stream:
diff --git a/api/src/main/models/llama3/model.py b/api/src/main/models/llama3/model.py
index cd9f618..1da961c 100644
--- a/api/src/main/models/llama3/model.py
+++ b/api/src/main/models/llama3/model.py
@@ -47,21 +47,22 @@ def _get_runtime(self, backend: BackendType | None = None):
)
def chat(
- self,
- chat_history: ChatHistory,
- user_prompt: str,
- system_prompt: str = system_prompt,
- tools: Optional[List[Dict[str, str]]] = None,
- temperature: float = 0.2,
- top_p: float = 0.95,
- top_k: int = 40,
- min_p: float = 0.05,
- typical_p: float = 1.0,
- stream: bool = True,
- max_new_tokens: int = 0,
- repeat_penalty: float = 1.0,
- print_output: bool = False,
- **kwargs
+ self,
+ chat_history: ChatHistory,
+ user_prompt: str,
+ system_prompt: str = system_prompt,
+ tools: Optional[List[Dict[str, str]]] = None,
+ temperature: float = 0.2,
+ top_p: float = 0.95,
+ top_k: int = 40,
+ min_p: float = 0.05,
+ typical_p: float = 1.0,
+ stream: bool = True,
+ max_new_tokens: int = 0,
+ repeat_penalty: float = 1.0,
+ print_output: bool = False,
+ tool_call_caches: Optional[dict[str, str]] = None,
+ **kwargs
) -> Union[Generator[str, None, None], str]:
return super().chat(
chat_history=chat_history,
@@ -77,5 +78,6 @@ def chat(
max_new_tokens=max_new_tokens,
repeat_penalty=repeat_penalty,
print_output=print_output,
+ tool_call_caches=tool_call_caches,
**kwargs
)
diff --git a/api/src/main/models/qwen3/model.py b/api/src/main/models/qwen3/model.py
index bd085cc..be73b52 100644
--- a/api/src/main/models/qwen3/model.py
+++ b/api/src/main/models/qwen3/model.py
@@ -37,6 +37,14 @@
- When interpreting relative time expressions, always use calendar week boundaries (Monday-Sunday), not rolling periods from today
- Always prioritize accuracy over speed
+**[Cache-First Principle]**
+The primary goal of tool use is to maximize efficiency by actively utilizing cached data. Before invoking a new tool, your thought process **MUST** follow the steps below.
+1. **Relevance Check:** First, check if there is a previous tool call in the conversation history that could be helpful in answering the user's current query. (e.g., same tool, similar parameters)
+2. **Retrieve Cache Data:** If a relevant record exists, this is the **mandatory first step**. You must immediately call `get_cache_data()` with the `cache_id` to retrieve the full data.
+3. **Data Analysis:** Analyze the cached data you have retrieved. Is it **sufficient to fully answer** the user's current question?
+4. **Utilize Cache:** If the data is sufficient, use the cached results to generate your response. **Calling the original tool again is absolutely forbidden.**
+5. **New Tool Call:** You should only proceed with a new tool call if there is no relevant cache, or if the cached content is irrelevant or insufficient for the question.
+
Remember: Your role is to be a reliable, knowledgeable professional assistant who thinks carefully before responding and actively seeks current information when needed."""
print("INFO: Use default system prompt -", system_prompt)
@@ -88,6 +96,7 @@ def chat(
max_new_tokens: int = 0,
repeat_penalty: float = 1.0,
print_output: bool = False,
+ tool_call_caches: Optional[dict[str, str]] = None,
**kwargs
) -> Union[Generator[str, None, None], str]:
return super().chat(
@@ -108,5 +117,6 @@ def chat(
max_new_tokens=max_new_tokens,
repeat_penalty=repeat_penalty,
print_output=print_output,
+ tool_call_caches=tool_call_caches,
**kwargs
)
diff --git a/api/src/main/server.py b/api/src/main/server.py
index 2490b5e..f90e0dd 100644
--- a/api/src/main/server.py
+++ b/api/src/main/server.py
@@ -94,7 +94,11 @@ async def chat(request: Request, user_prompt: str, history: Optional[List[Messag
if not session_id:
raise HTTPException(status_code=400, detail="Session ID is required.")
- model = Session(session_id=session_id).model
+ session = Session(session_id=session_id)
+
+ model = session.model
+ tool_call_caches = session.tool_call_caches
+
except ValueError:
traceback.print_exc()
raise HTTPException(status_code=404, detail="The session is not found.")
@@ -103,7 +107,7 @@ async def chat(request: Request, user_prompt: str, history: Optional[List[Messag
if history:
chat_history.extend([h.model_dump() for h in history])
- response = model.chat(chat_history, user_prompt, stream=False, print_output=True)
+ response = model.chat(chat_history, user_prompt, tool_call_caches=tool_call_caches, stream=False, print_output=True)
del model
return response
@@ -115,7 +119,13 @@ async def chat_with_streaming(websocket: WebSocket):
try:
session_id = json.loads(await websocket.receive_text()).get("session_id")
- model = Session(session_id=session_id).model
+ session = Session(session_id=session_id)
+ model = session.model
+
+ tool_call_caches = session.tool_call_caches
+
+ print(f"Cache ID in WebSocket: {id(session.tool_call_caches)}")
+
except Exception:
traceback.print_exc()
await websocket.close(code=1008, reason="Invalid session ID or model not found.")
@@ -125,7 +135,7 @@ async def chat_with_streaming(websocket: WebSocket):
chat_history.extend(json.loads(await websocket.receive_text()))
user_prompt = await websocket.receive_text()
- for token in model.chat(chat_history, user_prompt, print_output=True):
+ for token in model.chat(chat_history, user_prompt, tool_call_caches=tool_call_caches, print_output=True):
await websocket.send_text(token)
await asyncio.sleep(0.0001) # 0.1ms delay between tokens
diff --git a/api/src/main/settings.py b/api/src/main/settings.py
index 6b2e0c3..fca9090 100644
--- a/api/src/main/settings.py
+++ b/api/src/main/settings.py
@@ -52,6 +52,7 @@ def __init__(self, model_id: str = None, session_id: str = None):
self.__sessions[self.session_id] = self
print("INFO: Current sessions:", list(self.__sessions))
self._model = None
+ self.tool_call_caches = {}
@property
def model(self):
diff --git a/api/src/main/utils/__init__.py b/api/src/main/utils/__init__.py
index 3d44e5e..8b3d060 100644
--- a/api/src/main/utils/__init__.py
+++ b/api/src/main/utils/__init__.py
@@ -12,6 +12,7 @@
from . import currency
from . import calculator
from . import web_search
+from . import cache
#from . import embedding
@@ -97,11 +98,12 @@ def state(self):
return "\n" + "\n".join(queue)
def finalize(
- self,
- history_list: list,
- tag: tuple[str, str] = ("", ""),
- print_output: bool = False
- ) -> Union[str, False]:
+ self,
+ history_list: list,
+ tag: tuple[str, str] = ("", ""),
+ print_output: bool = False,
+ tool_call_caches: dict[str, dict] = None
+ ) -> Union[str, bool]:
# Check if there are any pending tool calls
with self.__queue_mutex:
if len(self.job_list) != self.__completed_jobs:
@@ -117,12 +119,18 @@ def finalize(
# Dump client-side tool call history
state = ""
+ caches : dict[str, str] = {}
if self.__message_queue:
state = "\n" + "\n".join(self.__message_queue)
for data in self[1:]:
if 'tool_call_id' in data:
+ caches[data['tool_call_id']] = data['content']
data['content'] = f""
- result = state + "\n" + tag[0] + "\n" + dumps(dict(history=self, ensure_ascii=False)) + "\n" + tag[1]
+
+ if tool_call_caches is not None and caches:
+ tool_call_caches.update(caches)
+
+ result = state + "\n" + tag[0] + "\n" + dumps(dict(history=self), ensure_ascii=False) + "\n" + tag[1]
# Clear the job list and message queue
self.job_list = []
@@ -131,7 +139,7 @@ def finalize(
self.__message_queue = []
return result
- def stage(self, calling: str, tag: tuple[str, str] = ("", "")):
+ def stage(self, calling: str, tag: tuple[str, str] = ("", ""), tool_call_caches: dict[str, str] = None):
with self.__queue_mutex:
job_id = datetime.now().strftime("call_%Y%m%d%H%M%S")
try:
@@ -148,20 +156,26 @@ def stage(self, calling: str, tag: tuple[str, str] = ("", "", "")
+ self,
+ job_id: str,
+ name: str,
+ arguments: dict,
+ tag: tuple[str, str] = ("", ""),
+ tool_call_caches: dict[str, str] = None
):
# Execute the function
try:
if name not in self.implementations:
raise ValueError(f"Function '{name}' is not registered.")
- result = self.implementations[name](**arguments)
+
+ if name in ["get_cache_data"]:
+ result = self.implementations[name](**arguments, tool_call_caches=tool_call_caches)
+ else:
+ result = self.implementations[name](**arguments)
+
except Exception as e:
result = str(e)
@@ -375,6 +389,20 @@ def do(
},
"required": ["url"]
}
+ ),
+ FunctionSchema(
+ name="get_cache_data",
+ description="Get results for previous tool_call",
+ parameters={
+ "type": "object",
+ "properties": {
+ "tool_call_cache_id": {
+ "type": "string",
+ "description": "The id of the previous tool_call to retrieve from cache"
+ }
+ },
+ "required": ["tool_call_cache_id"]
+ }
)
],
@@ -387,6 +415,7 @@ def do(
calculate=calculator.calculate,
search_web=web_search.search_web,
search_website=web_search.search_website,
- fetch_webpage=web_search.fetch_webpage
+ fetch_webpage=web_search.fetch_webpage,
+ get_cache_data=cache.get_cache_data
)
)
diff --git a/api/src/main/utils/cache.py b/api/src/main/utils/cache.py
new file mode 100644
index 0000000..122d510
--- /dev/null
+++ b/api/src/main/utils/cache.py
@@ -0,0 +1,12 @@
+import json
+import threading
+import datetime
+from typing import Union
+
+
+def get_cache_data(tool_call_cache_id: str, tool_call_caches: dict[str, str]) -> str: # 함수명 오타 수정
+ """ Get a specific cache data by name """
+ if tool_call_caches and tool_call_cache_id in tool_call_caches:
+ return json.dumps(tool_call_caches[tool_call_cache_id], default=str, ensure_ascii=False)
+ else:
+ return f"Cache '{tool_call_cache_id}' not found"
\ No newline at end of file