diff --git a/index.html b/index.html index 654a579..69eb215 100644 --- a/index.html +++ b/index.html @@ -9,6 +9,8 @@ + +
+
INPUT:
@@ -272,6 +378,12 @@
GPU Memory
+ + +
+
Conversation History
+
+
@@ -280,10 +392,80 @@ const chatContainer = document.getElementById('chat-container'); const userInput = document.getElementById('user-input'); const sendButton = document.getElementById('send-button'); + const chatTabs = document.getElementById('chat-tabs'); - let thinkingElement = null; - let thinkingDetails = null; - let thinkingStartTime = null; + let currentChatId = null; + let chats = {}; + + function createNewChat() { + const chatId = Date.now().toString(); + chats[chatId] = { + messages: [], + thinkingSections: [] + }; + addChatTab(chatId); + switchToChat(chatId); + saveChats(); + } + + function addChatTab(chatId) { + const tab = document.createElement('button'); + tab.classList.add('chat-tab'); + tab.textContent = `Chat ${Object.keys(chats).length}`; + tab.onclick = () => switchToChat(chatId); + + const closeButton = document.createElement('span'); + closeButton.classList.add('close-tab'); + closeButton.textContent = '×'; + closeButton.onclick = (e) => { + e.stopPropagation(); + closeChat(chatId); + }; + + tab.appendChild(closeButton); + chatTabs.insertBefore(tab, chatTabs.lastElementChild); + } + + function switchToChat(chatId) { + currentChatId = chatId; + document.querySelectorAll('.chat-tab').forEach(tab => tab.classList.remove('active')); + document.querySelector(`.chat-tab:nth-child(${Object.keys(chats).indexOf(chatId) + 1})`).classList.add('active'); + renderChat(chatId); + } + + function closeChat(chatId) { + delete chats[chatId]; + saveChats(); + const tabToRemove = Array.from(chatTabs.children).find(tab => tab.textContent.includes(`Chat ${Object.keys(chats).indexOf(chatId) + 1}`)); + if (tabToRemove) { + chatTabs.removeChild(tabToRemove); + } + if (currentChatId === chatId) { + const remainingChatIds = Object.keys(chats); + if (remainingChatIds.length > 0) { + switchToChat(remainingChatIds[0]); + } else { + createNewChat(); + } + } + } + + function renderChat(chatId) { + chatContainer.innerHTML = ''; + const chat = chats[chatId]; + chat.messages.forEach(message => addMessage(message.content, message.isUser)); + chat.thinkingSections.forEach(section => { + const thinkingSection = createThinkingSection(); + section.thoughts.forEach(thought => addThought(thought.type, thought.content, thought.details, thinkingSection)); + }); + } + + function createThinkingSection() { + const section = document.createElement('div'); + section.classList.add('thinking-section'); + chatContainer.appendChild(section); + return section; + } function addMessage(message, isUser) { const messageElement = document.createElement('div'); @@ -292,65 +474,40 @@ messageElement.innerHTML = isUser ? message : marked.parse(message); chatContainer.appendChild(messageElement); chatContainer.scrollTop = chatContainer.scrollHeight; + + if (currentChatId) { + chats[currentChatId].messages.push({ content: message, isUser: isUser }); + saveChats(); + } } - function startThinking() { - thinkingElement = document.createElement('div'); - thinkingElement.classList.add('thought-summary', 'collapsible'); + function addThought(type, content, details = '', thinkingSection) { + const stepElement = document.createElement('div'); + stepElement.classList.add('thought-summary', 'collapsible', type); + stepElement.textContent = type.charAt(0).toUpperCase() + type.slice(1).replace('_', ' ') + ':'; + stepElement.onclick = toggleStepDetails; + + const stepDetails = document.createElement('div'); + stepDetails.classList.add('thought-details'); - const led = document.createElement('div'); - led.classList.add('led', 'blinking'); - - const textNode = document.createTextNode('Thinking...'); - - thinkingElement.appendChild(led); - thinkingElement.appendChild(textNode); - thinkingElement.onclick = toggleThinkingDetails; - - thinkingDetails = document.createElement('div'); - thinkingDetails.classList.add('thought-details'); - - chatContainer.appendChild(thinkingElement); - chatContainer.appendChild(thinkingDetails); - - thinkingStartTime = Date.now(); + if (type === 'error') { + stepElement.classList.add('error-message'); + if (content.includes('retrying')) { + stepElement.classList.add('retrying'); + } + stepDetails.innerHTML = marked.parse(content + '\n\nDetails:\n```\n' + details + '\n```'); + } else { + stepDetails.innerHTML = marked.parse(content); + } + + thinkingSection.appendChild(stepElement); + thinkingSection.appendChild(stepDetails); chatContainer.scrollTop = chatContainer.scrollHeight; - } - function addThought(step, content) { - if (thinkingDetails) { - const stepElement = document.createElement('div'); - stepElement.classList.add('thought-summary', 'collapsible'); - stepElement.textContent = step; - stepElement.onclick = toggleStepDetails; - - const stepDetails = document.createElement('div'); - stepDetails.classList.add('thought-details'); - stepDetails.innerHTML = content; - - thinkingDetails.appendChild(stepElement); - thinkingDetails.appendChild(stepDetails); - chatContainer.scrollTop = chatContainer.scrollHeight; - } - } - - function endThinking(thinkingTime) { - if (thinkingElement) { - const textNode = thinkingElement.childNodes[1]; - textNode.nodeValue = `Thinking... (${thinkingTime}s)`; - const led = thinkingElement.querySelector('.led'); - led.classList.remove('blinking'); - led.style.backgroundColor = '#0f0'; - led.style.boxShadow = '0 0 10px #0f0'; - thinkingStartTime = null; - } - } - - function toggleThinkingDetails() { - this.classList.toggle('open'); - const details = this.nextElementSibling; - if (details) { - details.style.display = details.style.display === 'none' ? 'block' : 'none'; + if (currentChatId) { + const currentThinkingSection = chats[currentChatId].thinkingSections[chats[currentChatId].thinkingSections.length - 1]; + currentThinkingSection.thoughts.push({ type, content, details }); + saveChats(); } } @@ -362,34 +519,71 @@ } } - socket.on('thinking', (data) => { - if (!thinkingElement) startThinking(); - addThought(data.step, 'Started'); - }); + function saveChats() { + localStorage.setItem('chats', JSON.stringify(chats)); + } - socket.on('thought', (data) => { - addThought('Result', data.content); - }); - - socket.on('chat_response', (data) => { - endThinking(data.thinking_time); - addMessage(data.response, false); - }); - - socket.on('error', (data) => { - endThinking(data.thinking_time); - addMessage(`Error: ${data.message}`, false); - }); + function loadChats() { + const storedChats = localStorage.getItem('chats'); + if (storedChats) { + chats = JSON.parse(storedChats); + Object.keys(chats).forEach(chatId => addChatTab(chatId)); + if (Object.keys(chats).length > 0) { + switchToChat(Object.keys(chats)[0]); + } else { + createNewChat(); + } + } else { + createNewChat(); + } + } function sendMessage() { const message = userInput.value.trim(); - if (message) { + if (message && currentChatId) { addMessage(message, true); - socket.emit('chat_request', { message: message }); + chats[currentChatId].thinkingSections.push({ thoughts: [] }); + socket.emit('chat_request', { + message: message, + conversation_history: chats[currentChatId].messages.filter(m => !m.isUser).map(m => ({ role: 'assistant', content: m.content })) + .concat(chats[currentChatId].messages.filter(m => m.isUser).map(m => ({ role: 'user', content: m.content }))) + }); userInput.value = ''; } } + socket.on('thinking', (data) => { + if (currentChatId) { + const newThinkingSection = createThinkingSection(); + chats[currentChatId].thinkingSections.push({ thoughts: [] }); + addThought(data.step, 'Started', '', newThinkingSection); + } + }); + + socket.on('thought', (data) => { + if (currentChatId) { + const currentThinkingSection = chatContainer.querySelector('.thinking-section:last-child'); + addThought(data.type, data.content, data.details, currentThinkingSection); + } + }); + + socket.on('chat_response', (data) => { + if (currentChatId) { + addMessage(data.response, false); + } + }); + + socket.on('error', (data) => { + if (currentChatId) { + const currentThinkingSection = chatContainer.querySelector('.thinking-section:last-child'); + if (data.type === 'retrying') { + addThought('error', data.content, '', currentThinkingSection); + } else { + addThought('error', data.message, '', currentThinkingSection); + } + } + }); + sendButton.addEventListener('click', sendMessage); userInput.addEventListener('keypress', function(e) { if (e.key === 'Enter' && !e.shiftKey) { @@ -398,6 +592,16 @@ } }); + // Add new chat button + const newChatButton = document.createElement('button'); + newChatButton.id = 'new-chat-button'; + newChatButton.textContent = '+ New Chat'; + newChatButton.onclick = createNewChat; + chatTabs.appendChild(newChatButton); + + // Load chats when the page loads + loadChats(); + const chartOptions = { type: 'line', options: { @@ -570,6 +774,41 @@ window.addEventListener('resize', checkWindowSize); checkWindowSize(); // Initial check + + // Add this new function to update the conversation history + function updateConversationHistory(history) { + const conversationHistoryElement = document.getElementById('conversation-history'); + conversationHistoryElement.innerHTML = ''; + + history.forEach(item => { + const card = document.createElement('div'); + card.classList.add('history-card'); + + const role = document.createElement('div'); + role.classList.add('history-role'); + role.textContent = item.role.charAt(0).toUpperCase() + item.role.slice(1); + + const content = document.createElement('pre'); + content.classList.add('history-content'); + content.innerHTML = hljs.highlightAuto(item.content).value; + + card.appendChild(role); + card.appendChild(content); + conversationHistoryElement.appendChild(card); + }); + } + + // Add this new socket listener + socket.on('conversation_history', (data) => { + updateConversationHistory(data.history); + }); + + // Add event listener for the clear history button + clearHistoryButton.addEventListener('click', () => { + if (confirm('Are you sure you want to clear the conversation history?')) { + clearConversationHistory(); + } + }); \ No newline at end of file diff --git a/main.py b/main.py index 176661e..47ceccf 100644 --- a/main.py +++ b/main.py @@ -1,4 +1,4 @@ -from flask import Flask, send_from_directory +from flask import Flask, send_from_directory, request from flask_socketio import SocketIO, emit from flask_openapi3 import OpenAPI, Info from pydantic import BaseModel @@ -10,15 +10,20 @@ import psutil import GPUtil import threading import os - - - +from tools import DefaultToolManager +import ollama +import re +import json +from datetime import datetime +import pprint logger = structlog.get_logger() openapi = OpenAPI(__name__, info=Info(title="LLM Chat Server", version="1.0.0")) app = openapi socketio = SocketIO(app, cors_allowed_origins="*") +tool_manager = DefaultToolManager() + @app.route('/') def index(): logger.info("Serving index.html") @@ -33,51 +38,13 @@ class ChatResponse(BaseModel): @socketio.on('chat_request') def handle_chat_request(data): user_input = data['message'] - logger.info("Received chat request", user_input=user_input) + conversation_history = data.get('conversation_history', []) + conversation_history = [{"role": "system", "content": ANSWER_QUESTION_PROMPT}] + conversation_history + logger.info("Received chat request", user_input=user_input, conversation_history=conversation_history) start_time = time.time() - full_context = "" try: - # Step 1: Generate a plan using the initial LLM - emit('thinking', {'step': 'Generating plan'}) - plan, plan_generation = generate_plan(user_input) - full_context += f"Plan Thinking:\n{plan_generation}" - full_context += f"Plan:\n{plan}" - emit('thought', {'content': f"Plan Thinking:\n{plan_generation}"}) - emit('thought', {'content': f"Plan:\n{plan}"}) - - if plan[0].strip().lower() == "direct_answer": - final_response = plan[1] - thinking_time = round(time.time() - start_time, 2) - emit('chat_response', { - 'response': final_response, - 'thinking_time': thinking_time - }) - return - - # Step 2: Execute each step of the plan - step_results = [] - for i, step in enumerate(plan): - emit('thinking', {'step': f'Executing step {i+1}'}) - while True: - best_model, model_selection = select_best_model(step, step_results, full_context) - if best_model in model_manager.model_capabilities: - break - logger.warning(f"Selected model {best_model} is not in the list of available models. Retrying...") - emit('thought', {'content': f"Selected model for step {i+1}:\n{model_selection}"}) - # summary, summary_generation = summarize_context(f"Plan: {plan}\n\nSteps: {step_results}") - # emit('thought', {'content': f"Context summary:\n{summary_generation}"}) - step_result, step_execution = execute_step(step, best_model, step_results, full_context) - emit('thought', {'content': f"Step {i+1} result:\n{step_execution}"}) - emit('thought', {'content': f"Result {i+1}:\n{step_result}"}) - step_results.append(step_result) - full_context += f"Step {i+1} result:\n{step_execution}" - - # Step 3: Generate final response - emit('thinking', {'step': 'Generating final response'}) - final_response, final_generation = generate_final_response(user_input, plan, step_results) - emit('thought', {'content': f"Final response generation:\n{final_generation}"}) - + final_response = answer_question_tools(user_input, conversation_history) end_time = time.time() thinking_time = round(end_time - start_time, 2) @@ -94,146 +61,98 @@ def handle_chat_request(data): 'thinking_time': thinking_time }) -PLAN_GENERATE_PROMPT = """ -You are building a "chain of thought" workflow for a series of LLMs to complete a task provided by a user. -Your first task is to "think" through the problem provided by the user. Probe what it would take to complete the task, see if there are hidden nuances, what constrains might be relevant, how to be efficient. -This thinking should set question the premise of the task, and sets the scene for a plan of attack to be created. -Verbalize your thoughts out loud, allow the user to see your thought process. This thought process will also be used as context for processing the generated plan. -This thought process should mimic the process of a human, and not be a simple list of steps, but should be a narrative of thought that a human would have. -Each step in the formulated plan is a step that a seperate LLM will complete. The LLM that will complete the step will be selected based on the scope of the step and the capabilities of the available models. -There are models that are good at coding and math, and there are models that are good at reasoning and planning. Some models that are generalists, multilingual, or conversational. And even some that are vision models. -Use this context of the possible models to shape each step such that a LLM can complete the step given the step and some context. -Steps should follow a logical "chain of thought" in order to best complete the overall task. -Steps should be self contained and be designed such that the results of one step can be passed on to the next step. -Steps should be phrased in such a way that it acts as a prompt or instruction to the LLM that will complete the step. -Each step will return a result, and a thought process. The thought process is extremely important, it is the "chain of thought" that the LLM went through to complete the step. This thought process is critical for the next step in the plan. -Consider how results from one step can be combined with results from another step and consider how the chain of thought from one step can inform the next step when designing each step. -Try and minimize the number of steps required to complete the task since running a lot of steps is expensive. -Your output should be your thought process, followed by a single line titled "STEPS", followed by each step to take, one step per line. -Do not add any sort of markdown formatting, code formatting, or any other formatting. -Do not add any preamble, postamble, or other text, only the thought process and the steps. +def answer_question_tools(user_input: str, conversation_history: List[dict], max_retries: int = 100): + global tool_manager + + # If conversation_history is empty, initialize it with the system prompt + if not conversation_history: + conversation_history = [ + {"role": "system", "content": ANSWER_QUESTION_PROMPT}, + ] + + logger.info("Starting chat", user_input=user_input, conversation_history=conversation_history) + # Add the new user input to the conversation history + conversation_history.append({"role": "user", "content": user_input}) + + emit('thinking', {'step': 'Starting'}) + emit('conversation_history', {'history': conversation_history}) -Consider the following example: + for iteration in range(max_retries): + response = ollama.chat(model=PRIMARY_MODEL, messages=conversation_history, tools=tool_manager.get_tools_for_ollama_dict(), stream=False) + assistant_message = response['message'] + + conversation_history.append(assistant_message) + emit('conversation_history', {'history': conversation_history}) + pprint.pp(assistant_message) -Prompt: Write a program to reverse a string, then output ASCII block art of that reversed string. Do this in python. + if 'tool_calls' in assistant_message: + emit('thought', {'type': 'decision', 'content': "Tool Call\n\n" + assistant_message['content']}) + for tool_call in assistant_message['tool_calls']: + tool_name = tool_call['function']['name'] + tool_args = tool_call['function']['arguments'] + emit('thought', {'type': 'tool_call', 'content': f"Tool: {tool_name}\nArguments: {tool_args}"}) + tool_response = tool_manager.get_tool(tool_name).execute(tool_args) + conversation_history.append({ + "role": "tool", + "content": tool_response + }) + emit('conversation_history', {'history': conversation_history}) + emit('thought', {'type': 'tool_result', 'content': tool_response}) -So there are two parts to this task. First, we need to reverse the input string. Then we need to print the ASCII block art for each character in the reversed string. -We should be able to reverse the string using either a simple loop, or a python slice. Slicing is simpler, so we should use that. -For the ASCII block art, the challenge is in creating a mapping between each character and its block art representation. There are a few ways to go about this: - - Find a library that converts text to block art - - Create our own mapping from characters to block art - - Create a procedurally generated mapping from characters to block art -Procedural generation could be done with an algorithm, but coming up with a good algorithm could be challenging. -Generating a dictionary could be a good approach, but there are 26 letters in the alphabet, and 10 digits, so we would need 36 different outputs for the block art. -We should search for a library that already does this, import it, and call it on the result of the string reversal. We would also need to tell the user to install the library. + reflection_prompt = "Reflect on the tool results. If there were any errors, propose multiple alternative approaches to solve the problem. If successful, consider if the result fully answers the user's query or if additional steps are needed." + conversation_history.append({ + "role": "assistant", + "content": reflection_prompt + }) + emit('conversation_history', {'history': conversation_history}) + else: + if "" in assistant_message['content'].lower(): + answer_content = re.search(r'(.*?)', assistant_message['content'], re.DOTALL) + if answer_content: + final_answer = answer_content.group(1).strip() + emit('thought', {'type': 'answer', 'content': final_answer}) + return final_answer + else: + emit('thought', {'type': 'decision', 'content': "Think/Plan/Decision/Action\n\n" + assistant_message['content']}) + reflection_prompt = "Your last response didn't provide a final answer. Please reflect on your current understanding of the problem and consider if you need to use any tools or if you can now provide a final answer. If you're ready to give a final answer, put your response in tags " + conversation_history.append({"role": "assistant", "content": reflection_prompt}) + emit('conversation_history', {'history': conversation_history}) -We're now ready to create our plan. + return f"Max iterations reached. Last response: {assistant_message['content']}" -STEPS -1. Write a function that takes a string and reverses it. -2. Write a function that takes a string and returns the ASCII block art for each character in the string, this must be done using a library. -3. Combine the two functions into a single program. +ANSWER_QUESTION_PROMPT = f""" +The current date is {datetime.now().strftime("%A, %B %d, %Y")}, your knowledge cutoff was December 2023. +You are Dewey, an AI assistant with access to external tools and the ability to think through complex problems. Your role is to assist users by leveraging tools when necessary, thinking deeply about problems, and providing accurate and helpful information, all with a cheerful, but witty personality. Here are the tools available to you: ---- +{tool_manager.get_tools_and_descriptions_for_prompt()} -Now you try. -""" -_REMINADER_PT =""" -Each task you create should be should be self contained and be designed such that the results of one step can be passed on to the next step. -Try and minimize the number of steps required to complete the task. -Output only a numbered list of steps, each step should be a seperate line. -Do not output any preamble or other text, only the list of steps. -If you think a task can be completed by a single step, then you can output a single step. -If you can directly answer the question, you must begin your response with a single line containing the text "DIRECT_ANSWER" and then provide the answer to the question on the next line. +When addressing a query, follow these steps: -Here are some samples: +1. Analyze: Thoroughly analyze the query and consider multiple approaches to solving it. -Input: Write a program to reverse a string, then output the ASCII art of that reversed string. Do this in python. -Steps: -1. Define a template for a program that prints the ASCII art of the reversed string. -2. Fill in the logic to reverse the string. -3. Fill in the logic to print the ASCII art of the reversed string. -4. Output the final program. +2. Plan: Develop a plan of action, considering whether you need to use any tools or if you can answer directly. -Input: What are the oceans of the world? -Steps: -1. Use the encyclopedia tool to get the page on the oceans of the world, parse, and output the results. +3. Execute: If you need to use a tool, call it as you would a function. If not, proceed with your reasoning. -Input: What is the perfect gas law? -Steps: -DIRECT_ANSWER -The perfect gas law is the equation of state of a hypothetical ideal gas. The formula is $$PV = nRT$$ where P is pressure, V is volume, n is the number of moles, R is the ideal gas constant, and T is temperature. +4. Reflect: After each step or tool use, reflect on the results: + - If successful, consider if the result fully answers the user's query or if additional steps are needed. + - If there were errors or the result is unsatisfactory, don't give up! Use Tree of Thoughts reasoning: + a) Generate multiple alternative approaches or modifications to your previous approach. + b) Briefly evaluate the potential of each alternative. + c) Choose the most promising alternative and execute it. + d) Repeat this process if needed, building upon your growing understanding of the problem. + e) You cannot return a final answer after an error using a tool, you must try again. + +5. Iterate: Continue this process of execution and reflection, exploring different branches of thought as needed. + +6. Conclude: When you believe you have a comprehensive answer to the user's query, provide your final answer. + +Always explain your thought process, including your reasoning for each decision and how you arrived at your conclusions. If you're providing a final answer, put your response in tags . + +Remember, complex problems often require multiple steps and iterations. Don't hesitate to break down the problem, use tools multiple times, or explore different approaches to arrive at the best solution. """ -def generate_plan(user_input: str) -> tuple[List[str], str]: - logger.debug("Generating plan", prompt=user_input, system=PLAN_GENERATE_PROMPT) - response = model_manager.generate_text("qwen2.5:7b", user_input, max_length=1024, system=PLAN_GENERATE_PROMPT) - plan = response.split("STEPS")[1].strip() - response_no_steps = response.split("STEPS")[0].strip() - return [step.strip() for step in plan.split("\n") if step.strip()], response_no_steps - - -SELECT_BEST_MODEL_PROMPT = f""" -You are a large language model whos job it is to evaluate a step that is part of a larger plan, and determine what LLM would be best suited to complete the step based on the capabilities of the LLM. - -The LLMs and their capabilities are as follows: -{"\n".join([f"{k}: {','.join(v)}" for k,v in model_manager.model_capabilities.items()])} - -You will be provided with the current step of execution, the results of the previous steps in order, and the current chain of thought so far. -If the chain of thought is too long, a summary of the current chain of thought will be provided. -Your job is to use all this information to determine which of the provided LLMs would be best suited to complete the provided step given the capabilities of the LLM. -Your response should be the full name of the LLM that should complete the step. -Reply with only one of the following values: \n{'\n'.join(list(model_manager.model_capabilities.keys()))} -""" - -def select_best_model(step: str, results: List[str], context: str) -> tuple[str, str]: - prompt = f"Current Step: {step}\n\nResults So Far: {results}\n\nCurrent Chain of Thought: {context}" - logger.debug("Selecting best model", prompt=prompt, system=SELECT_BEST_MODEL_PROMPT) - response = model_manager.generate_text("llama3.2:3b", prompt, max_length=50, system=SELECT_BEST_MODEL_PROMPT) - model_name = response.strip().lower() - return model_name, response - - -def summarize_context(context: str) -> tuple[str, str]: - prompt = f"Summarize the following context: {context}" - logger.debug("Summarizing context", prompt=prompt) - response = model_manager.generate_text("llama3.2:3b", prompt, max_length=300) - return response, response - -EXECUTE_STEP_PROMPT = """ -You are a large language model that has been selected to complete a step within a larger task. -You have been selected to complete this step due to your specific capabilities. -You will be provided with the job to do in this current step, the results of the previous steps in order, and the current chain of thought so far. -If the chain of thought is too long, a summary of the current chain of thought will be provided. -Your job is to use all this information to complete the step. -Your response should be in two parts. The first part should be your thought process in completing the step, how you went about solving the step, assumptions made, relation to previous steps, and challenges faced. -You must then output a line with the word "RESPONSE". -The second part should be the result of completing your step. -The second part should contain nothing except the result of completing your step. -Only complete your part of the step. Do not extrapolate beyond the bounds of the step. Do not trample on the results of previous steps. Build on the results of previous steps, and use them to inform your work. -Do not include any preamble or other text, only the result of completing your step. -Do not use any markdown formatting, code formatting, or any other formatting. -""" - -def execute_step(step: str, model: str, results: List[str], context: str) -> tuple[str, str]: - prompt = f"Current Step: {step}\n\nResults So Far: {results}\n\nCurrent Chain of Thought: {context}" - logger.debug("Executing step", step=step, model=model, prompt=prompt) - response = model_manager.generate_text(model, prompt, max_length=1024, system=EXECUTE_STEP_PROMPT) - response_step = response.split("RESPONSE")[1].strip() - response_thinking = response.split("RESPONSE")[0].strip() - return response_step, response_thinking - -def generate_final_response(user_input: str, plan: List[str], step_results: List[str]) -> tuple[str, str]: - prompt = f"Question: {user_input}\n\nPlan:\n" - for i, step in enumerate(plan): - prompt += f"{i+1}. {step}\n" - prompt += "\nResults:\n" - for i, result in enumerate(step_results): - prompt += f"Step {i+1} result: {result}\n" - prompt += "\nBased on the above information, provide a comprehensive answer to the original question." - logger.debug("Generating final response", prompt=prompt) - response = model_manager.generate_text("qwen2.5:7b", prompt, max_length=500) - return response, response +PRIMARY_MODEL = "llama3.1:8b" UPDATE_INTERVAL = 0.1 # 100ms, configurable @@ -284,4 +203,4 @@ def send_system_resources(): if __name__ == "__main__": logger.info("Starting LLM Chat Server") threading.Thread(target=send_system_resources, daemon=True).start() - socketio.run(app, debug=True, host="0.0.0.0", port=5000) \ No newline at end of file + socketio.run(app, debug=True, host="0.0.0.0", port=5001) \ No newline at end of file diff --git a/models.py b/models.py index bd43282..366b91f 100644 --- a/models.py +++ b/models.py @@ -6,9 +6,9 @@ logger = structlog.get_logger() class ModelManager: def __init__(self): self.model_capabilities = { - "qwen2.5:7b": ["general_knowledge", "structured_output", "multilingual", "instruction_following", "structured_data"], + "ajindal/llama3.1-storm:8b": ["general_knowledge", "reasoning", "tool_calling", "conversation", "multilingual", "instruction_following"], "llama3.1:8b": ["general_knowledge", "reasoning", "tool_calling", "conversation", "multilingual", "instruction_following"], - "qwen2.5-coder:7b": ["code_generation", "code_analysis", "instruction_following", "math_reasoning"], + "qwen2.5:7b": ["general_knowledge", "reasoning", "tool_calling", "conversation", "multilingual", "instruction_following"], "llama3.2:3b": ["summarization", "instruction_following", "tool_calling", "multilingual"], "llava:7b": ["visual_reasoning", "visual_conversation", "visual_tool_calling", "vision", "ocr", "multimodal"], } @@ -25,8 +25,7 @@ class ModelManager: logger.info("Selected best model", required_capability=required_capability, selected_model=selected_model) return selected_model - def generate_text(self, model_name, prompt, max_length=100, system="You are a helpful assistant."): - logger.debug("Generating text", model=model_name, prompt=prompt, max_length=max_length) + def generate_text(self, model_name, prompt, max_length=100, system="You are a helpful assistant.", tools=[]): # Check if model exists try: ollama.pull(model_name) @@ -38,7 +37,9 @@ class ModelManager: else: logger.exception("Error pulling model", model=model_name, error=str(e)) raise e - response = ollama.generate(model=model_name, prompt=prompt, system=system) + + + response = ollama.generate(model=model_name, prompt=prompt, system=system, tools=tools, max_tokens=max_length) logger.debug("Text generated", model=model_name, response=response['response']) return response['response'] diff --git a/tools.py b/tools.py new file mode 100644 index 0000000..5693504 --- /dev/null +++ b/tools.py @@ -0,0 +1,116 @@ +import duckduckgo_search +import requests +from readability.readability import Document +from markdownify import markdownify as md +import sys +import time +import io +import subprocess +class Tool: + def __init__(self, name: str, description: str, arguments: dict, returns: str): + self.name = name + self.description = description + self.arguments = arguments + self.returns = returns + + def execute(self, arguments: dict) -> str: + pass + + +class ToolManager: + def __init__(self): + self.tools = [] + + def add_tool(self, tool: Tool): + self.tools.append(tool) + + def get_tool(self, name: str) -> Tool: + for tool in self.tools: + if tool.name == name: + return tool + return None + + def get_tools_and_descriptions_for_prompt(self): + return "\n".join([f"{tool.name}: {tool.description}" for tool in self.tools]) + + def get_tools_for_ollama_dict(self): + return [{'type': 'function', 'function': {'name': tool.name, 'description': tool.description, 'parameters': tool.arguments}} for tool in self.tools] + + +class DefaultToolManager(ToolManager): + def __init__(self): + super().__init__() + self.add_tool(SearchTool()) + self.add_tool(GetReadablePageContentsTool()) + self.add_tool(CalculatorTool()) + self.add_tool(PythonCodeTool()) + + +class SearchTool(Tool): + def __init__(self): + super().__init__("search_web", "Search the internet for information", {'type': 'object', 'properties': {'query': {'type': 'string', 'description': 'The search query'}}}, "results:list[string]") + + def execute(self, arg: dict) -> str: + res = duckduckgo_search.DDGS().text(arg['query'], max_results=5) + return '\n\n'.join([f"{r['title']}\n{r['body']}\n{r['href']}" for r in res]) + + +def get_readable_page_contents(url: str) -> str: + try: + response = requests.get(url) + response.raise_for_status() + doc = Document(response.content) + content = doc.summary() + return md(content) + except Exception as e: + return f"Error fetching readable content: {str(e)}" + + + +class GetReadablePageContentsTool(Tool): + def __init__(self): + super().__init__("get_readable_page_contents", "Get the contents of a web page in a readable format", {'type': 'object', 'properties': {'url': {'type': 'string', 'description': 'The url of the web page'}}}, "contents:string") + + def execute(self, arg: dict) -> str: + return get_readable_page_contents(arg['url']) + + +class CalculatorTool(Tool): + def __init__(self): + super().__init__("calculator", "Perform a calculation", {'type': 'object', 'properties': {'expression': {'type': 'string', 'description': 'The mathematical expression to evaluate, should be a python mathematical expression'}}}, "result:string") + + def execute(self, arg: dict) -> str: + try: + return str(exec(arg["expression"])) + except Exception as e: + return f"Error executing code: {str(e)}" + + +class PythonCodeTool(Tool): + def __init__(self): + super().__init__("python_code", "Execute python code", {'type': 'object', 'properties': {'code': {'type': 'string', 'description': 'The python code to execute, should be a single line of valid python'}}}, "result:string") + + def execute(self, arg: dict) -> str: + try: + start_time = time.time() + process = subprocess.Popen(['python', '-c', arg['code']], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True) + stdout, stderr = process.communicate(timeout=10) # 10 second timeout + end_time = time.time() + execution_time = end_time - start_time + + result = { + 'stdout': stdout, + 'stderr': stderr, + 'return_value': process.returncode, + 'execution_time': execution_time + } + except subprocess.TimeoutExpired: + process.kill() + return "Error: Code execution timed out after 10 seconds" + except Exception as e: + return f"Error executing code: {str(e)}" + + return '\n'.join([f"{k}: {v}" for k, v in result.items()]) \ No newline at end of file