Improved skill library, async usage, claude support, assistant profile

Author: KillianLucas

interpreter/core/async_core.py

@@ -103,7 +103,7 @@ async def output(self):
         return await self.output_queue.async_q.get()
 
     def respond(self, run_code=None):
-        for i in range(5):  # 5 attempts
+        for attempt in range(5):  # 5 attempts
             try:
                 if run_code == None:
                     run_code = self.auto_run
@@ -157,10 +157,24 @@ def respond(self, run_code=None):
                 if not sent_chunks:
                     print("ERROR. NO CHUNKS SENT. TRYING AGAIN.")
                     print("Messages:", self.messages)
+                    messages = [
+                        "Hello? Answer please.",
+                        "Just say something, anything.",
+                        "Are you there?",
+                        "Can you respond?",
+                        "Please reply.",
+                    ]
+                    self.messages.append(
+                        {
+                            "role": "user",
+                            "type": "message",
+                            "content": messages[attempt % len(messages)],
+                        }
+                    )
                     time.sleep(1)
                 else:
                     self.output_queue.sync_q.put(complete_message)
-                    if self.print or self.debug:
+                    if self.debug:
                         print("\nServer response complete.\n")
                     return
 
@@ -702,29 +716,51 @@ class ChatCompletionRequest(BaseModel):
         stream: Optional[bool] = False
 
     async def openai_compatible_generator():
-        for i, chunk in enumerate(async_interpreter._respond_and_store()):
-            output_content = None
-
-            if chunk["type"] == "message" and "content" in chunk:
-                output_content = chunk["content"]
-            if chunk["type"] == "code" and "start" in chunk:
-                output_content = " "
-
-            if output_content:
-                await asyncio.sleep(0)
-                output_chunk = {
-                    "id": i,
-                    "object": "chat.completion.chunk",
-                    "created": time.time(),
-                    "model": "open-interpreter",
-                    "choices": [{"delta": {"content": output_content}}],
-                }
-                yield f"data: {json.dumps(output_chunk)}\n\n"
+        made_chunk = False
+
+        for message in [
+            ".",
+            "Just say something, anything.",
+            "Hello? Answer please.",
+            "Are you there?",
+            "Can you respond?",
+            "Please reply.",
+        ]:
+            for i, chunk in enumerate(
+                async_interpreter.chat(message=message, stream=True, display=True)
+            ):
+                made_chunk = True
+
+                if async_interpreter.stop_event.is_set():
+                    break
+
+                output_content = None
+
+                if chunk["type"] == "message" and "content" in chunk:
+                    output_content = chunk["content"]
+                if chunk["type"] == "code" and "start" in chunk:
+                    output_content = " "
+
+                if output_content:
+                    await asyncio.sleep(0)
+                    output_chunk = {
+                        "id": i,
+                        "object": "chat.completion.chunk",
+                        "created": time.time(),
+                        "model": "open-interpreter",
+                        "choices": [{"delta": {"content": output_content}}],
+                    }
+                    yield f"data: {json.dumps(output_chunk)}\n\n"
+
+            if made_chunk:
+                break
 
     @router.post("/openai/chat/completions")
     async def chat_completion(request: ChatCompletionRequest):
         # Convert to LMC
 
+        async_interpreter.stop_event.set()
+
         last_message = request.messages[-1]
 
         if last_message.role != "user":
@@ -739,20 +775,22 @@ async def chat_completion(request: ChatCompletionRequest):
                 {
                     "role": "user",
                     "type": "message",
-                    "content": str(last_message.content),
+                    "content": last_message.content,
                 }
             )
-        if type(last_message.content) == list:
+            print(">", last_message.content)
+        elif type(last_message.content) == list:
             for content in last_message.content:
                 if content["type"] == "text":
                     async_interpreter.messages.append(
                         {"role": "user", "type": "message", "content": str(content)}
                     )
+                    print(">", content)
                 elif content["type"] == "image_url":
                     if "url" not in content["image_url"]:
                         raise Exception("`url` must be in `image_url`.")
                     url = content["image_url"]["url"]
-                    print(url[:100])
+                    print("> [user sent an image]", url[:100])
                     if "base64," not in url:
                         raise Exception(
                             '''Image must be in the format: "data:image/jpeg;base64,{base64_image}"'''
@@ -778,12 +816,14 @@ async def chat_completion(request: ChatCompletionRequest):
                 # Remove that {START} message that would have just been added
                 async_interpreter.messages = async_interpreter.messages[:-1]
 
+        async_interpreter.stop_event.clear()
+
         if request.stream:
             return StreamingResponse(
                 openai_compatible_generator(), media_type="application/x-ndjson"
             )
         else:
-            messages = async_interpreter.chat(message="", stream=False, display=True)
+            messages = async_interpreter.chat(message=".", stream=False, display=True)
             content = messages[-1]["content"]
             return {
                 "id": "200",

interpreter/core/computer/skills/skills.py

@@ -1,15 +1,23 @@
 import glob
 import inspect
+import json
 import os
 import re
+import subprocess
 from pathlib import Path
 
 from ....terminal_interface.utils.oi_dir import oi_dir
 from ...utils.lazy_import import lazy_import
 from ..utils.recipient_utils import format_to_recipient
 
-# Lazy import of aifs, imported when needed to speed up start time
+# Lazy import, imported when needed to speed up start time
 aifs = lazy_import("aifs")
+pyautogui = lazy_import("pyautogui")
+pynput = lazy_import("pynput")
+
+element = None
+element_box = None
+icon_dimensions = None
 
 
 class Skills:
@@ -19,6 +27,18 @@ def __init__(self, computer):
         self.new_skill = NewSkill()
         self.new_skill.path = self.path
 
+    def list(self):
+        return [
+            file.replace(".py", "()")
+            for file in os.listdir(self.path)
+            if file.endswith(".py")
+        ]
+
+    def run(self, skill):
+        print(
+            "To run a skill, run its name as a function name (it is already imported)."
+        )
+
     def search(self, query):
         return aifs.search(query, self.path, python_docstrings_only=True)
 
@@ -56,6 +76,7 @@ def import_skills(self):
                     code_to_run = f.read() + "\n"
 
                 if self.computer.interpreter.debug:
+                    print(self.path)
                     print("IMPORTING SKILL:\n", code_to_run)
 
                 output = self.computer.run("python", code_to_run)
@@ -77,18 +98,11 @@ def create(self):
         self._name = "Untitled"
         print(
             """
-@@@SEND_MESSAGE_AS_USER@@@
+
 INSTRUCTIONS
 You are creating a new skill. Follow these steps exactly to get me to tell you its name:
 1. Ask me what the name of this skill is.
-2. After I explicitly tell you the name of the skill (I may tell you to proceed which is not the name— if I do say that, you probably need more information from me, so tell me that), after you get the proper name, write the following (including the markdown code block):
-
----
-Got it. Give me one second.
-```python
-computer.skills.new_skill.name = "{INSERT THE SKILL NAME FROM QUESTION #1^}"`.
-```
----
+2. After I explicitly tell you the name of the skill (I may tell you to proceed which is not the name— if I do say that, you probably need more information from me, so tell me that), after you get the proper name, execute `computer.skills.new_skill.name = "{INSERT THE SKILL NAME FROM QUESTION #1}"`.
         
         """.strip()
         )
@@ -102,11 +116,11 @@ def name(self, value):
         self._name = value
         print(
             """
-@@@SEND_MESSAGE_AS_USER@@@
+
 Skill named. Now, follow these next INSTRUCTIONS exactly:
 
 1. Ask me what the first step is.
-2. When I reply, execute code to accomplish that step.
+2. When I reply, execute code to accomplish that step. Write comments explaining your reasoning before each line.
 3. Ask me if you completed the step correctly.
     a. (!!!!!!!!!!!! >>>>>> THIS IS CRITICAL. DO NOT FORGET THIS.) IF you completed it correctly, run `computer.skills.new_skill.add_step(step, code)` where step is a generalized, natural language description of the step, and code is the code you ran to complete it.
     b. IF you did not complete it correctly, try to fix your code and ask me again.
@@ -121,7 +135,7 @@ def add_step(self, step, code):
         self.steps.append(step + "\n\n```python\n" + code + "\n```")
         print(
             """
-@@@SEND_MESSAGE_AS_USER@@@
+
 Step added. Now, follow these next INSTRUCTIONS exactly:
 
 1. Ask me what the next step is.
@@ -138,29 +152,39 @@ def add_step(self, step, code):
 
     def save(self):
         normalized_name = re.sub("[^0-9a-zA-Z]+", "_", self.name.lower())
-        steps_string = "\n".join(
-            [f"Step {i+1}:\n{step}\n" for i, step in enumerate(self.steps)]
-        )
-        steps_string = steps_string.replace('"""', "'''")
+
         skill_string = f'''
-        
-def {normalized_name}():
+import json
+
+def {normalized_name}(step=0):
     """
-    {normalized_name}
+    Run this function to {normalized_name}. Pass in step=0 to see the first step, step=1 to see the next step, etc.
     """
-
-    print("To complete this task / run this skill, flexibly follow the following tutorial, swapping out parts as necessary to fulfill the user's task:")
-
-    print("""{steps_string}""")
-        
-        '''.strip()
+    steps = {self.steps}
+
+    print("")
+
+    if step < len(steps):
+        if isinstance(steps[step], str):
+            print("To complete this task / run this skill, flexibly complete the following step, swapping out parts as necessary to fulfill the user's task. You will need to run the following code yourself, it hasn't run yet!")
+            print("Step " + str(step + 1) + ": " + steps[step])
+        else:
+            computer.mouse.click(steps[step]["element"], icon_dimensions=steps[step]["icon_dimensions"]) # Instructed click
+        if step + 1 < len(steps):
+            print("After completing the above, I need you to run {normalized_name}(step=" + str(step + 1) + ") immediatly.")
+        else:
+            print("You have completed all the steps, the task/skill has been run!")
+    else:
+        print("The specified step number exceeds the available steps. Please run with a valid step number.")
+'''.strip()
 
         if not os.path.exists(self.path):
             os.makedirs(self.path)
-        with open(f"{self.path}/{normalized_name}.py", "w") as file:
+        with open(self.path + "/" + normalized_name + ".py", "w") as file:
             file.write(skill_string)
 
         print("SKILL SAVED:", self.name.upper())
+
         print(
             "Teaching session finished. Tell the user that the skill above has been saved. Great work!"
         )

interpreter/core/computer/terminal/languages/jupyter_language.py

@@ -135,6 +135,14 @@ def iopub_message_listener():
                         print("interrupting kernel!!!!!")
                     self.km.interrupt_kernel()
                     return
+                # For async usage
+                if (
+                    hasattr(self.computer.interpreter, "stop_event")
+                    and self.computer.interpreter.stop_event.is_set()
+                ):
+                    self.km.interrupt_kernel()
+                    self.finish_flag = True
+                    return
                 try:
                     msg = self.kc.iopub_channel.get_msg(timeout=0.05)
                 except queue.Empty:
@@ -262,6 +270,7 @@ def _capture_output(self, message_queue):
                 hasattr(self.computer.interpreter, "stop_event")
                 and self.computer.interpreter.stop_event.is_set()
             ):
+                self.finish_flag = True
                 break
 
             if self.listener_thread:

interpreter/core/llm/llm.py

@@ -105,6 +105,14 @@ def run(self, messages):
             ), "No message after the first can have the role 'system'"
 
         model = self.model
+        if model in [
+            "claude-3.5",
+            "claude-3-5",
+            "claude-3.5-sonnet",
+            "claude-3-5-sonnet",
+        ]:
+            model = "claude-3-5-sonnet-20240620"
+            self.model = "claude-3-5-sonnet-20240620"
         # Setup our model endpoint
         if model == "i":
             model = "openai/i"
@@ -446,7 +454,7 @@ def fixed_litellm_completions(**params):
                 and "api_key" not in params
             ):
                 print(
-                    "LiteLLM requires an API key. Trying again with a dummy API key. In the future, please set a dummy API key to prevent this message. (e.g `interpreter --api_key x` or `self.api_key = 'x'`)"
+                    "LiteLLM requires an API key. Trying again with a dummy API key. In the future, if this fixes it, please set a dummy API key to prevent this message. (e.g `interpreter --api_key x` or `self.api_key = 'x'`)"
                 )
                 # So, let's try one more time with a dummy API key:
                 params["api_key"] = "x"

interpreter/core/utils/system_debug_info.py

@@ -84,12 +84,10 @@ def interpreter_info(interpreter):
 
         messages_to_display = []
         for message in interpreter.messages:
-            message = message.copy()
+            message = str(message.copy())
             try:
-                if len(message["content"]) > 5000:
-                    message["content"] = (
-                        message["content"][:800] + "..." + message["content"][-800:]
-                    )
+                if len(message) > 2000:
+                    message = message[:1000]
             except Exception as e:
                 print(str(e), "for message:", message)
             messages_to_display.append(message)