Changes after beamline test

pyproject.toml

@@ -27,6 +27,7 @@ dynamic = ["version"]
 aps_mic = [
     "bluesky",
     "mic-instrument@git+https://github.com/BCDA-APS/bluesky-mic.git",
+    "mic-vis@git+https://github.com/grace227/mic-vis",
     "h5py",
 ]
 asksage = [

src/eaa/maths.py

@@ -47,11 +47,24 @@ def fit_gaussian_1d(
     Returns
     -------
     tuple[float, float, float]
-        The amplitude, mean, and standard deviation of the Gaussian.
+        The amplitude, mean, standard deviation, and constant offset of the Gaussian.
     """
     y_max, y_min = np.max(y), np.min(y)
     x_max = x[np.argmax(y)]
+    offset = x_max
+    x = x - offset
+    x_max = 0
     mask = y >= y_min + y_threshold * (y_max - y_min)
-    p0 = [y_max - y_min, x_max, np.count_nonzero(mask) / (x[-1] - x[0]) / 2, y_min]
+    a_guess = y_max - y_min
+    mu_guess = x_max
+    x_above_thresh = x[y > y_min + a_guess * 0.2]
+    if len(x_above_thresh) >= 3:
+        sigma_guess = (x_above_thresh.max() - x_above_thresh.min()) / 2
+    else:
+        sigma_guess = (x.max() - x.min()) / 2
+    c_guess = y_min
+    p0 = [a_guess, mu_guess, sigma_guess, c_guess]
     popt, _ = scipy.optimize.curve_fit(gaussian_1d, x[mask], y[mask], p0=p0)
+    popt[1] += offset
     return tuple(popt)
+

src/eaa/task_managers/base.py

@@ -441,6 +441,8 @@ def launch_task_manager(self, task_request: str) -> None:
             f"tools:\n{tool_catalog_json}\n"
             f"User request: {request_text}"
         )
+        # Escape image tag
+        parsing_prompt = parsing_prompt.replace("<img", "<img\\")
 
         local_context: List[Dict[str, Any]] = []
         parsed_result: Dict[str, Any] = {}
@@ -451,7 +453,6 @@ def launch_task_manager(self, task_request: str) -> None:
                 context=local_context,
                 return_outgoing_message=True,
             )
-            self.update_message_history(outgoing, update_context=False, update_full_history=True)
             self.update_message_history(response, update_context=False, update_full_history=True)
             local_context.append(outgoing)
             local_context.append(response)
@@ -560,6 +561,12 @@ def launch_task_manager(self, task_request: str) -> None:
             init_kwargs[key] = value
 
         try:
+            log_message = generate_openai_message(
+                content=f"Instantiating task manager '{manager_name}' with init_kwargs: {init_kwargs}",
+                role="system",
+            )
+            self.update_message_history(log_message, update_context=True, update_full_history=True)
+            print_message(log_message)
             sub_manager = manager_class(**init_kwargs)
         except Exception as exc:  # noqa: BLE001 - surface configuration errors
             logger.exception("Failed to instantiate task manager '%s'", manager_name)
@@ -577,6 +584,16 @@ def launch_task_manager(self, task_request: str) -> None:
                 result = sub_manager.run_conversation()
             else:
                 method_callable = getattr(sub_manager, resolved_method_name)
+                log_message = generate_openai_message(
+                    content=f"Running method '{resolved_method_name}' with method_kwargs: {method_kwargs}. Proceed? (yes/no)",
+                    role="system",
+                )
+                proceed = self.get_user_input(
+                    prompt=log_message["content"],
+                    display_prompt_in_webui=bool(self.message_db_conn),
+                )
+                if proceed.strip().lower() != "yes":
+                    return
                 result = method_callable(**method_kwargs)
             last_of_sub_manager_context = sub_manager.context[-1] if len(sub_manager.context) > 0 else None
         except Exception as exc:  # noqa: BLE001
@@ -625,77 +642,90 @@ def run_conversation(
         """
         response = None
         while True:
-            if response is None or (response is not None and not has_tool_call(response)):
-                message = self.get_user_input(
-                    prompt=(
-                        "Enter a message (/exit: exit; /return: return to upper level task; "
-                        "/help: show command help): "
+            try:
+                if response is None or (response is not None and not has_tool_call(response)):
+                    message = self.get_user_input(
+                        prompt=(
+                            "Enter a message (/exit: exit; /return: return to upper level task; "
+                            "/help: show command help): "
+                        )
                     )
-                )
-                stripped_message = message.strip()
-                command, _, remainder = stripped_message.partition(" ")
-                command_lower = command.lower()
+                    stripped_message = message.strip()
+                    command, _, remainder = stripped_message.partition(" ")
+                    command_lower = command.lower()
 
-                if command_lower == "/exit" and remainder == "":
-                    break
-                elif command_lower == "/return" and remainder == "":
-                    return
-                elif command_lower == "/monitor":
-                    if len(remainder.strip()) == 0:
-                        logger.info("Monitoring command requires a task description.")
-                    else:
-                        self.enter_monitoring_mode(remainder.strip())
-                    continue
-                elif command_lower == "/subtask":
-                    self.launch_task_manager(remainder.strip())
-                    continue
-                elif command_lower == "/help" and remainder == "":
-                    self.display_command_help()
-                    continue
-
-                # Send message and get response
-                response, outgoing_message = self.agent.receive(
-                    message, 
-                    context=self.context, 
-                    return_outgoing_message=True
+                    if command_lower == "/exit" and remainder == "":
+                        break
+                    elif command_lower == "/return" and remainder == "":
+                        return
+                    elif command_lower == "/monitor":
+                        if len(remainder.strip()) == 0:
+                            logger.info("Monitoring command requires a task description.")
+                        else:
+                            self.enter_monitoring_mode(remainder.strip())
+                        continue
+                    elif command_lower == "/subtask":
+                        self.launch_task_manager(remainder.strip())
+                        continue
+                    elif command_lower == "/help" and remainder == "":
+                        self.display_command_help()
+                        continue
+
+                    # Send message and get response
+                    response, outgoing_message = self.agent.receive(
+                        message, 
+                        context=self.context, 
+                        return_outgoing_message=True
+                    )
+                    # If message DB is used, user input should come from WebUI which writes
+                    # to the DB, so we don't update DB again.
+                    self.update_message_history(
+                        outgoing_message, 
+                        update_context=True, 
+                        update_full_history=True, 
+                        update_db=(self.message_db_conn is None)
+                    )
+                    self.update_message_history(response, update_context=True, update_full_history=True)
+
+                # Handle tool calls
+                tool_responses, tool_response_types = self.agent.handle_tool_call(response, return_tool_return_types=True)
+                for tool_response, tool_response_type in zip(tool_responses, tool_response_types):
+                    print_message(tool_response)
+                    self.update_message_history(tool_response, update_context=True, update_full_history=True)
+
+                if len(tool_responses) >= 1:
+                    for tool_response, tool_response_type in zip(tool_responses, tool_response_types):
+                        # If the tool returns an image path, load the image and send it to 
+                        # the assistant in a follow-up message as user.
+                        if tool_response_type == ToolReturnType.IMAGE_PATH:
+                            image_path = tool_response["content"]
+                            image_message = generate_openai_message(
+                                content="Here is the image the tool returned.",
+                                image_path=image_path,
+                                role="user",
+                            )
+                            self.update_message_history(
+                                image_message, update_context=store_all_images_in_context, update_full_history=True
+                            )
+                    # Send tool responses stored in the context
+                    response = self.agent.receive(
+                        message=None, 
+                        context=self.context, 
+                        return_outgoing_message=False
+                    )
+                    self.update_message_history(response, update_context=True, update_full_history=True)
+            except KeyboardInterrupt:
+                self.context = complete_unresponded_tool_calls(self.context)
+                response = generate_openai_message(
+                    content="Workflow interrupted by keyboard interrupt. TERMINATE",
+                    role="system"
                 )
-                # If message DB is used, user input should come from WebUI which writes
-                # to the DB, so we don't update DB again.
                 self.update_message_history(
-                    outgoing_message, 
+                    response, 
                     update_context=True, 
-                    update_full_history=True, 
-                    update_db=(self.message_db_conn is None)
-                )
-                self.update_message_history(response, update_context=True, update_full_history=True)
-
-            # Handle tool calls
-            tool_responses, tool_response_types = self.agent.handle_tool_call(response, return_tool_return_types=True)
-            for tool_response, tool_response_type in zip(tool_responses, tool_response_types):
-                print_message(tool_response)
-                self.update_message_history(tool_response, update_context=True, update_full_history=True)
-
-            if len(tool_responses) >= 1:
-                for tool_response, tool_response_type in zip(tool_responses, tool_response_types):
-                    # If the tool returns an image path, load the image and send it to 
-                    # the assistant in a follow-up message as user.
-                    if tool_response_type == ToolReturnType.IMAGE_PATH:
-                        image_path = tool_response["content"]
-                        image_message = generate_openai_message(
-                            content="Here is the image the tool returned.",
-                            image_path=image_path,
-                            role="user",
-                        )
-                        self.update_message_history(
-                            image_message, update_context=store_all_images_in_context, update_full_history=True
-                        )
-                # Send tool responses stored in the context
-                response = self.agent.receive(
-                    message=None, 
-                    context=self.context, 
-                    return_outgoing_message=False
+                    update_full_history=True
                 )
-                self.update_message_history(response, update_context=True, update_full_history=True)
+                continue
 
     def enter_monitoring_mode(
         self,

src/eaa/task_managers/imaging/feature_tracking.py

@@ -121,7 +121,9 @@ def run_fov_search(
         Parameters
         ----------
         feature_description : str
-            A text description of the feature to search for.
+            A text description of the feature to search for. The message
+            can contain the <img /path/to/image.png> tag to include a 
+            reference image of the feature.
         y_range : tuple[float, float]
             The range of y coordinates to search for the feature.
         x_range : tuple[float, float]
@@ -157,10 +159,11 @@ def run_fov_search(
                 f"but go back to this size when you find the feature and acquire a "
                 f"final image of it.\n"
                 f"- Start from position (y={y_range[0]}, x={x_range[0]}), and gradually "
-                f"move the FOV to find the feature. Positions should not go beyond "
-                f"y={y_range[1]} and x={x_range[1]}. When moving the FOV, you can start "
-                f"with the step size of {step_size[0]} in the y direction and {step_size[1]} "
-                f"in the x direction. You can change the step sizes during the process.\n"
+                f"move the FOV to find the feature. Positions should stay in the range of "
+                f"y={y_range[0]} to {y_range[1]} and x={x_range[0]} to {x_range[1]}. \n"
+                f"- Use a regular grid search pattern at the beginning. Use a step size of {step_size[0]} "
+                f"in the y direction and {step_size[1]} in the x direction. When you see the\n"
+                f"feature, you can move the FOV more arbitrarily to make it better centered.\n"
                 f"- When you find the feature, adjust the positions of the FOV to make the "
                 f"feature centered in the FOV. If the feature is off to the left, move "
                 f"the FOV to the left; if the feature is off to the top, move the FOV "
@@ -170,6 +173,7 @@ def run_fov_search(
                 f"stop the process.\n"
                 f"- When you find the feature of interest, report the coordinates of the "
                 f"FOV.\n" 
+                f"- Explain every tool call you make."
                 f"- When calling tools, make only one call at a time. Do not make "
                 f"another call before getting the response of a previous one. \n"
                 f"- When you finish the search or need user response, say 'TERMINATE'.\n"