cleanup and switch to no stream

Author: aprilk-ms

src/api/main.py

@@ -17,7 +17,6 @@
 from .routes import get_targeting_context
 
 from azure.identity import DefaultAzureCredential
-from azure.appconfiguration.provider import load
 
 from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
 
@@ -60,24 +59,29 @@ async def lifespan(app: fastapi.FastAPI):
             logger.error("Enable it via the 'Tracing' tab in your AI Foundry project page.")
             exit()
         else:
-            from azure.monitor.opentelemetry import configure_azure_monitor
-            from featuremanagement import FeatureManager
-            from featuremanagement.azuremonitor import publish_telemetry, TargetingSpanProcessor
-
-            configure_azure_monitor(connection_string=application_insights_connection_string, span_processors=[TargetingSpanProcessor(targeting_context_accessor=get_targeting_context)])
-
-            # Inititalize the feature manager / TODO: Add null check
-            app_config_conn_str = os.getenv("APP_CONFIGURATION_ENDPOINT") # this will become: project.experiments.get_connection_string()
-
-            app_config = load(
-                endpoint=app_config_conn_str,
-                credential=DefaultAzureCredential(),
-                feature_flag_enabled=True,
-                feature_flag_refresh_enabled=True,
-                refresh_interval=30,  # 30 seconds
-            )
-            feature_manager = FeatureManager(app_config, targeting_context_accessor=get_targeting_context, on_feature_evaluated=publish_telemetry)
-            app.state.feature_manager = feature_manager
+            from azure.monitor.opentelemetry import configure_azure_monitor            
+            app_config_conn_str = os.getenv("APP_CONFIGURATION_ENDPOINT")
+            if app_config_conn_str:
+                from azure.appconfiguration.provider import load
+                from featuremanagement import FeatureManager
+                from featuremanagement.azuremonitor import publish_telemetry, TargetingSpanProcessor
+                logger.info("Configured Application Insights with App Configuration feature flag support")
+                configure_azure_monitor(
+                    connection_string=application_insights_connection_string, 
+                    span_processors=[TargetingSpanProcessor(targeting_context_accessor=get_targeting_context)])
+                app_config = load(
+                    endpoint=app_config_conn_str,
+                    credential=DefaultAzureCredential(),
+                    feature_flag_enabled=True,
+                    feature_flag_refresh_enabled=True,
+                    refresh_interval=30,  # 30 seconds
+                )
+                feature_manager = FeatureManager(app_config, targeting_context_accessor=get_targeting_context, on_feature_evaluated=publish_telemetry)
+                app.state.app_config = app_config
+                app.state.feature_manager = feature_manager
+            else: 
+                logger.info("Configured Application Insights.")
+                configure_azure_monitor(connection_string=application_insights_connection_string)            
 
     chat = await project.inference.get_chat_completions_client()
     embed = await project.inference.get_embeddings_client()
@@ -106,8 +110,7 @@ async def lifespan(app: fastapi.FastAPI):
 
     app.state.chat = chat
     app.state.search_index_manager = search_index_manager
-    app.state.chat_model = os.environ["AZURE_AI_CHAT_DEPLOYMENT_NAME"]
-    
+    app.state.chat_model = os.environ["AZURE_AI_CHAT_DEPLOYMENT_NAME"]    
 
     yield
 
@@ -166,6 +169,8 @@ def create_app():
     else:
         logger.info("Tracing is not enabled")
 
+    # TODO: enable_app_config and make sure libaries are installed
+
     app = fastapi.FastAPI(lifespan=lifespan)
 
     static_dir = os.path.join(os.path.dirname(__file__), "static")

src/api/routes.py

@@ -10,8 +10,6 @@
 from fastapi.responses import HTMLResponse
 from fastapi.templating import Jinja2Templates
 
-from featuremanagement.azuremonitor import track_event
-
 import uuid
 import pathlib
 from azure.ai.inference.prompts import PromptTemplate
@@ -31,12 +29,14 @@
 )
 
 from opentelemetry.baggage import get_baggage
-from azure.ai.evaluation import CoherenceEvaluator, FluencyEvaluator, RelevanceEvaluator, ViolenceEvaluator, SexualEvaluator, HateUnfairnessEvaluator, ProtectedMaterialEvaluator, ContentSafetyEvaluator
-import asyncio
 from opentelemetry.baggage import set_baggage, get_baggage
 from opentelemetry.context import attach
 from featuremanagement import TargetingContext, FeatureManager
-from azure.identity import DefaultAzureCredential
+from azure.appconfiguration.provider import AzureAppConfigurationProvider
+
+# import asyncio
+# from azure.ai.evaluation import CoherenceEvaluator, FluencyEvaluator, RelevanceEvaluator, ViolenceEvaluator, SexualEvaluator, HateUnfairnessEvaluator, ProtectedMaterialEvaluator, ContentSafetyEvaluator
+# from azure.identity import DefaultAzureCredential
 
 router = fastapi.APIRouter()
 templates = Jinja2Templates(directory="api/templates")
@@ -51,9 +51,12 @@ def get_chat_model(request: Request) -> str:
 def get_search_index_namager(request: Request) -> SearchIndexManager:
     return request.app.state.search_index_manager
 
-def get_feature_manager(request: Request) -> str:
+def get_feature_manager(request: Request) -> FeatureManager:
     return request.app.state.feature_manager
 
+def get_app_config(request: Request) -> AzureAppConfigurationProvider:
+    return request.app.state.app_config
+
 class Message(pydantic.BaseModel):
     content: str
     role: str = "user"
@@ -78,16 +81,20 @@ async def chat_stream_handler(
     chat_client: ChatCompletionsClient = Depends(get_chat_client),
     model_deployment_name: str = Depends(get_chat_model),
     search_index_manager: SearchIndexManager = Depends(get_search_index_namager),
-    feature_manager: FeatureManager = Depends(get_feature_manager)
+    feature_manager: FeatureManager = Depends(get_feature_manager),
+    app_config: AzureAppConfigurationProvider = Depends(get_app_config),
 ) -> fastapi.responses.StreamingResponse:
     if chat_client is None:
         raise Exception("Chat client not initialized")
 
     async def response_stream():
         messages = [{"role": message.role, "content": message.content} for message in chat_request.messages]
 
-        targeting_id = chat_request.sessionState.get('sessionId', str(uuid.uuid4()))
-        attach(set_baggage("Microsoft.TargetingId", targeting_id))
+        # Refresh config and set targeting context for analysis
+        if app_config and feature_manager:
+            app_config.refresh()
+            targeting_id = chat_request.sessionState.get('sessionId', str(uuid.uuid4()))
+            attach(set_baggage("Microsoft.TargetingId", targeting_id))
 
         # figure out which prompty template to use
         prompt_template = "prompt.v1.prompty"
@@ -172,54 +179,69 @@ async def response_stream():
                 + "\n"
             )
 
+    # TODO: add variant to response
+
     return fastapi.responses.StreamingResponse(response_stream())
 
 
 def get_targeting_context() -> TargetingContext:
     return TargetingContext(user_id=get_baggage("Microsoft.TargetingId"))
 
-# @router.post("/chat")
-# async def chat_nostream_handler(
-#     chat_request: ChatRequest,
-#     request: Request
-# ):
-#     chat_client = globals["chat"]
-#     if chat_client is None:
-#         raise Exception("Chat client not initialized")
+@router.post("/chat")
+async def chat_nostream_handler(
+    chat_request: ChatRequest,
+    chat_client: ChatCompletionsClient = Depends(get_chat_client),
+    model_deployment_name: str = Depends(get_chat_model),
+    search_index_manager: SearchIndexManager = Depends(get_search_index_namager),
+    feature_manager: FeatureManager = Depends(get_feature_manager),
+    app_config: AzureAppConfigurationProvider = Depends(get_app_config),
+):  
+    messages = [{"role": message.role, "content": message.content} for message in chat_request.messages]
 
-#     messages = [{"role": message.role, "content": message.content} for message in chat_request.messages]
-#     model_deployment_name = globals["chat_model"]
-#     feature_manager = globals["feature_manager"] 
-
-#     targeting_id = chat_request.sessionState.get('sessionId', str(uuid.uuid4()))
-#     attach(set_baggage("Microsoft.TargetingId", targeting_id))
+    # Refresh config and set targeting context for analysis
+    if app_config and feature_manager:
+        app_config.refresh()
+        targeting_id = chat_request.sessionState.get('sessionId', str(uuid.uuid4()))
+        attach(set_baggage("Microsoft.TargetingId", targeting_id))
 
-#     # figure out which prompty template to use (replace file to API)
-#     variant = "none"
-#     if chat_request.prompt_override:
-#         prompt = PromptTemplate.from_prompty(pathlib.Path(__file__).parent.resolve() / chat_request.prompt_override)
-#         variant = chat_request.prompt_override
-#     else:                       
-#         prompt_variant = feature_manager.get_variant("prompty_file") # replace this with prompt_asset
-#         if prompt_variant and prompt_variant.configuration:
-#             prompt = PromptTemplate.from_prompty(pathlib.Path(__file__).parent.resolve() / prompt_variant.configuration)
-#             variant = prompt_variant.name
-#         else:
-#             prompt = globals["prompt"]
-
-#     prompt_messages = prompt.create_messages()
-
-#     try:
-#         response = await chat_client.complete(
-#             model=model_deployment_name, messages=prompt_messages + messages, stream=False
-#         )
-#         track_event("RequestMade", targeting_id)
-#         answer = response.choices[0].message.content
-#     except Exception as e:
-#         error = {"Error": str(e)}
-#         track_event("ErrorLLM", targeting_id, error)       
-#         return { "answer": str(e), "variant": variant }    
+    # figure out which prompty template to use
+    prompt_template = "prompt.v1.prompty"
+    if chat_request.prompt_override:
+        prompt_template = chat_request.prompt_override
+    elif feature_manager is not None:                       
+        prompt_variant = feature_manager.get_variant("prompty_file") # replace this with prompt_asset
+        if prompt_variant and prompt_variant.configuration: # TODO: check file exists
+            prompt_template = prompt_variant.configuration
+
+    prompt = PromptTemplate.from_prompty(pathlib.Path(__file__).parent.resolve() / prompt_template)
+    prompt_messages = prompt.create_messages()
+
+    # Use RAG model, only if we were provided index and we have found a context there.
+    if search_index_manager is not None:
+        context = await search_index_manager.search(chat_request)
+        if context:
+            prompt_messages = PromptTemplate.from_string(
+                'You are a helpful assistant that answers some questions '
+                'with the help of some context data.\n\nHere is '
+                'the context data:\n\n{{context}}').create_messages(data=dict(context=context))
+            logger.info(f"{prompt_messages=}")
+        else:
+            logger.info("Unable to find the relevant information in the index for the request.")
+                
+    try:
+        response = await chat_client.complete(
+            model=model_deployment_name, messages=prompt_messages + messages, stream=False
+        )
+        answer = response.choices[0].message.content        
+    except Exception as e:
+        error = {"Error": str(e)}
+        #track_event("ErrorLLM", targeting_id, error)       
+        answer = error
+    
+    return { "answer": answer, "variant": prompt_variant.name if prompt_variant else None }
+
 
+# Inline Evaluation Prototype
 
     # conversation = {}
 
@@ -247,28 +269,28 @@ def get_targeting_context() -> TargetingContext:
 
     # asyncio.create_task(run_evals(eval_input, targeting_id, project.scope, DefaultAzureCredential()))
 
-    return { "answer": answer, "variant": variant }
+    # return { "answer": answer, "variant": variant }
 
 
-async def run_evals(eval_input, targeting_id, ai_project_scope, credential):
-    run_eval(FluencyEvaluator, eval_input, targeting_id)
-    run_eval(RelevanceEvaluator, eval_input, targeting_id)
-    run_eval(CoherenceEvaluator, eval_input, targeting_id)
-
-    run_safety_eval(ViolenceEvaluator, eval_input, targeting_id, ai_project_scope, credential)
-    run_safety_eval(SexualEvaluator, eval_input, targeting_id, ai_project_scope, credential)
-    run_safety_eval(HateUnfairnessEvaluator, eval_input, targeting_id, ai_project_scope, credential)
-    run_safety_eval(ProtectedMaterialEvaluator, eval_input, targeting_id, ai_project_scope, credential)
-    run_safety_eval(ContentSafetyEvaluator, eval_input, targeting_id, ai_project_scope, credential)
-
-def run_safety_eval(evaluator, eval_input, targeting_id, ai_project_scope, credential):
-    eval = evaluator(credential=credential, azure_ai_project=ai_project_scope)
-    score = eval(**eval_input)
-    score.update({"evaluator_id": eval.id})
-    track_event("gen.ai." + type(eval).__name__, targeting_id, score)
-
-def run_eval(evaluator, eval_input, targeting_id):
-    eval = evaluator(globals["model_config"])
-    score = eval(**eval_input)
-    score.update({"evaluator_id": evaluator.id})
-    track_event("gen.ai." + evaluator.__name__, targeting_id, score)
+# async def run_evals(eval_input, targeting_id, ai_project_scope, credential):
+#     run_eval(FluencyEvaluator, eval_input, targeting_id)
+#     run_eval(RelevanceEvaluator, eval_input, targeting_id)
+#     run_eval(CoherenceEvaluator, eval_input, targeting_id)
+
+#     run_safety_eval(ViolenceEvaluator, eval_input, targeting_id, ai_project_scope, credential)
+#     run_safety_eval(SexualEvaluator, eval_input, targeting_id, ai_project_scope, credential)
+#     run_safety_eval(HateUnfairnessEvaluator, eval_input, targeting_id, ai_project_scope, credential)
+#     run_safety_eval(ProtectedMaterialEvaluator, eval_input, targeting_id, ai_project_scope, credential)
+#     run_safety_eval(ContentSafetyEvaluator, eval_input, targeting_id, ai_project_scope, credential)
+
+# def run_safety_eval(evaluator, eval_input, targeting_id, ai_project_scope, credential):
+#     eval = evaluator(credential=credential, azure_ai_project=ai_project_scope)
+#     score = eval(**eval_input)
+#     score.update({"evaluator_id": eval.id})
+#     track_event("gen.ai." + type(eval).__name__, targeting_id, score)
+
+# def run_eval(evaluator, eval_input, targeting_id):
+#     eval = evaluator(globals["model_config"])
+#     score = eval(**eval_input)
+#     score.update({"evaluator_id": evaluator.id})
+#     track_event("gen.ai." + evaluator.__name__, targeting_id, score)

src/api/templates/index.html

@@ -146,43 +146,45 @@
 
             try {
 
-                const result = await client.getStreamedCompletion(messages);
-
-                let answer = "";
-                for await (const response of result) {
-                    if (!response.delta) {
-                        continue;
-                    }
-                    if (response.delta.content) {
-                        // Clear out the DIV if its the first answer chunk we've received
-                        if (answer == "") {
-                            messageDiv.innerHTML = "";
-                        }
-                        answer += response.delta.content;
-                        messageDiv.innerHTML = converter.makeHtml(answer);
-                        messageDiv.scrollIntoView();
-                    }
-                    if (response.error) {
-                        messageDiv.innerHTML = "Error: " + response.error;
-                    }
-                }
-
+                // Uncomment the following lines if you want to use the streaming version
 
+                // const result = await client.getStreamedCompletion(messages, { "sessionState": { "sessionId": sessionId }});
+
+                // let answer = "";
+                // for await (const response of result) {
+                //     if (!response.delta) {
+                //         continue;
+                //     }
+                //     if (response.delta.content) {
+                //         // Clear out the DIV if its the first answer chunk we've received
+                //         if (answer == "") {
+                //             messageDiv.innerHTML = "";
+                //         }
+                //         answer += response.delta.content;
+                //         messageDiv.innerHTML = converter.makeHtml(answer);
+                //         messageDiv.scrollIntoView();
+                //     }
+                //     if (response.error) {
+                //         messageDiv.innerHTML = "Error: " + response.error;
+                //     }
+                // }
 
-                // const response = await client.getCompletion(messages, { "sessionState": { "sessionId": sessionId }});
-                // const answer = response.answer;
+                
+                // Uncomment the following lines if you want to use the non-streaming version
+                const response = await client.getCompletion(messages, { "sessionState": { "sessionId": sessionId }});
+                const answer = response.answer;
 
-                // messageDiv.innerHTML = converter.makeHtml(answer);
-                // messageDiv.scrollIntoView();
+                messageDiv.innerHTML = converter.makeHtml(answer);
+                messageDiv.scrollIntoView();
 
                 messages.push({
                     "role": "assistant",
                     "content": answer
                 });
 
-                // if (response.variant) {
-                //     messageTitleDiv.innerHTML += ` (Prompt Variant: ${response.variant})`;
-                // }
+                if (response.variant) {
+                    messageTitleDiv.innerHTML += ` (Prompt Variant: ${response.variant})`;
+                }
 
                 messageInput.value = "";
             } catch (error) {