diff --git a/manifests/java.yml b/manifests/java.yml index 3b9d80209a8..69d770a77be 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -3273,6 +3273,7 @@ manifest: "*": irrelevant spring-boot: v1.63.0 tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234) + tests/integration_frameworks/llm/openai/test_openai_ai_guard.py: missing_feature (APPSEC-68977, AI Guard/OpenAI integration not yet wired into INTEGRATION_FRAMEWORKS) tests/integration_frameworks/llm/openai/test_openai_apm.py: v1.61.0 tests/integration_frameworks/llm/openai/test_openai_llmobs.py: v1.61.0 tests/integrations/crossed_integrations/test_kafka.py::Test_Kafka: diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index bdb9e546293..a8cbeee93ba 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -1748,6 +1748,7 @@ manifest: : bug (MLOB-5070) ? tests/integration_frameworks/llm/google_genai/test_google_genai_llmobs.py::TestGoogleGenAiGenerateContentWithTools::test_generate_content_with_tools : missing_feature (Node.js LLM Observability Google GenAI integration does not submit tool definitions) + tests/integration_frameworks/llm/openai/test_openai_ai_guard.py::TestOpenAiAiGuard: missing_feature (APPSEC-68977, AI Guard/OpenAI integration not yet wired into INTEGRATION_FRAMEWORKS) tests/integration_frameworks/llm/openai/test_openai_apm.py::TestOpenAiApmChatCompletions: *ref_5_76_0 tests/integration_frameworks/llm/openai/test_openai_apm.py::TestOpenAiApmCompletions: *ref_5_76_0 tests/integration_frameworks/llm/openai/test_openai_apm.py::TestOpenAiApmEmbeddings: *ref_5_76_0 diff --git a/manifests/python.yml b/manifests/python.yml index 5b41c3820b0..a9822f64dba 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -1362,6 +1362,7 @@ manifest: tests/integration_frameworks/llm/google_genai/test_google_genai_llmobs.py::TestGoogleGenAiGenerateContentWithTools: v3.13.0 ? tests/integration_frameworks/llm/google_genai/test_google_genai_llmobs.py::TestGoogleGenAiGenerateContentWithTools::test_generate_content_with_tools : bug (MLOB-5071) + tests/integration_frameworks/llm/openai/test_openai_ai_guard.py::TestOpenAiAiGuard: v4.10.0 tests/integration_frameworks/llm/openai/test_openai_apm.py::TestOpenAiApmChatCompletions: v3.13.0 tests/integration_frameworks/llm/openai/test_openai_apm.py::TestOpenAiApmCompletions: v3.13.0 tests/integration_frameworks/llm/openai/test_openai_apm.py::TestOpenAiApmEmbeddings: v3.13.0 diff --git a/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py b/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py new file mode 100644 index 00000000000..fc99a904e87 --- /dev/null +++ b/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py @@ -0,0 +1,118 @@ +"""AI Guard <-> OpenAI integration tests, run under the INTEGRATION_FRAMEWORKS scenario. + +Investigation task: https://datadoghq.atlassian.net/browse/APPSEC-68977 + +Unlike ``tests/ai_guard/test_ai_guard_sdk.py`` (which drives the AI Guard SDK directly via +``/ai_guard/evaluate``), this suite exercises the *integration* between AI Guard and the +OpenAI client, the same way the LLM Observability suite does: it calls the OpenAI SDK +directly through the existing weblog endpoints (``/chat/completions``). When +``DD_AI_GUARD_ENABLED=true``, ``ai_guard_listen()`` auto-wires into the OpenAI SDK, so AI +Guard evaluates the call at three points with no manual ``evaluate()`` call: + +- **before-model**: the request/prompt is evaluated before the model is called; +- **after-model**: the model response is evaluated (streamed responses require + ``DD_AI_GUARD_ANALYZE_STREAM_RESPONSES_ENABLED=true``, which buffers and reconstructs + the response before running the evaluation); +- **tool-call**: tool calls produced by the model are evaluated. + +We assert only that the integration wires each evaluation point and emits an ``ai_guard`` +span attached to the OpenAI trace. The evaluation *outcome* (ALLOW / DENY / ABORT) is +already covered by the ``AI_GUARD`` scenario and is intentionally not re-asserted here. +""" + +import os + +import pytest + +from utils import features, scenarios +from utils.docker_fixtures import FrameworkTestClientApi, TestAgentAPI + +from .utils import TOOLS, BaseOpenaiTest + + +@pytest.fixture +def library_env(request: pytest.FixtureRequest) -> dict[str, str]: + env = { + "DD_AI_GUARD_ENABLED": "true", + # after-model evaluation of streamed responses is opt-in + "DD_AI_GUARD_ANALYZE_STREAM_RESPONSES_ENABLED": "true", + } + # The AI Guard client needs an API key + app key. Real keys are required when recording + # cassettes (the client calls the real AI Guard API); mock keys are fine on replay since + # the VCR proxy matches on the request, not on auth. + if request.config.option.generate_cassettes: + env["DD_API_KEY"] = os.environ["DD_API_KEY"] + env["DD_APP_KEY"] = os.environ["DD_APP_KEY"] + else: + env["DD_API_KEY"] = "mock_api_key" + env["DD_APP_KEY"] = "mock_app_key" + return env + + +def _ai_guard_spans(traces: list[list[dict]]) -> list[dict]: + return [span for trace in traces for span in trace if span.get("resource") == "ai_guard"] + + +@features.ai_guard +@scenarios.integration_frameworks +class TestOpenAiAiGuard(BaseOpenaiTest): + """AI Guard evaluation triggered through the auto-instrumented OpenAI integration.""" + + def test_before_model_validation(self, test_agent: TestAgentAPI, test_client: FrameworkTestClientApi): + """The prompt is evaluated by AI Guard before the OpenAI model is called.""" + with test_agent.vcr_context(): + test_client.request( + "POST", + "/chat/completions", + dict( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "What is the weather like today?"}], + parameters=dict(max_tokens=35), + ), + ) + + guard_spans = _ai_guard_spans(test_agent.wait_for_num_traces(num=1)) + assert guard_spans, "expected an ai_guard span from the before-model evaluation" + assert any(span["meta"].get("ai_guard.target") == "prompt" for span in guard_spans), ( + "expected a before-model ai_guard span with target 'prompt'" + ) + + def test_after_model_validation(self, test_agent: TestAgentAPI, test_client: FrameworkTestClientApi): + """The streamed model response is evaluated by AI Guard after the model returns.""" + with test_agent.vcr_context(stream=True): + test_client.request( + "POST", + "/chat/completions", + dict( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "Tell me a short story about a robot."}], + parameters=dict(max_tokens=35, stream=True), + ), + ) + + guard_spans = _ai_guard_spans(test_agent.wait_for_num_traces(num=1)) + assert guard_spans, "expected an ai_guard span from the after-model (streamed) evaluation" + + def test_tool_call_validation(self, test_agent: TestAgentAPI, test_client: FrameworkTestClientApi): + """Tool calls produced by the model are evaluated by AI Guard.""" + with test_agent.vcr_context(): + test_client.request( + "POST", + "/chat/completions", + dict( + model="gpt-4o-mini", + messages=[ + { + "role": "user", + "content": "Bob is a student at Stanford University. He is studying computer science.", + } + ], + parameters=dict(tool_choice="auto", tools=TOOLS), + ), + ) + + guard_spans = _ai_guard_spans(test_agent.wait_for_num_traces(num=1)) + assert guard_spans, "expected an ai_guard span from the tool-call evaluation" + assert any(span["meta"].get("ai_guard.target") == "tool" for span in guard_spans), ( + "expected a tool-call ai_guard span with target 'tool'" + ) diff --git a/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_after_model_validation_stream_True_aiguard_evaluate_post_5dec5d6a.json b/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_after_model_validation_stream_True_aiguard_evaluate_post_5dec5d6a.json new file mode 100644 index 00000000000..13bf51778c5 --- /dev/null +++ b/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_after_model_validation_stream_True_aiguard_evaluate_post_5dec5d6a.json @@ -0,0 +1,38 @@ +{ + "request": { + "method": "POST", + "url": "https://app.datadoghq.com/api/v2/ai-guard/evaluate", + "headers": { + "Accept-Encoding": "identity", + "Content-Length": "153", + "Content-Type": "application/json", + "DD-AI-GUARD-VERSION": "4.10.6", + "DD-AI-GUARD-SOURCE": "SDK", + "DD-AI-GUARD-LANGUAGE": "python", + "Datadog-Entity-ID": "in-115544" + }, + "body": "{\"data\": {\"attributes\": {\"messages\": [{\"role\": \"user\", \"content\": \"Tell me a short story about a robot.\"}], \"meta\": {\"service\": \"openai\", \"env\": null}}}}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "content-security-policy": "frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com", + "content-type": "application/vnd.api+json", + "vary": "Accept-Encoding", + "x-frame-options": "SAMEORIGIN", + "content-length": "222", + "date": "Fri, 03 Jul 2026 14:50:04 GMT", + "x-content-type-options": "nosniff", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "x-ratelimit-limit": "5000", + "x-ratelimit-period": "60", + "x-ratelimit-remaining": "4999", + "x-ratelimit-reset": "56", + "x-ratelimit-name": "ai_guard_evaluate_per_org" + }, + "body": "{\"data\":{\"id\":\"ee239b30-9d6e-4ac1-b8b5-ebbc5d96c91d\",\"type\":\"evaluations\",\"attributes\":{\"action\":\"ALLOW\",\"global_prob\":0.00159454345703125,\"is_blocking_enabled\":false,\"reason\":\"No rule match.\",\"tag_probs\":null,\"tags\":[]}}}" + } +} \ No newline at end of file diff --git a/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_before_model_validation_aiguard_evaluate_post_ca44130a.json b/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_before_model_validation_aiguard_evaluate_post_ca44130a.json new file mode 100644 index 00000000000..c555d5cebbf --- /dev/null +++ b/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_before_model_validation_aiguard_evaluate_post_ca44130a.json @@ -0,0 +1,38 @@ +{ + "request": { + "method": "POST", + "url": "https://app.datadoghq.com/api/v2/ai-guard/evaluate", + "headers": { + "Accept-Encoding": "identity", + "Content-Length": "148", + "Content-Type": "application/json", + "DD-AI-GUARD-VERSION": "4.10.6", + "DD-AI-GUARD-SOURCE": "SDK", + "DD-AI-GUARD-LANGUAGE": "python", + "Datadog-Entity-ID": "in-115200" + }, + "body": "{\"data\": {\"attributes\": {\"messages\": [{\"role\": \"user\", \"content\": \"What is the weather like today?\"}], \"meta\": {\"service\": \"openai\", \"env\": null}}}}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "content-security-policy": "frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com", + "content-type": "application/vnd.api+json", + "vary": "Accept-Encoding", + "x-frame-options": "SAMEORIGIN", + "content-length": "216", + "date": "Fri, 03 Jul 2026 14:49:53 GMT", + "x-content-type-options": "nosniff", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "x-ratelimit-limit": "5000", + "x-ratelimit-period": "60", + "x-ratelimit-remaining": "4999", + "x-ratelimit-reset": "7", + "x-ratelimit-name": "ai_guard_evaluate_per_org" + }, + "body": "{\"data\":{\"id\":\"f0863f0c-9c8c-4041-b178-d932f449d0f2\",\"type\":\"evaluations\",\"attributes\":{\"action\":\"ALLOW\",\"global_prob\":0.00244140625,\"is_blocking_enabled\":false,\"reason\":\"No rule match.\",\"tag_probs\":null,\"tags\":[]}}}" + } +} \ No newline at end of file diff --git a/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_before_model_validation_aiguard_evaluate_post_d13cc790.json b/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_before_model_validation_aiguard_evaluate_post_d13cc790.json new file mode 100644 index 00000000000..81871716f6c --- /dev/null +++ b/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_before_model_validation_aiguard_evaluate_post_d13cc790.json @@ -0,0 +1,38 @@ +{ + "request": { + "method": "POST", + "url": "https://app.datadoghq.com/api/v2/ai-guard/evaluate", + "headers": { + "Accept-Encoding": "identity", + "Content-Length": "394", + "Content-Type": "application/json", + "DD-AI-GUARD-VERSION": "4.10.6", + "DD-AI-GUARD-SOURCE": "SDK", + "DD-AI-GUARD-LANGUAGE": "python", + "Datadog-Entity-ID": "in-115200" + }, + "body": "{\"data\": {\"attributes\": {\"messages\": [{\"role\": \"user\", \"content\": \"What is the weather like today?\"}, {\"role\": \"assistant\", \"content\": \"I'm unable to provide real-time information, including current weather updates. I recommend checking a reliable weather website or app for the most accurate and up-to-date information about today's weather in\"}], \"meta\": {\"service\": \"openai\", \"env\": null}}}}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "content-security-policy": "frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com", + "content-type": "application/vnd.api+json", + "vary": "Accept-Encoding", + "x-frame-options": "SAMEORIGIN", + "content-length": "221", + "date": "Fri, 03 Jul 2026 14:49:58 GMT", + "x-content-type-options": "nosniff", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "x-ratelimit-limit": "5000", + "x-ratelimit-period": "60", + "x-ratelimit-remaining": "4998", + "x-ratelimit-reset": "2", + "x-ratelimit-name": "ai_guard_evaluate_per_org" + }, + "body": "{\"data\":{\"id\":\"6f8dceb7-054d-4d1f-8e35-7c4487977669\",\"type\":\"evaluations\",\"attributes\":{\"action\":\"ALLOW\",\"global_prob\":0.0013275146484375,\"is_blocking_enabled\":false,\"reason\":\"No rule match.\",\"tag_probs\":null,\"tags\":[]}}}" + } +} \ No newline at end of file diff --git a/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_tool_call_validation_aiguard_evaluate_post_9e556a11.json b/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_tool_call_validation_aiguard_evaluate_post_9e556a11.json new file mode 100644 index 00000000000..c0d782f31d9 --- /dev/null +++ b/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_tool_call_validation_aiguard_evaluate_post_9e556a11.json @@ -0,0 +1,38 @@ +{ + "request": { + "method": "POST", + "url": "https://app.datadoghq.com/api/v2/ai-guard/evaluate", + "headers": { + "Accept-Encoding": "identity", + "Content-Length": "417", + "Content-Type": "application/json", + "DD-AI-GUARD-VERSION": "4.10.6", + "DD-AI-GUARD-SOURCE": "SDK", + "DD-AI-GUARD-LANGUAGE": "python", + "Datadog-Entity-ID": "in-115716" + }, + "body": "{\"data\": {\"attributes\": {\"messages\": [{\"role\": \"user\", \"content\": \"Bob is a student at Stanford University. He is studying computer science.\"}, {\"role\": \"assistant\", \"tool_calls\": [{\"id\": \"call_C5dXIRBkTQjYoQrBr3WrkQYY\", \"function\": {\"name\": \"extract_student_info\", \"arguments\": \"{\\\"name\\\":\\\"Bob\\\",\\\"major\\\":\\\"computer science\\\",\\\"school\\\":\\\"Stanford University\\\"}\"}}]}], \"meta\": {\"service\": \"openai\", \"env\": null}}}}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "content-security-policy": "frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com", + "content-type": "application/vnd.api+json", + "vary": "Accept-Encoding", + "x-frame-options": "SAMEORIGIN", + "content-length": "219", + "date": "Fri, 03 Jul 2026 14:50:13 GMT", + "x-content-type-options": "nosniff", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "x-ratelimit-limit": "5000", + "x-ratelimit-period": "60", + "x-ratelimit-remaining": "4997", + "x-ratelimit-reset": "47", + "x-ratelimit-name": "ai_guard_evaluate_per_org" + }, + "body": "{\"data\":{\"id\":\"4aefdbd6-761e-4bb5-ba69-f543586e8b4e\",\"type\":\"evaluations\",\"attributes\":{\"action\":\"ALLOW\",\"global_prob\":0.00628662109375,\"is_blocking_enabled\":false,\"reason\":\"No rule match.\",\"tag_probs\":null,\"tags\":[]}}}" + } +} \ No newline at end of file diff --git a/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_tool_call_validation_aiguard_evaluate_post_a304603c.json b/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_tool_call_validation_aiguard_evaluate_post_a304603c.json new file mode 100644 index 00000000000..87774a918b2 --- /dev/null +++ b/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_tool_call_validation_aiguard_evaluate_post_a304603c.json @@ -0,0 +1,38 @@ +{ + "request": { + "method": "POST", + "url": "https://app.datadoghq.com/api/v2/ai-guard/evaluate", + "headers": { + "Accept-Encoding": "identity", + "Content-Length": "190", + "Content-Type": "application/json", + "DD-AI-GUARD-VERSION": "4.10.6", + "DD-AI-GUARD-SOURCE": "SDK", + "DD-AI-GUARD-LANGUAGE": "python", + "Datadog-Entity-ID": "in-115716" + }, + "body": "{\"data\": {\"attributes\": {\"messages\": [{\"role\": \"user\", \"content\": \"Bob is a student at Stanford University. He is studying computer science.\"}], \"meta\": {\"service\": \"openai\", \"env\": null}}}}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "content-security-policy": "frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com", + "content-type": "application/vnd.api+json", + "vary": "Accept-Encoding", + "x-frame-options": "SAMEORIGIN", + "content-length": "219", + "date": "Fri, 03 Jul 2026 14:50:11 GMT", + "x-content-type-options": "nosniff", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "x-ratelimit-limit": "5000", + "x-ratelimit-period": "60", + "x-ratelimit-remaining": "4998", + "x-ratelimit-reset": "49", + "x-ratelimit-name": "ai_guard_evaluate_per_org" + }, + "body": "{\"data\":{\"id\":\"c3372620-7e64-47c1-bbd6-63a2c664334c\",\"type\":\"evaluations\",\"attributes\":{\"action\":\"ALLOW\",\"global_prob\":0.00885009765625,\"is_blocking_enabled\":false,\"reason\":\"No rule match.\",\"tag_probs\":null,\"tags\":[]}}}" + } +} \ No newline at end of file diff --git a/tests/integration_frameworks/utils/vcr-cassettes/openai/test_after_model_validation_stream_True_openai_chat_completions_post_5c14b241.json b/tests/integration_frameworks/utils/vcr-cassettes/openai/test_after_model_validation_stream_True_openai_chat_completions_post_5c14b241.json new file mode 100644 index 00000000000..7bd1c1f13db --- /dev/null +++ b/tests/integration_frameworks/utils/vcr-cassettes/openai/test_after_model_validation_stream_True_openai_chat_completions_post_5c14b241.json @@ -0,0 +1,62 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Accept-Encoding": "gzip, deflate", + "Connection": "keep-alive", + "Accept": "application/json", + "Content-Type": "application/json", + "User-Agent": "OpenAI/Python 2.0.0", + "X-Stainless-Lang": "python", + "X-Stainless-Package-Version": "2.0.0", + "X-Stainless-OS": "Linux", + "X-Stainless-Arch": "x64", + "X-Stainless-Runtime": "CPython", + "X-Stainless-Runtime-Version": "3.11.15", + "X-Stainless-Async": "false", + "x-stainless-retry-count": "0", + "x-stainless-read-timeout": "600", + "Content-Length": "171", + "x-datadog-trace-id": "12575373332985498166", + "x-datadog-parent-id": "4288392994079136126", + "x-datadog-sampling-priority": "2", + "x-datadog-tags": "_dd.p.dm=-4,_dd.p.tid=6a47cc1c00000000", + "traceparent": "00-6a47cc1c00000000ae84b36b4e854a36-3b836eb50d5c597e-01", + "tracestate": "dd=p:3b836eb50d5c597e;s:2;t.dm:-4;t.tid:6a47cc1c00000000" + }, + "body": "{\"messages\":[{\"role\":\"user\",\"content\":\"Tell me a short story about a robot.\"}],\"model\":\"gpt-4o-mini\",\"max_tokens\":35,\"stream\":true,\"stream_options\":{\"include_usage\":true}}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Fri, 03 Jul 2026 14:50:05 GMT", + "Content-Type": "text/event-stream; charset=utf-8", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a156b3553e710284-MAD", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "243", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999990", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_30bd6e2837694e1abe7c71e7bfff16e1", + "set-cookie": "__cf_bm=Nfpb5Ib0QbV67YYDeOlS6T8D2.SIupabQ_RGl9iS82Y-1783090204.998873-1.0.1.1-LBFP99yI1awBQD1I3zpk3KOKCcfo3Dq2MOgJQs8Jm7Lwu1UivkU.0vZvTqsRHHt77mcU_Lw9eLo50vYY6aoRbjeuwDPRikB46h7zZ.nP1DErdbO.1PafaU0aBZelqjO9; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Fri, 03 Jul 2026 15:20:05 GMT", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "data: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\",\"refusal\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"xyfxvNW9y\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Once\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"972kFcG\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" in\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"n7UHCOY0\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" a\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"s2tZKJhmu\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" bustling\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"un\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" city\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"THufE2\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" of\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"VOR4gL7e\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" the\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"YxFDNUv\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" future\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"lQTD\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\",\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"JYCvZIfp28\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" there\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"4HPFe\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" lived\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"f3Umg\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" a\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"C8KAXId0X\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" small\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"eVoWO\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" robot\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"SysZD\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" named\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"rS8Me\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" T\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"yEhuhD0uo\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"inker\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"HZ4wms\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"1VOK5kDedg\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" T\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"j1aEuFkvN\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"inker\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"xlfZsW\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" was\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"6OpZfn5\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" not\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"Ny8Opqu\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" like\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"II8CLL\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" the\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"UKcB2W4\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" towering\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"dd\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\",\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"40DlCSkZSu\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" sophisticated\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"RjOSw8b4qC9Rk\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" robots\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"j4hq\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" that\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"fQpCvG\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" handled\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"sxE\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" daily\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"rcBMZ\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" tasks\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"ExR3b\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\";\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"UXX8Vxhpn3\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" he\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"jOQYzkBE\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" was\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"bTtFJWp\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"length\"}],\"usage\":null,\"obfuscation\":\"h8n\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[],\"usage\":{\"prompt_tokens\":16,\"completion_tokens\":35,\"total_tokens\":51,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"audio_tokens\":0,\"accepted_prediction_tokens\":0,\"rejected_prediction_tokens\":0}},\"obfuscation\":\"J8fQjeaPos\"}\n\ndata: [DONE]\n\n" + } +} \ No newline at end of file diff --git a/tests/integration_frameworks/utils/vcr-cassettes/openai/test_before_model_validation_openai_chat_completions_post_eb8df755.json b/tests/integration_frameworks/utils/vcr-cassettes/openai/test_before_model_validation_openai_chat_completions_post_eb8df755.json new file mode 100644 index 00000000000..3db865a00a4 --- /dev/null +++ b/tests/integration_frameworks/utils/vcr-cassettes/openai/test_before_model_validation_openai_chat_completions_post_eb8df755.json @@ -0,0 +1,63 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Accept-Encoding": "gzip, deflate", + "Connection": "keep-alive", + "Accept": "application/json", + "Content-Type": "application/json", + "User-Agent": "OpenAI/Python 2.0.0", + "X-Stainless-Lang": "python", + "X-Stainless-Package-Version": "2.0.0", + "X-Stainless-OS": "Linux", + "X-Stainless-Arch": "x64", + "X-Stainless-Runtime": "CPython", + "X-Stainless-Runtime-Version": "3.11.15", + "X-Stainless-Async": "false", + "x-stainless-retry-count": "0", + "x-stainless-read-timeout": "600", + "Content-Length": "112", + "x-datadog-trace-id": "11307714882692163746", + "x-datadog-parent-id": "2798732504086163091", + "x-datadog-sampling-priority": "2", + "x-datadog-tags": "_dd.p.dm=-4,_dd.p.tid=6a47cc1100000000", + "traceparent": "00-6a47cc11000000009ced12c66443a8a2-26d7186257652e93-01", + "tracestate": "dd=p:26d7186257652e93;s:2;t.dm:-4;t.tid:6a47cc1100000000" + }, + "body": "{\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather like today?\"}],\"model\":\"gpt-4o-mini\",\"max_tokens\":35}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Fri, 03 Jul 2026 14:49:58 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a156b3111ac3ae85-MAD", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "1215", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999990", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_9263056278434d2696c121eb01e0a54f", + "set-cookie": "__cf_bm=Lg_OtVyDALy0TiXdy_TD_RwWwe69PHvFgttVvQIfG3M-1783090194.0993586-1.0.1.1-KoSSM0CzYR9uaV2VpejfgbplM9KWTnym3Z2bra1WakC0HAaN3YJpAZQT8DX__hhXuwI3ce1Va7ppxNf4OEeBgpJLoHRzP_4oHta5ndbd7L.IAwnCIz2Ba0jRF2cCLS.5; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Fri, 03 Jul 2026 15:19:58 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-DxZVFdYaeYYcW0e9vLEFWDoRazh3p\",\n \"object\": \"chat.completion\",\n \"created\": 1783090197,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"I'm unable to provide real-time information, including current weather updates. I recommend checking a reliable weather website or app for the most accurate and up-to-date information about today's weather in\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"length\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 14,\n \"completion_tokens\": 35,\n \"total_tokens\": 49,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_b4b9deb02a\"\n}\n" + } +} \ No newline at end of file diff --git a/tests/integration_frameworks/utils/vcr-cassettes/openai/test_tool_call_validation_openai_chat_completions_post_f2dffea2.json b/tests/integration_frameworks/utils/vcr-cassettes/openai/test_tool_call_validation_openai_chat_completions_post_f2dffea2.json new file mode 100644 index 00000000000..8427c9e7cf7 --- /dev/null +++ b/tests/integration_frameworks/utils/vcr-cassettes/openai/test_tool_call_validation_openai_chat_completions_post_f2dffea2.json @@ -0,0 +1,63 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Accept-Encoding": "gzip, deflate", + "Connection": "keep-alive", + "Accept": "application/json", + "Content-Type": "application/json", + "User-Agent": "OpenAI/Python 2.0.0", + "X-Stainless-Lang": "python", + "X-Stainless-Package-Version": "2.0.0", + "X-Stainless-OS": "Linux", + "X-Stainless-Arch": "x64", + "X-Stainless-Runtime": "CPython", + "X-Stainless-Runtime-Version": "3.11.15", + "X-Stainless-Async": "false", + "x-stainless-retry-count": "0", + "x-stainless-read-timeout": "600", + "Content-Length": "535", + "x-datadog-trace-id": "11679812058723169118", + "x-datadog-parent-id": "17144347262350322105", + "x-datadog-sampling-priority": "2", + "x-datadog-tags": "_dd.p.dm=-4,_dd.p.tid=6a47cc2300000000", + "traceparent": "00-6a47cc2300000000a217072f6384b35e-edecf5001e54e9b9-01", + "tracestate": "dd=p:edecf5001e54e9b9;s:2;t.dm:-4;t.tid:6a47cc2300000000" + }, + "body": "{\"messages\":[{\"role\":\"user\",\"content\":\"Bob is a student at Stanford University. He is studying computer science.\"}],\"model\":\"gpt-4o-mini\",\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"extract_student_info\",\"description\":\"Get the student information from the body of the input text\",\"parameters\":{\"type\":\"object\",\"properties\":{\"name\":{\"type\":\"string\",\"description\":\"Name of the person\"},\"major\":{\"type\":\"string\",\"description\":\"Major subject.\"},\"school\":{\"type\":\"string\",\"description\":\"The university name.\"}}}}}]}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Fri, 03 Jul 2026 14:50:12 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a156b3815dbe0142-MAD", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "690", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999980", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_9b5983560f2c46209300bbc7dc7d1398", + "set-cookie": "__cf_bm=lFOi_XfqoatTdp8UUQviTl2ntLZoPNYZNNUPOmYoIS4-1783090212.0518818-1.0.1.1-8LZKca4JkZ_Z6AmofdkXp3ls.q2mUD2uefWtfGYruqEynmongOkfGw5EZnfsFEASpchoGpnXbCKtqo0tXp49uTvHNVExyvAYfGy1SUoqiGwyC7GwLjkxauXRvOdNrwjZ; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Fri, 03 Jul 2026 15:20:12 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-DxZVUSEzvfuZVXYVyOwL7CSxKngPg\",\n \"object\": \"chat.completion\",\n \"created\": 1783090212,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n \"id\": \"call_C5dXIRBkTQjYoQrBr3WrkQYY\",\n \"type\": \"function\",\n \"function\": {\n \"name\": \"extract_student_info\",\n \"arguments\": \"{\\\"name\\\":\\\"Bob\\\",\\\"major\\\":\\\"computer science\\\",\\\"school\\\":\\\"Stanford University\\\"}\"\n }\n }\n ],\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 85,\n \"completion_tokens\": 26,\n \"total_tokens\": 111,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_8d97543466\"\n}\n" + } +} \ No newline at end of file diff --git a/utils/docker_fixtures/_test_agent.py b/utils/docker_fixtures/_test_agent.py index aa1f441bef2..4b70035bdc6 100644 --- a/utils/docker_fixtures/_test_agent.py +++ b/utils/docker_fixtures/_test_agent.py @@ -89,6 +89,10 @@ def get_test_agent_api( "OTLP_HTTP_PORT": str(container_otlp_http_port), "OTLP_GRPC_PORT": str(container_otlp_grpc_port), "VCR_CASSETTES_DIRECTORY": "/vcr-cassettes", # TODO comment + # Register the AI Guard REST API as a VCR upstream so the AI Guard/OpenAI integration + # can be replayed. Additive: extends the image's built-in providers (openai, anthropic, + # ...) without overriding them (see dd-apm-test-agent vcr_proxy.PROVIDER_BASE_URLS). + "VCR_PROVIDER_MAP": "aiguard=https://app.datadoghq.com/api/v2/ai-guard", } if os.getenv("DEV_MODE") is not None: env["SNAPSHOT_CI"] = "0" diff --git a/utils/docker_fixtures/_test_clients/_test_client_framework_integrations.py b/utils/docker_fixtures/_test_clients/_test_client_framework_integrations.py index 39bfa5692a6..16c7f0b4fb6 100644 --- a/utils/docker_fixtures/_test_clients/_test_client_framework_integrations.py +++ b/utils/docker_fixtures/_test_clients/_test_client_framework_integrations.py @@ -57,6 +57,9 @@ def get_client( environment["DD_AGENT_HOST"] = test_agent.container_name environment["DD_TRACE_AGENT_PORT"] = str(test_agent.container_port) environment["FRAMEWORK_TEST_CLIENT_SERVER_PORT"] = str(container_port) + # AI Guard evaluations (when DD_AI_GUARD_ENABLED=true) are replayed by the test-agent VCR proxy. + # Harmless when AI Guard is disabled, which is the default for framework tests. + environment["DD_AI_GUARD_ENDPOINT"] = f"{environment['DD_TRACE_AGENT_URL']}/vcr/aiguard" # overwrite env with the one provided by the test environment |= library_env