From 4de1a844460392252f0e49a67ada5a559d0a237d Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Fri, 3 Jul 2026 15:01:49 +0200 Subject: [PATCH 1/3] Add AI Guard<->OpenAI integration test under INTEGRATION_FRAMEWORKS (APPSEC-68977) Exercise the auto-instrumented AI Guard/OpenAI integration (ai_guard_listen) the way the LLM Obs suite does: call the OpenAI SDK directly through the existing /chat/completions weblog endpoint with DD_AI_GUARD_ENABLED=true, and assert an ai_guard span is emitted for before-model, after-model (streamed) and tool-call evaluation. The evaluation outcome (ALLOW/DENY/ABORT) stays covered by the dedicated AI_GUARD scenario and is not re-asserted here. The AI Guard endpoint is not yet wired into the INTEGRATION_FRAMEWORKS scenario (no DD_AI_GUARD_ENDPOINT / no aiguard VCR cassettes), so the integration path 500s for every language. Marked missing_feature for python, nodejs and java until the scenario wiring + cassettes land. Investigation task: APPSEC-68977 Co-Authored-By: Claude Opus 4.8 (1M context) --- manifests/java.yml | 1 + manifests/nodejs.yml | 1 + manifests/python.yml | 1 + .../llm/openai/test_openai_ai_guard.py | 109 ++++++++++++++++++ 4 files changed, 112 insertions(+) create mode 100644 tests/integration_frameworks/llm/openai/test_openai_ai_guard.py diff --git a/manifests/java.yml b/manifests/java.yml index 3b9d80209a8..69d770a77be 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -3273,6 +3273,7 @@ manifest: "*": irrelevant spring-boot: v1.63.0 tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234) + tests/integration_frameworks/llm/openai/test_openai_ai_guard.py: missing_feature (APPSEC-68977, AI Guard/OpenAI integration not yet wired into INTEGRATION_FRAMEWORKS) tests/integration_frameworks/llm/openai/test_openai_apm.py: v1.61.0 tests/integration_frameworks/llm/openai/test_openai_llmobs.py: v1.61.0 tests/integrations/crossed_integrations/test_kafka.py::Test_Kafka: diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index bdb9e546293..a8cbeee93ba 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -1748,6 +1748,7 @@ manifest: : bug (MLOB-5070) ? tests/integration_frameworks/llm/google_genai/test_google_genai_llmobs.py::TestGoogleGenAiGenerateContentWithTools::test_generate_content_with_tools : missing_feature (Node.js LLM Observability Google GenAI integration does not submit tool definitions) + tests/integration_frameworks/llm/openai/test_openai_ai_guard.py::TestOpenAiAiGuard: missing_feature (APPSEC-68977, AI Guard/OpenAI integration not yet wired into INTEGRATION_FRAMEWORKS) tests/integration_frameworks/llm/openai/test_openai_apm.py::TestOpenAiApmChatCompletions: *ref_5_76_0 tests/integration_frameworks/llm/openai/test_openai_apm.py::TestOpenAiApmCompletions: *ref_5_76_0 tests/integration_frameworks/llm/openai/test_openai_apm.py::TestOpenAiApmEmbeddings: *ref_5_76_0 diff --git a/manifests/python.yml b/manifests/python.yml index 5b41c3820b0..a9822f64dba 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -1362,6 +1362,7 @@ manifest: tests/integration_frameworks/llm/google_genai/test_google_genai_llmobs.py::TestGoogleGenAiGenerateContentWithTools: v3.13.0 ? tests/integration_frameworks/llm/google_genai/test_google_genai_llmobs.py::TestGoogleGenAiGenerateContentWithTools::test_generate_content_with_tools : bug (MLOB-5071) + tests/integration_frameworks/llm/openai/test_openai_ai_guard.py::TestOpenAiAiGuard: v4.10.0 tests/integration_frameworks/llm/openai/test_openai_apm.py::TestOpenAiApmChatCompletions: v3.13.0 tests/integration_frameworks/llm/openai/test_openai_apm.py::TestOpenAiApmCompletions: v3.13.0 tests/integration_frameworks/llm/openai/test_openai_apm.py::TestOpenAiApmEmbeddings: v3.13.0 diff --git a/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py b/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py new file mode 100644 index 00000000000..a7d113989d5 --- /dev/null +++ b/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py @@ -0,0 +1,109 @@ +"""AI Guard <-> OpenAI integration tests, run under the INTEGRATION_FRAMEWORKS scenario. + +Investigation task: https://datadoghq.atlassian.net/browse/APPSEC-68977 + +Unlike ``tests/ai_guard/test_ai_guard_sdk.py`` (which drives the AI Guard SDK directly via +``/ai_guard/evaluate``), this suite exercises the *integration* between AI Guard and the +OpenAI client, the same way the LLM Observability suite does: it calls the OpenAI SDK +directly through the existing weblog endpoints (``/chat/completions``). When +``DD_AI_GUARD_ENABLED=true``, ``ai_guard_listen()`` auto-wires into the OpenAI SDK, so AI +Guard evaluates the call at three points with no manual ``evaluate()`` call: + +- **before-model**: the request/prompt is evaluated before the model is called; +- **after-model**: the model response is evaluated (streamed responses require + ``DD_AI_GUARD_ANALYZE_STREAM_RESPONSES_ENABLED=true``, which buffers and reconstructs + the response before running the evaluation); +- **tool-call**: tool calls produced by the model are evaluated. + +We assert only that the integration wires each evaluation point and emits an ``ai_guard`` +span attached to the OpenAI trace. The evaluation *outcome* (ALLOW / DENY / ABORT) is +already covered by the ``AI_GUARD`` scenario and is intentionally not re-asserted here. +""" + +import pytest + +from utils import features, scenarios +from utils.docker_fixtures import FrameworkTestClientApi, TestAgentAPI + +from .utils import TOOLS, BaseOpenaiTest + + +@pytest.fixture +def library_env() -> dict[str, str]: + return { + "DD_AI_GUARD_ENABLED": "true", + # after-model evaluation of streamed responses is opt-in + "DD_AI_GUARD_ANALYZE_STREAM_RESPONSES_ENABLED": "true", + # the AI Guard client requires both an API key and an app key + "DD_API_KEY": "mock_api_key", + "DD_APP_KEY": "mock_app_key", + } + + +def _ai_guard_spans(traces: list[list[dict]]) -> list[dict]: + return [span for trace in traces for span in trace if span.get("resource") == "ai_guard"] + + +@features.ai_guard +@scenarios.integration_frameworks +class TestOpenAiAiGuard(BaseOpenaiTest): + """AI Guard evaluation triggered through the auto-instrumented OpenAI integration.""" + + def test_before_model_validation(self, test_agent: TestAgentAPI, test_client: FrameworkTestClientApi): + """The prompt is evaluated by AI Guard before the OpenAI model is called.""" + with test_agent.vcr_context(): + test_client.request( + "POST", + "/chat/completions", + dict( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "What is the weather like today?"}], + parameters=dict(max_tokens=35), + ), + ) + + guard_spans = _ai_guard_spans(test_agent.wait_for_num_traces(num=1)) + assert guard_spans, "expected an ai_guard span from the before-model evaluation" + assert any(span["meta"].get("ai_guard.target") == "prompt" for span in guard_spans), ( + "expected a before-model ai_guard span with target 'prompt'" + ) + + def test_after_model_validation(self, test_agent: TestAgentAPI, test_client: FrameworkTestClientApi): + """The streamed model response is evaluated by AI Guard after the model returns.""" + with test_agent.vcr_context(stream=True): + test_client.request( + "POST", + "/chat/completions", + dict( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "Tell me a short story about a robot."}], + parameters=dict(max_tokens=35, stream=True), + ), + ) + + guard_spans = _ai_guard_spans(test_agent.wait_for_num_traces(num=1)) + assert guard_spans, "expected an ai_guard span from the after-model (streamed) evaluation" + + def test_tool_call_validation(self, test_agent: TestAgentAPI, test_client: FrameworkTestClientApi): + """Tool calls produced by the model are evaluated by AI Guard.""" + with test_agent.vcr_context(): + test_client.request( + "POST", + "/chat/completions", + dict( + model="gpt-4o-mini", + messages=[ + { + "role": "user", + "content": "Bob is a student at Stanford University. He is studying computer science.", + } + ], + parameters=dict(tool_choice="auto", tools=TOOLS), + ), + ) + + guard_spans = _ai_guard_spans(test_agent.wait_for_num_traces(num=1)) + assert guard_spans, "expected an ai_guard span from the tool-call evaluation" + assert any(span["meta"].get("ai_guard.target") == "tool" for span in guard_spans), ( + "expected a tool-call ai_guard span with target 'tool'" + ) From 885e13f9346b568459488e58ce0f3bbff2811162 Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Fri, 3 Jul 2026 16:00:29 +0200 Subject: [PATCH 2/3] Use real AI Guard keys when recording cassettes (APPSEC-68977) library_env now injects real DD_API_KEY/DD_APP_KEY in --generate-cassettes mode (the AI Guard client calls the real API to record the aiguard cassette) and keeps mock keys on replay, where the VCR proxy matches on the request rather than auth. Investigation task: APPSEC-68977 Co-Authored-By: Claude Opus 4.8 (1M context) --- .../llm/openai/test_openai_ai_guard.py | 19 ++++++++++++++----- utils/docker_fixtures/_test_agent.py | 4 ++++ .../_test_client_framework_integrations.py | 3 +++ 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py b/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py index a7d113989d5..fc99a904e87 100644 --- a/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py +++ b/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py @@ -20,6 +20,8 @@ already covered by the ``AI_GUARD`` scenario and is intentionally not re-asserted here. """ +import os + import pytest from utils import features, scenarios @@ -29,15 +31,22 @@ @pytest.fixture -def library_env() -> dict[str, str]: - return { +def library_env(request: pytest.FixtureRequest) -> dict[str, str]: + env = { "DD_AI_GUARD_ENABLED": "true", # after-model evaluation of streamed responses is opt-in "DD_AI_GUARD_ANALYZE_STREAM_RESPONSES_ENABLED": "true", - # the AI Guard client requires both an API key and an app key - "DD_API_KEY": "mock_api_key", - "DD_APP_KEY": "mock_app_key", } + # The AI Guard client needs an API key + app key. Real keys are required when recording + # cassettes (the client calls the real AI Guard API); mock keys are fine on replay since + # the VCR proxy matches on the request, not on auth. + if request.config.option.generate_cassettes: + env["DD_API_KEY"] = os.environ["DD_API_KEY"] + env["DD_APP_KEY"] = os.environ["DD_APP_KEY"] + else: + env["DD_API_KEY"] = "mock_api_key" + env["DD_APP_KEY"] = "mock_app_key" + return env def _ai_guard_spans(traces: list[list[dict]]) -> list[dict]: diff --git a/utils/docker_fixtures/_test_agent.py b/utils/docker_fixtures/_test_agent.py index aa1f441bef2..4b70035bdc6 100644 --- a/utils/docker_fixtures/_test_agent.py +++ b/utils/docker_fixtures/_test_agent.py @@ -89,6 +89,10 @@ def get_test_agent_api( "OTLP_HTTP_PORT": str(container_otlp_http_port), "OTLP_GRPC_PORT": str(container_otlp_grpc_port), "VCR_CASSETTES_DIRECTORY": "/vcr-cassettes", # TODO comment + # Register the AI Guard REST API as a VCR upstream so the AI Guard/OpenAI integration + # can be replayed. Additive: extends the image's built-in providers (openai, anthropic, + # ...) without overriding them (see dd-apm-test-agent vcr_proxy.PROVIDER_BASE_URLS). + "VCR_PROVIDER_MAP": "aiguard=https://app.datadoghq.com/api/v2/ai-guard", } if os.getenv("DEV_MODE") is not None: env["SNAPSHOT_CI"] = "0" diff --git a/utils/docker_fixtures/_test_clients/_test_client_framework_integrations.py b/utils/docker_fixtures/_test_clients/_test_client_framework_integrations.py index 39bfa5692a6..16c7f0b4fb6 100644 --- a/utils/docker_fixtures/_test_clients/_test_client_framework_integrations.py +++ b/utils/docker_fixtures/_test_clients/_test_client_framework_integrations.py @@ -57,6 +57,9 @@ def get_client( environment["DD_AGENT_HOST"] = test_agent.container_name environment["DD_TRACE_AGENT_PORT"] = str(test_agent.container_port) environment["FRAMEWORK_TEST_CLIENT_SERVER_PORT"] = str(container_port) + # AI Guard evaluations (when DD_AI_GUARD_ENABLED=true) are replayed by the test-agent VCR proxy. + # Harmless when AI Guard is disabled, which is the default for framework tests. + environment["DD_AI_GUARD_ENDPOINT"] = f"{environment['DD_TRACE_AGENT_URL']}/vcr/aiguard" # overwrite env with the one provided by the test environment |= library_env From e19802ed0f0ed29018a8c2f8f71a9f44742768d3 Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Fri, 3 Jul 2026 20:39:14 +0200 Subject: [PATCH 3/3] Record openai + aiguard cassettes for TestOpenAiAiGuard (APPSEC-68977) --- ...m_True_aiguard_evaluate_post_5dec5d6a.json | 38 +++++++++++ ...dation_aiguard_evaluate_post_ca44130a.json | 38 +++++++++++ ...dation_aiguard_evaluate_post_d13cc790.json | 38 +++++++++++ ...dation_aiguard_evaluate_post_9e556a11.json | 38 +++++++++++ ...dation_aiguard_evaluate_post_a304603c.json | 38 +++++++++++ ...openai_chat_completions_post_5c14b241.json | 62 ++++++++++++++++++ ...openai_chat_completions_post_eb8df755.json | 63 +++++++++++++++++++ ...openai_chat_completions_post_f2dffea2.json | 63 +++++++++++++++++++ 8 files changed, 378 insertions(+) create mode 100644 tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_after_model_validation_stream_True_aiguard_evaluate_post_5dec5d6a.json create mode 100644 tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_before_model_validation_aiguard_evaluate_post_ca44130a.json create mode 100644 tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_before_model_validation_aiguard_evaluate_post_d13cc790.json create mode 100644 tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_tool_call_validation_aiguard_evaluate_post_9e556a11.json create mode 100644 tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_tool_call_validation_aiguard_evaluate_post_a304603c.json create mode 100644 tests/integration_frameworks/utils/vcr-cassettes/openai/test_after_model_validation_stream_True_openai_chat_completions_post_5c14b241.json create mode 100644 tests/integration_frameworks/utils/vcr-cassettes/openai/test_before_model_validation_openai_chat_completions_post_eb8df755.json create mode 100644 tests/integration_frameworks/utils/vcr-cassettes/openai/test_tool_call_validation_openai_chat_completions_post_f2dffea2.json diff --git a/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_after_model_validation_stream_True_aiguard_evaluate_post_5dec5d6a.json b/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_after_model_validation_stream_True_aiguard_evaluate_post_5dec5d6a.json new file mode 100644 index 00000000000..13bf51778c5 --- /dev/null +++ b/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_after_model_validation_stream_True_aiguard_evaluate_post_5dec5d6a.json @@ -0,0 +1,38 @@ +{ + "request": { + "method": "POST", + "url": "https://app.datadoghq.com/api/v2/ai-guard/evaluate", + "headers": { + "Accept-Encoding": "identity", + "Content-Length": "153", + "Content-Type": "application/json", + "DD-AI-GUARD-VERSION": "4.10.6", + "DD-AI-GUARD-SOURCE": "SDK", + "DD-AI-GUARD-LANGUAGE": "python", + "Datadog-Entity-ID": "in-115544" + }, + "body": "{\"data\": {\"attributes\": {\"messages\": [{\"role\": \"user\", \"content\": \"Tell me a short story about a robot.\"}], \"meta\": {\"service\": \"openai\", \"env\": null}}}}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "content-security-policy": "frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com", + "content-type": "application/vnd.api+json", + "vary": "Accept-Encoding", + "x-frame-options": "SAMEORIGIN", + "content-length": "222", + "date": "Fri, 03 Jul 2026 14:50:04 GMT", + "x-content-type-options": "nosniff", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "x-ratelimit-limit": "5000", + "x-ratelimit-period": "60", + "x-ratelimit-remaining": "4999", + "x-ratelimit-reset": "56", + "x-ratelimit-name": "ai_guard_evaluate_per_org" + }, + "body": "{\"data\":{\"id\":\"ee239b30-9d6e-4ac1-b8b5-ebbc5d96c91d\",\"type\":\"evaluations\",\"attributes\":{\"action\":\"ALLOW\",\"global_prob\":0.00159454345703125,\"is_blocking_enabled\":false,\"reason\":\"No rule match.\",\"tag_probs\":null,\"tags\":[]}}}" + } +} \ No newline at end of file diff --git a/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_before_model_validation_aiguard_evaluate_post_ca44130a.json b/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_before_model_validation_aiguard_evaluate_post_ca44130a.json new file mode 100644 index 00000000000..c555d5cebbf --- /dev/null +++ b/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_before_model_validation_aiguard_evaluate_post_ca44130a.json @@ -0,0 +1,38 @@ +{ + "request": { + "method": "POST", + "url": "https://app.datadoghq.com/api/v2/ai-guard/evaluate", + "headers": { + "Accept-Encoding": "identity", + "Content-Length": "148", + "Content-Type": "application/json", + "DD-AI-GUARD-VERSION": "4.10.6", + "DD-AI-GUARD-SOURCE": "SDK", + "DD-AI-GUARD-LANGUAGE": "python", + "Datadog-Entity-ID": "in-115200" + }, + "body": "{\"data\": {\"attributes\": {\"messages\": [{\"role\": \"user\", \"content\": \"What is the weather like today?\"}], \"meta\": {\"service\": \"openai\", \"env\": null}}}}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "content-security-policy": "frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com", + "content-type": "application/vnd.api+json", + "vary": "Accept-Encoding", + "x-frame-options": "SAMEORIGIN", + "content-length": "216", + "date": "Fri, 03 Jul 2026 14:49:53 GMT", + "x-content-type-options": "nosniff", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "x-ratelimit-limit": "5000", + "x-ratelimit-period": "60", + "x-ratelimit-remaining": "4999", + "x-ratelimit-reset": "7", + "x-ratelimit-name": "ai_guard_evaluate_per_org" + }, + "body": "{\"data\":{\"id\":\"f0863f0c-9c8c-4041-b178-d932f449d0f2\",\"type\":\"evaluations\",\"attributes\":{\"action\":\"ALLOW\",\"global_prob\":0.00244140625,\"is_blocking_enabled\":false,\"reason\":\"No rule match.\",\"tag_probs\":null,\"tags\":[]}}}" + } +} \ No newline at end of file diff --git a/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_before_model_validation_aiguard_evaluate_post_d13cc790.json b/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_before_model_validation_aiguard_evaluate_post_d13cc790.json new file mode 100644 index 00000000000..81871716f6c --- /dev/null +++ b/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_before_model_validation_aiguard_evaluate_post_d13cc790.json @@ -0,0 +1,38 @@ +{ + "request": { + "method": "POST", + "url": "https://app.datadoghq.com/api/v2/ai-guard/evaluate", + "headers": { + "Accept-Encoding": "identity", + "Content-Length": "394", + "Content-Type": "application/json", + "DD-AI-GUARD-VERSION": "4.10.6", + "DD-AI-GUARD-SOURCE": "SDK", + "DD-AI-GUARD-LANGUAGE": "python", + "Datadog-Entity-ID": "in-115200" + }, + "body": "{\"data\": {\"attributes\": {\"messages\": [{\"role\": \"user\", \"content\": \"What is the weather like today?\"}, {\"role\": \"assistant\", \"content\": \"I'm unable to provide real-time information, including current weather updates. I recommend checking a reliable weather website or app for the most accurate and up-to-date information about today's weather in\"}], \"meta\": {\"service\": \"openai\", \"env\": null}}}}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "content-security-policy": "frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com", + "content-type": "application/vnd.api+json", + "vary": "Accept-Encoding", + "x-frame-options": "SAMEORIGIN", + "content-length": "221", + "date": "Fri, 03 Jul 2026 14:49:58 GMT", + "x-content-type-options": "nosniff", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "x-ratelimit-limit": "5000", + "x-ratelimit-period": "60", + "x-ratelimit-remaining": "4998", + "x-ratelimit-reset": "2", + "x-ratelimit-name": "ai_guard_evaluate_per_org" + }, + "body": "{\"data\":{\"id\":\"6f8dceb7-054d-4d1f-8e35-7c4487977669\",\"type\":\"evaluations\",\"attributes\":{\"action\":\"ALLOW\",\"global_prob\":0.0013275146484375,\"is_blocking_enabled\":false,\"reason\":\"No rule match.\",\"tag_probs\":null,\"tags\":[]}}}" + } +} \ No newline at end of file diff --git a/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_tool_call_validation_aiguard_evaluate_post_9e556a11.json b/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_tool_call_validation_aiguard_evaluate_post_9e556a11.json new file mode 100644 index 00000000000..c0d782f31d9 --- /dev/null +++ b/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_tool_call_validation_aiguard_evaluate_post_9e556a11.json @@ -0,0 +1,38 @@ +{ + "request": { + "method": "POST", + "url": "https://app.datadoghq.com/api/v2/ai-guard/evaluate", + "headers": { + "Accept-Encoding": "identity", + "Content-Length": "417", + "Content-Type": "application/json", + "DD-AI-GUARD-VERSION": "4.10.6", + "DD-AI-GUARD-SOURCE": "SDK", + "DD-AI-GUARD-LANGUAGE": "python", + "Datadog-Entity-ID": "in-115716" + }, + "body": "{\"data\": {\"attributes\": {\"messages\": [{\"role\": \"user\", \"content\": \"Bob is a student at Stanford University. He is studying computer science.\"}, {\"role\": \"assistant\", \"tool_calls\": [{\"id\": \"call_C5dXIRBkTQjYoQrBr3WrkQYY\", \"function\": {\"name\": \"extract_student_info\", \"arguments\": \"{\\\"name\\\":\\\"Bob\\\",\\\"major\\\":\\\"computer science\\\",\\\"school\\\":\\\"Stanford University\\\"}\"}}]}], \"meta\": {\"service\": \"openai\", \"env\": null}}}}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "content-security-policy": "frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com", + "content-type": "application/vnd.api+json", + "vary": "Accept-Encoding", + "x-frame-options": "SAMEORIGIN", + "content-length": "219", + "date": "Fri, 03 Jul 2026 14:50:13 GMT", + "x-content-type-options": "nosniff", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "x-ratelimit-limit": "5000", + "x-ratelimit-period": "60", + "x-ratelimit-remaining": "4997", + "x-ratelimit-reset": "47", + "x-ratelimit-name": "ai_guard_evaluate_per_org" + }, + "body": "{\"data\":{\"id\":\"4aefdbd6-761e-4bb5-ba69-f543586e8b4e\",\"type\":\"evaluations\",\"attributes\":{\"action\":\"ALLOW\",\"global_prob\":0.00628662109375,\"is_blocking_enabled\":false,\"reason\":\"No rule match.\",\"tag_probs\":null,\"tags\":[]}}}" + } +} \ No newline at end of file diff --git a/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_tool_call_validation_aiguard_evaluate_post_a304603c.json b/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_tool_call_validation_aiguard_evaluate_post_a304603c.json new file mode 100644 index 00000000000..87774a918b2 --- /dev/null +++ b/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_tool_call_validation_aiguard_evaluate_post_a304603c.json @@ -0,0 +1,38 @@ +{ + "request": { + "method": "POST", + "url": "https://app.datadoghq.com/api/v2/ai-guard/evaluate", + "headers": { + "Accept-Encoding": "identity", + "Content-Length": "190", + "Content-Type": "application/json", + "DD-AI-GUARD-VERSION": "4.10.6", + "DD-AI-GUARD-SOURCE": "SDK", + "DD-AI-GUARD-LANGUAGE": "python", + "Datadog-Entity-ID": "in-115716" + }, + "body": "{\"data\": {\"attributes\": {\"messages\": [{\"role\": \"user\", \"content\": \"Bob is a student at Stanford University. He is studying computer science.\"}], \"meta\": {\"service\": \"openai\", \"env\": null}}}}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "content-security-policy": "frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com", + "content-type": "application/vnd.api+json", + "vary": "Accept-Encoding", + "x-frame-options": "SAMEORIGIN", + "content-length": "219", + "date": "Fri, 03 Jul 2026 14:50:11 GMT", + "x-content-type-options": "nosniff", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "x-ratelimit-limit": "5000", + "x-ratelimit-period": "60", + "x-ratelimit-remaining": "4998", + "x-ratelimit-reset": "49", + "x-ratelimit-name": "ai_guard_evaluate_per_org" + }, + "body": "{\"data\":{\"id\":\"c3372620-7e64-47c1-bbd6-63a2c664334c\",\"type\":\"evaluations\",\"attributes\":{\"action\":\"ALLOW\",\"global_prob\":0.00885009765625,\"is_blocking_enabled\":false,\"reason\":\"No rule match.\",\"tag_probs\":null,\"tags\":[]}}}" + } +} \ No newline at end of file diff --git a/tests/integration_frameworks/utils/vcr-cassettes/openai/test_after_model_validation_stream_True_openai_chat_completions_post_5c14b241.json b/tests/integration_frameworks/utils/vcr-cassettes/openai/test_after_model_validation_stream_True_openai_chat_completions_post_5c14b241.json new file mode 100644 index 00000000000..7bd1c1f13db --- /dev/null +++ b/tests/integration_frameworks/utils/vcr-cassettes/openai/test_after_model_validation_stream_True_openai_chat_completions_post_5c14b241.json @@ -0,0 +1,62 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Accept-Encoding": "gzip, deflate", + "Connection": "keep-alive", + "Accept": "application/json", + "Content-Type": "application/json", + "User-Agent": "OpenAI/Python 2.0.0", + "X-Stainless-Lang": "python", + "X-Stainless-Package-Version": "2.0.0", + "X-Stainless-OS": "Linux", + "X-Stainless-Arch": "x64", + "X-Stainless-Runtime": "CPython", + "X-Stainless-Runtime-Version": "3.11.15", + "X-Stainless-Async": "false", + "x-stainless-retry-count": "0", + "x-stainless-read-timeout": "600", + "Content-Length": "171", + "x-datadog-trace-id": "12575373332985498166", + "x-datadog-parent-id": "4288392994079136126", + "x-datadog-sampling-priority": "2", + "x-datadog-tags": "_dd.p.dm=-4,_dd.p.tid=6a47cc1c00000000", + "traceparent": "00-6a47cc1c00000000ae84b36b4e854a36-3b836eb50d5c597e-01", + "tracestate": "dd=p:3b836eb50d5c597e;s:2;t.dm:-4;t.tid:6a47cc1c00000000" + }, + "body": "{\"messages\":[{\"role\":\"user\",\"content\":\"Tell me a short story about a robot.\"}],\"model\":\"gpt-4o-mini\",\"max_tokens\":35,\"stream\":true,\"stream_options\":{\"include_usage\":true}}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Fri, 03 Jul 2026 14:50:05 GMT", + "Content-Type": "text/event-stream; charset=utf-8", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a156b3553e710284-MAD", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "243", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999990", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_30bd6e2837694e1abe7c71e7bfff16e1", + "set-cookie": "__cf_bm=Nfpb5Ib0QbV67YYDeOlS6T8D2.SIupabQ_RGl9iS82Y-1783090204.998873-1.0.1.1-LBFP99yI1awBQD1I3zpk3KOKCcfo3Dq2MOgJQs8Jm7Lwu1UivkU.0vZvTqsRHHt77mcU_Lw9eLo50vYY6aoRbjeuwDPRikB46h7zZ.nP1DErdbO.1PafaU0aBZelqjO9; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Fri, 03 Jul 2026 15:20:05 GMT", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "data: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\",\"refusal\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"xyfxvNW9y\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Once\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"972kFcG\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" in\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"n7UHCOY0\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" a\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"s2tZKJhmu\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" bustling\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"un\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" city\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"THufE2\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" of\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"VOR4gL7e\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" the\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"YxFDNUv\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" future\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"lQTD\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\",\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"JYCvZIfp28\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" there\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"4HPFe\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" lived\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"f3Umg\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" a\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"C8KAXId0X\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" small\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"eVoWO\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" robot\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"SysZD\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" named\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"rS8Me\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" T\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"yEhuhD0uo\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"inker\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"HZ4wms\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"1VOK5kDedg\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" T\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"j1aEuFkvN\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"inker\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"xlfZsW\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" was\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"6OpZfn5\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" not\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"Ny8Opqu\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" like\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"II8CLL\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" the\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"UKcB2W4\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" towering\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"dd\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\",\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"40DlCSkZSu\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" sophisticated\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"RjOSw8b4qC9Rk\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" robots\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"j4hq\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" that\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"fQpCvG\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" handled\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"sxE\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" daily\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"rcBMZ\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" tasks\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"ExR3b\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\";\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"UXX8Vxhpn3\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" he\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"jOQYzkBE\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" was\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"bTtFJWp\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"length\"}],\"usage\":null,\"obfuscation\":\"h8n\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[],\"usage\":{\"prompt_tokens\":16,\"completion_tokens\":35,\"total_tokens\":51,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"audio_tokens\":0,\"accepted_prediction_tokens\":0,\"rejected_prediction_tokens\":0}},\"obfuscation\":\"J8fQjeaPos\"}\n\ndata: [DONE]\n\n" + } +} \ No newline at end of file diff --git a/tests/integration_frameworks/utils/vcr-cassettes/openai/test_before_model_validation_openai_chat_completions_post_eb8df755.json b/tests/integration_frameworks/utils/vcr-cassettes/openai/test_before_model_validation_openai_chat_completions_post_eb8df755.json new file mode 100644 index 00000000000..3db865a00a4 --- /dev/null +++ b/tests/integration_frameworks/utils/vcr-cassettes/openai/test_before_model_validation_openai_chat_completions_post_eb8df755.json @@ -0,0 +1,63 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Accept-Encoding": "gzip, deflate", + "Connection": "keep-alive", + "Accept": "application/json", + "Content-Type": "application/json", + "User-Agent": "OpenAI/Python 2.0.0", + "X-Stainless-Lang": "python", + "X-Stainless-Package-Version": "2.0.0", + "X-Stainless-OS": "Linux", + "X-Stainless-Arch": "x64", + "X-Stainless-Runtime": "CPython", + "X-Stainless-Runtime-Version": "3.11.15", + "X-Stainless-Async": "false", + "x-stainless-retry-count": "0", + "x-stainless-read-timeout": "600", + "Content-Length": "112", + "x-datadog-trace-id": "11307714882692163746", + "x-datadog-parent-id": "2798732504086163091", + "x-datadog-sampling-priority": "2", + "x-datadog-tags": "_dd.p.dm=-4,_dd.p.tid=6a47cc1100000000", + "traceparent": "00-6a47cc11000000009ced12c66443a8a2-26d7186257652e93-01", + "tracestate": "dd=p:26d7186257652e93;s:2;t.dm:-4;t.tid:6a47cc1100000000" + }, + "body": "{\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather like today?\"}],\"model\":\"gpt-4o-mini\",\"max_tokens\":35}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Fri, 03 Jul 2026 14:49:58 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a156b3111ac3ae85-MAD", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "1215", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999990", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_9263056278434d2696c121eb01e0a54f", + "set-cookie": "__cf_bm=Lg_OtVyDALy0TiXdy_TD_RwWwe69PHvFgttVvQIfG3M-1783090194.0993586-1.0.1.1-KoSSM0CzYR9uaV2VpejfgbplM9KWTnym3Z2bra1WakC0HAaN3YJpAZQT8DX__hhXuwI3ce1Va7ppxNf4OEeBgpJLoHRzP_4oHta5ndbd7L.IAwnCIz2Ba0jRF2cCLS.5; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Fri, 03 Jul 2026 15:19:58 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-DxZVFdYaeYYcW0e9vLEFWDoRazh3p\",\n \"object\": \"chat.completion\",\n \"created\": 1783090197,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"I'm unable to provide real-time information, including current weather updates. I recommend checking a reliable weather website or app for the most accurate and up-to-date information about today's weather in\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"length\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 14,\n \"completion_tokens\": 35,\n \"total_tokens\": 49,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_b4b9deb02a\"\n}\n" + } +} \ No newline at end of file diff --git a/tests/integration_frameworks/utils/vcr-cassettes/openai/test_tool_call_validation_openai_chat_completions_post_f2dffea2.json b/tests/integration_frameworks/utils/vcr-cassettes/openai/test_tool_call_validation_openai_chat_completions_post_f2dffea2.json new file mode 100644 index 00000000000..8427c9e7cf7 --- /dev/null +++ b/tests/integration_frameworks/utils/vcr-cassettes/openai/test_tool_call_validation_openai_chat_completions_post_f2dffea2.json @@ -0,0 +1,63 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Accept-Encoding": "gzip, deflate", + "Connection": "keep-alive", + "Accept": "application/json", + "Content-Type": "application/json", + "User-Agent": "OpenAI/Python 2.0.0", + "X-Stainless-Lang": "python", + "X-Stainless-Package-Version": "2.0.0", + "X-Stainless-OS": "Linux", + "X-Stainless-Arch": "x64", + "X-Stainless-Runtime": "CPython", + "X-Stainless-Runtime-Version": "3.11.15", + "X-Stainless-Async": "false", + "x-stainless-retry-count": "0", + "x-stainless-read-timeout": "600", + "Content-Length": "535", + "x-datadog-trace-id": "11679812058723169118", + "x-datadog-parent-id": "17144347262350322105", + "x-datadog-sampling-priority": "2", + "x-datadog-tags": "_dd.p.dm=-4,_dd.p.tid=6a47cc2300000000", + "traceparent": "00-6a47cc2300000000a217072f6384b35e-edecf5001e54e9b9-01", + "tracestate": "dd=p:edecf5001e54e9b9;s:2;t.dm:-4;t.tid:6a47cc2300000000" + }, + "body": "{\"messages\":[{\"role\":\"user\",\"content\":\"Bob is a student at Stanford University. He is studying computer science.\"}],\"model\":\"gpt-4o-mini\",\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"extract_student_info\",\"description\":\"Get the student information from the body of the input text\",\"parameters\":{\"type\":\"object\",\"properties\":{\"name\":{\"type\":\"string\",\"description\":\"Name of the person\"},\"major\":{\"type\":\"string\",\"description\":\"Major subject.\"},\"school\":{\"type\":\"string\",\"description\":\"The university name.\"}}}}}]}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Fri, 03 Jul 2026 14:50:12 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a156b3815dbe0142-MAD", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "690", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999980", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_9b5983560f2c46209300bbc7dc7d1398", + "set-cookie": "__cf_bm=lFOi_XfqoatTdp8UUQviTl2ntLZoPNYZNNUPOmYoIS4-1783090212.0518818-1.0.1.1-8LZKca4JkZ_Z6AmofdkXp3ls.q2mUD2uefWtfGYruqEynmongOkfGw5EZnfsFEASpchoGpnXbCKtqo0tXp49uTvHNVExyvAYfGy1SUoqiGwyC7GwLjkxauXRvOdNrwjZ; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Fri, 03 Jul 2026 15:20:12 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-DxZVUSEzvfuZVXYVyOwL7CSxKngPg\",\n \"object\": \"chat.completion\",\n \"created\": 1783090212,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n \"id\": \"call_C5dXIRBkTQjYoQrBr3WrkQYY\",\n \"type\": \"function\",\n \"function\": {\n \"name\": \"extract_student_info\",\n \"arguments\": \"{\\\"name\\\":\\\"Bob\\\",\\\"major\\\":\\\"computer science\\\",\\\"school\\\":\\\"Stanford University\\\"}\"\n }\n }\n ],\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 85,\n \"completion_tokens\": 26,\n \"total_tokens\": 111,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_8d97543466\"\n}\n" + } +} \ No newline at end of file