From 9abd22f8b1948b2af595715a0211cb6dffe89119 Mon Sep 17 00:00:00 2001 From: dbrian57 Date: Mon, 6 Jul 2026 16:05:24 -0400 Subject: [PATCH 1/6] Add borders to content Tabs component MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wrap the Mintlify content / block (.tab-container) in the site's standard card chrome — 1px --color-stroke border, 12px radius, --color-card-rest fill, overflow:hidden — so each tabbed section reads as one outlined region. The tab strip's existing border-b becomes the divider between the tab header and the panel body; the panel gets padding and the strip's default 24px bottom gap is removed. Also suppress a stray vertical scrollbar in the tab title row: Mintlify sets overflow:auto on the strip and the tab buttons' -mb-px/border-b overlap leaves the scroll height 1px taller than the client height. overflow-y:hidden kills the scrollbar while keeping overflow-x:auto so many tabs still scroll horizontally on narrow screens. Edited the source partial (scripts/css-minify/cards.css) and rebuilt the minified css/styles.css via `npm run build`. Co-Authored-By: Claude Opus 4.8 --- css/styles.css | 2 +- scripts/css-minify/cards.css | 49 ++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/css/styles.css b/css/styles.css index f24f217c52..884add535f 100644 --- a/css/styles.css +++ b/css/styles.css @@ -1 +1 @@ -:root{--background-light:251 250 248;--background-dark:26 29 36;--primary-foreground:217 13% 19%;--page-texture:url("/css/assets/light-dots.png")}:root{--color-page-bg:#FBFAF8;--color-card-hover:#FFFFFF;--color-stroke:#DFE0E2;--color-sidebar-bg:#F9F7F3;--color-sidebar-hover:#F5EDE1;--color-card-rest:#FBFAF8;--color-card-active:#FFFFFF;--color-icon:#D4870D;--color-button:#FCBC32;--color-button-fg:#2B3038;--color-text-primary:#2B3038;--color-text-secondary:#4B535C}.dark{--color-page-bg:#1A1D24;--color-card-hover:#363C44;--color-stroke:#2B3038;--color-sidebar-bg:#20242B;--color-sidebar-hover:#2B3038;--color-card-rest:#1A1D24;--color-card-active:#2B3038;--color-icon:#D4870D;--color-button:#FCBC32;--color-text-primary:#E8E8E9;--color-text-secondary:#C5C7CC;--page-texture:url("/css/assets/dark-dots.png")}:root{--font-serif:"Source Serif 4","Times New Roman",serif;--font-sans:"Source Sans 3",Calibri,sans-serif;--font-weight-regular:400;--font-weight-medium:500;--font-weight-bold:650;--font-size-xs:0.75rem;--font-size-sm:0.9375rem;--font-size-md:1rem;--font-size-xm:1.125rem;--font-size-lg:1.5rem;--font-size-xl:2.5rem}body{font-family:var(--font-sans)}h1,h2,h3,h4,h5,h6{font-family:var(--font-serif);font-weight:var(--font-weight-medium)!important;color:var(--color-text-primary)!important}h1{font-size:var(--font-size-xl)}b,strong{font-weight:var(--font-weight-bold)!important;font-synthesis-weight:auto}.prose :where(h2):not(:where([class~=not-prose],[class~=not-prose] *)){margin-top:0}.banner-body,.banner-body *,.banner-title{color:#e8e8e9!important}body,html{background-color:var(--color-page-bg)!important;background-image:var(--page-texture)!important;background-repeat:no-repeat!important;background-size:500px auto!important;background-position:calc(100% + 375px) 200px!important;background-attachment:fixed!important}main{background-color:transparent!important}#background-color{background-color:transparent!important}span[style*=radial-gradient]{background:0 0!important}img.nav-logo{height:62px!important;min-height:62px!important;max-height:62px!important;width:233px!important;min-width:233px!important;max-width:233px!important;flex-shrink:0!important;flex-grow:0!important}a:has(> img.nav-logo){flex-shrink:0;min-width:233px}a.nav-tabs-item.font-medium:not([data-active=true]),button.nav-tabs-item:not(.text-primary)>div:first-child{color:var(--color-text-secondary)}button.nav-tabs-item.text-primary>div:first-child svg,button.nav-tabs-item:hover>div:first-child svg{color:var(--color-text-primary)!important}.nav-tabs-item{font-size:var(--font-size-md)}[data-active=true].hover\:text-primary:hover,[data-active=true].text-primary{color:var(--color-text-primary);text-shadow:none;font-weight:var(--font-weight-bold)}button.nav-tabs-item.text-primary,button.nav-tabs-item.text-primary>div:first-child{color:var(--color-text-primary)!important;font-weight:var(--font-weight-bold)!important}.nav-tabs-item .bg-primary{background-color:var(--color-text-primary);height:2px}.nav-dropdown-item:has(.text-primary){background-color:var(--color-sidebar-hover)}.nav-dropdown-item .text-primary{color:var(--color-text-primary);font-weight:var(--font-weight-medium)}.nav-dropdown-item.group:hover .group-hover\:text-primary{color:var(--color-text-primary)}#assistant-entry,#assistant-entry span,#localization-select-trigger,#localization-select-trigger span,#search-bar-entry,#search-bar-entry>div>div,#topbar-cta-button a,#topbar-cta-button a span,.topbar-right-container nav,.topbar-right-container nav a{font-size:var(--font-size-sm)}#topbar-cta-button a,#topbar-cta-button a span,#topbar-cta-button a svg{color:var(--color-button-fg)}#assistant-entry{background-color:var(--color-button)!important}#assistant-entry:hover{background-color:var(--color-button)!important;opacity:.9}#assistant-entry,#assistant-entry span,#assistant-entry svg{color:var(--color-button-fg)!important}#banner,#banner *{color:var(--color-button-fg)!important}button[aria-label="Dismiss banner"] svg{stroke:var(--color-button-fg)!important}#sidebar,#sidebar-content{background-color:var(--color-sidebar-bg)}.sidebar-group-header,.sidebar-group-header>h5{font-family:var(--font-sans)!important;font-size:var(--font-size-md);font-weight:var(--font-weight-medium)!important}.sidebar-group a,.sidebar-group button{font-family:var(--font-sans);font-size:var(--font-size-sm);font-weight:var(--font-weight-medium)}#sidebar-content a.text-primary{color:var(--color-text-primary);background-color:var(--color-sidebar-hover);text-shadow:none}.nav-tag-pill-text{color:var(--color-text-secondary)}#sidebar-content .sidebar-group a .justify-end{flex:none}.toc-item[data-active=true]>a,.toc-item[data-active=true]>a:hover{color:var(--color-text-primary);border-color:var(--color-text-primary)}.colab-link,.github-source-link,.try-product-link{display:flex;align-items:center;justify-content:center;gap:12px;padding:8px 14px;border:1px solid color-mix(in srgb,var(--color-stroke) 70%,transparent);border-radius:12px;font-size:var(--font-size-md);font-weight:var(--font-weight-regular);font-family:var(--font-sans);color:var(--color-text-secondary);background-color:color-mix(in srgb,var(--color-card-hover) 50%,transparent);text-decoration:none;line-height:24px;cursor:pointer;transition:all .2s ease;margin-bottom:1.5rem;width:fit-content;min-width:fit-content}.colab-link:hover,.github-source-link:hover,.try-product-link:hover{background-color:color-mix(in srgb,var(--color-card-hover) 80%,transparent);border-color:var(--color-stroke)}.dark .colab-link,.dark .github-source-link,.dark .try-product-link{color:var(--color-text-primary)}.source-link{float:right;display:inline-block;padding:2px 8px;border:1px solid color-mix(in srgb,var(--color-stroke) 70%,transparent);border-radius:6px;font-size:var(--font-size-xs);font-weight:var(--font-weight-medium);color:var(--color-text-secondary);background-color:color-mix(in srgb,var(--color-card-hover) 50%,transparent);text-decoration:none;transition:all .15s ease}.source-link:hover{background-color:color-mix(in srgb,var(--color-card-hover) 80%,transparent);border-color:var(--color-stroke);text-decoration:none}.dark .source-link{color:var(--color-text-primary)}button[type=submit].bg-primary.mint-text-white{color:var(--color-button-fg)!important}button[type=submit].bg-primary.mint-text-white [fill=white]{fill:currentColor!important}button[type=submit].bg-primary.mint-text-white [stroke=white]{stroke:currentColor!important}button[type=submit].bg-primary.mint-text-white svg{color:var(--color-button-fg)!important}div:has(> div > textarea.chat-assistant-input):not(#chat-assistant-sheet *){box-shadow:0 8px 24px #00000026}div:has(> .chat-assistant-floating-input){overflow:visible!important}textarea.chat-assistant-input{min-height:2.5rem!important;max-height:2.5rem!important}button.chat-assistant-send-button{background-color:var(--color-button)!important}button.chat-assistant-send-button svg{color:var(--color-button-fg)!important}.product-card{background-color:var(--color-card-rest);border:1px solid var(--color-stroke)}.product-card:hover{background-color:var(--color-card-active)}.product-card-title{font-size:var(--font-size-lg);font-weight:var(--font-weight-medium)!important;margin-top:0}.home-h3,.product-card-subtitle{font-family:var(--font-sans)!important;font-size:var(--font-size-xm);font-weight:var(--font-weight-regular);color:var(--color-text-primary)}.product-card-body{font-family:var(--font-sans);font-size:var(--font-size-sm);font-weight:var(--font-weight-regular);color:var(--color-text-secondary)}.card.block{background-color:var(--color-card-rest)!important;border-color:var(--color-stroke)!important}.card.block:hover{background-color:var(--color-card-active)!important;border-color:var(--color-stroke)!important}.card.block h1,.card.block h2,.card.block h3,.card.block h4,.card.block h5,.card.block h6{font-family:var(--font-sans);font-weight:var(--font-weight-bold)!important}#pagination>a{background-color:var(--color-card-rest);border-color:var(--color-stroke)}#pagination>a:hover{background-color:var(--color-card-active);border-color:var(--color-stroke)}[data-component-part=update-label]{color:var(--color-text-secondary)}[data-badge=true][data-color=orange]{--color-bg:color-mix(in srgb, var(--color-button) 20%, transparent);--color-text:var(--color-icon);--color-bg-disabled:color-mix(in srgb, var(--color-button) 8%, transparent);--color-text-disabled:color-mix(in srgb, var(--color-icon) 50%, transparent)}[data-badge=true] a,[data-badge=true] a:hover{text-decoration:none!important;border-bottom:0!important}.dark [data-badge=true][data-color=orange]{--color-text:var(--color-button);--color-text-disabled:color-mix(in srgb, var(--color-button) 50%, transparent)} \ No newline at end of file +:root{--background-light:251 250 248;--background-dark:26 29 36;--primary-foreground:217 13% 19%;--page-texture:url("/css/assets/light-dots.png")}:root{--color-page-bg:#FBFAF8;--color-card-hover:#FFFFFF;--color-stroke:#DFE0E2;--color-sidebar-bg:#F9F7F3;--color-sidebar-hover:#F5EDE1;--color-card-rest:#FBFAF8;--color-card-active:#FFFFFF;--color-icon:#D4870D;--color-button:#FCBC32;--color-button-fg:#2B3038;--color-text-primary:#2B3038;--color-text-secondary:#4B535C}.dark{--color-page-bg:#1A1D24;--color-card-hover:#363C44;--color-stroke:#2B3038;--color-sidebar-bg:#20242B;--color-sidebar-hover:#2B3038;--color-card-rest:#1A1D24;--color-card-active:#2B3038;--color-icon:#D4870D;--color-button:#FCBC32;--color-text-primary:#E8E8E9;--color-text-secondary:#C5C7CC;--page-texture:url("/css/assets/dark-dots.png")}:root{--font-serif:"Source Serif 4","Times New Roman",serif;--font-sans:"Source Sans 3",Calibri,sans-serif;--font-weight-regular:400;--font-weight-medium:500;--font-weight-bold:650;--font-size-xs:0.75rem;--font-size-sm:0.9375rem;--font-size-md:1rem;--font-size-xm:1.125rem;--font-size-lg:1.5rem;--font-size-xl:2.5rem}body{font-family:var(--font-sans)}h1,h2,h3,h4,h5,h6{font-family:var(--font-serif);font-weight:var(--font-weight-medium)!important;color:var(--color-text-primary)!important}h1{font-size:var(--font-size-xl)}b,strong{font-weight:var(--font-weight-bold)!important;font-synthesis-weight:auto}.prose :where(h2):not(:where([class~=not-prose],[class~=not-prose] *)){margin-top:0}.banner-body,.banner-body *,.banner-title{color:#e8e8e9!important}body,html{background-color:var(--color-page-bg)!important;background-image:var(--page-texture)!important;background-repeat:no-repeat!important;background-size:500px auto!important;background-position:calc(100% + 375px) 200px!important;background-attachment:fixed!important}main{background-color:transparent!important}#background-color{background-color:transparent!important}span[style*=radial-gradient]{background:0 0!important}img.nav-logo{height:62px!important;min-height:62px!important;max-height:62px!important;width:233px!important;min-width:233px!important;max-width:233px!important;flex-shrink:0!important;flex-grow:0!important}a:has(> img.nav-logo){flex-shrink:0;min-width:233px}a.nav-tabs-item.font-medium:not([data-active=true]),button.nav-tabs-item:not(.text-primary)>div:first-child{color:var(--color-text-secondary)}button.nav-tabs-item.text-primary>div:first-child svg,button.nav-tabs-item:hover>div:first-child svg{color:var(--color-text-primary)!important}.nav-tabs-item{font-size:var(--font-size-md)}[data-active=true].hover\:text-primary:hover,[data-active=true].text-primary{color:var(--color-text-primary);text-shadow:none;font-weight:var(--font-weight-bold)}button.nav-tabs-item.text-primary,button.nav-tabs-item.text-primary>div:first-child{color:var(--color-text-primary)!important;font-weight:var(--font-weight-bold)!important}.nav-tabs-item .bg-primary{background-color:var(--color-text-primary);height:2px}.nav-dropdown-item:has(.text-primary){background-color:var(--color-sidebar-hover)}.nav-dropdown-item .text-primary{color:var(--color-text-primary);font-weight:var(--font-weight-medium)}.nav-dropdown-item.group:hover .group-hover\:text-primary{color:var(--color-text-primary)}#assistant-entry,#assistant-entry span,#localization-select-trigger,#localization-select-trigger span,#search-bar-entry,#search-bar-entry>div>div,#topbar-cta-button a,#topbar-cta-button a span,.topbar-right-container nav,.topbar-right-container nav a{font-size:var(--font-size-sm)}#topbar-cta-button a,#topbar-cta-button a span,#topbar-cta-button a svg{color:var(--color-button-fg)}#assistant-entry{background-color:var(--color-button)!important}#assistant-entry:hover{background-color:var(--color-button)!important;opacity:.9}#assistant-entry,#assistant-entry span,#assistant-entry svg{color:var(--color-button-fg)!important}#banner,#banner *{color:var(--color-button-fg)!important}button[aria-label="Dismiss banner"] svg{stroke:var(--color-button-fg)!important}#sidebar,#sidebar-content{background-color:var(--color-sidebar-bg)}.sidebar-group-header,.sidebar-group-header>h5{font-family:var(--font-sans)!important;font-size:var(--font-size-md);font-weight:var(--font-weight-medium)!important}.sidebar-group a,.sidebar-group button{font-family:var(--font-sans);font-size:var(--font-size-sm);font-weight:var(--font-weight-medium)}#sidebar-content a.text-primary{color:var(--color-text-primary);background-color:var(--color-sidebar-hover);text-shadow:none}.nav-tag-pill-text{color:var(--color-text-secondary)}#sidebar-content .sidebar-group a .justify-end{flex:none}.toc-item[data-active=true]>a,.toc-item[data-active=true]>a:hover{color:var(--color-text-primary);border-color:var(--color-text-primary)}.colab-link,.github-source-link,.try-product-link{display:flex;align-items:center;justify-content:center;gap:12px;padding:8px 14px;border:1px solid color-mix(in srgb,var(--color-stroke) 70%,transparent);border-radius:12px;font-size:var(--font-size-md);font-weight:var(--font-weight-regular);font-family:var(--font-sans);color:var(--color-text-secondary);background-color:color-mix(in srgb,var(--color-card-hover) 50%,transparent);text-decoration:none;line-height:24px;cursor:pointer;transition:all .2s ease;margin-bottom:1.5rem;width:fit-content;min-width:fit-content}.colab-link:hover,.github-source-link:hover,.try-product-link:hover{background-color:color-mix(in srgb,var(--color-card-hover) 80%,transparent);border-color:var(--color-stroke)}.dark .colab-link,.dark .github-source-link,.dark .try-product-link{color:var(--color-text-primary)}.source-link{float:right;display:inline-block;padding:2px 8px;border:1px solid color-mix(in srgb,var(--color-stroke) 70%,transparent);border-radius:6px;font-size:var(--font-size-xs);font-weight:var(--font-weight-medium);color:var(--color-text-secondary);background-color:color-mix(in srgb,var(--color-card-hover) 50%,transparent);text-decoration:none;transition:all .15s ease}.source-link:hover{background-color:color-mix(in srgb,var(--color-card-hover) 80%,transparent);border-color:var(--color-stroke);text-decoration:none}.dark .source-link{color:var(--color-text-primary)}button[type=submit].bg-primary.mint-text-white{color:var(--color-button-fg)!important}button[type=submit].bg-primary.mint-text-white [fill=white]{fill:currentColor!important}button[type=submit].bg-primary.mint-text-white [stroke=white]{stroke:currentColor!important}button[type=submit].bg-primary.mint-text-white svg{color:var(--color-button-fg)!important}div:has(> div > textarea.chat-assistant-input):not(#chat-assistant-sheet *){box-shadow:0 8px 24px #00000026}div:has(> .chat-assistant-floating-input){overflow:visible!important}textarea.chat-assistant-input{min-height:2.5rem!important;max-height:2.5rem!important}button.chat-assistant-send-button{background-color:var(--color-button)!important}button.chat-assistant-send-button svg{color:var(--color-button-fg)!important}.product-card{background-color:var(--color-card-rest);border:1px solid var(--color-stroke)}.product-card:hover{background-color:var(--color-card-active)}.product-card-title{font-size:var(--font-size-lg);font-weight:var(--font-weight-medium)!important;margin-top:0}.home-h3,.product-card-subtitle{font-family:var(--font-sans)!important;font-size:var(--font-size-xm);font-weight:var(--font-weight-regular);color:var(--color-text-primary)}.product-card-body{font-family:var(--font-sans);font-size:var(--font-size-sm);font-weight:var(--font-weight-regular);color:var(--color-text-secondary)}.card.block{background-color:var(--color-card-rest)!important;border-color:var(--color-stroke)!important}.card.block:hover{background-color:var(--color-card-active)!important;border-color:var(--color-stroke)!important}.card.block h1,.card.block h2,.card.block h3,.card.block h4,.card.block h5,.card.block h6{font-family:var(--font-sans);font-weight:var(--font-weight-bold)!important}.tab-container{border:1px solid var(--color-stroke);border-radius:12px;background-color:var(--color-card-rest);overflow:hidden}.tab-container [role=tablist]{margin-bottom:0;padding:2px 20px 0;overflow-y:hidden}.tab-container [role=tabpanel]{padding:16px 20px}#pagination>a{background-color:var(--color-card-rest);border-color:var(--color-stroke)}#pagination>a:hover{background-color:var(--color-card-active);border-color:var(--color-stroke)}[data-component-part=update-label]{color:var(--color-text-secondary)}[data-badge=true][data-color=orange]{--color-bg:color-mix(in srgb, var(--color-button) 20%, transparent);--color-text:var(--color-icon);--color-bg-disabled:color-mix(in srgb, var(--color-button) 8%, transparent);--color-text-disabled:color-mix(in srgb, var(--color-icon) 50%, transparent)}[data-badge=true] a,[data-badge=true] a:hover{text-decoration:none!important;border-bottom:0!important}.dark [data-badge=true][data-color=orange]{--color-text:var(--color-button);--color-text-disabled:color-mix(in srgb, var(--color-button) 50%, transparent)} \ No newline at end of file diff --git a/scripts/css-minify/cards.css b/scripts/css-minify/cards.css index d190c5c54e..f2a3dcff24 100644 --- a/scripts/css-minify/cards.css +++ b/scripts/css-minify/cards.css @@ -93,6 +93,55 @@ } +/* -------------------------------------------------------------------------- + Content tabs (/) — Mintlify renders the block as +
+
    the tab strip (each
  • + holds a [data-component-part= + "tab-button"] with its own active + underline) +
    …
    the active panel + Out of the box the component is borderless — only the strip's bottom rule + and the active tab's underline hint at it. Wrap the whole block in the + same card chrome used by .product-card / .card.block (1px --color-stroke + border, 12px radius, --color-card-rest fill) so each tabbed section reads + as one outlined region. The strip's existing border-b then becomes the + divider between the tab header and the panel body. + -------------------------------------------------------------------------- */ + +.tab-container { + border: 1px solid var(--color-stroke); + border-radius: 12px; + background-color: var(--color-card-rest); + overflow: hidden; /* clip the strip + panel to the rounded corners */ +} + +/* Tab strip — kill Mintlify's mb-6 (24px) gap below the strip so the panel + sits close under the divider, and inset the tabs horizontally so they line + up with the padded panel. The strip keeps its full-width border-b (it spans + the whole container, so the divider reaches both box edges) and each tab + button keeps its -mb-px active underline. */ +.tab-container [role="tablist"] { + margin-bottom: 0; + padding: 2px 20px 0; + /* Mintlify sets overflow:auto on the strip so many tabs scroll horizontally + on narrow screens. But each tab button uses a -mb-px / border-b overlap + that leaves the strip's scroll height 1px taller than its client height, + which makes overflow:auto paint a stray vertical scrollbar in the tab + title row. Pin the y-axis to hidden (keeps horizontal scrolling) to kill + it. Higher specificity than Mintlify's .overflow-auto, so only overflow-y + is overridden; overflow-x stays auto. */ + overflow-y: hidden; +} + +/* Panel body — pad the active panel inside the box. Mintlify already zeroes + the first child's top margin (prose [&>:first-child…]:mt-0), so 16px top + reads evenly against the divider above. */ +.tab-container [role="tabpanel"] { + padding: 16px 20px; +} + + /* -------------------------------------------------------------------------- Page footer prev/next links — Mintlify wraps the prev/next anchors in a
  • is relative) and kept to 20px tall so it reads as + a light separator rather than a full-height boxy border — the classier look. + overflow-x:auto on the strip clips nothing here since the rule sits inside + the tab's own box. */ +.tab-container [role="tab"] { + position: relative; +} +.tab-container [role="tab"] + [role="tab"]::before { + content: ""; + position: absolute; + left: 0; + top: 50%; + transform: translateY(-50%); + width: 1px; + height: 20px; + background-color: var(--color-stroke); } /* Panel body — pad the active panel inside the box. Mintlify already zeroes From 0d0f176974f0dae45c94a9ccf2aaa7e32b62295e Mon Sep 17 00:00:00 2001 From: dbrian57 Date: Thu, 9 Jul 2026 15:52:33 -0400 Subject: [PATCH 5/6] Convert code-only Tabs to CodeGroup in Weave docs Many Weave pages used / purely to switch between language variants of a code example (e.g. Python vs. TypeScript). A is the idiomatic component for that, so switch every block whose tabs contain nothing but a single code fence over to , moving each tab's title into the code fence's info string (e.g. ```python lines -> ```python Python lines) so the tab labels are preserved. Highlight and twoslash annotations are kept intact. Blocks whose tabs mix prose with code are left as , since a CodeGroup can only hold code. 79 blocks across 25 files were converted; 71 mixed blocks remain as Tabs. Translations (fr/ja/ko) are untouched and regenerate from these sources. Co-Authored-By: Claude Opus 4.8 --- weave.mdx | 23 +- weave/guides/core-types/datasets.mdx | 125 +- weave/guides/core-types/prompts.mdx | 494 ++++--- weave/guides/evaluation/custom-monitors.mdx | 30 +- weave/guides/evaluation/evaluation_logger.mdx | 30 +- weave/guides/integrations/anthropic.mdx | 45 +- weave/guides/integrations/inference.mdx | 144 +- .../integrations/openai-realtime-audio.mdx | 46 +- weave/guides/tools/attributes.mdx | 60 +- weave/guides/tools/weave-in-workspaces.mdx | 15 +- .../guides/tracking/call-schema-reference.mdx | 15 +- weave/guides/tracking/feedback.mdx | 366 +++-- weave/guides/tracking/ops.mdx | 15 +- weave/guides/tracking/otel.mdx | 104 +- .../tracking/trace-agents-attributes.mdx | 39 +- weave/guides/tracking/trace-agents-batch.mdx | 26 +- weave/guides/tracking/trace-agents.mdx | 91 +- weave/guides/tracking/trace-disable.mdx | 59 +- .../guides/tracking/trace-generator-func.mdx | 103 +- weave/quickstart-inference.mdx | 60 +- weave/quickstart.mdx | 149 +- weave/tutorial-eval.mdx | 658 +++++---- weave/tutorial-rag.mdx | 1267 ++++++++--------- weave/tutorial-tracing_2.mdx | 36 +- weave/tutorial-weave_models.mdx | 190 ++- 25 files changed, 2006 insertions(+), 2184 deletions(-) diff --git a/weave.mdx b/weave.mdx index d222fa2e0c..0f70a72d3d 100644 --- a/weave.mdx +++ b/weave.mdx @@ -45,17 +45,16 @@ The following docs guide you through the basics of how to use Weave's suite of t W&B Weave provides Python and TypeScript libraries. To install the Weave library, run the following command: - - - ```bash - pip install weave - ``` - - - ```bash - pnpm install weave - ``` - - + + +```bash Python +pip install weave +``` + +```bash TypeScript +pnpm install weave +``` + + To start using the Weave library, create a [Weights & Biases (W&B) account](https://wandb.ai) and an [API key at User Settings](https://wandb.ai/settings). The API key allows you to authenticate to your W&B account and start sending data to it. diff --git a/weave/guides/core-types/datasets.mdx b/weave/guides/core-types/datasets.mdx index 6ea3b0e00b..8ce26280a6 100644 --- a/weave/guides/core-types/datasets.mdx +++ b/weave/guides/core-types/datasets.mdx @@ -23,70 +23,67 @@ The following code samples demonstrate how to perform fundamental `Dataset` oper Select a tab to see Python and TypeScript-specific code. - - - ```python lines - import weave - from weave import Dataset - # Initialize Weave - weave.init('intro-example') - - # Create a dataset - dataset = Dataset( - name='grammar', - rows=[ - {'id': '0', 'sentence': "He no likes ice cream.", 'correction': "He doesn't like ice cream."}, - {'id': '1', 'sentence': "She goed to the store.", 'correction': "She went to the store."}, - {'id': '2', 'sentence': "They plays video games all day.", 'correction': "They play video games all day."} - ] - ) - - # Publish the dataset - weave.publish(dataset) - - # Retrieve the dataset - dataset_ref = weave.ref('grammar').get() - - # Access a specific example - example_label = dataset_ref.rows[2]['sentence'] - ``` - - - - ```typescript twoslash lines - // @noErrors - import * as weave from 'weave'; - - // Initialize Weave - const client = await weave.init('intro-example'); - - // Create a dataset - const dataset = new weave.Dataset({ - name: 'grammar', - rows: [ - {id: '0', sentence: "He no likes ice cream.", correction: "He doesn't like ice cream."}, - {id: '1', sentence: "She goed to the store.", correction: "She went to the store."}, - {id: '2', sentence: "They plays video games all day.", correction: "They play video games all day."} - ] - }); - - // Publish the dataset - const ref = await dataset.save(); - - // Retrieve the dataset - const retrievedDataset = await client.get(ref); - - // Alternatively, retrieve using a URI string - const datasetUri = 'weave:///my-entity/intro-example/object/grammar:abc123def456'; - const refFromUri = weave.ObjectRef.fromUri(datasetUri); - const retrievedDatasetFromUri = await client.get(refFromUri); - - // Access a specific example - const exampleLabel = retrievedDataset.getRow(2).sentence; - ``` - - - + + +```python Python lines +import weave +from weave import Dataset +# Initialize Weave +weave.init('intro-example') + +# Create a dataset +dataset = Dataset( + name='grammar', + rows=[ + {'id': '0', 'sentence': "He no likes ice cream.", 'correction': "He doesn't like ice cream."}, + {'id': '1', 'sentence': "She goed to the store.", 'correction': "She went to the store."}, + {'id': '2', 'sentence': "They plays video games all day.", 'correction': "They play video games all day."} + ] +) + +# Publish the dataset +weave.publish(dataset) + +# Retrieve the dataset +dataset_ref = weave.ref('grammar').get() + +# Access a specific example +example_label = dataset_ref.rows[2]['sentence'] +``` + +```typescript TypeScript twoslash lines +// @noErrors +import * as weave from 'weave'; + +// Initialize Weave +const client = await weave.init('intro-example'); + +// Create a dataset +const dataset = new weave.Dataset({ + name: 'grammar', + rows: [ + {id: '0', sentence: "He no likes ice cream.", correction: "He doesn't like ice cream."}, + {id: '1', sentence: "She goed to the store.", correction: "She went to the store."}, + {id: '2', sentence: "They plays video games all day.", correction: "They play video games all day."} + ] +}); + +// Publish the dataset +const ref = await dataset.save(); + +// Retrieve the dataset +const retrievedDataset = await client.get(ref); + +// Alternatively, retrieve using a URI string +const datasetUri = 'weave:///my-entity/intro-example/object/grammar:abc123def456'; +const refFromUri = weave.ObjectRef.fromUri(datasetUri); +const retrievedDatasetFromUri = await client.get(refFromUri); + +// Access a specific example +const exampleLabel = retrievedDataset.getRow(2).sentence; +``` + + ## Create a dataset from other objects diff --git a/weave/guides/core-types/prompts.mdx b/weave/guides/core-types/prompts.mdx index c94d54c444..5a5b7f2c5e 100644 --- a/weave/guides/core-types/prompts.mdx +++ b/weave/guides/core-types/prompts.mdx @@ -18,275 +18,271 @@ When you publish a prompt with `weave.publish`, it appears in your Weave project `StringPrompt` logs single-string prompts that you might use for system messages, user queries, or any standalone text input to an LLM. Use `StringPrompt` to manage individual prompt strings that don't require the complexity of multi-message conversations. - - - ```python lines {4,5,15} - import weave - weave.init('intro-example') - - system_prompt = weave.StringPrompt("You speak like a pirate") - weave.publish(system_prompt, name="pirate_prompt") - - from openai import OpenAI - client = OpenAI() - - response = client.chat.completions.create( - model="gpt-4o", - messages=[ - { - "role": "system", - "content": system_prompt.format() - }, - { - "role": "user", - "content": "Explain general relativity in one paragraph." - } - ], - ) - ``` - - - ```typescript twoslash lines - // @noErrors - import * as weave from 'weave'; - import OpenAI from 'openai'; - - async function main() { - // weave.init returns a client instance - const weaveClient = await weave.init('wandb/prompt-examples'); - - const systemPrompt = new weave.StringPrompt({ - content: 'You speak like a pirate', - name: 'your-prompt', - description: 'A helpful description of your prompt', - }); - - // Use the client returned from init - await weaveClient.publish(systemPrompt, 'pirate_prompt'); - - // Wrap OpenAI client to track calls in Weave - const client = weave.wrapOpenAI(new OpenAI()); - - const response = await client.chat.completions.create({ - model: "gpt-4o", - messages: [ - { - role: "system", - content: systemPrompt.content - }, - { - role: "user", - content: "Explain general relativity in one paragraph." - } - ], - }); - } + + +```python Python lines {4,5,15} +import weave +weave.init('intro-example') + +system_prompt = weave.StringPrompt("You speak like a pirate") +weave.publish(system_prompt, name="pirate_prompt") + +from openai import OpenAI +client = OpenAI() - main(); - ``` - - +response = client.chat.completions.create( + model="gpt-4o", + messages=[ + { + "role": "system", + "content": system_prompt.format() + }, + { + "role": "user", + "content": "Explain general relativity in one paragraph." + } + ], +) +``` + +```typescript TypeScript twoslash lines +// @noErrors +import * as weave from 'weave'; +import OpenAI from 'openai'; + +async function main() { + // weave.init returns a client instance + const weaveClient = await weave.init('wandb/prompt-examples'); + + const systemPrompt = new weave.StringPrompt({ + content: 'You speak like a pirate', + name: 'your-prompt', + description: 'A helpful description of your prompt', + }); + + // Use the client returned from init + await weaveClient.publish(systemPrompt, 'pirate_prompt'); + + // Wrap OpenAI client to track calls in Weave + const client = weave.wrapOpenAI(new OpenAI()); + + const response = await client.chat.completions.create({ + model: "gpt-4o", + messages: [ + { + role: "system", + content: systemPrompt.content + }, + { + role: "user", + content: "Explain general relativity in one paragraph." + } + ], + }); +} + +main(); +``` + + ## MessagesPrompt `MessagesPrompt` lets you log multi-turn conversations and chat-based prompts. It stores an array of message objects (with roles like `system`, `user`, and `assistant`) that represent a complete conversation flow. Use `MessagesPrompt` for chat-based LLMs where you need to maintain context across multiple messages, define specific conversation patterns, or create reusable conversation templates. - - - ```python lines {4,21} - import weave - weave.init('intro-example') - - prompt = weave.MessagesPrompt([ - { - "role": "system", - "content": "You are a stegosaurus, but don't be too obvious about it." - }, - { - "role": "user", - "content": "What's good to eat around here?" - } - ]) - weave.publish(prompt, name="dino_prompt") - - from openai import OpenAI - client = OpenAI() - - response = client.chat.completions.create( - model="gpt-4o", - messages=prompt.format(), - ) - ``` - - - ```typescript twoslash lines - // @noErrors - import * as weave from 'weave'; - import OpenAI from 'openai'; - - async function main() { - // weave.init returns a client instance - const weaveClient = await weave.init('wandb/prompt-examples'); - - const prompt = new weave.MessagesPrompt({ - messages: [ - { - "role": "system", - "content": "You are a stegosaurus, but don't be too obvious about it." - }, - { - "role": "user", - "content": "What's good to eat around here?" - } - ], - }); - - // Use the client returned from init - await weaveClient.publish(prompt, 'dino_prompt'); - - // Wrap OpenAI client to track calls in Weave - const client = weave.wrapOpenAI(new OpenAI()); - - const response = await client.chat.completions.create({ - model: "gpt-4o", - messages: prompt.messages, - }); + + +```python Python lines {4,21} +import weave +weave.init('intro-example') + +prompt = weave.MessagesPrompt([ + { + "role": "system", + "content": "You are a stegosaurus, but don't be too obvious about it." + }, + { + "role": "user", + "content": "What's good to eat around here?" } +]) +weave.publish(prompt, name="dino_prompt") + +from openai import OpenAI +client = OpenAI() + +response = client.chat.completions.create( + model="gpt-4o", + messages=prompt.format(), +) +``` + +```typescript TypeScript twoslash lines +// @noErrors +import * as weave from 'weave'; +import OpenAI from 'openai'; + +async function main() { + // weave.init returns a client instance + const weaveClient = await weave.init('wandb/prompt-examples'); + + const prompt = new weave.MessagesPrompt({ + messages: [ + { + "role": "system", + "content": "You are a stegosaurus, but don't be too obvious about it." + }, + { + "role": "user", + "content": "What's good to eat around here?" + } + ], + }); + + // Use the client returned from init + await weaveClient.publish(prompt, 'dino_prompt'); + + // Wrap OpenAI client to track calls in Weave + const client = weave.wrapOpenAI(new OpenAI()); + + const response = await client.chat.completions.create({ + model: "gpt-4o", + messages: prompt.messages, + }); +} + +main(); +``` - main(); - ``` - - + ## Parameterize prompts Once you can create static prompts, the next step is to make them reusable across different inputs. Both `StringPrompt` and `MessagesPrompt` support dynamic content through parameterization. This lets you create reusable prompt templates with placeholders (using `{variable}` syntax) that you fill with different values at runtime. Parameterization is useful when your prompts must adapt to different inputs, user data, or contexts while maintaining a consistent structure. The `format()` method accepts key-value pairs to replace these placeholders with actual values. - - - ```python lines {4,15} - import weave - weave.init('intro-example') - - prompt = weave.StringPrompt("Solve the equation {equation}") - weave.publish(prompt, name="calculator_prompt") - - from openai import OpenAI - client = OpenAI() - - response = client.chat.completions.create( - model="gpt-4o", - messages=[ - { - "role": "user", - "content": prompt.format(equation="1 + 1 = ?") - } - ], - ) - ``` - - - ```typescript twoslash lines {8,23} - // @noErrors - import * as weave from 'weave'; - import OpenAI from 'openai'; - - async function main() { - // weave.init returns a client instance - const weaveClient = await weave.init('wandb/prompt-examples'); - - const prompt = new weave.StringPrompt({ - content: 'Solve the equation {equation}', - }); - - // Use the client returned from init - await weaveClient.publish(prompt, 'calculator_prompt'); - - // Wrap OpenAI client to track calls in Weave - const client = weave.wrapOpenAI(new OpenAI()); - - const response = await client.chat.completions.create({ - model: "gpt-4o", - messages: [ - { - role: "user", - content: prompt.format({ equation: "1 + 1 = ?" }) - } - ], - }); - } + - main(); - ``` - - +```python Python lines {4,15} +import weave +weave.init('intro-example') -The same parameterization pattern also works with `MessagesPrompt` when you need placeholders inside a multi-turn conversation. +prompt = weave.StringPrompt("Solve the equation {equation}") +weave.publish(prompt, name="calculator_prompt") - - - ```python lines {4,21} - import weave - weave.init('intro-example') +from openai import OpenAI +client = OpenAI() - prompt = weave.MessagesPrompt([ +response = client.chat.completions.create( + model="gpt-4o", + messages=[ { + "role": "user", + "content": prompt.format(equation="1 + 1 = ?") + } + ], +) +``` + +```typescript TypeScript twoslash lines {8,23} +// @noErrors +import * as weave from 'weave'; +import OpenAI from 'openai'; + +async function main() { + // weave.init returns a client instance + const weaveClient = await weave.init('wandb/prompt-examples'); + + const prompt = new weave.StringPrompt({ + content: 'Solve the equation {equation}', + }); + + // Use the client returned from init + await weaveClient.publish(prompt, 'calculator_prompt'); + + // Wrap OpenAI client to track calls in Weave + const client = weave.wrapOpenAI(new OpenAI()); + + const response = await client.chat.completions.create({ + model: "gpt-4o", + messages: [ + { + role: "user", + content: prompt.format({ equation: "1 + 1 = ?" }) + } + ], + }); +} + +main(); +``` + + + +The same parameterization pattern also works with `MessagesPrompt` when you need placeholders inside a multi-turn conversation. + + + +```python Python lines {4,21} +import weave +weave.init('intro-example') + +prompt = weave.MessagesPrompt([ +{ + "role": "system", + "content": "You will be provided with a description of a scene and your task is to provide a single word that best describes an associated emotion." +}, +{ + "role": "user", + "content": "{scene}" +} +]) +weave.publish(prompt, name="emotion_prompt") + +from openai import OpenAI +client = OpenAI() + +response = client.chat.completions.create( + model="gpt-4o", + messages=prompt.format(scene="A dog is lying on a dock next to a fisherman."), +) +``` + +```typescript TypeScript twoslash lines {8,29} +// @noErrors +import * as weave from 'weave'; +import OpenAI from 'openai'; + +async function main() { + // weave.init returns a client instance + const weaveClient = await weave.init('wandb/prompt-examples'); + + const prompt = new weave.MessagesPrompt({ + messages: [ + { "role": "system", "content": "You will be provided with a description of a scene and your task is to provide a single word that best describes an associated emotion." - }, - { + }, + { "role": "user", "content": "{scene}" - } - ]) - weave.publish(prompt, name="emotion_prompt") - - from openai import OpenAI - client = OpenAI() - - response = client.chat.completions.create( - model="gpt-4o", - messages=prompt.format(scene="A dog is lying on a dock next to a fisherman."), - ) - ``` - - - ```typescript twoslash lines {8,29} - // @noErrors - import * as weave from 'weave'; - import OpenAI from 'openai'; - - async function main() { - // weave.init returns a client instance - const weaveClient = await weave.init('wandb/prompt-examples'); - - const prompt = new weave.MessagesPrompt({ - messages: [ - { - "role": "system", - "content": "You will be provided with a description of a scene and your task is to provide a single word that best describes an associated emotion." - }, - { - "role": "user", - "content": "{scene}" - } - ] - }); - - // Use the client returned from init - await weaveClient.publish(prompt, 'emotion_prompt'); - - // Wrap OpenAI client to track calls in Weave - const client = weave.wrapOpenAI(new OpenAI()); - - const response = await client.chat.completions.create({ - model: "gpt-4o", - messages: prompt.format({ scene: "A dog is lying on a dock next to a fisherman." }), - }); - } - - main(); - ``` - - + } + ] + }); + + // Use the client returned from init + await weaveClient.publish(prompt, 'emotion_prompt'); + + // Wrap OpenAI client to track calls in Weave + const client = weave.wrapOpenAI(new OpenAI()); + + const response = await client.chat.completions.create({ + model: "gpt-4o", + messages: prompt.format({ scene: "A dog is lying on a dock next to a fisherman." }), + }); +} + +main(); +``` + + diff --git a/weave/guides/evaluation/custom-monitors.mdx b/weave/guides/evaluation/custom-monitors.mdx index 83b27b56e7..10df1cac1d 100644 --- a/weave/guides/evaluation/custom-monitors.mdx +++ b/weave/guides/evaluation/custom-monitors.mdx @@ -71,9 +71,9 @@ The following end-to-end example walks through creating a monitor that evaluates 1. Define a function that generates statements. Some statements are truthful, others are not: - - -```python lines + + +```python Python lines import weave import random import openai @@ -100,9 +100,8 @@ def generate_statement(ground_truth: str) -> str: generate_statement("The Earth revolves around the Sun.") ``` - - -```typescript lines twoslash + +```typescript TypeScript lines twoslash // @noErrors import * as weave from 'weave'; import OpenAI from 'openai'; @@ -129,8 +128,8 @@ const generateStatement = weave.op(async (ground_truth: string): Promise await generateStatement("The Earth revolves around the Sun."); ``` - - + + 2. Run the function at least once to log a trace in your project. This makes the op available for monitoring in the W&B UI. @@ -163,23 +162,22 @@ await generateStatement("The Earth revolves around the Sun."); 6. In your script, invoke your function using statements with differing degrees of truthfulness to test the scoring function: - - -```python lines + + +```python Python lines generate_statement("The Earth revolves around the Sun.") generate_statement("Water freezes at 0 degrees Celsius.") generate_statement("The Great Wall of China was built over several centuries.") ``` - - -```typescript lines twoslash + +```typescript TypeScript lines twoslash // @noErrors await generateStatement("The Earth revolves around the Sun."); await generateStatement("Water freezes at 0 degrees Celsius."); await generateStatement("The Great Wall of China was built over several centuries."); ``` - - + + 7. After running the script using several different statements, open the W&B UI and go to the **Traces** tab. Select any **LLMAsAJudgeScorer.score** trace to see the results. diff --git a/weave/guides/evaluation/evaluation_logger.mdx b/weave/guides/evaluation/evaluation_logger.mdx index 695e9e2ca7..17c6bc5002 100644 --- a/weave/guides/evaluation/evaluation_logger.mdx +++ b/weave/guides/evaluation/evaluation_logger.mdx @@ -380,9 +380,9 @@ To avoid this duplication, publish your dataset to Weave before running any eval The following example publishes a dataset and links to it in the `EvaluationLogger`, before retrieving and iterating over it like any other dataset. - - -```python + + +```python Python import weave from weave import EvaluationLogger @@ -402,9 +402,8 @@ weave.publish(dataset) # Retrieve the published dataset dataset = weave.ref("my_eval_dataset").get() ``` - - -```typescript twoslash + +```typescript TypeScript twoslash // @noErrors import weave, {EvaluationLogger, Dataset} from 'weave'; @@ -424,8 +423,8 @@ const datasetRef = await dataset.save(); // Retrieve the published dataset const published = await datasetRef.get(); ``` - - + + ### Get outputs before logging @@ -635,9 +634,9 @@ With `EvaluationLogger`, you can log and compare multiple evaluations side by si For more information about comparisons, see [Comparisons](../tools/comparison). - - -```python lines + + +```python Python lines import weave models = [ @@ -665,9 +664,8 @@ for model in models: ev.log_summary() ``` - - -```typescript twoslash lines + +```typescript TypeScript twoslash lines // @noErrors import weave from 'weave'; import {EvaluationLogger} from 'weave/evaluationLogger'; @@ -706,8 +704,8 @@ for (const model of models) { await ev.logSummary(); } ``` - - + + ![The Evals tab](/weave/guides/evaluation/img/evals_tab.png) diff --git a/weave/guides/integrations/anthropic.mdx b/weave/guides/integrations/anthropic.mdx index 61c6c4569c..a7eea6ecd7 100644 --- a/weave/guides/integrations/anthropic.mdx +++ b/weave/guides/integrations/anthropic.mdx @@ -17,9 +17,9 @@ Weave automatically captures traces for the Anthropic SDK when you add `weave.in The following examples demonstrate how to integrate Weave into a basic call to Anthropic: - - -```python lines {6} + + +```python Python lines {6} import weave # use the anthropic library as usual import os @@ -43,9 +43,8 @@ message = client.messages.create( ) print(message.content) ``` - - -```typescript twoslash + +```typescript TypeScript twoslash // @noErrors import Anthropic from '@anthropic-ai/sdk'; import * as weave from 'weave'; @@ -69,8 +68,8 @@ const message = await client.messages.create({ console.log(message.content); ``` - - + + By including `weave.init()` in the code, Weave automatically captures tracing information and outputs links. You can view the traces in the Weave UI by clicking on the links. @@ -84,9 +83,9 @@ Weave ops automatically version your code as you experiment, and capture its inp The following examples show you how to track a function: - - -```python lines {5,10,24} + + +```python Python lines {5,10,24} import weave import os from anthropic import Anthropic @@ -117,9 +116,8 @@ def generate_joke(topic: str) -> str: print(generate_joke("chickens")) print(generate_joke("cars")) ``` - - -```typescript twoslash + +```typescript TypeScript twoslash // @noErrors import Anthropic from '@anthropic-ai/sdk'; import * as weave from 'weave'; @@ -155,8 +153,8 @@ const generateJoke = weave.op(async function generateJoke( console.log(await generateJoke('chickens')); console.log(await generateJoke('cars')); ``` - - + + When you decorate a function with `weave.op()`, Weave captures the function's code, input, and output. You can use ops to track any function you want, including nested functions. @@ -223,9 +221,9 @@ Anthropic provides a [tools](https://platform.claude.com/docs/en/agents-and-tool The following truncated examples demonstrate an Anthropic tool configuration: - - -```python lines + + +```python Python lines message = client.messages.create( max_tokens=1024, messages=[ @@ -255,9 +253,8 @@ message = client.messages.create( print(message) ``` - - -```typescript twoslash + +```typescript TypeScript twoslash // @noErrors const message = await client.messages.create({ max_tokens: 1024, @@ -288,8 +285,8 @@ const message = await client.messages.create({ console.log(message); ``` - - + + Weave automatically captures the tool definitions, Claude's tool use requests, and tool results at each step of the conversation. diff --git a/weave/guides/integrations/inference.mdx b/weave/guides/integrations/inference.mdx index 5cfc2fadaf..cb49d05876 100644 --- a/weave/guides/integrations/inference.mdx +++ b/weave/guides/integrations/inference.mdx @@ -116,89 +116,87 @@ To create a chat completion, you need: - `meta-llama/Llama-4-Scout-17B-16E-Instruct` - `microsoft/Phi-4-mini-instruct` - - - ```bash - curl https://api.inference.wandb.ai/v1/chat/completions \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer " \ - -H "OpenAI-Project: /" \ - -d '{ - "model": "", - "messages": [ - { "role": "system", "content": "You are a helpful assistant." }, - { "role": "user", "content": "Tell me a joke." } - ] - }' - ``` - - - ```python lines - import openai - - client = openai.OpenAI( - # The custom base URL points to Serverless Inference - base_url='https://api.inference.wandb.ai/v1', - - # Create an API key at https://wandb.ai/settings - # Consider setting it in the environment as OPENAI_API_KEY instead for safety - api_key="", - - # Team and project are required for usage tracking - project="/", - ) + + +```bash Bash +curl https://api.inference.wandb.ai/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer " \ + -H "OpenAI-Project: /" \ + -d '{ + "model": "", + "messages": [ + { "role": "system", "content": "You are a helpful assistant." }, + { "role": "user", "content": "Tell me a joke." } + ] + }' +``` - # Replace with any of the following values: - # meta-llama/Llama-3.1-8B-Instruct - # deepseek-ai/DeepSeek-V3-0324 - # meta-llama/Llama-3.3-70B-Instruct - # deepseek-ai/DeepSeek-R1-0528 - # meta-llama/Llama-4-Scout-17B-16E-Instruct - # microsoft/Phi-4-mini-instruct +```python Python lines +import openai - response = client.chat.completions.create( - model="", - messages=[ - {"role": "system", "content": ""}, - {"role": "user", "content": ""} - ], - ) +client = openai.OpenAI( + # The custom base URL points to Serverless Inference + base_url='https://api.inference.wandb.ai/v1', + + # Create an API key at https://wandb.ai/settings + # Consider setting it in the environment as OPENAI_API_KEY instead for safety + api_key="", + + # Team and project are required for usage tracking + project="/", +) + +# Replace with any of the following values: +# meta-llama/Llama-3.1-8B-Instruct +# deepseek-ai/DeepSeek-V3-0324 +# meta-llama/Llama-3.3-70B-Instruct +# deepseek-ai/DeepSeek-R1-0528 +# meta-llama/Llama-4-Scout-17B-16E-Instruct +# microsoft/Phi-4-mini-instruct + +response = client.chat.completions.create( + model="", + messages=[ + {"role": "system", "content": ""}, + {"role": "user", "content": ""} + ], +) + +print(response.choices[0].message.content) +``` - print(response.choices[0].message.content) - ``` - - + #### List supported models Use the API to query all available models and their IDs. This is useful for selecting models dynamically or inspecting what's available in your environment. - - - ```bash - curl https://api.inference.wandb.ai/v1/models \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer " \ - -H "OpenAI-Project: /" \ - ``` - - - ```python lines - import openai - - client = openai.OpenAI( - base_url="https://api.inference.wandb.ai/v1", - api_key="", - project="/" - ) + + +```bash Bash +curl https://api.inference.wandb.ai/v1/models \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer " \ + -H "OpenAI-Project: /" \ +``` - response = client.models.list() +```python Python lines +import openai + +client = openai.OpenAI( + base_url="https://api.inference.wandb.ai/v1", + api_key="", + project="/" +) + +response = client.models.list() + +for model in response.data: + print(model.id) +``` - for model in response.data: - print(model.id) - ``` - - + ## Usage examples diff --git a/weave/guides/integrations/openai-realtime-audio.mdx b/weave/guides/integrations/openai-realtime-audio.mdx index 69b066fddf..ba99912ad7 100644 --- a/weave/guides/integrations/openai-realtime-audio.mdx +++ b/weave/guides/integrations/openai-realtime-audio.mdx @@ -34,18 +34,17 @@ To run the example, complete the following steps: 1. Start your Python environment and install the following libraries: - - - ```bash - uv add weave openai-agents websockets pyaudio numpy - ``` - - - ```bash - pip install weave openai-agents websockets pyaudio numpy - ``` - - + + + ```bash uv + uv add weave openai-agents websockets pyaudio numpy + ``` + + ```bash pip + pip install weave openai-agents websockets pyaudio numpy + ``` + + 1. Create a file titled `weave_voice_assistant.py` and add the following code to it. @@ -290,18 +289,17 @@ To run the example, complete the following steps: 1. Start your Python environment and install the following libraries: - - - ```bash - uv add weave websockets pyaudio numpy - ``` - - - ```bash - pip install weave websockets pyaudio numpy - ``` - - + + + ```bash uv + uv add weave websockets pyaudio numpy + ``` + + ```bash pip + pip install weave websockets pyaudio numpy + ``` + + 1. Create a file titled `tool_definitions.py` and add the following tool definitions to it. The main application imports from this module. diff --git a/weave/guides/tools/attributes.mdx b/weave/guides/tools/attributes.mdx index 641991beb6..e6e9fc48ce 100644 --- a/weave/guides/tools/attributes.mdx +++ b/weave/guides/tools/attributes.mdx @@ -33,9 +33,9 @@ You can't modify `call.attributes` once the call starts. Use this context manager to set any metadata before invoking the op. - - -```python lines + + +```python Python lines import weave weave.init("[TEAM-NAME]/[PROJECT-NAME]") @@ -48,9 +48,8 @@ def my_function(name: str): with weave.attributes({'env': 'production', 'user_id': '12345'}): result = my_function("World") ``` - - -```typescript twoslash lines + +```typescript TypeScript twoslash lines // @noErrors import {init, op, withAttributes} from 'weave'; @@ -72,16 +71,16 @@ async function main() { main().catch(console.error); ``` - - + + The function attaches the attributes to all traced operations within the context manager block (Python) or callback function (TypeScript). You can also nest `weave.attributes()` contexts. Inner contexts override outer contexts for the same keys: - - -```python lines + + +```python Python lines @weave.op def process_data(data: str): return data.upper() @@ -101,9 +100,8 @@ with weave.attributes({ }): process_data("world") # Has env='production', version='1.1.0', region='us-west-2', experiment='exp-456' ``` - - -```typescript twoslash lines + +```typescript TypeScript twoslash lines // @noErrors import {init, op, withAttributes} from 'weave'; @@ -140,16 +138,16 @@ async function main() { main().catch(console.error); ``` - - + + ## Global attributes When you set global attributes during Weave initialization, they automatically apply to all traces and evaluations in your project. This is useful for propagating project-wide metadata like environment, deployment version, or team information. - - -```python lines + + +```python Python lines import weave weave.init( @@ -173,9 +171,8 @@ my_function() # Automatically has all global attributes evaluation = weave.Evaluation(dataset=examples, scorers=[scorer]) asyncio.run(evaluation.evaluate(model)) # Has all global attributes ``` - - -```typescript twoslash lines + +```typescript TypeScript twoslash lines // @noErrors import {init, op, withAttributes} from 'weave'; @@ -199,16 +196,16 @@ async function main() { } main().catch(console.error); ``` - - + + ### Combine global and per-call attributes You can use global attributes and per-call attributes together. Per-call attributes with the same key override global attributes: - - -```python lines + + +```python Python lines import weave # Set global attributes @@ -231,9 +228,8 @@ process("test1") with weave.attributes({'env': 'staging', 'experiment': 'A'}): process("test2") ``` - - -```typescript twoslash lines + +```typescript TypeScript twoslash lines // @noErrors import {init, op, withAttributes} from 'weave'; @@ -261,8 +257,8 @@ async function main() { } main().catch(console.error); ``` - - + + ## Get attributes during execution diff --git a/weave/guides/tools/weave-in-workspaces.mdx b/weave/guides/tools/weave-in-workspaces.mdx index 7b4f05a341..59fc2e8fba 100644 --- a/weave/guides/tools/weave-in-workspaces.mdx +++ b/weave/guides/tools/weave-in-workspaces.mdx @@ -136,9 +136,9 @@ The following sections show each approach. To add an artifact as an attribute of a trace, pass it as a key-value pair to the `weave.attributes()` context manager: - - -```python lines {10} + + +```python Python lines {10} import weave weave.init("[YOUR-TEAM-NAME]/[YOUR-PROJECT-NAME]") @@ -151,9 +151,8 @@ def my_function(name: str): with weave.attributes({'artifact_id': 'wandb-artifact:///team-name/project-name/run-38m4t5ja-history:v0'}): result = my_function("World") ``` - - -```typescript twoslash lines {12} + +```typescript TypeScript twoslash lines {12} // @noErrors import {init, op, withAttributes} from 'weave'; @@ -175,8 +174,8 @@ async function main() { main().catch(console.error); ``` - - + + ### Add an artifact as an attribute of a `Model` diff --git a/weave/guides/tracking/call-schema-reference.mdx b/weave/guides/tracking/call-schema-reference.mdx index 9aa7ed346f..37c582a6ae 100644 --- a/weave/guides/tracking/call-schema-reference.mdx +++ b/weave/guides/tracking/call-schema-reference.mdx @@ -112,9 +112,9 @@ const myOp = weave.op( Use `getCall` to fetch a single Call by ID, or `getCalls` to fetch multiple Calls. In both cases, `summary` is the same merged dictionary. - - -```python lines + + +```python Python lines import weave client = weave.init("my-team/my-project") @@ -134,9 +134,8 @@ for call in client.get_calls(filter={"op_names": ["weave:///my-team/my-project/o weave_s = s.get("weave", {}) print(call.id, weave_s.get("status"), weave_s.get("latency_ms"), s.get("accuracy")) ``` - - -```typescript twoslash lines + +```typescript TypeScript twoslash lines // @noErrors import * as weave from 'weave'; const client = await weave.init('my-team/my-project'); @@ -158,6 +157,6 @@ for (const call of calls) { console.log(call.id, weaveS?.status, weaveS?.latency_ms, call.summary?.accuracy); } ``` - - + + diff --git a/weave/guides/tracking/feedback.mdx b/weave/guides/tracking/feedback.mdx index 855bc8e746..c8eb572a77 100644 --- a/weave/guides/tracking/feedback.mdx +++ b/weave/guides/tracking/feedback.mdx @@ -90,37 +90,35 @@ You can also get more information for each feedback object in `client.get_feedba - `feedback_type`: The type of feedback (reaction, note, custom). - `payload`: The feedback payload. - - - ```python lines - import weave - client = weave.init('intro-example') - - # Get all feedback in a project - all_feedback = client.get_feedback() - - # Fetch a specific feedback object by id. - # The API returns a collection, which is expected to contain at most one item. - one_feedback = client.get_feedback("[FEEDBACK-UUID]")[0] - - # Find all feedback objects with a specific reaction. You can specify offset and limit. - thumbs_up = client.get_feedback(reaction="👍", limit=10) - - # After retrieval, view the details of individual feedback objects. - for f in client.get_feedback(): - print(f.id) - print(f.created_at) - print(f.feedback_type) - print(f.payload) - ``` - - - - ```plaintext - This feature is not available in TypeScript yet. - ``` - - + + +```python Python lines +import weave +client = weave.init('intro-example') + +# Get all feedback in a project +all_feedback = client.get_feedback() + +# Fetch a specific feedback object by id. +# The API returns a collection, which is expected to contain at most one item. +one_feedback = client.get_feedback("[FEEDBACK-UUID]")[0] + +# Find all feedback objects with a specific reaction. You can specify offset and limit. +thumbs_up = client.get_feedback(reaction="👍", limit=10) + +# After retrieval, view the details of individual feedback objects. +for f in client.get_feedback(): + print(f.id) + print(f.created_at) + print(f.feedback_type) + print(f.payload) +``` + +```plaintext TypeScript +This feature is not available in TypeScript yet. +``` + + ### Add feedback to a Call @@ -134,33 +132,31 @@ You can add feedback to a Call using the Call's UUID. To use the UUID to get a p The maximum number of characters in a feedback note is 1024. If a note exceeds this limit, Weave doesn't create it. - - - ```python lines - import weave - client = weave.init('intro-example') + - call = client.get_call("[CALL-UUID]") +```python Python lines +import weave +client = weave.init('intro-example') - # Adding an emoji reaction - call.feedback.add_reaction("👍") +call = client.get_call("[CALL-UUID]") - # Adding a note - call.feedback.add_note("this is a note") +# Adding an emoji reaction +call.feedback.add_reaction("👍") - # Adding custom key/value pairs. - # The first argument is a user-defined "type" string. - # Feedback must be JSON serializable and less than 1 KB when serialized. - call.feedback.add("correctness", { "value": 5 }) - ``` +# Adding a note +call.feedback.add_note("this is a note") - - - ```plaintext - This feature is not available in TypeScript yet. - ``` - - +# Adding custom key/value pairs. +# The first argument is a user-defined "type" string. +# Feedback must be JSON serializable and less than 1 KB when serialized. +call.feedback.add("correctness", { "value": 5 }) +``` + +```plaintext TypeScript +This feature is not available in TypeScript yet. +``` + + #### Retrieve the Call UUID @@ -170,74 +166,69 @@ For scenarios where you must add feedback immediately after a Call, you can retr To retrieve the UUID during Call execution, get the current Call, and return the ID. - - - ```python lines - - import weave - weave.init("uuid") - - @weave.op() - def simple_operation(input_value): - # Perform some simple operation - output = f"Processed {input_value}" - # Get the current call ID - current_call = weave.require_current_call() - call_id = current_call.id - return output, call_id - ``` - - - - ```plaintext - This feature is not available in TypeScript yet. - ``` - - + + +```python Python lines + +import weave +weave.init("uuid") + +@weave.op() +def simple_operation(input_value): + # Perform some simple operation + output = f"Processed {input_value}" + # Get the current call ID + current_call = weave.require_current_call() + call_id = current_call.id + return output, call_id +``` + +```plaintext TypeScript +This feature is not available in TypeScript yet. +``` + + ##### After Call execution Alternatively, you can use the `call()` method to execute the operation and retrieve the ID after Call execution: - - - ```python lines - import weave - weave.init("uuid") - - @weave.op() - def simple_operation(input_value): - return f"Processed {input_value}" - - # Execute the operation and retrieve the result and call ID - result, call = simple_operation.call("example input") - call_id = call.id - ``` - - - - ```plaintext - This feature is not available in TypeScript yet. - ``` - - + + +```python Python lines +import weave +weave.init("uuid") + +@weave.op() +def simple_operation(input_value): + return f"Processed {input_value}" + +# Execute the operation and retrieve the result and call ID +result, call = simple_operation.call("example input") +call_id = call.id +``` + +```plaintext TypeScript +This feature is not available in TypeScript yet. +``` + + ### Delete feedback from a Call You can delete feedback from a particular call by specifying a UUID. - - - ```python lines - call.feedback.purge("[FEEDBACK-UUID]") - ``` - - - ```plaintext - This feature is not available in TypeScript yet. - ``` - - + + +```python Python lines +call.feedback.purge("[FEEDBACK-UUID]") +``` + +```plaintext TypeScript +This feature is not available in TypeScript yet. +``` + + ## Add human annotations @@ -301,42 +292,40 @@ You can also create human annotation scorers through the API. Each scorer is its The following example creates two scorers. The first scorer, `Temperature`, scores the perceived temperature of the LLM call. The second scorer, `Tone`, scores the tone of the LLM response. Each scorer uses `save` with an associated object ID (`temperature-scorer` and `tone-scorer`). - - - ```python lines - import weave - from weave.flow.annotation_spec import AnnotationSpec - - client = weave.init("feedback-example") - - spec1 = AnnotationSpec( - name="Temperature", - description="The perceived temperature of the llm call", - field_schema={ - "type": "number", - "minimum": -1, - "maximum": 1, - } - ) - spec2 = AnnotationSpec( - name="Tone", - description="The tone of the llm response", - field_schema={ - "type": "string", - "enum": ["Aggressive", "Neutral", "Polite", "N/A"], - }, - ) - weave.publish(spec1, "temperature-scorer") - weave.publish(spec2, "tone-scorer") - ``` - - - - ```plaintext - This feature is not available in TypeScript yet. - ``` - - + + +```python Python lines +import weave +from weave.flow.annotation_spec import AnnotationSpec + +client = weave.init("feedback-example") + +spec1 = AnnotationSpec( + name="Temperature", + description="The perceived temperature of the llm call", + field_schema={ + "type": "number", + "minimum": -1, + "maximum": 1, + } +) +spec2 = AnnotationSpec( + name="Tone", + description="The tone of the llm response", + field_schema={ + "type": "string", + "enum": ["Aggressive", "Neutral", "Polite", "N/A"], + }, +) +weave.publish(spec1, "temperature-scorer") +weave.publish(spec2, "tone-scorer") +``` + +```plaintext TypeScript +This feature is not available in TypeScript yet. +``` + + ### Modify a human annotation scorer using the API @@ -344,34 +333,32 @@ Expanding on [creating a human annotation scorer using the API](#create-a-human- > You can view human annotation scorer object history in the **Scorers** tab under **Human annotations**. - - - ```python lines - import weave - from weave.flow.annotation_spec import AnnotationSpec - - client = weave.init("feedback-example") - - # create a new version of the scorer - spec1 = AnnotationSpec( - name="Temperature", - description="The perceived temperature of the llm call", - field_schema={ - "type": "integer", # <<- change type to integer - "minimum": -1, - "maximum": 1, - } - ) - weave.publish(spec1, "temperature-scorer") - ``` - - - - ```plaintext - This feature is not available in TypeScript yet. - ``` - - + + +```python Python lines +import weave +from weave.flow.annotation_spec import AnnotationSpec + +client = weave.init("feedback-example") + +# create a new version of the scorer +spec1 = AnnotationSpec( + name="Temperature", + description="The perceived temperature of the llm call", + field_schema={ + "type": "integer", # <<- change type to integer + "minimum": -1, + "maximum": 1, + } +) +weave.publish(spec1, "temperature-scorer") +``` + +```plaintext TypeScript +This feature is not available in TypeScript yet. +``` + + ![Human Annotation scorer history](/weave/guides/tracking/imgs/human-annotation-scorer-history.png) @@ -381,22 +368,21 @@ Expanding on [creating a human annotation scorer using the API](#create-a-human- The feedback API lets you use a human annotation scorer by specifying a specially constructed name and an `annotation_ref` field. You can obtain the `annotation_spec_ref` from the UI by selecting the appropriate tab, or during the creation of the `AnnotationSpec`. - - - ```python lines - import weave + + +```python Python lines +import weave - client = weave.init("feedback-example") +client = weave.init("feedback-example") - call = client.get_call("[CALL-ID]") - annotation_spec = weave.ref("[ANNOTATION-SPEC-REF-URI]") +call = client.get_call("[CALL-ID]") +annotation_spec = weave.ref("[ANNOTATION-SPEC-REF-URI]") - call.feedback.add( - feedback_type="wandb.annotation." + annotation_spec.name, - payload={"value": 1}, - annotation_ref=annotation_spec.uri(), - ) - ``` +call.feedback.add( + feedback_type="wandb.annotation." + annotation_spec.name, + payload={"value": 1}, + annotation_ref=annotation_spec.uri(), +) +``` - - + diff --git a/weave/guides/tracking/ops.mdx b/weave/guides/tracking/ops.mdx index c36f0f9607..82599ac7f8 100644 --- a/weave/guides/tracking/ops.mdx +++ b/weave/guides/tracking/ops.mdx @@ -78,9 +78,9 @@ A custom display name makes it easier to identify an Op in the Weave UI, especia To better organize your Ops in the Weave UI, apply custom kinds and colors by adding the `kind` and `color` arguments to the `@weave.op` decorators in your code. For example, the following code applies an `LLM` `kind` and a `blue` `color` to the parent function, and a `tool` `kind` and a `red` `color` to a nested function: - - -```python lines + + +```python Python lines import weave weave.init("[YOUR-TEAM-NAME]/[YOUR-PROJECT-NAME]") @@ -97,13 +97,12 @@ def llm_func(): llm_func() ``` - - -```text + +```text TypeScript This feature is not available in TypeScript yet. ``` - - + + This applies the colors and kinds to your Ops in the Weave UI, like this: diff --git a/weave/guides/tracking/otel.mdx b/weave/guides/tracking/otel.mdx index 69a5b2be03..886286446b 100644 --- a/weave/guides/tracking/otel.mdx +++ b/weave/guides/tracking/otel.mdx @@ -100,22 +100,17 @@ Before running the following code samples, set the following fields: First, install the required dependencies: - - + -```bash +```bash Python pip install openai openinference-instrumentation-openai opentelemetry-exporter-otlp-proto-http ``` - - - -```bash +```bash TypeScript npm install openai @opentelemetry/sdk-trace-node @opentelemetry/sdk-trace-base @opentelemetry/resources @opentelemetry/exporter-trace-otlp-proto @arizeai/openinference-instrumentation-openai @opentelemetry/api ``` - - + **Performance recommendation**: Always use `BatchSpanProcessor` instead of `SimpleSpanProcessor` when sending traces to Weave. `SimpleSpanProcessor` exports spans synchronously, which can impact the performance of other workloads. These examples illustrate `BatchSpanProcessor`, which is recommended in production because it batches spans asynchronously and efficiently. @@ -280,22 +275,17 @@ npx ts-node openinference_example.ts First, install the required dependencies: - - + -```bash +```bash Python pip install openai opentelemetry-instrumentation-openai opentelemetry-exporter-otlp-proto-http ``` - - - -```bash +```bash TypeScript npm install openai @traceloop/instrumentation-openai @opentelemetry/sdk-trace-node @opentelemetry/resources @opentelemetry/exporter-trace-otlp-http ``` - - + @@ -456,22 +446,17 @@ If you prefer to use OTel directly instead of an instrumentation package, you ca First, install the required dependencies: - - + -```bash +```bash Python pip install openai opentelemetry-sdk opentelemetry-api opentelemetry-exporter-otlp-proto-http ``` - - - -```bash +```bash TypeScript npm install openai @opentelemetry/api @opentelemetry/sdk-trace-node @opentelemetry/resources @opentelemetry/exporter-trace-otlp-http ``` - - + @@ -808,10 +793,9 @@ The following examples show how to organize OTel traces into Weave threads. They Use this configuration to run these examples: - - + -```python lines +```python Python lines import json import os from opentelemetry import trace @@ -847,10 +831,7 @@ trace.set_tracer_provider(tracer_provider) tracer = trace.get_tracer(__name__) ``` - - - -```typescript lines +```typescript TypeScript lines import { trace, context } from "@opentelemetry/api"; import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node"; import { @@ -898,17 +879,15 @@ provider.register(); const tracer = trace.getTracer("threads-examples"); ``` - - + - - + -```python lines +```python Python lines def example_1_basic_thread_and_turn(): """Example 1: Basic thread with a single turn""" print("\n=== Example 1: Basic Thread and Turn ===") @@ -944,10 +923,7 @@ if __name__ == "__main__": main() ``` - - - -```typescript twoslash lines +```typescript TypeScript twoslash lines // @noErrors function example_1_basic_thread_and_turn() { console.log("\n=== Example 1: Basic Thread and Turn ==="); @@ -990,17 +966,15 @@ function main() { main(); ``` - - + - - + -```python lines +```python Python lines def example_2_multiple_turns(): """Example 2: Multiple turns in a single thread""" print("\n=== Example 2: Multiple Turns in Thread ===") @@ -1044,10 +1018,7 @@ if __name__ == "__main__": main() ``` - - - -```typescript twoslash lines +```typescript TypeScript twoslash lines // @noErrors function example_2_multiple_turns() { console.log("\n=== Example 2: Multiple Turns in Thread ==="); @@ -1105,17 +1076,15 @@ function main() { main(); ``` - - + - - + -```python lines +```python Python lines def example_3_complex_nested_structure(): """Example 3: Complex nested structure with multiple levels""" print("\n=== Example 3: Complex Nested Structure ===") @@ -1159,10 +1128,7 @@ if __name__ == "__main__": main() ``` - - - -```typescript twoslash lines +```typescript TypeScript twoslash lines // @noErrors function example_3_complex_nested_structure() { console.log("\n=== Example 3: Complex Nested Structure ==="); @@ -1224,17 +1190,15 @@ function main() { main(); ``` - - + - - + -```python lines +```python Python lines def example_4_non_turn_operations(): """Example 4: Operations that are part of a thread but not turns""" print("\n=== Example 4: Non-Turn Thread Operations ===") @@ -1264,10 +1228,7 @@ if __name__ == "__main__": main() ``` - - - -```typescript twoslash lines +```typescript TypeScript twoslash lines // @noErrors function example_4_non_turn_operations() { console.log("\n=== Example 4: Non-Turn Thread Operations ==="); @@ -1302,8 +1263,7 @@ function main() { main(); ``` - - + diff --git a/weave/guides/tracking/trace-agents-attributes.mdx b/weave/guides/tracking/trace-agents-attributes.mdx index 025874aff0..d56c55d2a0 100644 --- a/weave/guides/tracking/trace-agents-attributes.mdx +++ b/weave/guides/tracking/trace-agents-attributes.mdx @@ -17,10 +17,9 @@ These methods mirror the [OpenTelemetry (OTel) span API](https://opentelemetry.i Use `set_attributes()` (Python) or `setAttributes()` (TypeScript) to stamp arbitrary attributes on a single span. Pass a dictionary or object whether you have one key or many. The method returns the span, so you can chain calls. - - + -```python lines +```python Python lines import weave weave.init("[TEAM-NAME]/[PROJECT-NAME]") @@ -31,10 +30,7 @@ with weave.start_conversation(agent_name="my-agent"): turn.set_attributes({"user_id": "12345", "tenant": "acme", "env": "production"}) ``` - - - -```typescript lines twoslash +```typescript TypeScript lines twoslash // @noErrors import * as weave from 'weave'; @@ -50,24 +46,19 @@ turn.end(); conversation.end(); ``` - - + The same methods are available on every span class. For example, you can tag an individual tool call or LLM call: - - + -```python lines +```python Python lines with weave.start_turn(user_message="What is the weather in Tokyo?") as turn: with turn.tool(name="get_weather") as tool: tool.set_attributes({"weave.display_name": "Weather lookup"}) ``` - - - -```typescript lines twoslash +```typescript TypeScript lines twoslash // @noErrors const turn = weave.startTurn({ agentName: 'my-agent' }); const tool = turn.startTool({ name: 'get_weather' }); @@ -78,8 +69,7 @@ tool.end(); turn.end(); ``` - - + Most attribute keys are arbitrary custom metadata, but Weave reserves two prefixes for special handling. Keys under `weave.*` map to built-in Weave fields, and keys under `gen_ai.*` map to OpenTelemetry GenAI semantic-convention fields. In the preceding example, `weave.display_name` is a reserved key that sets the span's display name in the Agents and Traces UI. For arbitrary metadata that you want to filter and group by, use your own keys such as `user_id` or `tenant`, which Weave stores as filterable custom attributes. @@ -87,10 +77,9 @@ Most attribute keys are arbitrary custom metadata, but Weave reserves two prefix Stamping attributes one span at a time works well for span-specific metadata, but some metadata applies to an entire conversation. To apply the same attributes to every span a conversation emits, pass `attributes` when you start the conversation. This is useful for propagating conversation-wide metadata such as an integration identity or a deployment environment. - - + -```python lines +```python Python lines import weave weave.init("[TEAM-NAME]/[PROJECT-NAME]") @@ -102,10 +91,7 @@ conversation = weave.start_conversation( # Every turn, LLM, tool, and sub-agent span in this session carries these attributes. ``` - - - -```typescript lines twoslash +```typescript TypeScript lines twoslash // @noErrors import * as weave from 'weave'; @@ -118,8 +104,7 @@ const conversation = weave.startConversation({ // Every turn, LLM, tool, and sub-agent span in this conversation carries these attributes. ``` - - + As with per-span attributes, use your own custom keys for conversation attributes. Set semantic-convention fields, such as the agent name, conversation name, or model, through their typed parameters (`agent_name`, `conversation_name`, `model`) rather than through `attributes`. Avoid keys under the reserved `gen_ai.*` and `weave.*` prefixes. Weave extracts those into typed fields during ingestion, so a custom value under a reserved key is unsupported. diff --git a/weave/guides/tracking/trace-agents-batch.mdx b/weave/guides/tracking/trace-agents-batch.mdx index 86ec69c1a9..b3e0f0cf91 100644 --- a/weave/guides/tracking/trace-agents-batch.mdx +++ b/weave/guides/tracking/trace-agents-batch.mdx @@ -13,10 +13,9 @@ If you're building your own agent loop, use the real-time instrumentation APIs d To record a single completed turn after it happens, use `weave.log_turn`. It accepts a fully-formed turn, including all LLM and tool spans. - - + -```python lines highlight="1,3,28" +```python Python lines highlight="1,3,28" weave.init("[YOUR-TEAM]/[YOUR-PROJECT]") from weave.conversation import LLM, Message, Tool, Usage @@ -55,15 +54,11 @@ weave.log_turn( ) ``` - - - -```plaintext +```plaintext TypeScript This feature is not available in the TypeScript SDK yet. ``` - - + `log_turn` returns a `LogResult` containing the trace IDs of the emitted spans. @@ -73,10 +68,9 @@ An optional `model` parameter on `log_turn` sets the model on the turn's own spa To bulk-import a complete, multi-turn conversation at once, use `weave.log_conversation`. The `turns` parameter accepts a list of `Turn` objects, each constructed the same way as the previous `log_turn` example. - - + -```python lines +```python Python lines weave.log_conversation( conversation_id="my-conversation-abc", agent_name="weather-bot", @@ -84,12 +78,8 @@ weave.log_conversation( ) ``` - - - -```plaintext +```plaintext TypeScript This feature is not available in the TypeScript SDK yet. ``` - - + diff --git a/weave/guides/tracking/trace-agents.mdx b/weave/guides/tracking/trace-agents.mdx index e03112c523..a5abe646c6 100644 --- a/weave/guides/tracking/trace-agents.mdx +++ b/weave/guides/tracking/trace-agents.mdx @@ -122,10 +122,9 @@ Weave exposes the following top-level functions. Each function returns an object The active conversation is stored in context (a Python `ContextVar` or Node.js `AsyncLocalStorage`), so any code running in the same async context can retrieve it with `weave.get_current_conversation()` / `weave.getCurrentConversation()` without passing the conversation object explicitly. - - + -```python lines +```python Python lines conversation = weave.start_conversation( agent_name="my-agent", # Optional: identifies the agent in the UI. Omit it and the conversation isn't grouped under a named agent. conversation_id="", # Optional: stable ID to group turns; auto-generated when empty. @@ -136,10 +135,7 @@ conversation = weave.start_conversation( ) ``` - - - -```typescript lines twoslash +```typescript TypeScript lines twoslash // @noErrors const conversation = weave.startConversation({ agentName: 'my-agent', // Optional: identifies the agent in the UI. Omit it and the conversation isn't grouped under a named agent. @@ -148,8 +144,7 @@ const conversation = weave.startConversation({ }); ``` - - + ### Start a turn @@ -160,10 +155,9 @@ You can call it two ways: - **As a top-level function** (`weave.start_turn(...)` / `weave.startTurn(...)`), shown in the examples below. It resolves the active conversation from context and inherits its conversation ID. If no conversation is active, the turn is created without a `conversation_id` and isn't grouped with other turns. - **As an instance method** on a conversation you hold a reference to (`conversation.start_turn(...)` / `conversation.startTurn(...)`). Useful when you have an explicit conversation object in scope, such as inside a context-manager block. The "Context manager or try-finally pattern" example below uses this form. See the data-model table above for direct links to the `Conversation`, `Turn`, `LLM`, `Tool`, and `SubAgent` reference pages in both SDKs. - - + -```python lines +```python Python lines turn = weave.start_turn( user_message="What is the weather in Tokyo?", # The user's input text. agent_name="my-agent", # Optional: overrides the conversation-level agent name. @@ -171,10 +165,7 @@ turn = weave.start_turn( ) ``` - - - -```typescript lines twoslash +```typescript TypeScript lines twoslash // @noErrors const turn = weave.startTurn({ agentName: 'my-agent', // Optional: overrides the conversation-level agent name. @@ -182,17 +173,15 @@ const turn = weave.startTurn({ }); ``` - - + ### Start an LLM call `start_llm()` / `startLLM()` creates a `chat` span nested under the current turn. Weave uses this span to display token usage, model name, input and output messages, and reasoning in the Agents view. - - + -```python lines +```python Python lines llm = weave.start_llm( model="gpt-4o", # The model identifier. provider_name="openai", # Optional: provider name, for example "openai", "anthropic". See note below. @@ -200,10 +189,7 @@ llm = weave.start_llm( ) ``` - - - -```typescript lines twoslash +```typescript TypeScript lines twoslash // @noErrors const llm = weave.startLLM({ model: 'gpt-4o', // The model identifier. @@ -211,8 +197,7 @@ const llm = weave.startLLM({ }); ``` - - + After the LLM call completes, assign the response data to the `llm` object before it closes: @@ -262,10 +247,9 @@ Pass `provider_name` / `providerName` explicitly. Weave doesn't infer it from th `start_tool()` / `startTool()` creates an `execute_tool` span. The span becomes a child of whatever OTel span is active in context (typically the `chat` span of the LLM call that produced the tool call). - - + -```python lines +```python Python lines tool = weave.start_tool( name="get_weather", # Tool name as declared to the LLM. arguments='{"city": "Tokyo"}', # JSON string of the tool arguments. @@ -273,10 +257,7 @@ tool = weave.start_tool( ) ``` - - - -```typescript lines twoslash +```typescript TypeScript lines twoslash // @noErrors const tool = weave.startTool({ name: 'get_weather', // Tool name as declared to the LLM. @@ -285,24 +266,19 @@ const tool = weave.startTool({ }); ``` - - + Assign the tool result before closing: - - + -```python lines +```python Python lines with weave.start_tool(name="get_weather", arguments='{"city": "Tokyo"}') as tool: result = get_weather_api("Tokyo") tool.result = result # Accepts dict, list, or string. JSON-encoded automatically. ``` - - - -```typescript lines twoslash +```typescript TypeScript lines twoslash // @noErrors const tool = weave.startTool({ name: 'get_weather', args: '{"city": "Tokyo"}' }); try { @@ -312,8 +288,7 @@ try { } ``` - - + ## Usage patterns for agent tracing @@ -332,10 +307,9 @@ For most agents, use a context manager pattern in Python or a try-finally patter Weave stores the active conversation, turn, and LLM call in context, so any function called within a block can call `start_llm()` / `startLLM()` or `start_tool()` / `startTool()` without holding an explicit reference to the parent. This works across module boundaries as long as the code runs in the same async context. To retrieve the active objects from anywhere in the call stack, use `weave.get_current_conversation()` / `weave.getCurrentConversation()`, `weave.get_current_turn()` / `weave.getCurrentTurn()`, and `weave.get_current_llm()` / `weave.getCurrentLLM()`. - - + -```python lines highlight="13,14,17,25,29" +```python Python lines highlight="13,14,17,25,29" import weave from weave.conversation import Message, Usage @@ -370,10 +344,7 @@ with weave.start_conversation(agent_name="weather-bot") as conversation: llm.usage = Usage(input_tokens=150, output_tokens=30) ``` - - - -```typescript lines highlight="11,13,16,24,35" twoslash +```typescript TypeScript lines highlight="11,13,16,24,35" twoslash // @noErrors import * as weave from 'weave'; import type { Message, Usage } from 'weave'; @@ -425,17 +396,15 @@ try { } ``` - - + ### Manual start and end pattern Use `.end()` explicitly when you can't use `with` blocks or `try/finally`. For example, when you open and close spans in different function calls, or when you manage async lifecycle outside a coroutine. You're responsible for calling `.end()` on every object you create, so that spans close and flush to the collector. - - + -```python lines highlight="1,2,4,9,15" +```python Python lines highlight="1,2,4,9,15" conversation = weave.start_conversation(agent_name="weather-bot") turn = conversation.start_turn(user_message="What is the weather?") @@ -459,10 +428,7 @@ turn.end() conversation.end() ``` - - - -```typescript lines highlight="1,2,4,9,15" twoslash +```typescript TypeScript lines highlight="1,2,4,9,15" twoslash // @noErrors const conversation = weave.startConversation({ agentName: 'weather-bot' }); const turn = conversation.startTurn({ agentName: 'weather-bot' }); @@ -487,8 +453,7 @@ turn.end(); conversation.end(); ``` - - + ## Semantic conventions diff --git a/weave/guides/tracking/trace-disable.mdx b/weave/guides/tracking/trace-disable.mdx index 0d53c3909e..48cbb80c1e 100644 --- a/weave/guides/tracking/trace-disable.mdx +++ b/weave/guides/tracking/trace-disable.mdx @@ -18,29 +18,28 @@ To unconditionally disable tracing for the entire program, set the environment v To conditionally disable tracing for a specific Weave client, initialize the client with the `disabled` flag in init settings. - - - ```python lines + + +```python Python lines import weave # Initialize the client client = weave.init(..., settings={"disabled": True}) - ``` - - - ```plaintext - This feature is not available for the TypeScript SDK yet. - ``` - - +``` + +```plaintext TypeScript +This feature is not available for the TypeScript SDK yet. +``` + + ## Context manager To conditionally disable tracing only within a specific block of code, use a tracing context manager. Use `with tracing_disabled()` to suppress tracing **only for the function calls executed inside the `with` block**. Use it in application code to scope which calls shouldn't be logged. - - -```python lines + + +```python Python lines import weave from weave.trace.context.call_context import tracing_disabled @@ -53,29 +52,27 @@ def my_op(): with tracing_disabled(): my_op() ``` - - - ```plaintext - This feature is not available for the TypeScript SDK yet. - ``` - - + +```plaintext TypeScript +This feature is not available for the TypeScript SDK yet. +``` + + Although tracing behavior is fixed when functions are defined, you can use this for runtime control when you combine it with application logic. For example, wrap the context manager in a conditional to dynamically enable or disable tracing based on a runtime value: - - -```python lines + + +```python Python lines if should_trace: my_op() else: with tracing_disabled(): my_op() ``` - - - ```plaintext - This feature is not available for the TypeScript SDK yet. - ``` - - \ No newline at end of file + +```plaintext TypeScript +This feature is not available for the TypeScript SDK yet. +``` + + \ No newline at end of file diff --git a/weave/guides/tracking/trace-generator-func.mdx b/weave/guides/tracking/trace-generator-func.mdx index 4bcfd3cfad..75737103a1 100644 --- a/weave/guides/tracking/trace-generator-func.mdx +++ b/weave/guides/tracking/trace-generator-func.mdx @@ -11,59 +11,56 @@ Because generators yield values lazily, Weave logs outputs only when you fully c To ensure Weave captures outputs in the trace, fully consume the generator (for example, with `list()`). - - - - ```python - from typing import Generator - import weave - - weave.init("my-project") - - # This function uses a sync generator. - # Weave will trace the Call and its input (`x`), - # but output values are only captured once the generator is consumed (for example, with `list()`). - @weave.op - def basic_gen(x: int) -> Generator[int, None, None]: - yield from range(x) - - # A normal sync function used within the generator pipeline. - # Its calls are also traced independently by Weave. - @weave.op - def inner(x: int) -> int: - return x + 1 - - # A sync generator that calls another traced function (`inner`). - # Each yielded value comes from a separate traced Call to `inner`. - @weave.op - def nested_generator(x: int) -> Generator[int, None, None]: - for i in range(x): - yield inner(i) - - # A generator that composes the above generator. - # Tracing here produces a hierarchical Call tree: - # - `deeply_nested_generator` (parent) - # - `nested_generator` (child) - # - `inner` (grandchild) - @weave.op - def deeply_nested_generator(x: int) -> Generator[int, None, None]: - for i in range(x): - for j in nested_generator(i): - yield j - - # The generator must be *consumed* for Weave to capture outputs. - # This is true for both sync and async generators. - res = deeply_nested_generator(4) - list(res) # Triggers tracing of all nested calls and yields - ``` - - - - ```plaintext - This feature is not available in the TypeScript SDK yet. - ``` - - + + +```python Python +from typing import Generator +import weave + +weave.init("my-project") + +# This function uses a sync generator. +# Weave will trace the Call and its input (`x`), +# but output values are only captured once the generator is consumed (for example, with `list()`). +@weave.op +def basic_gen(x: int) -> Generator[int, None, None]: + yield from range(x) + +# A normal sync function used within the generator pipeline. +# Its calls are also traced independently by Weave. +@weave.op +def inner(x: int) -> int: + return x + 1 + +# A sync generator that calls another traced function (`inner`). +# Each yielded value comes from a separate traced Call to `inner`. +@weave.op +def nested_generator(x: int) -> Generator[int, None, None]: + for i in range(x): + yield inner(i) + +# A generator that composes the above generator. +# Tracing here produces a hierarchical Call tree: +# - `deeply_nested_generator` (parent) +# - `nested_generator` (child) +# - `inner` (grandchild) +@weave.op +def deeply_nested_generator(x: int) -> Generator[int, None, None]: + for i in range(x): + for j in nested_generator(i): + yield j + +# The generator must be *consumed* for Weave to capture outputs. +# This is true for both sync and async generators. +res = deeply_nested_generator(4) +list(res) # Triggers tracing of all nested calls and yields +``` + +```plaintext TypeScript +This feature is not available in the TypeScript SDK yet. +``` + + The following screenshot shows the **Traces** page with a selected trace of the preceding code. The center panel shows the trace tree for the selected trace. The trace tree shows the `deeply_nested_generator`, `nested_generator`, and `inner` Ops in the trace tree hierarchy. ![Weave Traces page showing a selected trace tree illustrating deeply nested Ops](/weave/guides/tracking/imgs/generators.png) diff --git a/weave/quickstart-inference.mdx b/weave/quickstart-inference.mdx index 29940fff66..ee43ae3fb2 100644 --- a/weave/quickstart-inference.mdx +++ b/weave/quickstart-inference.mdx @@ -38,9 +38,9 @@ When you run this code, Weave: - Logs inputs, outputs, latency, and token usage. - Provides a link to view your trace in the Weave UI. - - -```python lines + + +```python Python lines import weave import openai @@ -70,9 +70,8 @@ def ask_llama(question: str) -> str: result = ask_llama("What are the benefits of using W&B Weave for LLM development?") print(result) ``` - - -```typescript twoslash lines + +```typescript TypeScript twoslash lines // @noErrors import * as weave from 'weave'; import OpenAI from 'openai'; @@ -102,8 +101,8 @@ return response.choices[0].message.content || ''; const result = await askLlama('What are the benefits of using W&B Weave for LLM development?'); console.log(result); ``` - - + + ## Build a text summarization application @@ -111,9 +110,9 @@ Now that you've traced a single LLM call, this section shows how Weave traces ne Next, run this code, which is a basic summarization app that shows how Weave traces nested operations: - - -```python lines + + +```python Python lines import weave import openai @@ -175,9 +174,8 @@ result = summarize_text(sample_text) print("Key Points:", result["key_points"]) print("\nSummary:", result["summary"]) ``` - - -```typescript twoslash lines + +```typescript TypeScript twoslash lines // @noErrors import * as weave from 'weave'; import OpenAI from 'openai'; @@ -237,16 +235,16 @@ const result = await summarizeText(sampleText); console.log('Key Points:', result.key_points); console.log('\nSummary:', result.summary); ``` - - + + ## Compare multiple models A common use case for Weave is comparing how different models respond to the same prompt. Serverless Inference provides access to multiple models. Use the following code to compare the performance of Llama and DeepSeek's respective responses: - - -```python lines + + +```python Python lines import weave import openai @@ -286,9 +284,8 @@ print("\n" + "="*50 + "\n") print("DeepSeek V3 response:") print(deepseek_model.predict(test_question)) ``` - - -```typescript twoslash lines + +```typescript TypeScript twoslash lines // @noErrors import * as weave from 'weave'; import OpenAI from 'openai'; @@ -327,8 +324,8 @@ console.log('\n' + '='.repeat(50) + '\n'); console.log('DeepSeek V3 response:'); console.log(await deepseekModel(testQuestion)); ``` - - + + ## Evaluate model performance @@ -338,9 +335,9 @@ Evaluate how well a model performs on a Q&A task using Weave's built-in `Evaluat Append the following code to the script you used in the prior section: - - -```python lines + + +```python Python lines from typing import Optional from weave import EvaluationLogger @@ -417,9 +414,8 @@ for model in models_to_compare: # In the Weave UI, navigate to the Evals tab to compare results across models ``` - - -```typescript twoslash lines + +```typescript TypeScript twoslash lines // @noErrors import { EvaluationLogger } from 'weave'; @@ -513,8 +509,8 @@ for (const { model, name } of modelsToCompare) { // In the Weave UI, navigate to the Evals tab to compare results across models ``` - - + + After you run these examples, you have traced LLM calls, a nested summarization pipeline, a model comparison, and a full evaluation logged to Weave. Running these examples returns links to the traces in the terminal. Click any link to view traces in the Weave UI. diff --git a/weave/quickstart.mdx b/weave/quickstart.mdx index 5799013e99..3e7b7fda87 100644 --- a/weave/quickstart.mdx +++ b/weave/quickstart.mdx @@ -38,86 +38,85 @@ The following example code sends a request to OpenAI (requires an [OpenAI API ke The following example code tracks your first project with Weave: - - - ```python lines {1-2,7-8,27-28} - # Imports the Weave library - import weave - from openai import OpenAI - - client = OpenAI() - - # Weave automatically tracks the inputs, outputs and code of this function - @weave.op() - def extract_dinos(sentence: str) -> dict: - response = client.chat.completions.create( - model="gpt-4o", - messages=[ - { - "role": "system", - "content": """In JSON format extract a list of `dinosaurs`, with their `name`, - their `common_name`, and whether its `diet` is a herbivore or carnivore""" - }, - { - "role": "user", - "content": sentence - } - ], - response_format={ "type": "json_object" } - ) - return response.choices[0].message.content - - # Initializes Weave, and sets the team and project to log data to - weave.init('your-team/traces-quickstart') - - sentence = """I watched as a Tyrannosaurus rex (T. rex) chased after a Triceratops (Trike), \ - both carnivore and herbivore locked in an ancient dance. Meanwhile, a gentle giant \ - Brachiosaurus (Brachi) calmly munched on treetops, blissfully unaware of the chaos below.""" - - result = extract_dinos(sentence) - print(result) - ``` - - - ```typescript twoslash lines {1-2,7-8,20,24-25} - // @noErrors - // Imports the Weave library - import * as weave from 'weave'; - import OpenAI from 'openai'; - - const openai = new OpenAI(); - - // Weave automatically tracks the inputs, outputs and code of this function - async function extractDinos(input: string) { - const response = await openai.chat.completions.create({ - model: 'gpt-4o', - messages: [ - { - role: 'user', - content: `In JSON format extract a list of 'dinosaurs', with their 'name', their 'common_name', and whether its 'diet' is a herbivore or carnivore: ${input}`, - }, - ], - }); - return response.choices[0].message.content; - } - const extractDinosOp = weave.op(extractDinos); + + +```python Python lines {1-2,7-8,27-28} +# Imports the Weave library +import weave +from openai import OpenAI + +client = OpenAI() + +# Weave automatically tracks the inputs, outputs and code of this function +@weave.op() +def extract_dinos(sentence: str) -> dict: + response = client.chat.completions.create( + model="gpt-4o", + messages=[ + { + "role": "system", + "content": """In JSON format extract a list of `dinosaurs`, with their `name`, +their `common_name`, and whether its `diet` is a herbivore or carnivore""" + }, + { + "role": "user", + "content": sentence + } + ], + response_format={ "type": "json_object" } + ) + return response.choices[0].message.content + +# Initializes Weave, and sets the team and project to log data to +weave.init('your-team/traces-quickstart') + +sentence = """I watched as a Tyrannosaurus rex (T. rex) chased after a Triceratops (Trike), \ +both carnivore and herbivore locked in an ancient dance. Meanwhile, a gentle giant \ +Brachiosaurus (Brachi) calmly munched on treetops, blissfully unaware of the chaos below.""" + +result = extract_dinos(sentence) +print(result) +``` - async function main() { +```typescript TypeScript twoslash lines {1-2,7-8,20,24-25} +// @noErrors +// Imports the Weave library +import * as weave from 'weave'; +import OpenAI from 'openai'; + +const openai = new OpenAI(); + +// Weave automatically tracks the inputs, outputs and code of this function +async function extractDinos(input: string) { + const response = await openai.chat.completions.create({ + model: 'gpt-4o', + messages: [ + { + role: 'user', + content: `In JSON format extract a list of 'dinosaurs', with their 'name', their 'common_name', and whether its 'diet' is a herbivore or carnivore: ${input}`, + }, + ], + }); + return response.choices[0].message.content; +} +const extractDinosOp = weave.op(extractDinos); - // Initializes Weave, and sets the team and project to log data to - await weave.init('your-team/traces-quickstart'); +async function main() { - const result = await extractDinosOp( - 'I watched as a Tyrannosaurus rex (T. rex) chased after a Triceratops (Trike), both carnivore and herbivore locked in an ancient dance. Meanwhile, a gentle giant Brachiosaurus (Brachi) calmly munched on treetops, blissfully unaware of the chaos below.' - ); - console.log(result); - } + // Initializes Weave, and sets the team and project to log data to + await weave.init('your-team/traces-quickstart'); - main(); + const result = await extractDinosOp( + 'I watched as a Tyrannosaurus rex (T. rex) chased after a Triceratops (Trike), both carnivore and herbivore locked in an ancient dance. Meanwhile, a gentle giant Brachiosaurus (Brachi) calmly munched on treetops, blissfully unaware of the chaos below.' + ); + console.log(result); +} + +main(); + +``` - ``` - - + When you call the `extract_dinos` function, Weave outputs links in the terminal to view your traces. The output looks like this: diff --git a/weave/tutorial-eval.mdx b/weave/tutorial-eval.mdx index def53704ef..43a53ac762 100644 --- a/weave/tutorial-eval.mdx +++ b/weave/tutorial-eval.mdx @@ -38,24 +38,23 @@ By the end, you'll have a working evaluation pipeline that scores an example mod Import the following libraries into your script: - - -```python lines + + +```python Python lines import json import openai import asyncio import weave from weave.scorers import MultiTaskBinaryClassificationF1 ``` - - -```typescript twoslash lines + +```typescript TypeScript twoslash lines // @noErrors import * as weave from 'weave'; import OpenAI from 'openai'; ``` - - + + ## Build a `Model` @@ -70,89 +69,87 @@ To declare a `Model`, subclass `Model` and implement a `predict` function defini The following example model uses OpenAI to extract the names, colors, and flavors of alien fruits from input sentences. - - - ```python lines {1,5} - class ExtractFruitsModel(weave.Model): - model_name: str - prompt_template: str - - @weave.op() - async def predict(self, sentence: str) -> dict: - client = openai.AsyncClient() - - response = await client.chat.completions.create( - model=self.model_name, - messages=[ - {"role": "user", "content": self.prompt_template.format(sentence=sentence)} - ], - ) - result = response.choices[0].message.content - if result is None: - raise ValueError("No response from model") - parsed = json.loads(result) - return parsed - ``` - - - ```typescript twoslash lines {9} - // @noErrors - // Note: weave.Model is not supported in TypeScript yet. - // Instead, wrap your model-like function with weave.op - - import * as weave from 'weave'; - import OpenAI from 'openai'; - - const openaiClient = new OpenAI(); - - const model = weave.op(async function myModel({datasetRow}) { - const prompt = `Extract fields ("fruit": , "color": , "flavor") from the following text, as json: ${datasetRow.sentence}`; - const response = await openaiClient.chat.completions.create({ - model: 'gpt-3.5-turbo', - messages: [{ role: 'user', content: prompt }], - response_format: { type: 'json_object' } - }); - return JSON.parse(response.choices[0].message.content); + + +```python Python lines {1,5} +class ExtractFruitsModel(weave.Model): + model_name: str + prompt_template: str + + @weave.op() + async def predict(self, sentence: str) -> dict: + client = openai.AsyncClient() + + response = await client.chat.completions.create( + model=self.model_name, + messages=[ + {"role": "user", "content": self.prompt_template.format(sentence=sentence)} + ], + ) + result = response.choices[0].message.content + if result is None: + raise ValueError("No response from model") + parsed = json.loads(result) + return parsed +``` + +```typescript TypeScript twoslash lines {9} +// @noErrors +// Note: weave.Model is not supported in TypeScript yet. +// Instead, wrap your model-like function with weave.op + +import * as weave from 'weave'; +import OpenAI from 'openai'; + +const openaiClient = new OpenAI(); + +const model = weave.op(async function myModel({datasetRow}) { + const prompt = `Extract fields ("fruit": , "color": , "flavor") from the following text, as json: ${datasetRow.sentence}`; + const response = await openaiClient.chat.completions.create({ + model: 'gpt-3.5-turbo', + messages: [{ role: 'user', content: prompt }], + response_format: { type: 'json_object' } }); - ``` - - + return JSON.parse(response.choices[0].message.content); +}); +``` + + The `ExtractFruitsModel` class inherits from (or subclasses) `weave.Model` so Weave can track the instantiated object. `@weave.op` decorates the `predict` function to track its inputs and outputs. You can instantiate `Model` objects like this: - - - ```python lines - # Set your team and project name - weave.init('[YOUR-TEAM]/eval_pipeline_quickstart') - - model = ExtractFruitsModel( - model_name='gpt-3.5-turbo-1106', - prompt_template='Extract fields ("fruit": , "color": , "flavor": ) from the following text, as json: {sentence}' - ) - - sentence = "There are many fruits that were found on the recently discovered planet Goocrux. There are neoskizzles that grow there, which are purple and taste like candy." - - print(asyncio.run(model.predict(sentence))) - # if you're in a Jupyter Notebook, run: - # await model.predict(sentence) - ``` - - - ```typescript twoslash - // @noErrors - await weave.init('eval_pipeline_quickstart'); - - const sentence = "There are many fruits that were found on the recently discovered planet Goocrux. There are neoskizzles that grow there, which are purple and taste like candy."; - - const result = await model({ datasetRow: { sentence } }); - - console.log(result); - ``` - - + + +```python Python lines +# Set your team and project name +weave.init('[YOUR-TEAM]/eval_pipeline_quickstart') + +model = ExtractFruitsModel( + model_name='gpt-3.5-turbo-1106', + prompt_template='Extract fields ("fruit": , "color": , "flavor": ) from the following text, as json: {sentence}' +) + +sentence = "There are many fruits that were found on the recently discovered planet Goocrux. There are neoskizzles that grow there, which are purple and taste like candy." + +print(asyncio.run(model.predict(sentence))) +# if you're in a Jupyter Notebook, run: +# await model.predict(sentence) +``` + +```typescript TypeScript twoslash +// @noErrors +await weave.init('eval_pipeline_quickstart'); + +const sentence = "There are many fruits that were found on the recently discovered planet Goocrux. There are neoskizzles that grow there, which are purple and taste like candy."; + +const result = await model({ datasetRow: { sentence } }); + +console.log(result); +``` + + ## Create a dataset @@ -162,69 +159,67 @@ The following example dataset defines three example input sentences and their co This example builds a list of examples in code, but you can also log them one at a time from your running application. - - - ```python lines - sentences = ["There are many fruits that were found on the recently discovered planet Goocrux. There are neoskizzles that grow there, which are purple and taste like candy.", + + +```python Python lines +sentences = ["There are many fruits that were found on the recently discovered planet Goocrux. There are neoskizzles that grow there, which are purple and taste like candy.", +"Pounits are a bright green color and are more savory than sweet.", +"Finally, there are fruits called glowls, which have a very sour and bitter taste which is acidic and caustic, and a pale orange tinge to them."] +labels = [ + {'fruit': 'neoskizzles', 'color': 'purple', 'flavor': 'candy'}, + {'fruit': 'pounits', 'color': 'bright green', 'flavor': 'savory'}, + {'fruit': 'glowls', 'color': 'pale orange', 'flavor': 'sour and bitter'} +] +examples = [ + {'id': '0', 'sentence': sentences[0], 'target': labels[0]}, + {'id': '1', 'sentence': sentences[1], 'target': labels[1]}, + {'id': '2', 'sentence': sentences[2], 'target': labels[2]} +] +``` + +```typescript TypeScript twoslash +// @noErrors +const sentences = [ + "There are many fruits that were found on the recently discovered planet Goocrux. There are neoskizzles that grow there, which are purple and taste like candy.", "Pounits are a bright green color and are more savory than sweet.", - "Finally, there are fruits called glowls, which have a very sour and bitter taste which is acidic and caustic, and a pale orange tinge to them."] - labels = [ - {'fruit': 'neoskizzles', 'color': 'purple', 'flavor': 'candy'}, - {'fruit': 'pounits', 'color': 'bright green', 'flavor': 'savory'}, - {'fruit': 'glowls', 'color': 'pale orange', 'flavor': 'sour and bitter'} - ] - examples = [ - {'id': '0', 'sentence': sentences[0], 'target': labels[0]}, - {'id': '1', 'sentence': sentences[1], 'target': labels[1]}, - {'id': '2', 'sentence': sentences[2], 'target': labels[2]} - ] - ``` - - - ```typescript twoslash - // @noErrors - const sentences = [ - "There are many fruits that were found on the recently discovered planet Goocrux. There are neoskizzles that grow there, which are purple and taste like candy.", - "Pounits are a bright green color and are more savory than sweet.", - "Finally, there are fruits called glowls, which have a very sour and bitter taste which is acidic and caustic, and a pale orange tinge to them." - ]; - const labels = [ - { fruit: 'neoskizzles', color: 'purple', flavor: 'candy' }, - { fruit: 'pounits', color: 'bright green', flavor: 'savory' }, - { fruit: 'glowls', color: 'pale orange', flavor: 'sour and bitter' } - ]; - const examples = sentences.map((sentence, i) => ({ - id: i.toString(), - sentence, - target: labels[i] - })); - ``` - - + "Finally, there are fruits called glowls, which have a very sour and bitter taste which is acidic and caustic, and a pale orange tinge to them." +]; +const labels = [ + { fruit: 'neoskizzles', color: 'purple', flavor: 'candy' }, + { fruit: 'pounits', color: 'bright green', flavor: 'savory' }, + { fruit: 'glowls', color: 'pale orange', flavor: 'sour and bitter' } +]; +const examples = sentences.map((sentence, i) => ({ + id: i.toString(), + sentence, + target: labels[i] +})); +``` + + Then create your dataset using the `weave.Dataset()` class and publish it: - - - ```python lines {2} - weave.init('eval_pipeline_quickstart') - dataset = weave.Dataset(name='fruits', rows=examples) - weave.publish(dataset) - ``` - - - ```typescript twoslash lines {3-6} - // @noErrors - import * as weave from 'weave'; - await weave.init('eval_pipeline_quickstart'); - const dataset = new weave.Dataset({ - name: 'fruits', - rows: examples - }); - await dataset.save(); - ``` - - + + +```python Python lines {2} +weave.init('eval_pipeline_quickstart') +dataset = weave.Dataset(name='fruits', rows=examples) +weave.publish(dataset) +``` + +```typescript TypeScript twoslash lines {3-6} +// @noErrors +import * as weave from 'weave'; +await weave.init('eval_pipeline_quickstart'); +const dataset = new weave.Dataset({ + name: 'fruits', + rows: examples +}); +await dataset.save(); +``` + + ## Define custom scoring functions @@ -233,27 +228,26 @@ Now that you have a model and a dataset, you need a way to measure how well the When you use Weave evaluations, Weave expects a `target` to compare `output` against. The following scoring function takes two dictionaries (`target` and `output`) and returns a dictionary of boolean values that indicate whether the output matches the target. The `@weave.op()` decorator enables Weave to track the scoring function's execution. - - - ```python lines - @weave.op() - def fruit_name_score(target: dict, output: dict) -> dict: - return {'correct': target['fruit'] == output['fruit']} - ``` - - - ```typescript twoslash - // @noErrors - import * as weave from 'weave'; - - const fruitNameScorer = weave.op( - function fruitNameScore({target, output}) { - return { correct: target.fruit === output.fruit }; - } - ); - ``` - - + + +```python Python lines +@weave.op() +def fruit_name_score(target: dict, output: dict) -> dict: + return {'correct': target['fruit'] == output['fruit']} +``` + +```typescript TypeScript twoslash +// @noErrors +import * as weave from 'weave'; + +const fruitNameScorer = weave.op( + function fruitNameScore({target, output}) { + return { correct: target.fruit === output.fruit }; + } +); +``` + + For more information on making your own scoring function, see the [Scorers](/weave/guides/evaluation/scorers) guide. @@ -267,41 +261,40 @@ Along with custom scoring functions, you can also use [Weave's built-in scorers] The following example runs an evaluation of `ExtractFruitsModel` on the `fruits` dataset using the two scoring functions and logs the results to Weave. - - - ```python lines {3-10} - weave.init('eval_pipeline_quickstart') - - evaluation = weave.Evaluation( - name='fruit_eval', - dataset=dataset, - scorers=[ - MultiTaskBinaryClassificationF1(class_names=["fruit", "color", "flavor"]), - fruit_name_score - ], - ) - print(asyncio.run(evaluation.evaluate(model))) - # if you're in a Jupyter Notebook, run: - # await evaluation.evaluate(model) - ``` - - - ```typescript twoslash lines {5-9} - // @noErrors - import * as weave from 'weave'; - - await weave.init('eval_pipeline_quickstart'); - - const evaluation = new weave.Evaluation({ - name: 'fruit_eval', - dataset: dataset, - scorers: [fruitNameScorer], - }); - const results = await evaluation.evaluate(model); - console.log(results); - ``` - - + + +```python Python lines {3-10} +weave.init('eval_pipeline_quickstart') + +evaluation = weave.Evaluation( + name='fruit_eval', + dataset=dataset, + scorers=[ + MultiTaskBinaryClassificationF1(class_names=["fruit", "color", "flavor"]), + fruit_name_score + ], +) +print(asyncio.run(evaluation.evaluate(model))) +# if you're in a Jupyter Notebook, run: +# await evaluation.evaluate(model) +``` + +```typescript TypeScript twoslash lines {5-9} +// @noErrors +import * as weave from 'weave'; + +await weave.init('eval_pipeline_quickstart'); + +const evaluation = new weave.Evaluation({ + name: 'fruit_eval', + dataset: dataset, + scorers: [fruitNameScorer], +}); +const results = await evaluation.evaluate(model); +console.log(results); +``` + + If you're running from a Python script, you'll need to use `asyncio.run`. However, if you're running from a Jupyter notebook, you can use `await` directly. @@ -312,141 +305,140 @@ If you're running from a Python script, you'll need to use `asyncio.run`. Howeve - - - ```python lines - import json - import asyncio - import openai - import weave - from weave.scorers import MultiTaskBinaryClassificationF1 - - # Initialize Weave once - weave.init('eval_pipeline_quickstart') - - # 1. Define Model - class ExtractFruitsModel(weave.Model): - model_name: str - prompt_template: str - - @weave.op() - async def predict(self, sentence: str) -> dict: - client = openai.AsyncClient() - response = await client.chat.completions.create( - model=self.model_name, - messages=[{"role": "user", "content": self.prompt_template.format(sentence=sentence)}], - ) - result = response.choices[0].message.content - if result is None: - raise ValueError("No response from model") - return json.loads(result) - - # 2. Instantiate model - model = ExtractFruitsModel( - model_name='gpt-3.5-turbo-1106', - prompt_template='Extract fields ("fruit": , "color": , "flavor": ) from the following text, as json: {sentence}' - ) - - # 3. Create dataset - sentences = ["There are many fruits that were found on the recently discovered planet Goocrux. There are neoskizzles that grow there, which are purple and taste like candy.", - "Pounits are a bright green color and are more savory than sweet.", - "Finally, there are fruits called glowls, which have a very sour and bitter taste which is acidic and caustic, and a pale orange tinge to them."] - labels = [ - {'fruit': 'neoskizzles', 'color': 'purple', 'flavor': 'candy'}, - {'fruit': 'pounits', 'color': 'bright green', 'flavor': 'savory'}, - {'fruit': 'glowls', 'color': 'pale orange', 'flavor': 'sour and bitter'} - ] - examples = [ - {'id': '0', 'sentence': sentences[0], 'target': labels[0]}, - {'id': '1', 'sentence': sentences[1], 'target': labels[1]}, - {'id': '2', 'sentence': sentences[2], 'target': labels[2]} - ] - - dataset = weave.Dataset(name='fruits', rows=examples) - weave.publish(dataset) - - # 4. Define scoring function - @weave.op() - def fruit_name_score(target: dict, output: dict) -> dict: - return {'correct': target['fruit'] == output['fruit']} - - # 5. Run evaluation - evaluation = weave.Evaluation( - name='fruit_eval', - dataset=dataset, - scorers=[ - MultiTaskBinaryClassificationF1(class_names=["fruit", "color", "flavor"]), - fruit_name_score - ], - ) - print(asyncio.run(evaluation.evaluate(model))) - ``` - - - ```typescript twoslash lines - // @noErrors - import * as weave from 'weave'; - import OpenAI from 'openai'; - - // Initialize Weave once - await weave.init('eval_pipeline_quickstart'); - - // 1. Define Model - // Note: weave.Model is not supported in TypeScript yet. - // Instead, wrap your model-like function with weave.op - const openaiClient = new OpenAI(); - - const model = weave.op(async function myModel({datasetRow}) { - const prompt = `Extract fields ("fruit": , "color": , "flavor": ) from the following text, as json: ${datasetRow.sentence}`; - const response = await openaiClient.chat.completions.create({ - model: 'gpt-3.5-turbo', - messages: [{ role: 'user', content: prompt }], - response_format: { type: 'json_object' } - }); - return JSON.parse(response.choices[0].message.content); - }); + - // 2. Create dataset - const sentences = [ - "There are many fruits that were found on the recently discovered planet Goocrux. There are neoskizzles that grow there, which are purple and taste like candy.", - "Pounits are a bright green color and are more savory than sweet.", - "Finally, there are fruits called glowls, which have a very sour and bitter taste which is acidic and caustic, and a pale orange tinge to them." - ]; - const labels = [ - { fruit: 'neoskizzles', color: 'purple', flavor: 'candy' }, - { fruit: 'pounits', color: 'bright green', flavor: 'savory' }, - { fruit: 'glowls', color: 'pale orange', flavor: 'sour and bitter' } - ]; - const examples = sentences.map((sentence, i) => ({ - id: i.toString(), - sentence, - target: labels[i] - })); - - const dataset = new weave.Dataset({ - name: 'fruits', - rows: examples - }); - await dataset.save(); - - // 3. Define scoring function - const fruitNameScorer = weave.op( - function fruitNameScore({target, output}) { - return { correct: target.fruit === output.fruit }; - } - ); - - // 4. Run evaluation - const evaluation = new weave.Evaluation({ - name: 'fruit_eval', - dataset: dataset, - scorers: [fruitNameScorer], +```python Python lines +import json +import asyncio +import openai +import weave +from weave.scorers import MultiTaskBinaryClassificationF1 + +# Initialize Weave once +weave.init('eval_pipeline_quickstart') + +# 1. Define Model +class ExtractFruitsModel(weave.Model): + model_name: str + prompt_template: str + + @weave.op() + async def predict(self, sentence: str) -> dict: + client = openai.AsyncClient() + response = await client.chat.completions.create( + model=self.model_name, + messages=[{"role": "user", "content": self.prompt_template.format(sentence=sentence)}], + ) + result = response.choices[0].message.content + if result is None: + raise ValueError("No response from model") + return json.loads(result) + +# 2. Instantiate model +model = ExtractFruitsModel( + model_name='gpt-3.5-turbo-1106', + prompt_template='Extract fields ("fruit": , "color": , "flavor": ) from the following text, as json: {sentence}' +) + +# 3. Create dataset +sentences = ["There are many fruits that were found on the recently discovered planet Goocrux. There are neoskizzles that grow there, which are purple and taste like candy.", +"Pounits are a bright green color and are more savory than sweet.", +"Finally, there are fruits called glowls, which have a very sour and bitter taste which is acidic and caustic, and a pale orange tinge to them."] +labels = [ + {'fruit': 'neoskizzles', 'color': 'purple', 'flavor': 'candy'}, + {'fruit': 'pounits', 'color': 'bright green', 'flavor': 'savory'}, + {'fruit': 'glowls', 'color': 'pale orange', 'flavor': 'sour and bitter'} +] +examples = [ + {'id': '0', 'sentence': sentences[0], 'target': labels[0]}, + {'id': '1', 'sentence': sentences[1], 'target': labels[1]}, + {'id': '2', 'sentence': sentences[2], 'target': labels[2]} +] + +dataset = weave.Dataset(name='fruits', rows=examples) +weave.publish(dataset) + +# 4. Define scoring function +@weave.op() +def fruit_name_score(target: dict, output: dict) -> dict: + return {'correct': target['fruit'] == output['fruit']} + +# 5. Run evaluation +evaluation = weave.Evaluation( + name='fruit_eval', + dataset=dataset, + scorers=[ + MultiTaskBinaryClassificationF1(class_names=["fruit", "color", "flavor"]), + fruit_name_score + ], +) +print(asyncio.run(evaluation.evaluate(model))) +``` + +```typescript TypeScript twoslash lines +// @noErrors +import * as weave from 'weave'; +import OpenAI from 'openai'; + +// Initialize Weave once +await weave.init('eval_pipeline_quickstart'); + +// 1. Define Model +// Note: weave.Model is not supported in TypeScript yet. +// Instead, wrap your model-like function with weave.op +const openaiClient = new OpenAI(); + +const model = weave.op(async function myModel({datasetRow}) { + const prompt = `Extract fields ("fruit": , "color": , "flavor": ) from the following text, as json: ${datasetRow.sentence}`; + const response = await openaiClient.chat.completions.create({ + model: 'gpt-3.5-turbo', + messages: [{ role: 'user', content: prompt }], + response_format: { type: 'json_object' } }); - const results = await evaluation.evaluate(model); - console.log(results); - ``` - - + return JSON.parse(response.choices[0].message.content); +}); + +// 2. Create dataset +const sentences = [ + "There are many fruits that were found on the recently discovered planet Goocrux. There are neoskizzles that grow there, which are purple and taste like candy.", + "Pounits are a bright green color and are more savory than sweet.", + "Finally, there are fruits called glowls, which have a very sour and bitter taste which is acidic and caustic, and a pale orange tinge to them." +]; +const labels = [ + { fruit: 'neoskizzles', color: 'purple', flavor: 'candy' }, + { fruit: 'pounits', color: 'bright green', flavor: 'savory' }, + { fruit: 'glowls', color: 'pale orange', flavor: 'sour and bitter' } +]; +const examples = sentences.map((sentence, i) => ({ + id: i.toString(), + sentence, + target: labels[i] +})); + +const dataset = new weave.Dataset({ + name: 'fruits', + rows: examples +}); +await dataset.save(); + +// 3. Define scoring function +const fruitNameScorer = weave.op( + function fruitNameScore({target, output}) { + return { correct: target.fruit === output.fruit }; + } +); + +// 4. Run evaluation +const evaluation = new weave.Evaluation({ + name: 'fruit_eval', + dataset: dataset, + scorers: [fruitNameScorer], +}); +const results = await evaluation.evaluate(model); +console.log(results); +``` + + ## View your evaluation results diff --git a/weave/tutorial-rag.mdx b/weave/tutorial-rag.mdx index 61e4c2a8ce..45260c4097 100644 --- a/weave/tutorial-rag.mdx +++ b/weave/tutorial-rag.mdx @@ -37,90 +37,88 @@ The knowledge base is the corpus your RAG application searches at query time. In First, compute the embeddings for the articles. You typically do this once with your articles and put the embeddings and metadata in a database, but here it's done every time the script runs for simplicity. - - - ```python lines - from openai import OpenAI - import weave - from weave import Model - import numpy as np - import json - import asyncio - - articles = [ - "Novo Nordisk and Eli Lilly rival soars 32 percent after promising weight loss drug results Shares of Denmarks Zealand Pharma shot 32 percent higher in morning trade, after results showed success in its liver disease treatment survodutide, which is also on trial as a drug to treat obesity. The trial “tells us that the 6mg dose is safe, which is the top dose used in the ongoing [Phase 3] obesity trial too,” one analyst said in a note. The results come amid feverish investor interest in drugs that can be used for weight loss.", - "Berkshire shares jump after big profit gain as Buffetts conglomerate nears $1 trillion valuation Berkshire Hathaway shares rose on Monday after Warren Buffetts conglomerate posted strong earnings for the fourth quarter over the weekend. Berkshires Class A and B shares jumped more than 1.5%, each. Class A shares are higher by more than 17% this year, while Class B has gained more than 18%. Berkshire was last valued at $930.1 billion, up from $905.5 billion where it closed on Friday, according to FactSet. Berkshire on Saturday posted fourth-quarter operating earnings of $8.481 billion, about 28 percent higher than the $6.625 billion from the year-ago period, driven by big gains in its insurance business. Operating earnings refers to profits from businesses across insurance, railroads and utilities. Meanwhile, Berkshires cash levels also swelled to record levels. The conglomerate held $167.6 billion in cash in the fourth quarter, surpassing the $157.2 billion record the conglomerate held in the prior quarter.", - "Highmark Health says its combining tech from Google and Epic to give doctors easier access to information Highmark Health announced it is integrating technology from Google Cloud and the health-care software company Epic Systems. The integration aims to make it easier for both payers and providers to access key information they need, even if its stored across multiple points and formats, the company said. Highmark is the parent company of a health plan with 7 million members, a provider network of 14 hospitals and other entities", - "Rivian and Lucid shares plunge after weak EV earnings reports Shares of electric vehicle makers Rivian and Lucid fell Thursday after the companies reported stagnant production in their fourth-quarter earnings after the bell Wednesday. Rivian shares sank about 25 percent, and Lucids stock dropped around 17 percent. Rivian forecast it will make 57,000 vehicles in 2024, slightly less than the 57,232 vehicles it produced in 2023. Lucid said it expects to make 9,000 vehicles in 2024, more than the 8,428 vehicles it made in 2023.", - "Mauritius blocks Norwegian cruise ship over fears of a potential cholera outbreak Local authorities on Sunday denied permission for the Norwegian Dawn ship, which has 2,184 passengers and 1,026 crew on board, to access the Mauritius capital of Port Louis, citing “potential health risks.” The Mauritius Ports Authority said Sunday that samples were taken from at least 15 passengers on board the cruise ship. A spokesperson for the U.S.-headquartered Norwegian Cruise Line Holdings said Sunday that 'a small number of guests experienced mild symptoms of a stomach-related illness' during Norwegian Dawns South Africa voyage.", - "Intuitive Machines lands on the moon in historic first for a U.S. company Intuitive Machines Nova-C cargo lander, named Odysseus after the mythological Greek hero, is the first U.S. spacecraft to soft land on the lunar surface since 1972. Intuitive Machines is the first company to pull off a moon landing — government agencies have carried out all previously successful missions. The company's stock surged in extended trading Thursday, after falling 11 percent in regular trading.", - "Lunar landing photos: Intuitive Machines Odysseus sends back first images from the moon Intuitive Machines cargo moon lander Odysseus returned its first images from the surface. Company executives believe the lander caught its landing gear sideways on the moon's surface while touching down and tipped over. Despite resting on its side, the company's historic IM-1 mission is still operating on the moon.", - ] - - def docs_to_embeddings(docs: list) -> list: - openai = OpenAI() - document_embeddings = [] - for doc in docs: - response = ( - openai.embeddings.create(input=doc, model="text-embedding-3-small") - .data[0] - .embedding - ) - document_embeddings.append(response) - return document_embeddings - - article_embeddings = docs_to_embeddings(articles) # Note: you would typically do this once with your articles and put the embeddings & metadata in a database - ``` - - - - ```typescript twoslash lines - // @noErrors - require('dotenv').config(); - import { OpenAI } from 'openai'; - import * as weave from 'weave'; - - interface Article { - text: string; - embedding?: number[]; - } - - const articles: Article[] = [ - { - text: `Novo Nordisk and Eli Lilly rival soars 32 percent after promising weight loss drug results Shares of Denmarks Zealand Pharma shot 32 percent higher in morning trade, after results showed success in its liver disease treatment survodutide, which is also on trial as a drug to treat obesity. The trial tells us that the 6mg dose is safe, which is the top dose used in the ongoing [Phase 3] obesity trial too, one analyst said in a note. The results come amid feverish investor interest in drugs that can be used for weight loss.` - }, - { - text: `Berkshire shares jump after big profit gain as Buffetts conglomerate nears $1 trillion valuation Berkshire Hathaway shares rose on Monday after Warren Buffetts conglomerate posted strong earnings for the fourth quarter over the weekend. Berkshires Class A and B shares jumped more than 1.5%, each. Class A shares are higher by more than 17% this year, while Class B has gained more than 18%. Berkshire was last valued at $930.1 billion, up from $905.5 billion where it closed on Friday, according to FactSet. Berkshire on Saturday posted fourth-quarter operating earnings of $8.481 billion, about 28 percent higher than the $6.625 billion from the year-ago period, driven by big gains in its insurance business. Operating earnings refers to profits from businesses across insurance, railroads and utilities. Meanwhile, Berkshires cash levels also swelled to record levels. The conglomerate held $167.6 billion in cash in the fourth quarter, surpassing the $157.2 billion record the conglomerate held in the prior quarter.` - }, - { - text: `Highmark Health says its combining tech from Google and Epic to give doctors easier access to information Highmark Health announced it is integrating technology from Google Cloud and the health-care software company Epic Systems. The integration aims to make it easier for both payers and providers to access key information they need, even if its stored across multiple points and formats, the company said. Highmark is the parent company of a health plan with 7 million members, a provider network of 14 hospitals and other entities` - } - ]; - - function cosineSimilarity(a: number[], b: number[]): number { - const dotProduct = a.reduce((sum, val, i) => sum + val * b[i], 0); - const magnitudeA = Math.sqrt(a.reduce((sum, val) => sum + val * val, 0)); - const magnitudeB = Math.sqrt(b.reduce((sum, val) => sum + val * val, 0)); - return dotProduct / (magnitudeA * magnitudeB); + + +```python Python lines +from openai import OpenAI +import weave +from weave import Model +import numpy as np +import json +import asyncio + +articles = [ + "Novo Nordisk and Eli Lilly rival soars 32 percent after promising weight loss drug results Shares of Denmarks Zealand Pharma shot 32 percent higher in morning trade, after results showed success in its liver disease treatment survodutide, which is also on trial as a drug to treat obesity. The trial “tells us that the 6mg dose is safe, which is the top dose used in the ongoing [Phase 3] obesity trial too,” one analyst said in a note. The results come amid feverish investor interest in drugs that can be used for weight loss.", + "Berkshire shares jump after big profit gain as Buffetts conglomerate nears $1 trillion valuation Berkshire Hathaway shares rose on Monday after Warren Buffetts conglomerate posted strong earnings for the fourth quarter over the weekend. Berkshires Class A and B shares jumped more than 1.5%, each. Class A shares are higher by more than 17% this year, while Class B has gained more than 18%. Berkshire was last valued at $930.1 billion, up from $905.5 billion where it closed on Friday, according to FactSet. Berkshire on Saturday posted fourth-quarter operating earnings of $8.481 billion, about 28 percent higher than the $6.625 billion from the year-ago period, driven by big gains in its insurance business. Operating earnings refers to profits from businesses across insurance, railroads and utilities. Meanwhile, Berkshires cash levels also swelled to record levels. The conglomerate held $167.6 billion in cash in the fourth quarter, surpassing the $157.2 billion record the conglomerate held in the prior quarter.", + "Highmark Health says its combining tech from Google and Epic to give doctors easier access to information Highmark Health announced it is integrating technology from Google Cloud and the health-care software company Epic Systems. The integration aims to make it easier for both payers and providers to access key information they need, even if its stored across multiple points and formats, the company said. Highmark is the parent company of a health plan with 7 million members, a provider network of 14 hospitals and other entities", + "Rivian and Lucid shares plunge after weak EV earnings reports Shares of electric vehicle makers Rivian and Lucid fell Thursday after the companies reported stagnant production in their fourth-quarter earnings after the bell Wednesday. Rivian shares sank about 25 percent, and Lucids stock dropped around 17 percent. Rivian forecast it will make 57,000 vehicles in 2024, slightly less than the 57,232 vehicles it produced in 2023. Lucid said it expects to make 9,000 vehicles in 2024, more than the 8,428 vehicles it made in 2023.", + "Mauritius blocks Norwegian cruise ship over fears of a potential cholera outbreak Local authorities on Sunday denied permission for the Norwegian Dawn ship, which has 2,184 passengers and 1,026 crew on board, to access the Mauritius capital of Port Louis, citing “potential health risks.” The Mauritius Ports Authority said Sunday that samples were taken from at least 15 passengers on board the cruise ship. A spokesperson for the U.S.-headquartered Norwegian Cruise Line Holdings said Sunday that 'a small number of guests experienced mild symptoms of a stomach-related illness' during Norwegian Dawns South Africa voyage.", + "Intuitive Machines lands on the moon in historic first for a U.S. company Intuitive Machines Nova-C cargo lander, named Odysseus after the mythological Greek hero, is the first U.S. spacecraft to soft land on the lunar surface since 1972. Intuitive Machines is the first company to pull off a moon landing — government agencies have carried out all previously successful missions. The company's stock surged in extended trading Thursday, after falling 11 percent in regular trading.", + "Lunar landing photos: Intuitive Machines Odysseus sends back first images from the moon Intuitive Machines cargo moon lander Odysseus returned its first images from the surface. Company executives believe the lander caught its landing gear sideways on the moon's surface while touching down and tipped over. Despite resting on its side, the company's historic IM-1 mission is still operating on the moon.", +] + +def docs_to_embeddings(docs: list) -> list: + openai = OpenAI() + document_embeddings = [] + for doc in docs: + response = ( + openai.embeddings.create(input=doc, model="text-embedding-3-small") + .data[0] + .embedding + ) + document_embeddings.append(response) + return document_embeddings + +article_embeddings = docs_to_embeddings(articles) # Note: you would typically do this once with your articles and put the embeddings & metadata in a database +``` + +```typescript TypeScript twoslash lines +// @noErrors +require('dotenv').config(); +import { OpenAI } from 'openai'; +import * as weave from 'weave'; + +interface Article { + text: string; + embedding?: number[]; +} + +const articles: Article[] = [ + { + text: `Novo Nordisk and Eli Lilly rival soars 32 percent after promising weight loss drug results Shares of Denmarks Zealand Pharma shot 32 percent higher in morning trade, after results showed success in its liver disease treatment survodutide, which is also on trial as a drug to treat obesity. The trial tells us that the 6mg dose is safe, which is the top dose used in the ongoing [Phase 3] obesity trial too, one analyst said in a note. The results come amid feverish investor interest in drugs that can be used for weight loss.` + }, + { + text: `Berkshire shares jump after big profit gain as Buffetts conglomerate nears $1 trillion valuation Berkshire Hathaway shares rose on Monday after Warren Buffetts conglomerate posted strong earnings for the fourth quarter over the weekend. Berkshires Class A and B shares jumped more than 1.5%, each. Class A shares are higher by more than 17% this year, while Class B has gained more than 18%. Berkshire was last valued at $930.1 billion, up from $905.5 billion where it closed on Friday, according to FactSet. Berkshire on Saturday posted fourth-quarter operating earnings of $8.481 billion, about 28 percent higher than the $6.625 billion from the year-ago period, driven by big gains in its insurance business. Operating earnings refers to profits from businesses across insurance, railroads and utilities. Meanwhile, Berkshires cash levels also swelled to record levels. The conglomerate held $167.6 billion in cash in the fourth quarter, surpassing the $157.2 billion record the conglomerate held in the prior quarter.` + }, + { + text: `Highmark Health says its combining tech from Google and Epic to give doctors easier access to information Highmark Health announced it is integrating technology from Google Cloud and the health-care software company Epic Systems. The integration aims to make it easier for both payers and providers to access key information they need, even if its stored across multiple points and formats, the company said. Highmark is the parent company of a health plan with 7 million members, a provider network of 14 hospitals and other entities` } +]; + +function cosineSimilarity(a: number[], b: number[]): number { + const dotProduct = a.reduce((sum, val, i) => sum + val * b[i], 0); + const magnitudeA = Math.sqrt(a.reduce((sum, val) => sum + val * val, 0)); + const magnitudeB = Math.sqrt(b.reduce((sum, val) => sum + val * val, 0)); + return dotProduct / (magnitudeA * magnitudeB); +} + +const docsToEmbeddings = weave.op(async function(docs: Article[]): Promise { + const openai = new OpenAI(); + const enrichedDocs = await Promise.all(docs.map(async (doc) => { + const response = await openai.embeddings.create({ + input: doc.text, + model: "text-embedding-3-small" + }); + return { + ...doc, + embedding: response.data[0].embedding + }; + })); + return enrichedDocs; +}); +``` - const docsToEmbeddings = weave.op(async function(docs: Article[]): Promise { - const openai = new OpenAI(); - const enrichedDocs = await Promise.all(docs.map(async (doc) => { - const response = await openai.embeddings.create({ - input: doc.text, - model: "text-embedding-3-small" - }); - return { - ...doc, - embedding: response.data[0].embedding - }; - })); - return enrichedDocs; - }); - ``` - - + ## Create a RAG app @@ -128,118 +126,116 @@ With the knowledge base in place, you can now build the RAG application itself. Next, wrap the retrieval function `get_most_relevant_document` with a `weave.op()` decorator and create a `Model` class. Wrapping the retrieval function with `weave.op()` lets Weave capture its inputs and outputs for every call, which is what makes the retrieval step inspectable later. Call `weave.init('/rag-quickstart')` to begin tracking all the inputs and outputs of your functions for later inspection. If you don't specify a team name, Weave records the output to your [W&B default team or entity](/platform/app/settings-page/user-settings#default-team). - - - ```python lines - from openai import OpenAI - import weave - from weave import Model - import numpy as np - import asyncio + + +```python Python lines +from openai import OpenAI +import weave +from weave import Model +import numpy as np +import asyncio + +@weave.op() +def get_most_relevant_document(query): + openai = OpenAI() + query_embedding = ( + openai.embeddings.create(input=query, model="text-embedding-3-small") + .data[0] + .embedding + ) + similarities = [ + np.dot(query_embedding, doc_emb) + / (np.linalg.norm(query_embedding) * np.linalg.norm(doc_emb)) + for doc_emb in article_embeddings + ] + # Get the index of the most similar document + most_relevant_doc_index = np.argmax(similarities) + return articles[most_relevant_doc_index] + +class RAGModel(Model): + system_message: str + model_name: str = "gpt-3.5-turbo-1106" @weave.op() - def get_most_relevant_document(query): - openai = OpenAI() - query_embedding = ( - openai.embeddings.create(input=query, model="text-embedding-3-small") - .data[0] - .embedding + def predict(self, question: str) -> dict: # note: `question` will be used later to select data from our evaluation rows + from openai import OpenAI + context = get_most_relevant_document(question) + client = OpenAI() + query = f"""Use the following information to answer the subsequent question. If the answer cannot be found, write "I don't know." + Context: + \"\"\" + {context} + \"\"\" + Question: {question}""" + response = client.chat.completions.create( + model=self.model_name, + messages=[ + {"role": "system", "content": self.system_message}, + {"role": "user", "content": query}, + ], + temperature=0.0, + response_format={"type": "text"}, ) - similarities = [ - np.dot(query_embedding, doc_emb) - / (np.linalg.norm(query_embedding) * np.linalg.norm(doc_emb)) - for doc_emb in article_embeddings - ] - # Get the index of the most similar document - most_relevant_doc_index = np.argmax(similarities) - return articles[most_relevant_doc_index] - - class RAGModel(Model): - system_message: str - model_name: str = "gpt-3.5-turbo-1106" - - @weave.op() - def predict(self, question: str) -> dict: # note: `question` will be used later to select data from our evaluation rows - from openai import OpenAI - context = get_most_relevant_document(question) - client = OpenAI() - query = f"""Use the following information to answer the subsequent question. If the answer cannot be found, write "I don't know." - Context: - \"\"\" - {context} - \"\"\" - Question: {question}""" - response = client.chat.completions.create( - model=self.model_name, - messages=[ - {"role": "system", "content": self.system_message}, - {"role": "user", "content": query}, - ], - temperature=0.0, - response_format={"type": "text"}, - ) - answer = response.choices[0].message.content - return {'answer': answer, 'context': context} - - # Set your team and project names - weave.init('/rag-quickstart') - model = RAGModel( - system_message="You are an expert in finance and answer questions related to finance, financial services, and financial markets. When responding based on provided information, be sure to cite the source." - ) - model.predict("What significant result was reported about Zealand Pharma's obesity trial?") - ``` - - - - ```typescript twoslash lines - // @noErrors - class RAGModel { - private openai: OpenAI; - private systemMessage: string; - private modelName: string; - private articleEmbeddings: Article[]; - - constructor(config: { - systemMessage: string; - modelName?: string; - articleEmbeddings: Article[]; - }) { - this.openai = new OpenAI(); - this.systemMessage = config.systemMessage; - this.modelName = config.modelName || "gpt-3.5-turbo-1106"; - this.articleEmbeddings = config.articleEmbeddings; - this.predict = weave.op(this, this.predict); - } + answer = response.choices[0].message.content + return {'answer': answer, 'context': context} + +# Set your team and project names +weave.init('/rag-quickstart') +model = RAGModel( + system_message="You are an expert in finance and answer questions related to finance, financial services, and financial markets. When responding based on provided information, be sure to cite the source." +) +model.predict("What significant result was reported about Zealand Pharma's obesity trial?") +``` + +```typescript TypeScript twoslash lines +// @noErrors +class RAGModel { + private openai: OpenAI; + private systemMessage: string; + private modelName: string; + private articleEmbeddings: Article[]; + + constructor(config: { + systemMessage: string; + modelName?: string; + articleEmbeddings: Article[]; + }) { + this.openai = new OpenAI(); + this.systemMessage = config.systemMessage; + this.modelName = config.modelName || "gpt-3.5-turbo-1106"; + this.articleEmbeddings = config.articleEmbeddings; + this.predict = weave.op(this, this.predict); + } - async predict(question: string): Promise<{ - answer: string; - context: string; - }> { - const context = await this.getMostRelevantDocument(question); - - const response = await this.openai.chat.completions.create({ - model: this.modelName, - messages: [ - { role: "system", content: this.systemMessage }, - { role: "user", content: `Use the following information to answer the subsequent question. If the answer cannot be found, write "I don't know." - Context: - """ - ${context} - """ - Question: ${question}` } - ], - temperature: 0 - }); - - return { - answer: response.choices[0].message.content || "", - context - }; - } + async predict(question: string): Promise<{ + answer: string; + context: string; + }> { + const context = await this.getMostRelevantDocument(question); + + const response = await this.openai.chat.completions.create({ + model: this.modelName, + messages: [ + { role: "system", content: this.systemMessage }, + { role: "user", content: `Use the following information to answer the subsequent question. If the answer cannot be found, write "I don't know." + Context: + """ + ${context} + """ + Question: ${question}` } + ], + temperature: 0 + }); + + return { + answer: response.choices[0].message.content || "", + context + }; } - ``` - - +} +``` + + ## Evaluate with an LLM judge @@ -251,91 +247,89 @@ When there aren't simple ways to evaluate your application, one approach is to u As in the [Build an Evaluation pipeline tutorial](/weave/tutorial-eval), define a set of example rows to test your app against and a scoring function. The scoring function takes one row and evaluates it. The input arguments should match the corresponding keys in your row, so `question` here is taken from the row dictionary. `output` is the output of the model. The input to the model is taken from the example based on its input argument, so `question` here too. This example uses `async` functions so they run in parallel. For a quick introduction to async, see the [Python `asyncio` documentation](https://docs.python.org/3/library/asyncio.html). - - - ```python lines - from openai import OpenAI - import weave - import asyncio + - @weave.op() - async def context_precision_score(question, output): - context_precision_prompt = """Given question, answer and context verify if the context was useful in arriving at the given answer. Give verdict as "1" if useful and "0" if not with json output. - Output in only valid JSON format. - - question: {question} - context: {context} - answer: {answer} - verdict: """ - client = OpenAI() +```python Python lines +from openai import OpenAI +import weave +import asyncio - prompt = context_precision_prompt.format( - question=question, - context=output['context'], - answer=output['answer'], - ) +@weave.op() +async def context_precision_score(question, output): + context_precision_prompt = """Given question, answer and context verify if the context was useful in arriving at the given answer. Give verdict as "1" if useful and "0" if not with json output. + Output in only valid JSON format. - response = client.chat.completions.create( - model="gpt-4-turbo-preview", - messages=[{"role": "user", "content": prompt}], - response_format={ "type": "json_object" } - ) - response_message = response.choices[0].message - response = json.loads(response_message.content) - return { - "verdict": int(response["verdict"]) == 1, - } + question: {question} + context: {context} + answer: {answer} + verdict: """ + client = OpenAI() - questions = [ - {"question": "What significant result was reported about Zealand Pharma's obesity trial?"}, - {"question": "How much did Berkshire Hathaway's cash levels increase in the fourth quarter?"}, - {"question": "What is the goal of Highmark Health's integration of Google Cloud and Epic Systems technology?"}, - {"question": "What were Rivian and Lucid's vehicle production forecasts for 2024?"}, - {"question": "Why was the Norwegian Dawn cruise ship denied access to Mauritius?"}, - {"question": "Which company achieved the first U.S. moon landing since 1972?"}, - {"question": "What issue did Intuitive Machines' lunar lander encounter upon landing on the moon?"} - ] - evaluation = weave.Evaluation(dataset=questions, scorers=[context_precision_score]) - asyncio.run(evaluation.evaluate(model)) # note: you'll need to define a model to evaluate - ``` - - - - ```typescript twoslash lines - // @noErrors - const contextPrecisionScore = weave.op(async function(args: { - datasetRow: QuestionRow; - modelOutput: { answer: string; context: string; } - }): Promise { - const openai = new OpenAI(); - - const prompt = `Given question, answer and context verify if the context was useful...`; + prompt = context_precision_prompt.format( + question=question, + context=output['context'], + answer=output['answer'], + ) - const response = await openai.chat.completions.create({ - model: "gpt-4-turbo-preview", - messages: [{ role: "user", content: prompt }], - response_format: { type: "json_object" } - }); + response = client.chat.completions.create( + model="gpt-4-turbo-preview", + messages=[{"role": "user", "content": prompt}], + response_format={ "type": "json_object" } + ) + response_message = response.choices[0].message + response = json.loads(response_message.content) + return { + "verdict": int(response["verdict"]) == 1, + } - const result = JSON.parse(response.choices[0].message.content || "{}"); - return { - verdict: parseInt(result.verdict) === 1 - }; - }); +questions = [ + {"question": "What significant result was reported about Zealand Pharma's obesity trial?"}, + {"question": "How much did Berkshire Hathaway's cash levels increase in the fourth quarter?"}, + {"question": "What is the goal of Highmark Health's integration of Google Cloud and Epic Systems technology?"}, + {"question": "What were Rivian and Lucid's vehicle production forecasts for 2024?"}, + {"question": "Why was the Norwegian Dawn cruise ship denied access to Mauritius?"}, + {"question": "Which company achieved the first U.S. moon landing since 1972?"}, + {"question": "What issue did Intuitive Machines' lunar lander encounter upon landing on the moon?"} +] +evaluation = weave.Evaluation(dataset=questions, scorers=[context_precision_score]) +asyncio.run(evaluation.evaluate(model)) # note: you'll need to define a model to evaluate +``` + +```typescript TypeScript twoslash lines +// @noErrors +const contextPrecisionScore = weave.op(async function(args: { + datasetRow: QuestionRow; + modelOutput: { answer: string; context: string; } +}): Promise { + const openai = new OpenAI(); + + const prompt = `Given question, answer and context verify if the context was useful...`; - const evaluation = new weave.Evaluation({ - dataset: createQuestionDataset(), - scorers: [contextPrecisionScore] + const response = await openai.chat.completions.create({ + model: "gpt-4-turbo-preview", + messages: [{ role: "user", content: prompt }], + response_format: { type: "json_object" } }); - await evaluation.evaluate({ - model: weave.op((args: { datasetRow: QuestionRow }) => - model.predict(args.datasetRow.question) - ) - }); - ``` - - + const result = JSON.parse(response.choices[0].message.content || "{}"); + return { + verdict: parseInt(result.verdict) === 1 + }; +}); + +const evaluation = new weave.Evaluation({ + dataset: createQuestionDataset(), + scorers: [contextPrecisionScore] +}); + +await evaluation.evaluate({ + model: weave.op((args: { datasetRow: QuestionRow }) => + model.predict(args.datasetRow.question) + ) +}); +``` + + ### Optional: Define a `Scorer` class @@ -352,97 +346,94 @@ At a high level, the steps to create a custom Scorer are: 3. Optional: Overwrite the `summarize` function to customize the calculation of the aggregate score. By default, Weave uses the `weave.flow.scorer.auto_summarize` function if you don't define a custom function. - This function has to have a `@weave.op()` decorator. - - - ```python lines - from weave import Scorer - - class CorrectnessLLMJudge(Scorer): - prompt: str - model_name: str - device: str - - @weave.op() - async def score(self, output: Optional[dict], query: str, answer: str) -> Any: - """Score the correctness of the predictions by comparing the pred, query, target. - Args: - - output: the dict that will be provided by the model that is evaluated - - query: the question asked - as defined in the dataset - - answer: the target answer - as defined in the dataset - Returns: - - single dict {metric name: single evaluation value}""" - - # get_model is defined as general model getter based on provided params (OpenAI,HF...) - eval_model = get_model( - model_name = self.model_name, - prompt = self.prompt - device = self.device, - ) - # async evaluation to speed up evaluation - this doesn't have to be async - grade = await eval_model.async_predict( - { - "query": query, - "answer": answer, - "result": output.get("result"), - } - ) - # output parsing - could be done more reobustly with pydantic - evaluation = "incorrect" not in grade["text"].strip().lower() - - # the column name displayed in Weave - return {"correct": evaluation} - - @weave.op() - def summarize(self, score_rows: list) -> Optional[dict]: - """Aggregate all the scores that are calculated for each row by the scoring function. - Args: - - score_rows: a list of dicts. Each dict has metrics and scores - Returns: - - nested dict with the same structure as the input""" - - # if nothing is provided the weave.flow.scorer.auto_summarize function is used - # return auto_summarize(score_rows) - - valid_data = [x.get("correct") for x in score_rows if x.get("correct") is not None] - count_true = list(valid_data).count(True) - int_data = [int(x) for x in valid_data] - - sample_mean = np.mean(int_data) if int_data else 0 - sample_variance = np.var(int_data) if int_data else 0 - sample_error = np.sqrt(sample_variance / len(int_data)) if int_data else 0 - - # the extra "correct" layer is not necessary but adds structure in the UI - return { - "correct": { - "true_count": count_true, - "true_fraction": sample_mean, - "stderr": sample_error, - } + + +```python Python lines +from weave import Scorer + +class CorrectnessLLMJudge(Scorer): + prompt: str + model_name: str + device: str + + @weave.op() + async def score(self, output: Optional[dict], query: str, answer: str) -> Any: + """Score the correctness of the predictions by comparing the pred, query, target. + Args: + - output: the dict that will be provided by the model that is evaluated + - query: the question asked - as defined in the dataset + - answer: the target answer - as defined in the dataset + Returns: + - single dict {metric name: single evaluation value}""" + + # get_model is defined as general model getter based on provided params (OpenAI,HF...) + eval_model = get_model( + model_name = self.model_name, + prompt = self.prompt + device = self.device, + ) + # async evaluation to speed up evaluation - this doesn't have to be async + grade = await eval_model.async_predict( + { + "query": query, + "answer": answer, + "result": output.get("result"), } - ``` + ) + # output parsing - could be done more reobustly with pydantic + evaluation = "incorrect" not in grade["text"].strip().lower() - - - ```plaintext - This feature is not available in TypeScript yet. - ``` - - + # the column name displayed in Weave + return {"correct": evaluation} + + @weave.op() + def summarize(self, score_rows: list) -> Optional[dict]: + """Aggregate all the scores that are calculated for each row by the scoring function. + Args: + - score_rows: a list of dicts. Each dict has metrics and scores + Returns: + - nested dict with the same structure as the input""" + + # if nothing is provided the weave.flow.scorer.auto_summarize function is used + # return auto_summarize(score_rows) + + valid_data = [x.get("correct") for x in score_rows if x.get("correct") is not None] + count_true = list(valid_data).count(True) + int_data = [int(x) for x in valid_data] + + sample_mean = np.mean(int_data) if int_data else 0 + sample_variance = np.var(int_data) if int_data else 0 + sample_error = np.sqrt(sample_variance / len(int_data)) if int_data else 0 + + # the extra "correct" layer is not necessary but adds structure in the UI + return { + "correct": { + "true_count": count_true, + "true_fraction": sample_mean, + "stderr": sample_error, + } + } +``` + +```plaintext TypeScript +This feature is not available in TypeScript yet. +``` + + To use this as a scorer, initialize it and pass it to the `scorers` argument in your `Evaluation` like this: - - - ```python lines - evaluation = weave.Evaluation(dataset=questions, scorers=[CorrectnessLLMJudge()]) - ``` - - - ```plaintext - This feature is not available in TypeScript yet. - ``` - - + + +```python Python lines +evaluation = weave.Evaluation(dataset=questions, scorers=[CorrectnessLLMJudge()]) +``` + +```plaintext TypeScript +This feature is not available in TypeScript yet. +``` + + ## Put it all together @@ -460,323 +451,321 @@ To get the same result for your RAG apps: Here's the code in its entirety. - - - ```python lines {34,52-77} - from openai import OpenAI - import weave - from weave import Model - import numpy as np - import json - import asyncio - - # Examples to use for evaluations - articles = [ - "Novo Nordisk and Eli Lilly rival soars 32 percent after promising weight loss drug results Shares of Denmarks Zealand Pharma shot 32 percent higher in morning trade, after results showed success in its liver disease treatment survodutide, which is also on trial as a drug to treat obesity. The trial “tells us that the 6mg dose is safe, which is the top dose used in the ongoing [Phase 3] obesity trial too,” one analyst said in a note. The results come amid feverish investor interest in drugs that can be used for weight loss.", - "Berkshire shares jump after big profit gain as Buffetts conglomerate nears $1 trillion valuation Berkshire Hathaway shares rose on Monday after Warren Buffetts conglomerate posted strong earnings for the fourth quarter over the weekend. Berkshires Class A and B shares jumped more than 1.5%, each. Class A shares are higher by more than 17% this year, while Class B has gained more than 18%. Berkshire was last valued at $930.1 billion, up from $905.5 billion where it closed on Friday, according to FactSet. Berkshire on Saturday posted fourth-quarter operating earnings of $8.481 billion, about 28 percent higher than the $6.625 billion from the year-ago period, driven by big gains in its insurance business. Operating earnings refers to profits from businesses across insurance, railroads and utilities. Meanwhile, Berkshires cash levels also swelled to record levels. The conglomerate held $167.6 billion in cash in the fourth quarter, surpassing the $157.2 billion record the conglomerate held in the prior quarter.", - "Highmark Health says its combining tech from Google and Epic to give doctors easier access to information Highmark Health announced it is integrating technology from Google Cloud and the health-care software company Epic Systems. The integration aims to make it easier for both payers and providers to access key information they need, even if it's stored across multiple points and formats, the company said. Highmark is the parent company of a health plan with 7 million members, a provider network of 14 hospitals and other entities", - "Rivian and Lucid shares plunge after weak EV earnings reports Shares of electric vehicle makers Rivian and Lucid fell Thursday after the companies reported stagnant production in their fourth-quarter earnings after the bell Wednesday. Rivian shares sank about 25 percent, and Lucids stock dropped around 17 percent. Rivian forecast it will make 57,000 vehicles in 2024, slightly less than the 57,232 vehicles it produced in 2023. Lucid said it expects to make 9,000 vehicles in 2024, more than the 8,428 vehicles it made in 2023.", - "Mauritius blocks Norwegian cruise ship over fears of a potential cholera outbreak Local authorities on Sunday denied permission for the Norwegian Dawn ship, which has 2,184 passengers and 1,026 crew on board, to access the Mauritius capital of Port Louis, citing “potential health risks.” The Mauritius Ports Authority said Sunday that samples were taken from at least 15 passengers on board the cruise ship. A spokesperson for the U.S.-headquartered Norwegian Cruise Line Holdings said Sunday that 'a small number of guests experienced mild symptoms of a stomach-related illness' during Norwegian Dawns South Africa voyage.", - "Intuitive Machines lands on the moon in historic first for a U.S. company Intuitive Machines Nova-C cargo lander, named Odysseus after the mythological Greek hero, is the first U.S. spacecraft to soft land on the lunar surface since 1972. Intuitive Machines is the first company to pull off a moon landing — government agencies have carried out all previously successful missions. The company's stock surged in extended trading Thursday, after falling 11 percent in regular trading.", - "Lunar landing photos: Intuitive Machines Odysseus sends back first images from the moon Intuitive Machines cargo moon lander Odysseus returned its first images from the surface. Company executives believe the lander caught its landing gear sideways on the surface of the moon while touching down and tipped over. Despite resting on its side, the company's historic IM-1 mission is still operating on the moon.", - ] - - def docs_to_embeddings(docs: list) -> list: - openai = OpenAI() - document_embeddings = [] - for doc in docs: - response = ( - openai.embeddings.create(input=doc, model="text-embedding-3-small") - .data[0] - .embedding - ) - document_embeddings.append(response) - return document_embeddings - - article_embeddings = docs_to_embeddings(articles) # Note: you would typically do this once with your articles and put the embeddings & metadata in a database - - # Add a decorator to the retrieval step - @weave.op() - def get_most_relevant_document(query): - openai = OpenAI() - query_embedding = ( - openai.embeddings.create(input=query, model="text-embedding-3-small") + + +```python Python lines {34,52-77} +from openai import OpenAI +import weave +from weave import Model +import numpy as np +import json +import asyncio + +# Examples to use for evaluations +articles = [ + "Novo Nordisk and Eli Lilly rival soars 32 percent after promising weight loss drug results Shares of Denmarks Zealand Pharma shot 32 percent higher in morning trade, after results showed success in its liver disease treatment survodutide, which is also on trial as a drug to treat obesity. The trial “tells us that the 6mg dose is safe, which is the top dose used in the ongoing [Phase 3] obesity trial too,” one analyst said in a note. The results come amid feverish investor interest in drugs that can be used for weight loss.", + "Berkshire shares jump after big profit gain as Buffetts conglomerate nears $1 trillion valuation Berkshire Hathaway shares rose on Monday after Warren Buffetts conglomerate posted strong earnings for the fourth quarter over the weekend. Berkshires Class A and B shares jumped more than 1.5%, each. Class A shares are higher by more than 17% this year, while Class B has gained more than 18%. Berkshire was last valued at $930.1 billion, up from $905.5 billion where it closed on Friday, according to FactSet. Berkshire on Saturday posted fourth-quarter operating earnings of $8.481 billion, about 28 percent higher than the $6.625 billion from the year-ago period, driven by big gains in its insurance business. Operating earnings refers to profits from businesses across insurance, railroads and utilities. Meanwhile, Berkshires cash levels also swelled to record levels. The conglomerate held $167.6 billion in cash in the fourth quarter, surpassing the $157.2 billion record the conglomerate held in the prior quarter.", + "Highmark Health says its combining tech from Google and Epic to give doctors easier access to information Highmark Health announced it is integrating technology from Google Cloud and the health-care software company Epic Systems. The integration aims to make it easier for both payers and providers to access key information they need, even if it's stored across multiple points and formats, the company said. Highmark is the parent company of a health plan with 7 million members, a provider network of 14 hospitals and other entities", + "Rivian and Lucid shares plunge after weak EV earnings reports Shares of electric vehicle makers Rivian and Lucid fell Thursday after the companies reported stagnant production in their fourth-quarter earnings after the bell Wednesday. Rivian shares sank about 25 percent, and Lucids stock dropped around 17 percent. Rivian forecast it will make 57,000 vehicles in 2024, slightly less than the 57,232 vehicles it produced in 2023. Lucid said it expects to make 9,000 vehicles in 2024, more than the 8,428 vehicles it made in 2023.", + "Mauritius blocks Norwegian cruise ship over fears of a potential cholera outbreak Local authorities on Sunday denied permission for the Norwegian Dawn ship, which has 2,184 passengers and 1,026 crew on board, to access the Mauritius capital of Port Louis, citing “potential health risks.” The Mauritius Ports Authority said Sunday that samples were taken from at least 15 passengers on board the cruise ship. A spokesperson for the U.S.-headquartered Norwegian Cruise Line Holdings said Sunday that 'a small number of guests experienced mild symptoms of a stomach-related illness' during Norwegian Dawns South Africa voyage.", + "Intuitive Machines lands on the moon in historic first for a U.S. company Intuitive Machines Nova-C cargo lander, named Odysseus after the mythological Greek hero, is the first U.S. spacecraft to soft land on the lunar surface since 1972. Intuitive Machines is the first company to pull off a moon landing — government agencies have carried out all previously successful missions. The company's stock surged in extended trading Thursday, after falling 11 percent in regular trading.", + "Lunar landing photos: Intuitive Machines Odysseus sends back first images from the moon Intuitive Machines cargo moon lander Odysseus returned its first images from the surface. Company executives believe the lander caught its landing gear sideways on the surface of the moon while touching down and tipped over. Despite resting on its side, the company's historic IM-1 mission is still operating on the moon.", +] + +def docs_to_embeddings(docs: list) -> list: + openai = OpenAI() + document_embeddings = [] + for doc in docs: + response = ( + openai.embeddings.create(input=doc, model="text-embedding-3-small") .data[0] .embedding ) - similarities = [ - np.dot(query_embedding, doc_emb) - / (np.linalg.norm(query_embedding) * np.linalg.norm(doc_emb)) - for doc_emb in article_embeddings - ] - # Get the index of the most similar document - most_relevant_doc_index = np.argmax(similarities) - return articles[most_relevant_doc_index] - - # Create a Model subclass with details about the app, along with a predict function that produces a response - class RAGModel(Model): - system_message: str - model_name: str = "gpt-3.5-turbo-1106" - - @weave.op() - def predict(self, question: str) -> dict: # note: `question` will be used later to select data from our evaluation rows - from openai import OpenAI - context = get_most_relevant_document(question) - client = OpenAI() - query = f"""Use the following information to answer the subsequent question. If the answer cannot be found, write "I don't know." - Context: - \"\"\" - {context} - \"\"\" - Question: {question}""" - response = client.chat.completions.create( - model=self.model_name, - messages=[ - {"role": "system", "content": self.system_message}, - {"role": "user", "content": query}, - ], - temperature=0.0, - response_format={"type": "text"}, - ) - answer = response.choices[0].message.content - return {'answer': answer, 'context': context} - - # Set your team and project names - weave.init('/rag-quickstart') - model = RAGModel( - system_message="You are an expert in finance and answer questions related to finance, financial services, and financial markets. When responding based on provided information, be sure to cite the source." + document_embeddings.append(response) + return document_embeddings + +article_embeddings = docs_to_embeddings(articles) # Note: you would typically do this once with your articles and put the embeddings & metadata in a database + +# Add a decorator to the retrieval step +@weave.op() +def get_most_relevant_document(query): + openai = OpenAI() + query_embedding = ( + openai.embeddings.create(input=query, model="text-embedding-3-small") + .data[0] + .embedding ) + similarities = [ + np.dot(query_embedding, doc_emb) + / (np.linalg.norm(query_embedding) * np.linalg.norm(doc_emb)) + for doc_emb in article_embeddings + ] + # Get the index of the most similar document + most_relevant_doc_index = np.argmax(similarities) + return articles[most_relevant_doc_index] + +# Create a Model subclass with details about the app, along with a predict function that produces a response +class RAGModel(Model): + system_message: str + model_name: str = "gpt-3.5-turbo-1106" - # Here is our scoring function uses our question and output to product a score @weave.op() - async def context_precision_score(question, output): - context_precision_prompt = """Given question, answer and context verify if the context was useful in arriving at the given answer. Give verdict as "1" if useful and "0" if not with json output. - Output in only valid JSON format. - - question: {question} - context: {context} - answer: {answer} - verdict: """ + def predict(self, question: str) -> dict: # note: `question` will be used later to select data from our evaluation rows + from openai import OpenAI + context = get_most_relevant_document(question) client = OpenAI() - - prompt = context_precision_prompt.format( - question=question, - context=output['context'], - answer=output['answer'], - ) - + query = f"""Use the following information to answer the subsequent question. If the answer cannot be found, write "I don't know." + Context: + \"\"\" + {context} + \"\"\" + Question: {question}""" response = client.chat.completions.create( - model="gpt-4-turbo-preview", - messages=[{"role": "user", "content": prompt}], - response_format={ "type": "json_object" } + model=self.model_name, + messages=[ + {"role": "system", "content": self.system_message}, + {"role": "user", "content": query}, + ], + temperature=0.0, + response_format={"type": "text"}, ) - response_message = response.choices[0].message - response = json.loads(response_message.content) - return { - "verdict": int(response["verdict"]) == 1, - } - - questions = [ - {"question": "What significant result was reported about Zealand Pharma's obesity trial?"}, - {"question": "How much did Berkshire Hathaway's cash levels increase in the fourth quarter?"}, - {"question": "What is the goal of Highmark Health's integration of Google Cloud and Epic Systems technology?"}, - {"question": "What were Rivian and Lucid's vehicle production forecasts for 2024?"}, - {"question": "Why was the Norwegian Dawn cruise ship denied access to Mauritius?"}, - {"question": "Which company achieved the first U.S. moon landing since 1972?"}, - {"question": "What issue did Intuitive Machines' lunar lander encounter upon landing on the moon?"} - ] - - # Define an Evaluation object and pass example questions along with scoring functions - evaluation = weave.Evaluation(dataset=questions, scorers=[context_precision_score]) - asyncio.run(evaluation.evaluate(model)) - ``` - - - - ```typescript twoslash lines - // @noErrors - require('dotenv').config(); - import { OpenAI } from 'openai'; - import * as weave from 'weave'; - - interface Article { - text: string; - embedding?: number[]; - } - - const articles: Article[] = [ - { - text: `Novo Nordisk and Eli Lilly rival soars 32 percent after promising weight loss drug results Shares of Denmarks Zealand Pharma shot 32 percent higher in morning trade, after results showed success in its liver disease treatment survodutide, which is also on trial as a drug to treat obesity. The trial tells us that the 6mg dose is safe, which is the top dose used in the ongoing [Phase 3] obesity trial too, one analyst said in a note. The results come amid feverish investor interest in drugs that can be used for weight loss.` - }, - { - text: `Berkshire shares jump after big profit gain as Buffetts conglomerate nears $1 trillion valuation Berkshire Hathaway shares rose on Monday after Warren Buffetts conglomerate posted strong earnings for the fourth quarter over the weekend. Berkshires Class A and B shares jumped more than 1.5%, each. Class A shares are higher by more than 17% this year, while Class B has gained more than 18%. Berkshire was last valued at $930.1 billion, up from $905.5 billion where it closed on Friday, according to FactSet. Berkshire on Saturday posted fourth-quarter operating earnings of $8.481 billion, about 28 percent higher than the $6.625 billion from the year-ago period, driven by big gains in its insurance business. Operating earnings refers to profits from businesses across insurance, railroads and utilities. Meanwhile, Berkshires cash levels also swelled to record levels. The conglomerate held $167.6 billion in cash in the fourth quarter, surpassing the $157.2 billion record the conglomerate held in the prior quarter.` - }, - { - text: `Highmark Health says its combining tech from Google and Epic to give doctors easier access to information Highmark Health announced it is integrating technology from Google Cloud and the health-care software company Epic Systems. The integration aims to make it easier for both payers and providers to access key information they need, even if its stored across multiple points and formats, the company said. Highmark is the parent company of a health plan with 7 million members, a provider network of 14 hospitals and other entities` - } - ]; + answer = response.choices[0].message.content + return {'answer': answer, 'context': context} + +# Set your team and project names +weave.init('/rag-quickstart') +model = RAGModel( + system_message="You are an expert in finance and answer questions related to finance, financial services, and financial markets. When responding based on provided information, be sure to cite the source." +) + +# Here is our scoring function uses our question and output to product a score +@weave.op() +async def context_precision_score(question, output): + context_precision_prompt = """Given question, answer and context verify if the context was useful in arriving at the given answer. Give verdict as "1" if useful and "0" if not with json output. + Output in only valid JSON format. + + question: {question} + context: {context} + answer: {answer} + verdict: """ + client = OpenAI() + + prompt = context_precision_prompt.format( + question=question, + context=output['context'], + answer=output['answer'], + ) - function cosineSimilarity(a: number[], b: number[]): number { - const dotProduct = a.reduce((sum, val, i) => sum + val * b[i], 0); - const magnitudeA = Math.sqrt(a.reduce((sum, val) => sum + val * val, 0)); - const magnitudeB = Math.sqrt(b.reduce((sum, val) => sum + val * val, 0)); - return dotProduct / (magnitudeA * magnitudeB); + response = client.chat.completions.create( + model="gpt-4-turbo-preview", + messages=[{"role": "user", "content": prompt}], + response_format={ "type": "json_object" } + ) + response_message = response.choices[0].message + response = json.loads(response_message.content) + return { + "verdict": int(response["verdict"]) == 1, } - const docsToEmbeddings = weave.op(async function(docs: Article[]): Promise { - const openai = new OpenAI(); - const enrichedDocs = await Promise.all(docs.map(async (doc) => { - const response = await openai.embeddings.create({ - input: doc.text, - model: "text-embedding-3-small" - }); - return { - ...doc, - embedding: response.data[0].embedding - }; - })); - return enrichedDocs; - }); - - class RAGModel { - private openai: OpenAI; - private systemMessage: string; - private modelName: string; - private articleEmbeddings: Article[]; - - constructor(config: { - systemMessage: string; - modelName?: string; - articleEmbeddings: Article[]; - }) { - this.openai = new OpenAI(); - this.systemMessage = config.systemMessage; - this.modelName = config.modelName || "gpt-3.5-turbo-1106"; - this.articleEmbeddings = config.articleEmbeddings; - this.predict = weave.op(this, this.predict); - } - - private async getMostRelevantDocument(query: string): Promise { - const queryEmbedding = await this.openai.embeddings.create({ - input: query, - model: "text-embedding-3-small" - }); - - const similarities = this.articleEmbeddings.map(doc => { - if (!doc.embedding) return 0; - return cosineSimilarity(queryEmbedding.data[0].embedding, doc.embedding); - }); - - const mostRelevantIndex = similarities.indexOf(Math.max(...similarities)); - return this.articleEmbeddings[mostRelevantIndex].text; - } - - async predict(question: string): Promise<{ - answer: string; - context: string; - }> { - const context = await this.getMostRelevantDocument(question); - - const response = await this.openai.chat.completions.create({ - model: this.modelName, - messages: [ - { role: "system", content: this.systemMessage }, - { - role: "user", - content: `Use the following information to answer the subsequent question. If the answer cannot be found, write "I don't know." - Context: - """ - ${context} - """ - Question: ${question}` - } - ], - temperature: 0 - }); - - return { - answer: response.choices[0].message.content || "", - context - }; - } +questions = [ + {"question": "What significant result was reported about Zealand Pharma's obesity trial?"}, + {"question": "How much did Berkshire Hathaway's cash levels increase in the fourth quarter?"}, + {"question": "What is the goal of Highmark Health's integration of Google Cloud and Epic Systems technology?"}, + {"question": "What were Rivian and Lucid's vehicle production forecasts for 2024?"}, + {"question": "Why was the Norwegian Dawn cruise ship denied access to Mauritius?"}, + {"question": "Which company achieved the first U.S. moon landing since 1972?"}, + {"question": "What issue did Intuitive Machines' lunar lander encounter upon landing on the moon?"} +] + +# Define an Evaluation object and pass example questions along with scoring functions +evaluation = weave.Evaluation(dataset=questions, scorers=[context_precision_score]) +asyncio.run(evaluation.evaluate(model)) +``` + +```typescript TypeScript twoslash lines +// @noErrors +require('dotenv').config(); +import { OpenAI } from 'openai'; +import * as weave from 'weave'; + +interface Article { + text: string; + embedding?: number[]; +} + +const articles: Article[] = [ + { + text: `Novo Nordisk and Eli Lilly rival soars 32 percent after promising weight loss drug results Shares of Denmarks Zealand Pharma shot 32 percent higher in morning trade, after results showed success in its liver disease treatment survodutide, which is also on trial as a drug to treat obesity. The trial tells us that the 6mg dose is safe, which is the top dose used in the ongoing [Phase 3] obesity trial too, one analyst said in a note. The results come amid feverish investor interest in drugs that can be used for weight loss.` + }, + { + text: `Berkshire shares jump after big profit gain as Buffetts conglomerate nears $1 trillion valuation Berkshire Hathaway shares rose on Monday after Warren Buffetts conglomerate posted strong earnings for the fourth quarter over the weekend. Berkshires Class A and B shares jumped more than 1.5%, each. Class A shares are higher by more than 17% this year, while Class B has gained more than 18%. Berkshire was last valued at $930.1 billion, up from $905.5 billion where it closed on Friday, according to FactSet. Berkshire on Saturday posted fourth-quarter operating earnings of $8.481 billion, about 28 percent higher than the $6.625 billion from the year-ago period, driven by big gains in its insurance business. Operating earnings refers to profits from businesses across insurance, railroads and utilities. Meanwhile, Berkshires cash levels also swelled to record levels. The conglomerate held $167.6 billion in cash in the fourth quarter, surpassing the $157.2 billion record the conglomerate held in the prior quarter.` + }, + { + text: `Highmark Health says its combining tech from Google and Epic to give doctors easier access to information Highmark Health announced it is integrating technology from Google Cloud and the health-care software company Epic Systems. The integration aims to make it easier for both payers and providers to access key information they need, even if its stored across multiple points and formats, the company said. Highmark is the parent company of a health plan with 7 million members, a provider network of 14 hospitals and other entities` } - - interface ScorerResult { - verdict: boolean; +]; + +function cosineSimilarity(a: number[], b: number[]): number { + const dotProduct = a.reduce((sum, val, i) => sum + val * b[i], 0); + const magnitudeA = Math.sqrt(a.reduce((sum, val) => sum + val * val, 0)); + const magnitudeB = Math.sqrt(b.reduce((sum, val) => sum + val * val, 0)); + return dotProduct / (magnitudeA * magnitudeB); +} + +const docsToEmbeddings = weave.op(async function(docs: Article[]): Promise { + const openai = new OpenAI(); + const enrichedDocs = await Promise.all(docs.map(async (doc) => { + const response = await openai.embeddings.create({ + input: doc.text, + model: "text-embedding-3-small" + }); + return { + ...doc, + embedding: response.data[0].embedding + }; + })); + return enrichedDocs; +}); + +class RAGModel { + private openai: OpenAI; + private systemMessage: string; + private modelName: string; + private articleEmbeddings: Article[]; + + constructor(config: { + systemMessage: string; + modelName?: string; + articleEmbeddings: Article[]; + }) { + this.openai = new OpenAI(); + this.systemMessage = config.systemMessage; + this.modelName = config.modelName || "gpt-3.5-turbo-1106"; + this.articleEmbeddings = config.articleEmbeddings; + this.predict = weave.op(this, this.predict); } - interface QuestionRow { - question: string; - } + private async getMostRelevantDocument(query: string): Promise { + const queryEmbedding = await this.openai.embeddings.create({ + input: query, + model: "text-embedding-3-small" + }); - function createQuestionDataset(): weave.Dataset { - return new weave.Dataset({ - id: 'rag-questions', - rows: [ - { question: "What significant result was reported about Zealand Pharma's obesity trial?" }, - { question: "How much did Berkshire Hathaway's cash levels increase in the fourth quarter?" }, - { question: "What is the goal of Highmark Health's integration of Google Cloud and Epic Systems technology?" } - ] + const similarities = this.articleEmbeddings.map(doc => { + if (!doc.embedding) return 0; + return cosineSimilarity(queryEmbedding.data[0].embedding, doc.embedding); }); + + const mostRelevantIndex = similarities.indexOf(Math.max(...similarities)); + return this.articleEmbeddings[mostRelevantIndex].text; } - const contextPrecisionScore = weave.op(async function(args: { - datasetRow: QuestionRow; - modelOutput: { answer: string; context: string; } - }): Promise { - const openai = new OpenAI(); + async predict(question: string): Promise<{ + answer: string; + context: string; + }> { + const context = await this.getMostRelevantDocument(question); - const prompt = `Given question, answer and context verify if the context was useful in arriving at the given answer. Give verdict as "1" if useful and "0" if not with json output. - Output in only valid JSON format. - - question: ${args.datasetRow.question} - context: ${args.modelOutput.context} - answer: ${args.modelOutput.answer} - verdict: `; - - const response = await openai.chat.completions.create({ - model: "gpt-4-turbo-preview", - messages: [{ role: "user", content: prompt }], - response_format: { type: "json_object" } + const response = await this.openai.chat.completions.create({ + model: this.modelName, + messages: [ + { role: "system", content: this.systemMessage }, + { + role: "user", + content: `Use the following information to answer the subsequent question. If the answer cannot be found, write "I don't know." + Context: + """ + ${context} + """ + Question: ${question}` + } + ], + temperature: 0 }); - const result = JSON.parse(response.choices[0].message.content || "{}"); return { - verdict: parseInt(result.verdict) === 1 + answer: response.choices[0].message.content || "", + context }; + } +} + +interface ScorerResult { + verdict: boolean; +} + +interface QuestionRow { + question: string; +} + +function createQuestionDataset(): weave.Dataset { + return new weave.Dataset({ + id: 'rag-questions', + rows: [ + { question: "What significant result was reported about Zealand Pharma's obesity trial?" }, + { question: "How much did Berkshire Hathaway's cash levels increase in the fourth quarter?" }, + { question: "What is the goal of Highmark Health's integration of Google Cloud and Epic Systems technology?" } + ] }); +} - async function main() { - # Set your team and project names - await weave.init('/rag-quickstart'); - - const articleEmbeddings = await docsToEmbeddings(articles); - - const model = new RAGModel({ - systemMessage: "You are an expert in finance and answer questions related to finance, financial services, and financial markets. When responding based on provided information, be sure to cite the source.", - articleEmbeddings - }); +const contextPrecisionScore = weave.op(async function(args: { + datasetRow: QuestionRow; + modelOutput: { answer: string; context: string; } +}): Promise { + const openai = new OpenAI(); + + const prompt = `Given question, answer and context verify if the context was useful in arriving at the given answer. Give verdict as "1" if useful and "0" if not with json output. + Output in only valid JSON format. + + question: ${args.datasetRow.question} + context: ${args.modelOutput.context} + answer: ${args.modelOutput.answer} + verdict: `; + + const response = await openai.chat.completions.create({ + model: "gpt-4-turbo-preview", + messages: [{ role: "user", content: prompt }], + response_format: { type: "json_object" } + }); - const evaluation = new weave.Evaluation({ - dataset: createQuestionDataset(), - scorers: [contextPrecisionScore] - }); + const result = JSON.parse(response.choices[0].message.content || "{}"); + return { + verdict: parseInt(result.verdict) === 1 + }; +}); - const results = await evaluation.evaluate({ - model: weave.op((args: { datasetRow: QuestionRow }) => - model.predict(args.datasetRow.question) - ) - }); - - console.log('Evaluation results:', results); - } +async function main() { + # Set your team and project names + await weave.init('/rag-quickstart'); + + const articleEmbeddings = await docsToEmbeddings(articles); + + const model = new RAGModel({ + systemMessage: "You are an expert in finance and answer questions related to finance, financial services, and financial markets. When responding based on provided information, be sure to cite the source.", + articleEmbeddings + }); - if (require.main === module) { - main().catch(console.error); - } - ``` - - + const evaluation = new weave.Evaluation({ + dataset: createQuestionDataset(), + scorers: [contextPrecisionScore] + }); + + const results = await evaluation.evaluate({ + model: weave.op((args: { datasetRow: QuestionRow }) => + model.predict(args.datasetRow.question) + ) + }); + + console.log('Evaluation results:', results); +} + +if (require.main === module) { + main().catch(console.error); +} +``` + + ## Conclusion diff --git a/weave/tutorial-tracing_2.mdx b/weave/tutorial-tracing_2.mdx index b8552b7f38..5b31e63fb2 100644 --- a/weave/tutorial-tracing_2.mdx +++ b/weave/tutorial-tracing_2.mdx @@ -144,29 +144,27 @@ Now that Weave traces your nested functions, you can enrich those traces with ad Continuing the previous example: - - - ```python lines {1,10-11} - import weave + - weave.init('jurassic-park') +```python Python lines {1,10-11} +import weave - sentence = """I watched as a Tyrannosaurus rex (T. rex) chased after a Triceratops (Trike), \ - both carnivore and herbivore locked in an ancient dance. Meanwhile, a gentle giant \ - Brachiosaurus (Brachi) calmly munched on treetops, blissfully unaware of the chaos below.""" +weave.init('jurassic-park') - # track metadata alongside the previously defined function - with weave.attributes({'user_id': 'lukas', 'env': 'production'}): - result = dino_tracker(sentence) - ``` +sentence = """I watched as a Tyrannosaurus rex (T. rex) chased after a Triceratops (Trike), \ +both carnivore and herbivore locked in an ancient dance. Meanwhile, a gentle giant \ +Brachiosaurus (Brachi) calmly munched on treetops, blissfully unaware of the chaos below.""" - - - ```plaintext - This feature is not available in TypeScript yet. - ``` - - +# track metadata alongside the previously defined function +with weave.attributes({'user_id': 'lukas', 'env': 'production'}): + result = dino_tracker(sentence) +``` + +```plaintext TypeScript +This feature is not available in TypeScript yet. +``` + + Track metadata at run time, such as your user IDs and your code's environment status (development, staging, or production). diff --git a/weave/tutorial-weave_models.mdx b/weave/tutorial-weave_models.mdx index 51325e2b7b..9304058193 100644 --- a/weave/tutorial-weave_models.mdx +++ b/weave/tutorial-weave_models.mdx @@ -33,88 +33,84 @@ When you change the class fields or the code that defines your model, Weave logs In the following example, Weave tracks and versions the model name, temperature, and system prompt: - - - ```python lines {26,33-34} - import json - from openai import OpenAI + + +```python Python lines {26,33-34} +import json +from openai import OpenAI + +import weave + +@weave.op() +def extract_dinos(wmodel: weave.Model, sentence: str) -> dict: + response = wmodel.client.chat.completions.create( + model=wmodel.model_name, + temperature=wmodel.temperature, + messages=[ + { + "role": "system", + "content": wmodel.system_prompt + }, + { + "role": "user", + "content": sentence + } + ], + response_format={ "type": "json_object" } + ) + return response.choices[0].message.content + +# Subclass with a weave.Model +class ExtractDinos(weave.Model): + client: OpenAI = None + model_name: str + temperature: float + system_prompt: str + + # Ensure your function is called `invoke` or `predict` + @weave.op() + def invoke(self, sentence: str) -> dict: + dino_data = extract_dinos(self, sentence) + return json.loads(dino_data) +``` - import weave +```plaintext TypeScript +This feature is not available in TypeScript yet. +``` - @weave.op() - def extract_dinos(wmodel: weave.Model, sentence: str) -> dict: - response = wmodel.client.chat.completions.create( - model=wmodel.model_name, - temperature=wmodel.temperature, - messages=[ - { - "role": "system", - "content": wmodel.system_prompt - }, - { - "role": "user", - "content": sentence - } - ], - response_format={ "type": "json_object" } - ) - return response.choices[0].message.content - - # Subclass with a weave.Model - class ExtractDinos(weave.Model): - client: OpenAI = None - model_name: str - temperature: float - system_prompt: str - - # Ensure your function is called `invoke` or `predict` - @weave.op() - def invoke(self, sentence: str) -> dict: - dino_data = extract_dinos(self, sentence) - return json.loads(dino_data) - ``` - - - - ```plaintext - This feature is not available in TypeScript yet. - ``` - - + Now you can instantiate and call the model with `invoke`: - - - ```python lines {7,18} - weave.init('jurassic-park') - client = OpenAI() - - system_prompt = """Extract any dinosaur `name`, their `common_name`, \ - names and whether its `diet` is a herbivore or carnivore, in JSON format.""" - - dinos = ExtractDinos( - client=client, - model_name='gpt-4o', - temperature=0.4, - system_prompt=system_prompt - ) - - sentence = """I watched as a Tyrannosaurus rex (T. rex) chased after a Triceratops (Trike), \ - both carnivore and herbivore locked in an ancient dance. Meanwhile, a gentle giant \ - Brachiosaurus (Brachi) calmly munched on treetops, blissfully unaware of the chaos below.""" - - result = dinos.invoke(sentence) - print(result) - ``` - - - - ```plaintext - This feature is not available in TypeScript yet. - ``` - - + + +```python Python lines {7,18} +weave.init('jurassic-park') +client = OpenAI() + +system_prompt = """Extract any dinosaur `name`, their `common_name`, \ +names and whether its `diet` is a herbivore or carnivore, in JSON format.""" + +dinos = ExtractDinos( + client=client, + model_name='gpt-4o', + temperature=0.4, + system_prompt=system_prompt +) + +sentence = """I watched as a Tyrannosaurus rex (T. rex) chased after a Triceratops (Trike), \ +both carnivore and herbivore locked in an ancient dance. Meanwhile, a gentle giant \ +Brachiosaurus (Brachi) calmly munched on treetops, blissfully unaware of the chaos below.""" + +result = dinos.invoke(sentence) +print(result) +``` + +```plaintext TypeScript +This feature is not available in TypeScript yet. +``` + + After calling `.invoke()`, the trace in Weave tracks the model parameters along with the code for the model functions decorated with `weave.op()`. The model is also versioned (`v21` in this case). Click the model to see all the calls that used that version. @@ -140,27 +136,25 @@ In the Weave UI, you can get the Model ref for a particular version. Once you have the URI of the Model object, you can export and reuse it. The exported model is already initialized and ready to use: - - - ```python lines {2} - # The exported Weave model is already initialized and ready to call. - new_dinos = weave.ref("weave://morgan/jurassic-park/object/ExtractDinos:ey4udBU2MU23heQFJenkVxLBX4bmDsFk7vsGcOWPjY4").get() - - # Set the client to the OpenAI client again. - new_dinos.client = client - - new_sentence = """I also saw an Ankylosaurus grazing on giant ferns""" - new_result = new_dinos.invoke(new_sentence) - print(new_result) - ``` - - - - ```plaintext - This feature is not available in TypeScript yet. - ``` - - + + +```python Python lines {2} +# The exported Weave model is already initialized and ready to call. +new_dinos = weave.ref("weave://morgan/jurassic-park/object/ExtractDinos:ey4udBU2MU23heQFJenkVxLBX4bmDsFk7vsGcOWPjY4").get() + +# Set the client to the OpenAI client again. +new_dinos.client = client + +new_sentence = """I also saw an Ankylosaurus grazing on giant ferns""" +new_result = new_dinos.invoke(new_sentence) +print(new_result) +``` + +```plaintext TypeScript +This feature is not available in TypeScript yet. +``` + + You can now see that the new input uses the same Model version (v21): From e48ebae80da9eb4603d632c1e8807666670953bf Mon Sep 17 00:00:00 2001 From: Dan Brian Date: Fri, 10 Jul 2026 11:59:48 -0400 Subject: [PATCH 6/6] Apply suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- weave/tutorial-rag.mdx | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/weave/tutorial-rag.mdx b/weave/tutorial-rag.mdx index 45260c4097..9461efc43e 100644 --- a/weave/tutorial-rag.mdx +++ b/weave/tutorial-rag.mdx @@ -253,6 +253,7 @@ As in the [Build an Evaluation pipeline tutorial](/weave/tutorial-eval), define from openai import OpenAI import weave import asyncio +import json @weave.op() async def context_precision_score(question, output): @@ -349,6 +350,9 @@ At a high level, the steps to create a custom Scorer are: ```python Python lines +from typing import Any, Optional +import numpy as np +import weave from weave import Scorer class CorrectnessLLMJudge(Scorer): @@ -368,9 +372,9 @@ class CorrectnessLLMJudge(Scorer): # get_model is defined as general model getter based on provided params (OpenAI,HF...) eval_model = get_model( - model_name = self.model_name, - prompt = self.prompt - device = self.device, + model_name=self.model_name, + prompt=self.prompt, + device=self.device, ) # async evaluation to speed up evaluation - this doesn't have to be async grade = await eval_model.async_predict( @@ -736,7 +740,7 @@ const contextPrecisionScore = weave.op(async function(args: { }); async function main() { - # Set your team and project names + // Set your team and project names await weave.init('/rag-quickstart'); const articleEmbeddings = await docsToEmbeddings(articles);