From b9a75f6b975c5330e3d35a2627a5e132b2e06e93 Mon Sep 17 00:00:00 2001 From: Mario Jack Vela Date: Thu, 23 Apr 2026 07:33:53 -0500 Subject: [PATCH 01/13] Add procedural memory layer and legacy comparison benchmarks --- AGENTS.md | 152 ++ ARCHITECTURE.md | 69 +- COGNITIVE_PROTOCOL.md | 16 + MCP_SERVER.md | 17 +- README.md | 30 +- benchmarks/__init__.py | 2 + benchmarks/brainctl_retrieval.py | 192 ++ benchmarks/compare_memory_engines.py | 254 +++ benchmarks/convomem_bench.py | 226 ++ benchmarks/datasets.py | 60 + benchmarks/framework.py | 384 ++++ benchmarks/legacy_refs.py | 242 +++ benchmarks/locomo_bench.py | 172 ++ benchmarks/longmemeval_bench.py | 211 ++ benchmarks/membench_bench.py | 193 ++ bin/intent_classifier.py | 23 +- db/init_schema.sql | 1885 ++++++++++++++++- db/migrations/052_procedural_memory_layer.sql | 510 +++++ src/agentmemory/_impl.py | 384 +++- src/agentmemory/brain.py | 245 ++- src/agentmemory/commands/procedure.py | 260 +++ src/agentmemory/db/init_schema.sql | 158 +- src/agentmemory/hippocampus.py | 30 + src/agentmemory/mcp_server.py | 253 +-- src/agentmemory/mcp_tools_meb.py | 6 +- src/agentmemory/mcp_tools_procedural.py | 324 +++ src/agentmemory/procedural.py | 1679 +++++++++++++++ src/agentmemory/retrieval/__init__.py | 18 + src/agentmemory/retrieval/answerability.py | 120 ++ .../retrieval/candidate_generation.py | 43 + src/agentmemory/retrieval/diagnostics.py | 33 + src/agentmemory/retrieval/evidence_graph.py | 55 + src/agentmemory/retrieval/late_reranker.py | 43 + src/agentmemory/retrieval/query_planner.py | 181 ++ tests/bench/baselines/search_quality.json | 72 +- tests/bench/eval.py | 93 +- tests/bench/fixtures.py | 141 +- tests/test_audit_2_4_11_regressions.py | 19 +- tests/test_cli.py | 73 + tests/test_mcp_tools_meb.py | 2 + tests/test_mcp_tools_procedural.py | 82 + tests/test_migrate.py | 5 + tests/test_procedural.py | 138 ++ tests/test_validation.py | 13 + 44 files changed, 8751 insertions(+), 357 deletions(-) create mode 100644 AGENTS.md create mode 100644 benchmarks/__init__.py create mode 100644 benchmarks/brainctl_retrieval.py create mode 100644 benchmarks/compare_memory_engines.py create mode 100644 benchmarks/convomem_bench.py create mode 100644 benchmarks/datasets.py create mode 100644 benchmarks/framework.py create mode 100644 benchmarks/legacy_refs.py create mode 100644 benchmarks/locomo_bench.py create mode 100644 benchmarks/longmemeval_bench.py create mode 100644 benchmarks/membench_bench.py create mode 100644 db/migrations/052_procedural_memory_layer.sql create mode 100644 src/agentmemory/commands/procedure.py create mode 100644 src/agentmemory/mcp_tools_procedural.py create mode 100644 src/agentmemory/procedural.py create mode 100644 src/agentmemory/retrieval/__init__.py create mode 100644 src/agentmemory/retrieval/answerability.py create mode 100644 src/agentmemory/retrieval/candidate_generation.py create mode 100644 src/agentmemory/retrieval/diagnostics.py create mode 100644 src/agentmemory/retrieval/evidence_graph.py create mode 100644 src/agentmemory/retrieval/late_reranker.py create mode 100644 src/agentmemory/retrieval/query_planner.py create mode 100644 tests/test_mcp_tools_procedural.py create mode 100644 tests/test_procedural.py diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..52e28af --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,152 @@ +# Codex — brainctl / agentmemory + +## What This Is +Unified agent memory system. SQLite-backed (brain.db) with FTS5, vector embeddings (sqlite-vec + Ollama nomic-embed-text), knowledge graph, affect tracking, belief collapse mechanics, and AGM conflict resolution. + +Published as `brainctl` on PyPI (v2.2.1+, current 2.4.10). + +## Key Paths +- **DB:** `db/brain.db` (WAL mode, foreign keys ON, 59 user-facing tables, 49 numbered migrations + one unnumbered V2-4 quantum-schema file). The numbered sequence has an intentional gap at 050 — the V2-4 quantum schema (`db/migrations/quantum_schema_migration_sqlite.sql`) occupies that slot without a number because it was applied ad-hoc during the V2-4 rollout and pre-dates the idempotent runner fix in 2.4.8. The runner only picks up files matching `^\d+_.+\.sql$` so the quantum file is a no-op for `brainctl migrate` on fresh installs — apply manually if you need the quantum columns on a new DB. (Audit I28 — 2026-04-19.) +- **CLI:** `bin/brainctl` — main CLI entry +- **MCP server:** canonical entry is `agentmemory.mcp_server:run` (201 tools across `mcp_server.py` + 29 `mcp_tools_*.py` modules). Installed as the `brainctl-mcp` console script via pip. The legacy standalone `bin/brainctl-mcp` only registers a subset and is being phased out. +- **Bench:** `bin/brainctl-bench` — retrieval eval harness (P@k / MRR / nDCG@k regression gate, fixtures under `tests/bench/`) +- **Source:** `src/agentmemory/` — Python package +- **Config:** `config/` — quiet hours, consolidation schedules +- **Agents:** `agents/` — per-agent config (pipeline, engram, etc.) + +## Build & Test +```bash +pip install -e . # dev install +brainctl stats # verify DB +brainctl search "test" # test search +python3 -m agentmemory.mcp_server --list-tools # full 199-tool MCP surface +python3 -m tests.bench.run # retrieval quality benchmark +python3 -m tests.bench.run --check # fail on >2% regression vs baseline +``` + +## Architecture +- Tables: memories, events, entities, decisions, context, knowledge_edges, affect_log, access_log, agent_state, agent_beliefs +- FTS5 indexes on memories, events, entities +- Vector embeddings via sqlite-vec extension +- Hybrid retrieval: FTS5 + vector via Reciprocal Rank Fusion, routed through a regex intent classifier (`bin/intent_classifier.py`) that normalises 10 intent labels onto 6 rerank profiles inside `cmd_search` +- Retrieval regression-gated by `tests/bench/` (P@1 / P@5 / Recall@5 / MRR / nDCG@5; >2% drop fails CI) +- W(m) worthiness gate on memory writes (surprise scoring + semantic dedup) +- PII recency gate (Proactive Interference Index) on supersedes +- Bayesian alpha/beta tracking on memory recall +- Entities carry a rewriteable `compiled_truth` synthesis, a 3-level `enrichment_tier`, and a first-class `aliases` JSON list (migrations 033–035) +- Knowledge-gap scanner (`brainctl gaps scan`) also detects orphan memories, broken knowledge_edges, and unreferenced entities (migration 036) + +## Conventions +- Agent IDs: use descriptive names like `my-agent`, `research-bot`, `code-reviewer` +- Memory categories: convention, decision, environment, identity, integration, lesson, preference, project, user +- Event types: artifact, decision, error, handoff, memory_promoted, memory_retired, observation, result, session_start, session_end, stale_context, task_update, warning +- Entity types: agent, concept, document, event, location, organization, other, person, project, service, tool + +## Don't Touch +- Migration files in `db/migrations/` — append-only +- The W(m) gate logic without understanding surprise scoring +- Quiet hours scripts — they're cron-scheduled + +## Signed exports (2.3.0+) + +`brainctl export --sign` and `brainctl verify` produce / check +portable, signed JSON bundles of memories using the user's Solana +keypair. Local-first by design — memories never leave the machine; +only the SHA-256 hash is ever pinned on-chain (opt-in via +`--pin-onchain`, ~$0.001 per pin). Implementation lives in +`src/agentmemory/signing.py` (offline + on-chain plumbing) and +`src/agentmemory/commands/sign.py` (CLI handlers, parser +registration). Optional dep: + + pip install 'brainctl[signing]' # pulls solders>=0.21 + +CLI surface: + + brainctl export --sign --keystore [--filter-agent X] + [--category Y] [--scope Z] [--created-after T] + [--created-before T] [--ids 1,2,3] [--pin-onchain] + [--rpc-url ] [-o bundle.json] [--json] + + brainctl verify [--check-onchain] + [--rpc-url ] [--json] + +Exit codes: `0` ok, `1` tamper / missing keystore / IO, `2` unsigned +export attempted or `--check-onchain` found no receipt. + +No token gating — anyone with brainctl + a Solana keypair can sign +their own memories (preference memory #1691). Threat model + bundle +format spec + a 30-line "verify without brainctl" recipe live in +`docs/SIGNED_EXPORTS.md`. + +## Managed wallet (2.3.2+) + +For users who don't already have a Solana wallet (most chat-bot +operators and agent builders), `brainctl wallet new` creates a +non-custodial keypair at `~/.brainctl/wallet.json` (chmod 0600). The +key is generated locally and **never** transmitted, copied, or backed +up by brainctl — the user owns the file. `brainctl export --sign` +auto-discovers the managed wallet when `--keystore` isn't passed +(precedence: `--keystore` > `~/.brainctl/wallet.json` > +`$BRAINCTL_SIGNING_KEY_PATH` > error). Implementation in +`src/agentmemory/commands/wallet.py`. Subcommands: `new` (with +`--force`/`--yes`), `address`, `balance`, `show`, `export `, +`import `, `rm`, `onboard`. Override default location via +`$BRAINCTL_WALLET_PATH` or `--path`. + +For AI agents: when a user wants to sign a bundle but doesn't have a +wallet, surface this flow. The MCP tools `wallet_show` and +`wallet_create` let you check / create on the user's behalf — but +`wallet_create` will REFUSE to overwrite an existing keystore unless +explicitly passed `force=true` (destructive — pass only after the +user explicitly consents). Or use `brainctl export --sign +--auto-setup-wallet` which creates a wallet inline and signs in one +shot. `--pin-onchain` with 0 SOL skips cleanly with `exit 0` and +`pin_skipped_reason="zero_balance"` — the offline signature is still +valid; pinning is opportunistic. + +## Code-aware ingestion (2.4.5+, optional `[code]` extra) + +`brainctl ingest code ` walks a source tree and writes file / +function / class entities plus `contains` and `imports` relations into +the existing entity graph. CPU-only via tree-sitter — no LLM, no GPU, +no network at ingest time. SHA256-cached in migration 051 so re-runs +on unchanged trees are metadata-only and finish in well under a +second for a ~100-file package. + +Ships three grammars on purpose (python, typescript, go) to keep the +wheel footprint around 4 MB. Adding a language means: + + 1. Add grammar to `[code]` extra in `pyproject.toml`. + 2. Add suffix(es) to `EXT_TO_LANG` in `src/agentmemory/code_ingest.py`. + 3. Write `extract_(path, src, relpath)` and register in `EXTRACTORS`. + +CLI surface: + + pip install 'brainctl[code]' + brainctl ingest code [--scope project:] + [--languages python,typescript,go] [--no-cache] + [--max-files N] [--verbose] [--json] + +Entity naming is prefixed so searches stay unambiguous: `file:`, +`fn::`, `class::`, +`module:`. The fine-grained kind lives in +`properties.kind` alongside `language`, `path`, `line`, `signature`, +`parent`. Provenance is encoded on `knowledge_edges.weight` — 1.0 for +direct-source (`contains`, local resolvable imports), 0.7 for +unresolved external imports. Re-ingest does **not** touch +`last_reinforced_at` / `co_activation_count`: those are synaptic +reinforcement signals owned by hippocampus, and re-parsing a file is +an idempotent state-sync, not an activation event. + +Inspired by `github.com/safishamsi/graphify` (the `{nodes, edges}` +extractor protocol + SHA256 skip-when-unchanged pattern). + +Known follow-ups (not blocking the extra): + + * No `mcp__brainctl__ingest_code` tool yet — agents that want to + trigger ingest must shell out via the CLI. MCP wrapper is a + separate PR. + * `init_schema.sql` won't include `code_ingest_cache` until it's + regenerated in a release commit. Until then, fresh installs that + want code-ingest need `brainctl migrate` applied after + `brainctl init`. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 5faa732..5f2cc17 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -5,7 +5,8 @@ brainctl is a persistent memory system for AI agents backed by a single SQLite database. No server process, no external dependencies beyond Python and SQLite. Multiple agents (or a single agent across sessions) share one `brain.db` file -for memories, events, entities, decisions, and a knowledge graph. +for episodic, semantic, and procedural memory plus events, entities, +decisions, and a knowledge graph. ## Project Structure @@ -18,7 +19,9 @@ src/agentmemory/ cli.py Entry point mcp_server.py MCP server entry hippocampus.py Consolidation engine (brainctl-consolidate entry point) - commands/ 24 command modules + procedural.py Canonical procedural memory service + heuristics + retrieval/ Query planner, candidate generation, evidence fusion, answerability + commands/ 25 command modules agent.py Agent registration and state memory.py Memory CRUD and search event.py Event logging and queries @@ -55,6 +58,11 @@ All state lives in a single `brain.db` file (SQLite, WAL mode). | Table | Purpose | |-------|---------| | `memories` | Durable facts, preferences, lessons, conventions | +| `procedures` | Canonical procedural memories linked 1:1 to bridge rows in `memories` | +| `procedure_steps` | Ordered step projection for procedures | +| `procedure_sources` | Provenance links from procedures back to memories/events/decisions/entities | +| `procedure_runs` | Execution/application feedback history for procedures | +| `procedure_candidates` | Repeat-pattern staging area before promotion to canonical procedures | | `events` | Timestamped event log (append-oriented) | | `entities` | Named entities (people, projects, tools, concepts) | | `knowledge_edges` | Typed, weighted edges between any two records | @@ -70,6 +78,15 @@ All state lives in a single `brain.db` file (SQLite, WAL mode). See `db/init_schema.sql` for full column definitions and migrations. +`memories.memory_type` is now a three-way core layer selector: +- `episodic` — specific events, traces, and observations +- `semantic` — distilled facts, preferences, and conventions +- `procedural` — reusable workflows, runbooks, troubleshooting sequences, rollback plans + +The canonical structured procedure lives in `procedures`; the linked +`memories` row keeps a human-readable synopsis so legacy memory search and +older interfaces continue to see something useful. + ### Vector Tables (optional, requires sqlite-vec) | Table | Purpose | @@ -94,17 +111,31 @@ natural-language queries ("what does Alice prefer?") match memories that contain *any* meaningful term, not only memories that contain every word. Stopwords are dropped before OR expansion. -### Hybrid Search + Reciprocal Rank Fusion +### Retrieval Executive + Hybrid Search + +`cmd_search` now acts as a compatibility shell around a retrieval executive: + +1. `retrieval.query_planner` inspects the query and emits a structured plan + (`normalized_intent`, `answer_type`, target entities, temporal anchors, + preferred memory layers, candidate tables, abstain policy). +2. `cmd_search` still performs the existing FTS5/sqlite-vec retrieval paths + for memories, events, and context. +3. `retrieval.candidate_generation` adds a first-class procedural candidate + path using `procedures_fts` plus structured fallback search. +4. `retrieval.evidence_graph` expands top procedures over + `procedure_sources` and `knowledge_edges` to gather supporting episodes, + decisions, events, tools, and rollback relations. +5. `retrieval.late_reranker` deterministically fuses direct lexical match, + procedural structure match, validation recency, execution history, and + evidence support. +6. `retrieval.answerability` decides whether to abstain instead of returning + ungrounded nearest-neighbor junk. + +The effective plan and answerability diagnostics surface in `_debug` / +`metacognition` so benchmark misses remain explainable. -`cmd_search` merges FTS5 and sqlite-vec results with Reciprocal Rank Fusion -(`rrf_score = 1/(60 + rank)`), applies temporal decay, category half-life, -and adaptive salience weighting, then runs a regex-based query intent -classifier (`bin/intent_classifier.py`) whose output is normalized inside -`cmd_search` onto six rerank profiles: `entity_lookup`, `event_lookup`, -`decision_lookup`, `graph_traversal`, `procedural`, `general`. The -classifier covers ~80% of real agent queries with zero latency; the -rerank branch applied to the blend is reported in the -`metacognition.rerank_branch` field of every response. +The old hybrid core is preserved: memories/events/context still merge FTS5 +and sqlite-vec via Reciprocal Rank Fusion when vector search is available. ### Vector Search (optional) @@ -123,14 +154,17 @@ Multi-hop neighbor queries across the knowledge graph via `brainctl graph`. ### Retrieval Regression Gate -`tests/bench/` ships a deterministic search-quality harness: 30 synthetic -memories + 8 events + 6 entities + 20 graded queries (3=primary, 2=related, -1=tangential) across seven query classes (entity / procedural / decision / -temporal / troubleshooting / negative / ambiguous). The runner reports +`tests/bench/` ships a deterministic search-quality harness: synthetic +memories + procedures + events + entities with graded queries (3=primary, +2=related, 1=tangential) across entity / procedural / decision / temporal / +troubleshooting / negative / ambiguous classes. The runner reports P@1, P@5, Recall@5, MRR, nDCG@5 against a committed baseline at `tests/bench/baselines/search_quality.json`. Any >2% drop on a headline metric fails the `test_search_quality_bench.py` pytest regression test. -Run with `python3 -m tests.bench.run` or `bin/brainctl-bench`. +The harness also records failure modes (`retrieval_failure`, +`utilization_failure`, `hallucination`, `correct_abstain`) and captures the +retrieval executive debug payload. Run with `python3 -m tests.bench.run` or +`bin/brainctl-bench`. ## Knowledge Graph @@ -190,6 +224,7 @@ Runs as part of the nightly consolidation cycle; results surface in | **Compression** | Merges clusters of related low-value memories into summaries | | **Dream** | Synthesizes new hypotheses from loosely connected memories | | **Hebbian** | Strengthens edges between frequently co-accessed records | +| **Procedural synthesis** | Promotes repeated successful action patterns into procedure candidates or canonical procedures | `bin/consolidation-cycle.sh` chains the hippocampus passes with: diff --git a/COGNITIVE_PROTOCOL.md b/COGNITIVE_PROTOCOL.md index c8f0cd9..07a6b6d 100644 --- a/COGNITIVE_PROTOCOL.md +++ b/COGNITIVE_PROTOCOL.md @@ -15,6 +15,7 @@ Before starting any task, check what's already known: ```bash brainctl -a myagent search "task keywords" --limit 10 +brainctl -a myagent procedure search "task keywords" --limit 5 brainctl event tail -n 10 brainctl decision list ``` @@ -35,9 +36,24 @@ When you find something non-obvious, save it right away: brainctl -a myagent memory add "what you discovered" -c CATEGORY -s SCOPE ``` +If what you learned is reusable execution knowledge rather than a plain fact, +store it as a procedure: + +```bash +brainctl -a myagent procedure add \ + --title "staging deploy runbook" \ + --goal "deploy to staging safely" \ + --step "run tests" \ + --step "brainctl migrate" \ + --step "deploy and verify health" +``` + **Good memories:** "The API rate-limits at 100 req/15s with Retry-After header" **Bad memories:** "I ran npm install" (trivial) / "The build passed" (transient) +**Good procedures:** rollback plans, troubleshooting sequences, migration +runbooks, validated tool-use recipes. + ### Categories | Category | Use for | diff --git a/MCP_SERVER.md b/MCP_SERVER.md index cd56b9c..68cd20f 100644 --- a/MCP_SERVER.md +++ b/MCP_SERVER.md @@ -50,7 +50,7 @@ docker run -v ~/.agentmemory:/data -e BRAIN_DB=/data/brain.db brainctl The `CMD` defaults to `brainctl-mcp`, so the container runs the MCP server over stdio. -## Available Tools (201) +## Available Tools (209) | Tool | Description | |------|-------------| @@ -69,12 +69,20 @@ server over stdio. | `trigger_update` | Update fields on an existing trigger | | `trigger_delete` | Cancel/delete a trigger by ID | | `decision_add` | Record a decision with rationale | +| `procedure_add` | Create a structured procedural memory with ordered steps | +| `procedure_get` | Fetch a canonical procedure with steps and provenance | +| `procedure_list` | List procedures with scope/status filters | +| `procedure_search` | Search procedural memories and return structured matches | +| `procedure_update` | Update a canonical procedure | +| `procedure_feedback` | Record execution outcome / validation against a procedure | +| `procedure_backfill` | Promote likely procedures from existing memories/events/decisions | +| `procedure_stats` | Show canonical procedure and candidate counts | | `handoff_add` | Create a structured handoff packet | | `handoff_latest` | Fetch the latest matching handoff packet | | `handoff_consume` | Mark a handoff packet consumed | | `handoff_pin` | Pin a handoff packet for preservation | | `handoff_expire` | Mark a handoff packet expired | -| `search` | Cross-table search (memories + events + entities) | +| `search` | Cross-table search with retrieval planning across memories + procedures + events + entities | | `pagerank` | Compute PageRank centrality over knowledge graph | | `stats` | Database statistics and health summary | | `resolve_conflict` | AGM credibility-weighted belief conflict resolution | @@ -114,6 +122,7 @@ server over stdio. **Store information:** - Durable fact/lesson/convention: `memory_add` (enforces W(m) write gate) +- Durable workflow/runbook: `procedure_add` or `memory_add(memory_type="procedural")` - What just happened: `event_add` (timestamped, no gate) - Why a choice was made: `decision_add` (with rationale) - Working state for next session: `handoff_add` @@ -121,6 +130,7 @@ server over stdio. **Find information:** - Everything about a topic: `search` (memories + events + entities) - Just memories: `memory_search` (supports category, scope, pagerank_boost) +- Just procedures: `procedure_search` - Just events: `event_search` (supports event_type, project) - A specific entity: `entity_get` - Entities matching a query: `entity_search` @@ -144,6 +154,7 @@ server over stdio. | Category | Tools | When to use | |----------|-------|-------------| +| Procedural memory | `procedure_add`, `procedure_search`, `procedure_feedback`, `procedure_backfill`, `procedure_stats` | Runbooks, rollback plans, troubleshooting routines, validated workflows | | Consolidation | `consolidation_run`, `replay_boost`, `replay_queue` | Memory maintenance | | Reconsolidation | `reconsolidation_check`, `reconsolidate` | Lability window mechanics | | Beliefs & Conflicts | `resolve_conflict`, `belief_collapse` | When memories contradict | @@ -165,6 +176,7 @@ What do you need? | +-- Store something? | +-- Durable fact ----------> memory_add +| +-- Durable runbook -------> procedure_add | +-- What just happened ----> event_add | +-- Why a choice was made -> decision_add | +-- State for next session > handoff_add @@ -172,6 +184,7 @@ What do you need? +-- Find something? | +-- Broad topic search ----> search | +-- Memories only ---------> memory_search +| +-- Procedures only -------> procedure_search | +-- Events only -----------> event_search | +-- Entity by name --------> entity_get | diff --git a/README.md b/README.md index 77fb74a..07adf19 100644 --- a/README.md +++ b/README.md @@ -2,14 +2,19 @@ **Forgetful agents, fixed by a SQLite file.** -One `brain.db` gives your agent durable memory across sessions — facts learned, decisions made, entities tracked, and state handed off. No server. No API keys. No LLM calls required. +One `brain.db` gives your agent durable memory across sessions — episodic evidence, semantic facts, procedural runbooks, decisions made, entities tracked, and state handed off. No server. No API keys. No LLM calls required. ```python from agentmemory import Brain brain = Brain(agent_id="my-agent") -ctx = brain.orient(project="api-v2") # session start: handoff + events + triggers + memories +ctx = brain.orient(project="api-v2") # handoff + events + triggers + memories + procedures brain.remember("rate-limit: 100/15s", category="integration") +brain.remember_procedure( + goal="Deploy to staging safely", + steps=["Run tests", "brainctl migrate", "Deploy", "Check health"], + title="Staging deploy runbook", +) brain.decide("use Retry-After for backoff", "server controls timing", project="api-v2") brain.wrap_up("auth module complete", project="api-v2") # session end: logs + handoff for next run ``` @@ -45,6 +50,7 @@ brain.relate("OpenAI", "provides", "GPT-4o") **Memory types** - `convention`, `decision`, `environment`, `identity`, `integration`, `lesson`, `preference`, `project`, `user` +- Core memory layers: episodic, semantic, and procedural - Category controls natural half-life: identity decays over ~1 year; integration details over ~1 month - Hard cap: 10,000 memories per agent. Emergency compression retires lowest-confidence entries. @@ -52,6 +58,7 @@ brain.relate("OpenAI", "provides", "GPT-4o") - FTS5 full-text search with stemming (default, zero dependencies) - Vector similarity via sqlite-vec + Ollama nomic-embed-text (`brainctl[vec]`) - Hybrid: Reciprocal Rank Fusion over FTS5 + vector results +- Retrieval executive above memories/events/context/decisions/procedures: query planning, candidate fusion, procedural evidence expansion, deterministic late reranking, grounded abstention - Context profiles: named search presets scoped to task type (`--profile ops`, `--profile research`, etc.) - `--benchmark` preset: flattens recency/salience for synthetic evaluation runs @@ -112,7 +119,7 @@ Trading bots: | `plugins/octobot/` | OctoBot | | `plugins/coinbase-agentkit/` | Coinbase AgentKit | -## MCP server (201 tools) +## MCP server (209 tools) ```json { @@ -130,7 +137,11 @@ Add to `~/.claude/claude_desktop_config.json`, `~/.cursor/mcp.json`, or equivale ```bash brainctl memory add "content" -c convention # store a memory +brainctl memory add "rollback steps..." -c convention --type procedural brainctl search "query" # FTS5 search +brainctl procedure add --goal "Deploy to staging safely" --step "Run tests" --step "brainctl migrate" +brainctl procedure search "how do I deploy to staging?" +brainctl procedure feedback 12 --success --validated --outcome "deploy completed cleanly" brainctl vsearch "semantic query" # vector search (requires [vec]) brainctl entity create "Alice" -t person # create entity brainctl entity relate Alice works_at Acme # link entities @@ -146,14 +157,17 @@ brainctl gaps scan # coverage + orphan + broken-edge brainctl consolidate cycle # full consolidation pass ``` -## Python API (22 methods) +## Python API | Method | What it does | |--------|--------------| | `orient(project)` | One-call session start: handoff + events + triggers + memories | | `wrap_up(summary)` | One-call session end: logs event + creates handoff | | `remember(content, category)` | Store a durable fact through the W(m) write gate | +| `remember(content, category, memory_type="procedural")` | Store free text and compile it into a structured procedure when appropriate | +| `remember_procedure(goal, steps, ...)` | Create a canonical procedural memory with structured fields | | `search(query)` | FTS5 full-text search with stemming | +| `search_procedures(query)` | Search structured procedures with deterministic procedural scoring | | `vsearch(query)` | Vector similarity search (optional) | | `think(query)` | Spreading-activation recall across the knowledge graph | | `forget(memory_id)` | Soft-delete a memory | @@ -167,6 +181,8 @@ brainctl consolidate cycle # full consolidation pass | `resume()` | Fetch and consume latest handoff | | `doctor()` | Diagnostic health check | | `consolidate()` | Promote high-importance memories | +| `procedure_feedback(procedure_id, ...)` | Record execution outcome, validation, and utility for a procedure | +| `backfill_procedures()` | Synthesize candidate/canonical procedures from existing memories, events, and decisions | | `tier_stats()` | Write-tier distribution | | `stats()` | Database overview | | `affect(text)` | Classify emotional state | @@ -177,9 +193,10 @@ brainctl consolidate cycle # full consolidation pass - **Write gate** (W(m)): surprise scoring rejects redundant writes. Bypass with `force=True`. - **Three-tier routing**: high-value memories get full indexing; low-value get lightweight storage. +- **Procedural compilation**: explicit runbooks live in dedicated procedure tables; `memory_type="procedural"` free text is heuristically compiled without deleting the original evidence. - **Duplicate suppression**: near-duplicates reinforce existing memories instead of creating new rows. - **Half-life decay**: unused memories fade at a rate set by category. Recalled memories are reinforced. -- **Consolidation**: Hebbian learning, temporal promotion, compression — runs on a cron schedule. +- **Consolidation**: Hebbian learning, temporal promotion, compression, and procedural candidate synthesis — runs on a cron schedule. ## Retrieval benchmarks @@ -187,7 +204,8 @@ Tested with default settings, no tuning for benchmark data. Two harnesses ship in the tree: * `tests/bench/` — single-system retrieval baselines for `Brain.search` - and `cmd_search`, gated against regression in CI. + and `cmd_search`, now covering procedural lookup, rollback/troubleshooting, + ambiguity, and abstention, gated against regression in CI. * `tests/bench/competitor_runs/` — same-fixture head-to-head harness with adapters for Mem0, Letta, Zep, Cognee, MemPalace, OpenAI Memory. Skip-not-fabricate contract: missing SDK / API key raises diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 0000000..f36516f --- /dev/null +++ b/benchmarks/__init__.py @@ -0,0 +1,2 @@ +"""Legacy benchmark comparison helpers for brainctl vs MemPalace.""" + diff --git a/benchmarks/brainctl_retrieval.py b/benchmarks/brainctl_retrieval.py new file mode 100644 index 0000000..b295105 --- /dev/null +++ b/benchmarks/brainctl_retrieval.py @@ -0,0 +1,192 @@ +from __future__ import annotations + +import os +import shutil +import sqlite3 +import tempfile +from dataclasses import dataclass +from pathlib import Path +from types import SimpleNamespace +from typing import Iterable + + +ROOT = Path(__file__).resolve().parent.parent +SRC = ROOT / "src" +if str(ROOT) not in os.sys.path: + os.sys.path.insert(0, str(ROOT)) +if str(SRC) not in os.sys.path: + os.sys.path.insert(0, str(SRC)) + +from agentmemory.brain import Brain + + +AGENT_ID = "legacy-compare-bench" + + +@dataclass +class SeededCorpus: + root_dir: Path + template_db_path: Path + rowid_to_doc_id: dict[int, str] + + def cleanup(self) -> None: + shutil.rmtree(self.root_dir, ignore_errors=True) + + +def init_empty_db(db_path: Path) -> None: + init_sql = ROOT / "src" / "agentmemory" / "db" / "init_schema.sql" + db_path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(str(db_path)) + try: + conn.executescript(init_sql.read_text(encoding="utf-8")) + now = "2026-01-01T00:00:00Z" + conn.execute( + """ + INSERT OR IGNORE INTO agents ( + id, display_name, agent_type, status, created_at, updated_at + ) VALUES (?, ?, 'bench', 'active', ?, ?) + """, + (AGENT_ID, AGENT_ID, now, now), + ) + conn.execute( + "INSERT OR IGNORE INTO workspace_config (key, value) VALUES ('enabled', '0')" + ) + conn.execute( + """ + INSERT OR IGNORE INTO neuromodulation_state ( + id, org_state, dopamine_signal, arousal_level, + confidence_boost_rate, confidence_decay_rate, retrieval_breadth_multiplier, + focus_level, temporal_lambda, context_window_depth + ) VALUES (1, 'normal', 0.0, 0.3, 0.1, 0.02, 1.0, 0.3, 0.03, 50) + """ + ) + conn.commit() + try: + conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") + except Exception: + pass + finally: + conn.close() + + +def _search_brain(db_path: Path, query: str, top_k: int) -> list[dict]: + brain = Brain(db_path=str(db_path), agent_id=AGENT_ID) + try: + return list(brain.search(query, limit=top_k)) + finally: + brain.close() + + +def _search_cmd(db_path: Path, query: str, top_k: int) -> list[dict]: + import agentmemory._impl as _impl + + _impl.DB_PATH = db_path + args = SimpleNamespace( + query=query, + limit=top_k, + output="return", + tables="memories", + profile=None, + no_recency=False, + no_graph=True, + budget=None, + min_salience=None, + mmr=False, + mmr_lambda=0.7, + explore=False, + pagerank_boost=0.0, + quantum=False, + benchmark=False, + agent=AGENT_ID, + format="json", + oneline=False, + verbose=False, + debug=False, + ) + payload = _impl.cmd_search(args, db=None, db_path=str(db_path)) + memories = list((payload or {}).get("memories") or []) + memories.sort(key=lambda row: row.get("final_score", row.get("rrf_score", 0.0)), reverse=True) + return memories[:top_k] + + +def seed_documents( + documents: Iterable[tuple[str, str]], + *, + category: str = "benchmark", +) -> SeededCorpus: + os.environ.setdefault("BRAINCTL_SILENT_MIGRATIONS", "1") + tmp_dir = Path(tempfile.mkdtemp(prefix="brainctl-legacy-seeded-")) + db_path = tmp_dir / "template_brain.db" + try: + init_empty_db(db_path) + rowid_to_doc_id: dict[int, str] = {} + brain = Brain(db_path=str(db_path), agent_id=AGENT_ID) + try: + for doc_id, text in documents: + rowid = brain.remember(text, category=category) + rowid_to_doc_id[int(rowid)] = doc_id + finally: + brain.close() + conn = sqlite3.connect(str(db_path)) + try: + try: + conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") + except Exception: + pass + conn.commit() + finally: + conn.close() + return SeededCorpus( + root_dir=tmp_dir, + template_db_path=db_path, + rowid_to_doc_id=rowid_to_doc_id, + ) + except Exception: + shutil.rmtree(tmp_dir, ignore_errors=True) + raise + + +def rank_seeded_documents( + query: str, + seeded: SeededCorpus, + *, + pipeline: str = "cmd", + top_k: int = 10, +) -> list[str]: + work_dir = Path(tempfile.mkdtemp(prefix="brainctl-legacy-query-")) + db_path = work_dir / "brain.db" + try: + shutil.copy2(seeded.template_db_path, db_path) + + if pipeline == "brain": + results = _search_brain(db_path, query, top_k) + elif pipeline == "cmd": + results = _search_cmd(db_path, query, top_k) + else: + raise ValueError(f"Unknown pipeline {pipeline!r}") + + ranked: list[str] = [] + seen: set[str] = set() + for result in results: + doc_id = seeded.rowid_to_doc_id.get(int(result["id"])) + if doc_id and doc_id not in seen: + ranked.append(doc_id) + seen.add(doc_id) + return ranked + finally: + shutil.rmtree(work_dir, ignore_errors=True) + + +def rank_documents( + query: str, + documents: Iterable[tuple[str, str]], + *, + pipeline: str = "cmd", + top_k: int = 10, + category: str = "benchmark", +) -> list[str]: + seeded = seed_documents(documents, category=category) + try: + return rank_seeded_documents(query, seeded, pipeline=pipeline, top_k=top_k) + finally: + seeded.cleanup() diff --git a/benchmarks/compare_memory_engines.py b/benchmarks/compare_memory_engines.py new file mode 100644 index 0000000..e33d582 --- /dev/null +++ b/benchmarks/compare_memory_engines.py @@ -0,0 +1,254 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) +SRC = ROOT / "src" +if str(SRC) not in sys.path: + sys.path.insert(0, str(SRC)) + +from benchmarks.convomem_bench import run_brainctl_convomem +from benchmarks.datasets import resolve_dataset_paths +from benchmarks.framework import ( + BenchmarkRunResult, + new_artifact_dir, + plot_aggregate_primary_chart, + plot_benchmark_chart, + plot_status_chart, + runtime_metadata, + write_bundle_summary, + write_json, + write_normalized_comparison, + write_normalized_comparison_csv, + write_run_payload, + write_summary_csv, + write_text, +) +from benchmarks.legacy_refs import AGGREGATE_BENCHMARKS, BENCHMARK_SPECS, COVERAGE_BENCHMARKS, load_historical_runs +from benchmarks.locomo_bench import run_brainctl_locomo +from benchmarks.longmemeval_bench import run_brainctl_longmemeval_pipeline +from benchmarks.membench_bench import run_brainctl_membench + + +def _git_commit() -> str | None: + try: + return ( + subprocess.check_output( + ["git", "rev-parse", "HEAD"], + cwd=ROOT, + text=True, + stderr=subprocess.DEVNULL, + ) + .strip() + ) + except Exception: + return None + + +def _write_run_artifact( + artifact_dir: Path, + run: BenchmarkRunResult, + rows: list[dict] | None = None, +) -> None: + run_path = artifact_dir / "runs" / f"{run.benchmark}_{run.series_name}_{run.mode}.json" + write_run_payload(run_path, run, rows=rows) + run.artifacts["run_json"] = str(run_path) + + +def _provenance_readme( + *, + artifact_dir: Path, + historical_source, + used_fallback: bool, + dataset_paths, + argv: list[str], + runs: list[BenchmarkRunResult], +) -> None: + limited = [run for run in runs if run.status in {"blocked", "partial"}] + text = "\n".join( + [ + "# Legacy BrainCTL vs MemPalace comparison bundle", + "", + f"- Current repo commit: `{_git_commit() or 'unknown'}`", + f"- Historical reference source: `{historical_source}`", + f"- Historical source mode: `{'fallback' if used_fallback else 'recovered summary bundle'}`", + f"- Command: `{' '.join(argv)}`", + "", + "## Datasets", + "", + f"- LongMemEval: `{dataset_paths.longmemeval_data}`", + f"- LoCoMo: `{dataset_paths.locomo_data}`", + f"- MemBench FirstAgent: `{dataset_paths.membench_data}`", + f"- ConvoMem cache: `{dataset_paths.convomem_cache}`", + "", + "## What is measured now", + "", + "- New BrainCTL reruns: LongMemEval `brain` and `cmd`, LoCoMo `cmd_session`, MemBench FirstAgent `cmd_turn`, and ConvoMem `cmd` coverage/status.", + "- Old BrainCTL and MemPalace are frozen historical reference series loaded from the recovered 2026-04-18 bundle.", + "", + "## Blocked or partial runs", + "", + ] + + ([f"- {run.benchmark} {run.series_name} {run.mode}: {' | '.join(run.caveats) or run.status}" for run in limited] if limited else ["- none"]) + + [ + "", + "## Output files", + "", + "- `summary.json` and `summary.csv`: all series in one table.", + "- `comparison_table.json` and `comparison_table.csv`: long-form metric rows.", + "- `runs/*.json`: per-run payloads.", + "- `charts/*.png`: regenerated charts with old BrainCTL, new BrainCTL, and MemPalace together.", + ] + ) + write_text(artifact_dir / "README.md", text + "\n") + + +def main() -> int: + parser = argparse.ArgumentParser(description="Rebuild the legacy BrainCTL vs MemPalace comparison charts.") + parser.add_argument( + "--artifact-dir", + type=Path, + default=Path(__file__).resolve().parent, + help="Base directory for results/charts output (default: benchmarks/)", + ) + parser.add_argument("--label", default="legacy_compare_refresh", help="Artifact directory prefix label.") + parser.add_argument("--longmemeval-limit", type=int, default=None) + parser.add_argument("--locomo-limit", type=int, default=None) + parser.add_argument("--membench-limit", type=int, default=None) + parser.add_argument("--membench-top-k", type=int, default=5) + parser.add_argument("--convomem-limit-per-category", type=int, default=1) + parser.add_argument("--convomem-top-k", type=int, default=10) + parser.add_argument("--skip-convomem", action="store_true") + args = parser.parse_args() + + os.environ.setdefault("BRAINCTL_SILENT_MIGRATIONS", "1") + artifact_dir = new_artifact_dir(args.artifact_dir, label=args.label) + dataset_paths = resolve_dataset_paths() + historical_runs, historical_source, used_fallback = load_historical_runs() + + measured_runs_with_rows = [ + run_brainctl_longmemeval_pipeline("brain", dataset_paths.longmemeval_data, limit=args.longmemeval_limit), + run_brainctl_longmemeval_pipeline("cmd", dataset_paths.longmemeval_data, limit=args.longmemeval_limit), + run_brainctl_locomo( + dataset_paths.locomo_data, + pipeline="cmd", + granularity="session", + limit=args.locomo_limit, + ), + run_brainctl_membench( + dataset_paths.membench_data, + pipeline="cmd", + top_k=args.membench_top_k, + limit=args.membench_limit, + ), + ] + if not args.skip_convomem: + measured_runs_with_rows.append( + run_brainctl_convomem( + limit_per_category=args.convomem_limit_per_category, + top_k=args.convomem_top_k, + cache_dir=dataset_paths.convomem_cache, + ) + ) + + measured_runs = [run for run, _rows in measured_runs_with_rows] + all_runs = historical_runs + measured_runs + + for run in historical_runs: + _write_run_artifact(artifact_dir, run, rows=None) + for run, rows in measured_runs_with_rows: + _write_run_artifact(artifact_dir, run, rows=rows) + + for benchmark_name, spec in BENCHMARK_SPECS.items(): + chart_path = plot_benchmark_chart( + artifact_dir / "charts" / spec["chart"], + benchmark_name, + [run for run in all_runs if run.benchmark == benchmark_name], + spec["metrics"], + ) + if chart_path is not None: + for run in all_runs: + if run.benchmark == benchmark_name: + run.artifacts["benchmark_chart"] = str(chart_path) + + aggregate_chart = plot_aggregate_primary_chart( + artifact_dir / "charts" / "aggregate_primary_metrics.png", + all_runs, + AGGREGATE_BENCHMARKS, + ) + if aggregate_chart is not None: + for run in all_runs: + if run.benchmark in AGGREGATE_BENCHMARKS: + run.artifacts["aggregate_chart"] = str(aggregate_chart) + + status_chart = plot_status_chart( + artifact_dir / "charts" / "coverage_status.png", + all_runs, + COVERAGE_BENCHMARKS, + ) + for run in all_runs: + run.artifacts["status_chart"] = str(status_chart) + + # Rewrite per-run payloads after chart paths are attached so every JSON + # artifact is self-contained. + for run in all_runs: + rows = None + for measured_run, measured_rows in measured_runs_with_rows: + if measured_run is run: + rows = measured_rows + break + _write_run_artifact(artifact_dir, run, rows=rows) + + metadata = runtime_metadata( + { + "git_commit": _git_commit(), + "cwd": str(ROOT), + "argv": sys.argv, + "historical_summary_path": str(historical_source), + "historical_summary_mode": "fallback" if used_fallback else "recovered", + "datasets": { + "longmemeval_data": str(dataset_paths.longmemeval_data) if dataset_paths.longmemeval_data else None, + "locomo_data": str(dataset_paths.locomo_data) if dataset_paths.locomo_data else None, + "membench_data": str(dataset_paths.membench_data) if dataset_paths.membench_data else None, + "convomem_cache": str(dataset_paths.convomem_cache) if dataset_paths.convomem_cache else None, + }, + } + ) + + write_bundle_summary( + artifact_dir / "summary.json", + all_runs, + notes=[ + "Historical old-BrainCTL and MemPalace series come from the recovered 2026-04-18 comparison bundle.", + "New BrainCTL series are rerun in the current checked-out repo using the legacy benchmark definitions.", + "MemBench remains intentionally partial because the legacy comparison only covered the FirstAgent slice.", + "ConvoMem remains a coverage/status benchmark here; it has no dedicated comparison chart in the legacy chart pack.", + ], + metadata=metadata, + ) + write_summary_csv(artifact_dir / "summary.csv", all_runs) + write_normalized_comparison(artifact_dir / "comparison_table.json", all_runs) + write_normalized_comparison_csv(artifact_dir / "comparison_table.csv", all_runs) + write_json(artifact_dir / "metadata.json", metadata) + _provenance_readme( + artifact_dir=artifact_dir, + historical_source=historical_source, + used_fallback=used_fallback, + dataset_paths=dataset_paths, + argv=sys.argv, + runs=all_runs, + ) + + print(artifact_dir) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/convomem_bench.py b/benchmarks/convomem_bench.py new file mode 100644 index 0000000..9b4f406 --- /dev/null +++ b/benchmarks/convomem_bench.py @@ -0,0 +1,226 @@ +from __future__ import annotations + +import json +import time +import urllib.request +from collections import defaultdict +from pathlib import Path +from typing import Any + +from benchmarks.brainctl_retrieval import rank_documents +from benchmarks.framework import BenchmarkRunResult, BLOCKED, PARTIAL + + +HF_BASE = "https://huggingface.co/datasets/Salesforce/ConvoMem/resolve/main/core_benchmark/evidence_questions" +HF_TREE = "https://huggingface.co/api/datasets/Salesforce/ConvoMem/tree/main/core_benchmark/evidence_questions" + +CATEGORIES = { + "user_evidence": "User Facts", + "assistant_facts_evidence": "Assistant Facts", + "changing_evidence": "Changing Facts", + "abstention_evidence": "Abstention", + "preference_evidence": "Preferences", + "implicit_connection_evidence": "Implicit Connections", +} + + +def _read_json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def _download_json(url: str, path: Path) -> Any: + path.parent.mkdir(parents=True, exist_ok=True) + if path.exists(): + return _read_json(path) + with urllib.request.urlopen(url, timeout=30) as response: + payload = response.read().decode("utf-8") + path.write_text(payload, encoding="utf-8") + return json.loads(payload) + + +def _discover_files(category: str, cache_dir: Path) -> list[str]: + cache_path = cache_dir / f"{category}_1_evidence_files.json" + url = f"{HF_TREE}/{category}/1_evidence" + payload = _download_json(url, cache_path) + paths = [] + for entry in payload: + raw_path = entry.get("path", "") + if raw_path.endswith(".json") and f"{category}/" in raw_path: + paths.append(raw_path.split(f"{category}/", 1)[1]) + return paths + + +def load_evidence_items( + *, + categories: list[str], + limit_per_category: int, + cache_dir: Path, +) -> list[dict[str, Any]]: + items: list[dict[str, Any]] = [] + for category in categories: + loaded = 0 + for subpath in _discover_files(category, cache_dir): + cache_path = cache_dir / category / subpath.replace("/", "_") + url = f"{HF_BASE}/{category}/{subpath}" + payload = _download_json(url, cache_path) + for item in payload.get("evidence_items", []): + item["_category_key"] = category + items.append(item) + loaded += 1 + if loaded >= limit_per_category: + break + if loaded >= limit_per_category: + break + return items + + +def _message_docs(item: dict[str, Any]) -> list[tuple[str, str]]: + docs: list[tuple[str, str]] = [] + index = 0 + for conversation in item.get("conversations", []): + for message in conversation.get("messages", []): + docs.append((f"msg_{index}", str(message.get("text", "")))) + index += 1 + return docs + + +def _evidence_texts(item: dict[str, Any]) -> set[str]: + texts = set() + for evidence in item.get("message_evidences", []): + text = str(evidence.get("text", "")).strip().lower() + if text: + texts.add(text) + return texts + + +def _recall_from_texts(retrieved_texts: list[str], evidence_texts: set[str]) -> float: + if not evidence_texts: + return 1.0 + found = 0 + lowered = [text.strip().lower() for text in retrieved_texts] + for evidence_text in evidence_texts: + if any(evidence_text in candidate or candidate in evidence_text for candidate in lowered): + found += 1 + return found / len(evidence_texts) + + +def _ranked_texts_from_ids(documents: list[tuple[str, str]], ranked_ids: list[str]) -> list[str]: + by_id = {doc_id: text for doc_id, text in documents} + return [by_id[doc_id] for doc_id in ranked_ids if doc_id in by_id] + + +def run_brainctl_convomem( + *, + categories: list[str] | None = None, + limit_per_category: int = 1, + top_k: int = 10, + pipeline: str = "cmd", + cache_dir: Path | None = None, +) -> tuple[BenchmarkRunResult, list[dict[str, Any]]]: + if cache_dir is None: + run = BenchmarkRunResult( + benchmark="convomem", + system_name="brainctl", + mode=pipeline, + status=BLOCKED, + example_count=0, + metrics={}, + primary_metric="avg_recall", + primary_metric_value=None, + dataset_path=None, + notes=[f"limit_per_category={limit_per_category}", f"top_k={top_k}"], + series_name="new_brainctl", + caveats=["ConvoMem cache directory is unavailable on this machine."], + ) + return run, [] + + try: + items = load_evidence_items( + categories=categories or list(CATEGORIES.keys()), + limit_per_category=limit_per_category, + cache_dir=cache_dir, + ) + except Exception as exc: + run = BenchmarkRunResult( + benchmark="convomem", + system_name="brainctl", + mode=pipeline, + status=BLOCKED, + example_count=0, + metrics={}, + primary_metric="avg_recall", + primary_metric_value=None, + dataset_path=str(cache_dir), + notes=[f"limit_per_category={limit_per_category}", f"top_k={top_k}"], + series_name="new_brainctl", + caveats=[f"Blocked while loading ConvoMem evidence data: {exc!s}"], + ) + return run, [] + + if not items: + run = BenchmarkRunResult( + benchmark="convomem", + system_name="brainctl", + mode=pipeline, + status=BLOCKED, + example_count=0, + metrics={}, + primary_metric="avg_recall", + primary_metric_value=None, + dataset_path=str(cache_dir), + notes=[f"limit_per_category={limit_per_category}", f"top_k={top_k}"], + series_name="new_brainctl", + caveats=["Blocked because no ConvoMem evidence items could be loaded for the requested categories."], + ) + return run, [] + + rows: list[dict[str, Any]] = [] + recalls: list[float] = [] + per_category: dict[str, list[float]] = defaultdict(list) + started = time.perf_counter() + + for item in items: + docs = _message_docs(item) + if not docs: + continue + evidence_texts = _evidence_texts(item) + ranked_ids = rank_documents(item["question"], docs, pipeline=pipeline, top_k=top_k) + retrieved_texts = _ranked_texts_from_ids(docs, ranked_ids) + recall = _recall_from_texts(retrieved_texts[:top_k], evidence_texts) + category = item.get("_category_key", "unknown") + recalls.append(recall) + per_category[category].append(recall) + rows.append( + { + "category": category, + "question": item["question"], + "recall": round(recall, 4), + "evidence_count": len(evidence_texts), + "retrieved_ids": ranked_ids[:top_k], + } + ) + + runtime_seconds = round(time.perf_counter() - started, 3) + example_count = len(rows) + avg_recall = round(sum(recalls) / len(recalls), 4) if recalls else 0.0 + perfect_rate = round(sum(1 for value in recalls if value >= 1.0) / len(recalls), 4) if recalls else 0.0 + metrics: dict[str, float | int] = {"avg_recall": avg_recall, "perfect_rate": perfect_rate, "top_k": top_k} + for category, values in sorted(per_category.items()): + metrics[f"{category}_recall"] = round(sum(values) / len(values), 4) + + run = BenchmarkRunResult( + benchmark="convomem", + system_name="brainctl", + mode=pipeline, + status=PARTIAL, + example_count=example_count, + metrics=metrics, + primary_metric="avg_recall", + primary_metric_value=avg_recall, + runtime_seconds=runtime_seconds, + dataset_path=str(cache_dir), + notes=[f"categories={len(categories or list(CATEGORIES.keys()))}", f"limit_per_category={limit_per_category}", f"top_k={top_k}"], + caveats=["ConvoMem comparison is partial because it uses a bounded same-machine sample, not the full benchmark."], + series_name="new_brainctl", + ) + return run, rows diff --git a/benchmarks/datasets.py b/benchmarks/datasets.py new file mode 100644 index 0000000..ed98222 --- /dev/null +++ b/benchmarks/datasets.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +GITHUB_ROOT = ROOT.parents[2] +HISTORICAL_REPO_ROOT = ROOT.parents[1] + + +@dataclass +class DatasetPaths: + longmemeval_data: Path | None + locomo_data: Path | None + membench_data: Path | None + convomem_cache: Path | None + + +def _env_path(name: str) -> Path | None: + raw = os.environ.get(name) + return Path(raw).expanduser() if raw else None + + +def _first_existing(candidates: list[Path | None]) -> Path | None: + for candidate in candidates: + if candidate and candidate.exists(): + return candidate + return None + + +def resolve_dataset_paths() -> DatasetPaths: + return DatasetPaths( + longmemeval_data=_first_existing( + [ + _env_path("BRAINCTL_LEGACY_LONGMEMEVAL_DATA"), + GITHUB_ROOT / "LongMemEval" / "data" / "longmemeval_s_cleaned.json", + ] + ), + locomo_data=_first_existing( + [ + _env_path("BRAINCTL_LEGACY_LOCOMO_DATA"), + GITHUB_ROOT / "locomo" / "data" / "locomo10.json", + ROOT / "tests" / "bench" / "locomo" / "locomo10.json", + ] + ), + membench_data=_first_existing( + [ + _env_path("BRAINCTL_LEGACY_MEMBENCH_DATA"), + GITHUB_ROOT / "Membench" / "MemData" / "FirstAgent", + ] + ), + convomem_cache=_first_existing( + [ + _env_path("BRAINCTL_LEGACY_CONVOMEM_CACHE"), + GITHUB_ROOT / "mempalace" / "benchmarks" / "convomem_cache", + ] + ), + ) diff --git a/benchmarks/framework.py b/benchmarks/framework.py new file mode 100644 index 0000000..44944b2 --- /dev/null +++ b/benchmarks/framework.py @@ -0,0 +1,384 @@ +from __future__ import annotations + +import csv +import json +import platform +import sys +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable + +import matplotlib + +matplotlib.use("Agg") + +import matplotlib.pyplot as plt + + +FULL_SAME_MACHINE = "full_same_machine" +PARTIAL = "partial" +BLOCKED = "blocked" + +SERIES_COLORS = { + "old_brainctl": "#4c78a8", + "new_brainctl": "#54a24b", + "mempalace": "#f58518", +} + +SERIES_ORDER = { + "old_brainctl": 0, + "new_brainctl": 1, + "mempalace": 2, +} + + +@dataclass +class BenchmarkRunResult: + benchmark: str + system_name: str + mode: str + status: str + example_count: int + metrics: dict[str, float | int | None] = field(default_factory=dict) + primary_metric: str | None = None + primary_metric_value: float | None = None + runtime_seconds: float | None = None + dataset_path: str | None = None + notes: list[str] = field(default_factory=list) + caveats: list[str] = field(default_factory=list) + artifacts: dict[str, str] = field(default_factory=dict) + reference_kind: str = "measured" + series_name: str | None = None + source_path: str | None = None + metadata: dict[str, Any] = field(default_factory=dict) + + def label(self) -> str: + series = self.series_name or self.system_name + return f"{series.replace('_', ' ')}\n{self.mode}" + + def to_dict(self) -> dict[str, Any]: + payload = asdict(self) + payload["measured"] = self.reference_kind == "measured" + return payload + + +def now_utc_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def new_artifact_dir(root: Path, label: str = "comparison") -> Path: + stamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") + path = root / "results" / f"{label}_{stamp}" + path.mkdir(parents=True, exist_ok=True) + (path / "runs").mkdir(exist_ok=True) + (path / "charts").mkdir(exist_ok=True) + return path + + +def write_json(path: Path, payload: Any) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8") + return path + + +def write_text(path: Path, text: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + +def _metric_fieldnames(runs: Iterable[BenchmarkRunResult]) -> list[str]: + keys: set[str] = set() + for run in runs: + keys.update(run.metrics.keys()) + return sorted(keys) + + +def write_summary_csv(path: Path, runs: list[BenchmarkRunResult]) -> Path: + fieldnames = [ + "benchmark", + "series_name", + "system_name", + "mode", + "reference_kind", + "status", + "example_count", + "primary_metric", + "primary_metric_value", + "runtime_seconds", + "dataset_path", + "source_path", + "notes", + "caveats", + ] + _metric_fieldnames(runs) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames) + writer.writeheader() + for run in runs: + row = { + "benchmark": run.benchmark, + "series_name": run.series_name, + "system_name": run.system_name, + "mode": run.mode, + "reference_kind": run.reference_kind, + "status": run.status, + "example_count": run.example_count, + "primary_metric": run.primary_metric, + "primary_metric_value": run.primary_metric_value, + "runtime_seconds": run.runtime_seconds, + "dataset_path": run.dataset_path, + "source_path": run.source_path, + "notes": " | ".join(run.notes), + "caveats": " | ".join(run.caveats), + } + row.update(run.metrics) + writer.writerow(row) + return path + + +def write_run_payload( + path: Path, + run: BenchmarkRunResult, + rows: list[dict[str, Any]] | None = None, +) -> Path: + payload = run.to_dict() + if rows is not None: + payload["rows"] = rows + return write_json(path, payload) + + +def write_normalized_comparison(path: Path, runs: list[BenchmarkRunResult]) -> Path: + rows: list[dict[str, Any]] = [] + for run in runs: + for metric, value in sorted(run.metrics.items()): + rows.append( + { + "benchmark": run.benchmark, + "metric": metric, + "series_name": run.series_name, + "system_name": run.system_name, + "mode": run.mode, + "reference_kind": run.reference_kind, + "status": run.status, + "value": value, + "example_count": run.example_count, + "dataset_path": run.dataset_path, + "source_path": run.source_path, + } + ) + return write_json(path, rows) + + +def write_normalized_comparison_csv(path: Path, runs: list[BenchmarkRunResult]) -> Path: + rows: list[dict[str, Any]] = [] + for run in runs: + for metric, value in sorted(run.metrics.items()): + rows.append( + { + "benchmark": run.benchmark, + "metric": metric, + "series_name": run.series_name, + "system_name": run.system_name, + "mode": run.mode, + "reference_kind": run.reference_kind, + "status": run.status, + "value": value, + "example_count": run.example_count, + "dataset_path": run.dataset_path, + "source_path": run.source_path, + } + ) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter( + handle, + fieldnames=[ + "benchmark", + "metric", + "series_name", + "system_name", + "mode", + "reference_kind", + "status", + "value", + "example_count", + "dataset_path", + "source_path", + ], + ) + writer.writeheader() + writer.writerows(rows) + return path + + +def _sort_runs_for_plot(runs: list[BenchmarkRunResult]) -> list[BenchmarkRunResult]: + return sorted( + runs, + key=lambda run: ( + SERIES_ORDER.get(run.series_name or run.system_name, 99), + (run.mode != "brain"), + run.mode, + ), + ) + + +def plot_benchmark_chart( + path: Path, + benchmark_name: str, + runs: list[BenchmarkRunResult], + metric_keys: list[str], +) -> Path | None: + plotted = [ + run + for run in _sort_runs_for_plot(runs) + if run.status != BLOCKED and any(run.metrics.get(key) is not None for key in metric_keys) + ] + if not plotted: + return None + + x = list(range(len(metric_keys))) + width = 0.8 / max(len(plotted), 1) + + fig, ax = plt.subplots(figsize=(max(8, len(metric_keys) * 2.0), 5)) + for idx, run in enumerate(plotted): + offsets = [pos + (idx - (len(plotted) - 1) / 2) * width for pos in x] + values = [float(run.metrics.get(key) or 0.0) for key in metric_keys] + color = SERIES_COLORS.get(run.series_name or run.system_name) + ax.bar(offsets, values, width=width, label=run.label(), color=color) + + ymax = max(float(run.metrics.get(key) or 0.0) for run in plotted for key in metric_keys) + ax.set_title(f"{benchmark_name} comparison") + ax.set_xticks(x) + ax.set_xticklabels(metric_keys, rotation=20, ha="right") + ax.set_ylim(0, max(1.0, ymax * 1.15)) + ax.set_ylabel("score") + ax.legend() + ax.grid(axis="y", alpha=0.25) + fig.tight_layout() + fig.savefig(path, dpi=160) + plt.close(fig) + return path + + +def plot_aggregate_primary_chart( + path: Path, + runs: list[BenchmarkRunResult], + benchmarks: list[str], +) -> Path | None: + measured = [ + run + for run in runs + if run.benchmark in benchmarks + and run.status != BLOCKED + and run.primary_metric_value is not None + ] + if not measured: + return None + + benchmark_order = [name for name in benchmarks if any(run.benchmark == name for run in measured)] + run_labels = [] + for run in _sort_runs_for_plot(measured): + label = f"{run.series_name}|{run.mode}" + if label not in run_labels: + run_labels.append(label) + + x = list(range(len(benchmark_order))) + width = 0.8 / max(len(run_labels), 1) + + fig, ax = plt.subplots(figsize=(max(9, len(benchmark_order) * 2.5), 5)) + for idx, run_label in enumerate(run_labels): + offsets = [pos + (idx - (len(run_labels) - 1) / 2) * width for pos in x] + values: list[float] = [] + color = None + pretty_label = run_label.replace("|", "\n").replace("_", " ") + for benchmark in benchmark_order: + match = next( + ( + run + for run in measured + if run.benchmark == benchmark and f"{run.series_name}|{run.mode}" == run_label + ), + None, + ) + values.append(float(match.primary_metric_value or 0.0) if match else 0.0) + if match and color is None: + color = SERIES_COLORS.get(match.series_name or match.system_name) + ax.bar(offsets, values, width=width, label=pretty_label, color=color) + + ymax = max(float(run.primary_metric_value or 0.0) for run in measured) + ax.set_title("Primary metric by benchmark") + ax.set_xticks(x) + ax.set_xticklabels(benchmark_order, rotation=15, ha="right") + ax.set_ylabel("primary score") + ax.set_ylim(0, max(1.0, ymax * 1.15)) + ax.legend() + ax.grid(axis="y", alpha=0.25) + fig.tight_layout() + fig.savefig(path, dpi=160) + plt.close(fig) + return path + + +def plot_status_chart(path: Path, runs: list[BenchmarkRunResult], benchmarks: list[str]) -> Path: + status_order = [FULL_SAME_MACHINE, PARTIAL, BLOCKED] + colors = { + FULL_SAME_MACHINE: "#4c78a8", + PARTIAL: "#f58518", + BLOCKED: "#e45756", + } + + counts: dict[str, list[int]] = {status: [] for status in status_order} + for benchmark in benchmarks: + benchmark_runs = [run for run in runs if run.benchmark == benchmark] + for status in status_order: + counts[status].append(sum(1 for run in benchmark_runs if run.status == status)) + + fig, ax = plt.subplots(figsize=(max(8, len(benchmarks) * 2.0), 5)) + bottom = [0] * len(benchmarks) + x = list(range(len(benchmarks))) + for status in status_order: + values = counts[status] + ax.bar(x, values, bottom=bottom, label=status, color=colors[status]) + bottom = [a + b for a, b in zip(bottom, values)] + + ax.set_title("Benchmark coverage status") + ax.set_xticks(x) + ax.set_xticklabels(benchmarks, rotation=15, ha="right") + ax.set_ylabel("run count") + ax.legend() + ax.grid(axis="y", alpha=0.25) + fig.tight_layout() + fig.savefig(path, dpi=160) + plt.close(fig) + return path + + +def runtime_metadata(extra: dict[str, Any] | None = None) -> dict[str, Any]: + payload = { + "generated_at_utc": now_utc_iso(), + "python_version": sys.version, + "platform": platform.platform(), + "machine": platform.machine(), + "processor": platform.processor(), + } + if extra: + payload.update(extra) + return payload + + +def write_bundle_summary( + path: Path, + runs: list[BenchmarkRunResult], + *, + notes: list[str] | None = None, + metadata: dict[str, Any] | None = None, +) -> Path: + payload = { + "generated_at_utc": now_utc_iso(), + "metadata": metadata or {}, + "runs": [run.to_dict() for run in runs], + "notes": notes or [], + } + return write_json(path, payload) diff --git a/benchmarks/legacy_refs.py b/benchmarks/legacy_refs.py new file mode 100644 index 0000000..7146ddf --- /dev/null +++ b/benchmarks/legacy_refs.py @@ -0,0 +1,242 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from benchmarks.framework import BenchmarkRunResult + + +ROOT = Path(__file__).resolve().parent.parent +HISTORICAL_RESULTS_DIR = ROOT.parents[1] / "benchmarks" / "results" + +BENCHMARK_SPECS = { + "locomo": { + "chart": "locomo_comparison.png", + "metrics": ["avg_recall", "perfect_rate", "zero_rate"], + "primary_metric": "avg_recall", + }, + "longmemeval": { + "chart": "longmemeval_comparison.png", + "metrics": ["r_at_5", "r_at_10", "ndcg_at_5", "ndcg_at_10"], + "primary_metric": "r_at_5", + }, + "membench": { + "chart": "membench_comparison.png", + "metrics": ["hit_at_5"], + "primary_metric": "hit_at_5", + }, +} + +AGGREGATE_BENCHMARKS = ["locomo", "longmemeval", "membench"] +COVERAGE_BENCHMARKS = ["convomem", "locomo", "longmemeval", "membench"] + + +def _coerce_run(payload: dict, *, source_path: Path, used_fallback: bool) -> BenchmarkRunResult: + system_name = str(payload["system_name"]) + series_name = "old_brainctl" if system_name == "brainctl" else "mempalace" + notes = list(payload.get("notes") or []) + if used_fallback: + notes.append("Loaded from hardcoded fallback because the historical summary bundle was unavailable.") + return BenchmarkRunResult( + benchmark=str(payload["benchmark"]), + system_name=system_name, + mode=str(payload["mode"]), + status=str(payload["status"]), + example_count=int(payload.get("example_count") or 0), + metrics=dict(payload.get("metrics") or {}), + primary_metric=payload.get("primary_metric"), + primary_metric_value=payload.get("primary_metric_value"), + runtime_seconds=payload.get("runtime_seconds"), + dataset_path=payload.get("dataset_path"), + notes=notes, + caveats=list(payload.get("caveats") or []), + artifacts=dict(payload.get("artifacts") or {}), + reference_kind="historical", + series_name=series_name, + source_path=str(source_path), + ) + + +def _candidate_summary_paths() -> list[Path]: + candidates: list[Path] = [] + exact = HISTORICAL_RESULTS_DIR / "full_compare_20260418_033425" / "summary.json" + if exact.exists(): + candidates.append(exact) + if HISTORICAL_RESULTS_DIR.exists(): + candidates.extend(sorted(HISTORICAL_RESULTS_DIR.glob("full_compare_*/summary.json"), reverse=True)) + seen: set[str] = set() + unique: list[Path] = [] + for candidate in candidates: + key = str(candidate.resolve()) + if key in seen: + continue + seen.add(key) + unique.append(candidate) + return unique + + +def _fallback_payload() -> dict: + return { + "generated_at_utc": "2026-04-18T04:48:04.326900+00:00", + "notes": [ + "LongMemEval and LoCoMo are configured for full same-machine retrieval comparisons when limits are unset.", + "MemBench is intentionally marked partial because this harness compares the FirstAgent slice only.", + "ConvoMem is intentionally marked partial because this harness uses a bounded same-machine sample per category.", + ], + "runs": [ + { + "benchmark": "longmemeval", + "system_name": "brainctl", + "mode": "brain", + "status": "full_same_machine", + "example_count": 470, + "metrics": {"r_at_5": 0.9681, "r_at_10": 0.9894, "ndcg_at_5": 0.9204, "ndcg_at_10": 0.9253}, + "primary_metric": "r_at_5", + "primary_metric_value": 0.9681, + "runtime_seconds": 85.439, + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "notes": ["top_k=10"], + "caveats": [], + "artifacts": {}, + }, + { + "benchmark": "longmemeval", + "system_name": "brainctl", + "mode": "cmd", + "status": "full_same_machine", + "example_count": 470, + "metrics": {"r_at_5": 0.9702, "r_at_10": 0.9894, "ndcg_at_5": 0.9206, "ndcg_at_10": 0.9247}, + "primary_metric": "r_at_5", + "primary_metric_value": 0.9702, + "runtime_seconds": 130.863, + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "notes": ["top_k=10"], + "caveats": [], + "artifacts": {}, + }, + { + "benchmark": "longmemeval", + "system_name": "mempalace", + "mode": "raw_session", + "status": "full_same_machine", + "example_count": 470, + "metrics": {"r_at_5": 0.9660, "r_at_10": 0.9830, "ndcg_at_5": 0.8930, "ndcg_at_10": 0.8948}, + "primary_metric": "r_at_5", + "primary_metric_value": 0.9660, + "runtime_seconds": 695.36, + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "notes": ["top_k=10", "Runs MemPalace benchmark module raw session retrieval logic directly."], + "caveats": [], + "artifacts": {}, + }, + { + "benchmark": "locomo", + "system_name": "brainctl", + "mode": "cmd_session", + "status": "full_same_machine", + "example_count": 1986, + "metrics": {"avg_recall": 0.9217, "perfect_rate": 0.8817, "zero_rate": 0.0438, "top_k": 10}, + "primary_metric": "avg_recall", + "primary_metric_value": 0.9217, + "runtime_seconds": 445.74, + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "notes": ["granularity=session", "top_k=10"], + "caveats": [], + "artifacts": {}, + }, + { + "benchmark": "locomo", + "system_name": "mempalace", + "mode": "raw_session", + "status": "full_same_machine", + "example_count": 1986, + "metrics": {"avg_recall": 0.6028, "perfect_rate": 0.5534, "zero_rate": 0.3499, "top_k": 10}, + "primary_metric": "avg_recall", + "primary_metric_value": 0.6028, + "runtime_seconds": 2106.411, + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "notes": ["granularity=session", "top_k=10"], + "caveats": [], + "artifacts": {}, + }, + { + "benchmark": "membench", + "system_name": "brainctl", + "mode": "cmd_turn", + "status": "partial", + "example_count": 200, + "metrics": {"hit_at_5": 0.9300, "top_k": 5}, + "primary_metric": "hit_at_5", + "primary_metric_value": 0.9300, + "runtime_seconds": 140.592, + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "notes": ["FirstAgent slice only", "turn-level retrieval", "topic=all"], + "caveats": ["MemBench comparison is partial because ThirdAgent and noise-extended slices are not included."], + "artifacts": {}, + }, + { + "benchmark": "membench", + "system_name": "mempalace", + "mode": "raw_turn", + "status": "partial", + "example_count": 200, + "metrics": {"hit_at_5": 0.8850, "top_k": 5}, + "primary_metric": "hit_at_5", + "primary_metric_value": 0.8850, + "runtime_seconds": 804.35, + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "notes": ["FirstAgent slice only", "turn-level retrieval", "topic=all"], + "caveats": ["MemBench comparison is partial because ThirdAgent and noise-extended slices are not included."], + "artifacts": {}, + }, + { + "benchmark": "convomem", + "system_name": "brainctl", + "mode": "cmd", + "status": "blocked", + "example_count": 0, + "metrics": {}, + "primary_metric": "avg_recall", + "primary_metric_value": None, + "runtime_seconds": None, + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "notes": ["limit_per_category=1", "top_k=10"], + "caveats": ["Blocked while loading ConvoMem evidence data: "], + "artifacts": {}, + }, + { + "benchmark": "convomem", + "system_name": "mempalace", + "mode": "raw", + "status": "blocked", + "example_count": 0, + "metrics": {}, + "primary_metric": "avg_recall", + "primary_metric_value": None, + "runtime_seconds": None, + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "notes": ["limit_per_category=1", "top_k=10"], + "caveats": ["Blocked while loading ConvoMem evidence data: "], + "artifacts": {}, + }, + ], + } + + +def load_historical_runs() -> tuple[list[BenchmarkRunResult], Path, bool]: + for candidate in _candidate_summary_paths(): + payload = json.loads(candidate.read_text(encoding="utf-8")) + runs = [ + _coerce_run(run_payload, source_path=candidate, used_fallback=False) + for run_payload in payload.get("runs", []) + ] + if runs: + return runs, candidate, False + + fallback_path = HISTORICAL_RESULTS_DIR / "full_compare_20260418_033425" / "summary.json" + payload = _fallback_payload() + runs = [ + _coerce_run(run_payload, source_path=fallback_path, used_fallback=True) + for run_payload in payload.get("runs", []) + ] + return runs, fallback_path, True diff --git a/benchmarks/locomo_bench.py b/benchmarks/locomo_bench.py new file mode 100644 index 0000000..6cce8ef --- /dev/null +++ b/benchmarks/locomo_bench.py @@ -0,0 +1,172 @@ +from __future__ import annotations + +import json +import time +from collections import defaultdict +from pathlib import Path +from typing import Any + +from benchmarks.brainctl_retrieval import rank_seeded_documents, seed_documents +from benchmarks.framework import BenchmarkRunResult, BLOCKED, FULL_SAME_MACHINE + + +CATEGORIES = { + 1: "Single-hop", + 2: "Temporal", + 3: "Temporal-inference", + 4: "Open-domain", + 5: "Adversarial", +} + + +def _load_samples(data_path: Path, limit: int | None = None) -> list[dict[str, Any]]: + payload = json.loads(data_path.read_text(encoding="utf-8-sig")) + samples = list(payload) + if limit: + samples = samples[:limit] + return samples + + +def _load_sessions(conversation: dict[str, Any]) -> list[dict[str, Any]]: + sessions: list[dict[str, Any]] = [] + session_num = 1 + while True: + key = f"session_{session_num}" + date_key = f"session_{session_num}_date_time" + if key not in conversation: + break + sessions.append( + { + "session_num": session_num, + "date": conversation.get(date_key, ""), + "dialogs": conversation[key], + } + ) + session_num += 1 + return sessions + + +def _build_corpus(sessions: list[dict[str, Any]], granularity: str) -> list[tuple[str, str]]: + corpus: list[tuple[str, str]] = [] + for session in sessions: + if granularity == "session": + texts = [] + for dialog in session["dialogs"]: + speaker = dialog.get("speaker", "?") + text = dialog.get("text", "") + texts.append(f'{speaker} said, "{text}"') + corpus.append((f"session_{session['session_num']}", "\n".join(texts))) + continue + + for dialog in session["dialogs"]: + dialog_id = dialog.get("dia_id", f"D{session['session_num']}:?") + speaker = dialog.get("speaker", "?") + text = dialog.get("text", "") + corpus.append((dialog_id, f'{speaker} said, "{text}"')) + return corpus + + +def _evidence_ids(evidence: list[str], granularity: str) -> set[str]: + if granularity == "dialog": + return set(evidence) + sessions: set[str] = set() + for evidence_id in evidence: + if evidence_id.startswith("D") and ":" in evidence_id: + sessions.add(f"session_{evidence_id[1:].split(':', 1)[0]}") + return sessions + + +def _recall(retrieved_ids: list[str], evidence_ids: set[str]) -> float: + if not evidence_ids: + return 1.0 + found = sum(1 for item in evidence_ids if item in retrieved_ids) + return found / len(evidence_ids) + + +def run_brainctl_locomo( + data_path: Path | None, + *, + pipeline: str = "cmd", + granularity: str = "session", + top_k: int = 10, + limit: int | None = None, +) -> tuple[BenchmarkRunResult, list[dict[str, Any]]]: + if data_path is None or not data_path.exists(): + run = BenchmarkRunResult( + benchmark="locomo", + system_name="brainctl", + mode=f"{pipeline}_{granularity}", + status=BLOCKED, + example_count=0, + metrics={}, + primary_metric="avg_recall", + primary_metric_value=None, + dataset_path=str(data_path) if data_path else None, + series_name="new_brainctl", + caveats=["LoCoMo dataset path is unavailable on this machine."], + ) + return run, [] + + samples = _load_samples(data_path, limit=limit) + rows: list[dict[str, Any]] = [] + per_category: dict[int, list[float]] = defaultdict(list) + recalls: list[float] = [] + started = time.perf_counter() + + for sample in samples: + sample_id = sample.get("sample_id", "unknown") + sessions = _load_sessions(sample["conversation"]) + corpus = _build_corpus(sessions, granularity=granularity) + seeded = seed_documents(corpus) + try: + for qa in sample["qa"]: + question = qa["question"] + evidence_ids = _evidence_ids(qa.get("evidence", []), granularity) + retrieved_ids = rank_seeded_documents(question, seeded, pipeline=pipeline, top_k=top_k) + recall = _recall(retrieved_ids, evidence_ids) + category = int(qa["category"]) + recalls.append(recall) + per_category[category].append(recall) + rows.append( + { + "sample_id": sample_id, + "question": question, + "category": category, + "category_name": CATEGORIES.get(category, str(category)), + "evidence_ids": sorted(evidence_ids), + "retrieved_ids": retrieved_ids, + "recall": round(recall, 4), + } + ) + finally: + seeded.cleanup() + + runtime_seconds = round(time.perf_counter() - started, 3) + example_count = len(rows) + avg_recall = round(sum(recalls) / len(recalls), 4) if recalls else 0.0 + perfect_rate = round(sum(1 for value in recalls if value >= 1.0) / len(recalls), 4) if recalls else 0.0 + zero_rate = round(sum(1 for value in recalls if value == 0.0) / len(recalls), 4) if recalls else 0.0 + metrics: dict[str, float | int] = { + "avg_recall": avg_recall, + "perfect_rate": perfect_rate, + "zero_rate": zero_rate, + "top_k": top_k, + } + for category, values in sorted(per_category.items()): + metrics[f"cat_{category}_recall"] = round(sum(values) / len(values), 4) + + run = BenchmarkRunResult( + benchmark="locomo", + system_name="brainctl", + mode=f"{pipeline}_{granularity}", + status=FULL_SAME_MACHINE, + example_count=example_count, + metrics=metrics, + primary_metric="avg_recall", + primary_metric_value=avg_recall, + runtime_seconds=runtime_seconds, + dataset_path=str(data_path), + notes=[f"granularity={granularity}", f"top_k={top_k}"], + series_name="new_brainctl", + ) + return run, rows diff --git a/benchmarks/longmemeval_bench.py b/benchmarks/longmemeval_bench.py new file mode 100644 index 0000000..32ffd7c --- /dev/null +++ b/benchmarks/longmemeval_bench.py @@ -0,0 +1,211 @@ +from __future__ import annotations + +import json +import math +import os +import random +import time +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable + +from benchmarks.brainctl_retrieval import rank_documents +from benchmarks.framework import BenchmarkRunResult, BLOCKED, FULL_SAME_MACHINE + + +@dataclass +class QuestionEntry: + question_id: str + question_type: str + question: str + answer_session_ids: list[str] + haystack_session_ids: list[str] + haystack_dates: list[str] + haystack_sessions: list[list[dict[str, Any]]] + + +def _is_abstention(raw: dict[str, Any]) -> bool: + qid = str(raw.get("question_id", "")) + return qid.endswith("_abs") or not raw.get("answer_session_ids") + + +def load_entries( + dataset_path: Path, + *, + include_abstention: bool = False, + limit: int | None = None, +) -> list[QuestionEntry]: + payload = json.loads(dataset_path.read_text(encoding="utf-8-sig")) + entries: list[QuestionEntry] = [] + for raw in payload: + if not include_abstention and _is_abstention(raw): + continue + entries.append( + QuestionEntry( + question_id=str(raw["question_id"]), + question_type=str(raw["question_type"]), + question=str(raw["question"]), + answer_session_ids=[str(x) for x in raw.get("answer_session_ids", [])], + haystack_session_ids=[str(x) for x in raw.get("haystack_session_ids", [])], + haystack_dates=[str(x) for x in raw.get("haystack_dates", [])], + haystack_sessions=list(raw.get("haystack_sessions", [])), + ) + ) + if limit is not None and len(entries) >= limit: + break + return entries + + +def session_document(session_id: str, session_date: str, turns: list[dict[str, Any]]) -> str: + lines = [f"Session ID: {session_id}", f"Session Date: {session_date}", "Conversation:"] + for turn in turns: + role = str(turn.get("role", "unknown")).strip().title() or "Unknown" + content = str(turn.get("content", "")).strip() + if content: + lines.append(f"{role}: {content}") + return "\n".join(lines) + + +def dcg(relevances: Iterable[float], k: int) -> float: + total = 0.0 + for i, rel in enumerate(list(relevances)[:k]): + total += rel / math.log2(i + 2) + return total + + +def ndcg(rankings: list[int], correct_ids: set[str], corpus_ids: list[str], k: int) -> float: + relevances = [1.0 if corpus_ids[idx] in correct_ids else 0.0 for idx in rankings[:k]] + ideal = sorted(relevances, reverse=True) + ideal_dcg = dcg(ideal, k) + if ideal_dcg == 0: + return 0.0 + return dcg(relevances, k) / ideal_dcg + + +def recall_any(rankings: list[int], correct_ids: set[str], corpus_ids: list[str], k: int) -> float: + top_ids = {corpus_ids[idx] for idx in rankings[:k]} + return float(any(cid in top_ids for cid in correct_ids)) + + +def recall_all(rankings: list[int], correct_ids: set[str], corpus_ids: list[str], k: int) -> float: + if not correct_ids: + return 1.0 + top_ids = {corpus_ids[idx] for idx in rankings[:k]} + return float(all(cid in top_ids for cid in correct_ids)) + + +def _mean(values: Iterable[float]) -> float: + bucket = list(values) + if not bucket: + return 0.0 + return round(sum(bucket) / len(bucket), 4) + + +def run_entry(entry: QuestionEntry, *, pipeline: str = "cmd", top_k: int = 10) -> dict[str, Any]: + docs = [ + (session_id, session_document(session_id, session_date, turns)) + for session_id, session_date, turns in zip( + entry.haystack_session_ids, + entry.haystack_dates, + entry.haystack_sessions, + ) + ] + ranked_session_ids = rank_documents(entry.question, docs, pipeline=pipeline, top_k=top_k) + seen = set(ranked_session_ids) + remaining = [sid for sid in entry.haystack_session_ids if sid not in seen] + corpus_ids = ranked_session_ids + remaining + ranked_indices = list(range(len(ranked_session_ids))) + correct_ids = set(entry.answer_session_ids) + return { + "question_id": entry.question_id, + "question_type": entry.question_type, + "r_at_5": recall_any(ranked_indices, correct_ids, corpus_ids, 5), + "r_at_10": recall_any(ranked_indices, correct_ids, corpus_ids, 10), + "r_all_at_5": recall_all(ranked_indices, correct_ids, corpus_ids, 5), + "r_all_at_10": recall_all(ranked_indices, correct_ids, corpus_ids, 10), + "ndcg_at_5": round(ndcg(ranked_indices, correct_ids, corpus_ids, 5), 4), + "ndcg_at_10": round(ndcg(ranked_indices, correct_ids, corpus_ids, 10), 4), + "answer_session_ids": entry.answer_session_ids, + "top_session_ids": ranked_session_ids[:top_k], + } + + +def aggregate_rows(rows: list[dict[str, Any]]) -> dict[str, Any]: + overall = { + "n_questions": len(rows), + "r_at_5": _mean(row["r_at_5"] for row in rows), + "r_at_10": _mean(row["r_at_10"] for row in rows), + "r_all_at_5": _mean(row["r_all_at_5"] for row in rows), + "r_all_at_10": _mean(row["r_all_at_10"] for row in rows), + "ndcg_at_5": _mean(row["ndcg_at_5"] for row in rows), + "ndcg_at_10": _mean(row["ndcg_at_10"] for row in rows), + } + by_question_type: dict[str, dict[str, float]] = {} + buckets: dict[str, list[dict[str, Any]]] = defaultdict(list) + for row in rows: + buckets[row["question_type"]].append(row) + for question_type, group in sorted(buckets.items()): + by_question_type[question_type] = { + "count": len(group), + "r_at_5": _mean(row["r_at_5"] for row in group), + "r_at_10": _mean(row["r_at_10"] for row in group), + "r_all_at_5": _mean(row["r_all_at_5"] for row in group), + "r_all_at_10": _mean(row["r_all_at_10"] for row in group), + "ndcg_at_5": _mean(row["ndcg_at_5"] for row in group), + "ndcg_at_10": _mean(row["ndcg_at_10"] for row in group), + } + return {"overall": overall, "by_question_type": by_question_type} + + +def run_brainctl_longmemeval_pipeline( + pipeline: str, + dataset_path: Path | None, + *, + limit: int | None = None, + include_abstention: bool = False, + top_k: int = 10, +) -> tuple[BenchmarkRunResult, list[dict[str, Any]]]: + if dataset_path is None or not dataset_path.exists(): + run = BenchmarkRunResult( + benchmark="longmemeval", + system_name="brainctl", + mode=pipeline, + status=BLOCKED, + example_count=0, + metrics={}, + primary_metric="r_at_5", + primary_metric_value=None, + dataset_path=str(dataset_path) if dataset_path else None, + series_name="new_brainctl", + caveats=["LongMemEval dataset path is unavailable on this machine."], + ) + return run, [] + + random.seed(42) + os.environ.setdefault("BRAINCTL_SILENT_MIGRATIONS", "1") + started = time.perf_counter() + entries = load_entries(dataset_path, include_abstention=include_abstention, limit=limit) + rows = [run_entry(entry, pipeline=pipeline, top_k=top_k) for entry in entries] + runtime_seconds = round(time.perf_counter() - started, 3) + overall = aggregate_rows(rows)["overall"] + run = BenchmarkRunResult( + benchmark="longmemeval", + system_name="brainctl", + mode=pipeline, + status=FULL_SAME_MACHINE, + example_count=int(overall["n_questions"]), + metrics={ + "r_at_5": overall["r_at_5"], + "r_at_10": overall["r_at_10"], + "ndcg_at_5": overall["ndcg_at_5"], + "ndcg_at_10": overall["ndcg_at_10"], + }, + primary_metric="r_at_5", + primary_metric_value=float(overall["r_at_5"]), + runtime_seconds=runtime_seconds, + dataset_path=str(dataset_path), + notes=[f"top_k={top_k}", "Legacy 470-question session-level slice."], + series_name="new_brainctl", + ) + return run, rows diff --git a/benchmarks/membench_bench.py b/benchmarks/membench_bench.py new file mode 100644 index 0000000..354d7b1 --- /dev/null +++ b/benchmarks/membench_bench.py @@ -0,0 +1,193 @@ +from __future__ import annotations + +import json +import time +from collections import defaultdict +from pathlib import Path +from typing import Any + +from benchmarks.brainctl_retrieval import rank_documents +from benchmarks.framework import BenchmarkRunResult, BLOCKED, PARTIAL + + +CATEGORY_FILES = { + "simple": "simple.json", + "highlevel": "highlevel.json", + "knowledge_update": "knowledge_update.json", + "comparative": "comparative.json", + "conditional": "conditional.json", + "noisy": "noisy.json", + "aggregative": "aggregative.json", + "highlevel_rec": "highlevel_rec.json", + "lowlevel_rec": "lowlevel_rec.json", + "RecMultiSession": "RecMultiSession.json", + "post_processing": "post_processing.json", +} + + +def _load_json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8-sig")) + + +def load_items( + data_dir: Path, + *, + categories: list[str] | None = None, + topic: str | None = None, + limit: int | None = None, +) -> list[dict[str, Any]]: + selected_categories = categories or list(CATEGORY_FILES.keys()) + items: list[dict[str, Any]] = [] + for category in selected_categories: + file_name = CATEGORY_FILES.get(category) + if not file_name: + continue + path = data_dir / file_name + if not path.exists(): + continue + raw = _load_json(path) + for key, topic_items in raw.items(): + if topic and key not in (topic, "roles", "events"): + continue + for item in topic_items: + turns = item.get("message_list", []) + qa = item.get("QA", {}) + if not turns or not qa: + continue + items.append( + { + "category": category, + "topic": key, + "tid": item.get("tid", 0), + "turns": turns, + "question": qa.get("question", ""), + "target_step_ids": qa.get("target_step_id", []), + } + ) + if limit and len(items) >= limit: + return items + return items + + +def _turn_text(turn: dict[str, Any]) -> str: + user = turn.get("user") or turn.get("user_message", "") + assistant = turn.get("assistant") or turn.get("assistant_message", "") + when = turn.get("time", "") + text = f"[User] {user} [Assistant] {assistant}" + return f"[{when}] {text}" if when else text + + +def _flatten_turns(message_list: list[Any], item_key: str) -> list[tuple[str, str]]: + docs: list[tuple[str, str]] = [] + sessions = [message_list] if message_list and isinstance(message_list[0], dict) else message_list + global_idx = 0 + for session_idx, session in enumerate(sessions): + if not isinstance(session, list): + continue + for turn_idx, turn in enumerate(session): + if not isinstance(turn, dict): + continue + sid = turn.get("sid", turn.get("mid", global_idx)) + doc_id = f"{item_key}|sid={sid}|g={global_idx}|s={session_idx}|t={turn_idx}" + docs.append((doc_id, _turn_text(turn))) + global_idx += 1 + return docs + + +def _target_ids(target_step_ids: list[Any]) -> set[str]: + targets: set[str] = set() + for step in target_step_ids: + if isinstance(step, list) and step: + targets.add(str(step[0])) + else: + targets.add(str(step)) + return targets + + +def _hit_at_k(retrieved_ids: list[str], targets: set[str]) -> bool: + if not targets: + return False + for retrieved in retrieved_ids: + for target in targets: + if f"sid={target}|" in retrieved or f"|g={target}|" in retrieved: + return True + return False + + +def run_brainctl_membench( + data_dir: Path | None, + *, + pipeline: str = "cmd", + categories: list[str] | None = None, + topic: str | None = None, + top_k: int = 5, + limit: int | None = None, +) -> tuple[BenchmarkRunResult, list[dict[str, Any]]]: + if data_dir is None or not data_dir.exists(): + run = BenchmarkRunResult( + benchmark="membench", + system_name="brainctl", + mode=f"{pipeline}_turn", + status=BLOCKED, + example_count=0, + metrics={}, + primary_metric=f"hit_at_{top_k}", + primary_metric_value=None, + dataset_path=str(data_dir) if data_dir else None, + series_name="new_brainctl", + caveats=["MemBench FirstAgent data is unavailable on this machine."], + ) + return run, [] + + items = load_items(data_dir, categories=categories, topic=topic, limit=limit) + rows: list[dict[str, Any]] = [] + by_category: dict[str, list[bool]] = defaultdict(list) + hits = 0 + started = time.perf_counter() + + for idx, item in enumerate(items): + item_key = f"{item['category']}_{item['topic']}_{idx}" + docs = _flatten_turns(item["turns"], item_key) + if not docs: + continue + retrieved_ids = rank_documents(item["question"], docs, pipeline=pipeline, top_k=top_k) + targets = _target_ids(item["target_step_ids"]) + hit = _hit_at_k(retrieved_ids, targets) + if hit: + hits += 1 + by_category[item["category"]].append(hit) + rows.append( + { + "category": item["category"], + "topic": item["topic"], + "tid": item["tid"], + "question": item["question"], + "retrieved_ids": retrieved_ids, + "target_ids": sorted(targets), + "hit_at_k": hit, + } + ) + + runtime_seconds = round(time.perf_counter() - started, 3) + example_count = len(rows) + hit_rate = round(hits / example_count, 4) if example_count else 0.0 + metrics: dict[str, float | int] = {f"hit_at_{top_k}": hit_rate, "top_k": top_k} + for category, values in sorted(by_category.items()): + metrics[f"{category}_hit_at_{top_k}"] = round(sum(1 for value in values if value) / len(values), 4) + + run = BenchmarkRunResult( + benchmark="membench", + system_name="brainctl", + mode=f"{pipeline}_turn", + status=PARTIAL, + example_count=example_count, + metrics=metrics, + primary_metric=f"hit_at_{top_k}", + primary_metric_value=hit_rate, + runtime_seconds=runtime_seconds, + dataset_path=str(data_dir), + notes=["FirstAgent slice only", "turn-level retrieval", f"topic={'all' if topic is None else topic}"], + caveats=["MemBench comparison is partial because ThirdAgent and noise-extended slices are not included."], + series_name="new_brainctl", + ) + return run, rows diff --git a/bin/intent_classifier.py b/bin/intent_classifier.py index 0d64474..c507752 100644 --- a/bin/intent_classifier.py +++ b/bin/intent_classifier.py @@ -42,16 +42,16 @@ class IntentResult: # Each entry is (primary_tables, secondary_tables). # The final merged list fed to --tables is primary + secondary (de-duped). _TABLE_ROUTES = { - "cross_reference": ["events", "memories", "context"], - "troubleshooting": ["events", "memories", "context"], + "cross_reference": ["events", "memories", "context", "procedures"], + "troubleshooting": ["procedures", "events", "memories", "context", "decisions"], "task_status": ["events", "context", "memories"], - "entity_lookup": ["memories", "context", "events"], # entities not in universal search pipeline - "historical_timeline":["events", "context", "memories"], - "how_to": ["memories", "context"], - "decision_rationale": ["memories", "context", "events"], - "research_concept": ["memories", "context"], - "orientation": ["memories", "events", "context"], - "factual_lookup": ["memories", "context", "events"], # same as default + "entity_lookup": ["memories", "procedures", "context", "events"], # entities not in universal search pipeline + "historical_timeline":["events", "memories", "context", "procedures"], + "how_to": ["procedures", "memories", "context", "events", "decisions"], + "decision_rationale": ["decisions", "memories", "context", "events", "procedures"], + "research_concept": ["memories", "procedures", "context"], + "orientation": ["memories", "events", "context", "procedures"], + "factual_lookup": ["memories", "procedures", "context", "events"], # same as default + procedures fallback } _FORMAT_HINTS = { @@ -81,6 +81,7 @@ class IntentResult: _WAVE_RE = re.compile(r'\bwave\s*\d+\b', re.IGNORECASE) _HOW_RE = re.compile(r'\bhow\s+(to|do|does|can|should)\b', re.IGNORECASE) _WHY_RE = re.compile(r'\bwhy\b', re.IGNORECASE) +_PROCEDURAL_RE = re.compile(r'\b(runbook|playbook|rollback|roll back|procedure|workflow|steps?|migrate|deployment?|troubleshoot|debug)\b', re.IGNORECASE) # First-person/identity statement (Hermes memory dumps stored as queries) _IDENTITY_STMT_RE = re.compile( r'^(I |My |The vault|Chief wakes|Continuity is|Tasks that|Learn the|' @@ -157,12 +158,12 @@ def classify_intent(query: str) -> IntentResult: ) # ---- Rule 4: How-to ---- - if _HOW_RE.search(q): + if _HOW_RE.search(q) or _PROCEDURAL_RE.search(q): return IntentResult( intent="how_to", confidence=0.88, tables=_TABLE_ROUTES["how_to"], - matched_rule="how_to_regex", + matched_rule="how_to_regex" if _HOW_RE.search(q) else "procedural_kw_regex", format_hint=_FORMAT_HINTS["how_to"], ) diff --git a/db/init_schema.sql b/db/init_schema.sql index 8edcb6e..33056a9 120000 --- a/db/init_schema.sql +++ b/db/init_schema.sql @@ -1 +1,1884 @@ -../src/agentmemory/db/init_schema.sql \ No newline at end of file +-- brainctl init_schema.sql -- Full production schema +-- Generated from brain.db +-- Use: brainctl init + +PRAGMA journal_mode = WAL; +PRAGMA synchronous = NORMAL; +PRAGMA foreign_keys = ON; + +-- Legacy tracking table. Ten migration files still write to this singular +-- form (`INSERT INTO schema_version ...`) for historical reasons. The +-- runner in src/agentmemory/migrate.py uses a separate `schema_versions` +-- (plural) table created lazily via `_ensure_schema_versions()`, which +-- is the authoritative "has this migration been applied?" source. The +-- singular table is preserved so legacy migration statements don't error +-- on fresh installs; nothing reads it. Audit I27 — kept as-is per the +-- "migrations are append-only" convention in CLAUDE.md. +CREATE TABLE schema_version ( + version INTEGER NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')), + description TEXT +); + +CREATE TABLE agents ( + id TEXT PRIMARY KEY, -- e.g. 'my-agent', 'data-pipeline', 'reviewer' + display_name TEXT NOT NULL, + agent_type TEXT NOT NULL, -- 'autonomous', 'pipeline', 'assistant', 'human' + adapter_info TEXT, -- JSON: connection details, model, etc + status TEXT NOT NULL DEFAULT 'active', -- active, paused, retired + last_seen_at TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + attention_class TEXT NOT NULL DEFAULT 'ic', + attention_budget_tier INTEGER NOT NULL DEFAULT 1 +); + +CREATE TABLE memories ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + agent_id TEXT NOT NULL REFERENCES agents(id), -- who wrote this + category TEXT NOT NULL, -- 'identity', 'user', 'environment', 'convention', + -- 'project', 'decision', 'lesson', 'preference' + scope TEXT NOT NULL DEFAULT 'global', -- 'global', 'project:', 'agent:' + content TEXT NOT NULL, -- the actual memory + confidence REAL NOT NULL DEFAULT 1.0, -- 0.0-1.0, decays or gets boosted + source_event_id INTEGER, -- event that spawned this memory + supersedes_id INTEGER REFERENCES memories(id), -- if this replaces an older memory + tags TEXT, -- JSON array of tags + expires_at TEXT, -- optional TTL + recalled_count INTEGER NOT NULL DEFAULT 0, -- how often this memory was retrieved + last_recalled_at TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + retired_at TEXT, -- soft delete + epoch_id INTEGER REFERENCES epochs(id), + temporal_class TEXT NOT NULL DEFAULT 'medium', + validation_agent_id TEXT REFERENCES agents(id), + validated_at TEXT, + trust_score REAL DEFAULT 1.0, + derived_from_ids TEXT, + retracted_at TEXT, + retraction_reason TEXT, + version INTEGER NOT NULL DEFAULT 1, + memory_type TEXT NOT NULL DEFAULT 'episodic' CHECK(memory_type IN ('episodic','semantic','procedural')), + protected INTEGER NOT NULL DEFAULT 0, + salience_score REAL NOT NULL DEFAULT 0.0, + gw_broadcast INTEGER NOT NULL DEFAULT 0, + visibility TEXT NOT NULL DEFAULT 'public', + read_acl TEXT, + ewc_importance REAL NOT NULL DEFAULT 0.0, + alpha REAL DEFAULT 1.0, + beta REAL DEFAULT 1.0, + confidence_alpha REAL GENERATED ALWAYS AS (alpha) VIRTUAL, + confidence_beta REAL GENERATED ALWAYS AS (beta) VIRTUAL, + confidence_phase REAL NOT NULL DEFAULT 0.0, + hilbert_projection BLOB DEFAULT NULL, + coherence_syndrome TEXT DEFAULT NULL, + decoherence_rate REAL DEFAULT NULL, + gated_from_memory_id INTEGER REFERENCES memories(id), + file_path TEXT, + file_line INTEGER, + write_tier TEXT NOT NULL DEFAULT 'full' CHECK(write_tier IN ('skip', 'construct', 'full')), + indexed INTEGER NOT NULL DEFAULT 1, + promoted_at TEXT DEFAULT NULL, + replay_priority REAL NOT NULL DEFAULT 0.0, + ripple_tags INTEGER NOT NULL DEFAULT 0, + labile_until TEXT DEFAULT NULL, + labile_agent_id TEXT DEFAULT NULL, + retrieval_prediction_error REAL DEFAULT NULL, + encoding_affect_id INTEGER REFERENCES affect_log(id) DEFAULT NULL, + tag_cycles_remaining INTEGER DEFAULT 0, + stability REAL DEFAULT 1.0, + encoding_task_context TEXT DEFAULT NULL, + encoding_context_hash TEXT DEFAULT NULL, + temporal_level TEXT NOT NULL DEFAULT 'moment' + CHECK(temporal_level IN ('moment','session','day','week','month','quarter')), + next_review_at TEXT DEFAULT NULL, + q_value REAL DEFAULT 0.5 +); + +CREATE INDEX idx_memories_agent ON memories(agent_id); + +CREATE INDEX idx_memories_category ON memories(category); + +CREATE INDEX idx_memories_scope ON memories(scope); + +CREATE INDEX idx_memories_active ON memories(retired_at) WHERE retired_at IS NULL; + +CREATE INDEX idx_memories_confidence ON memories(confidence DESC); + +CREATE INDEX idx_memories_agent_active_cat ON memories(agent_id, category) WHERE retired_at IS NULL; + +CREATE INDEX idx_memories_agent_time ON memories(agent_id, created_at DESC) WHERE retired_at IS NULL; + +CREATE INDEX IF NOT EXISTS idx_memories_encoding_affect + ON memories(encoding_affect_id) WHERE encoding_affect_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_memories_context_hash + ON memories(encoding_context_hash) WHERE encoding_context_hash IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_memories_next_review + ON memories(next_review_at) WHERE next_review_at IS NOT NULL AND retired_at IS NULL; + +CREATE VIRTUAL TABLE memories_fts USING fts5( + content, + category, + tags, + content=memories, + content_rowid=id, + tokenize='porter unicode61' +); + +CREATE TRIGGER memories_fts_insert AFTER INSERT ON memories WHEN new.indexed = 1 BEGIN + INSERT INTO memories_fts(rowid, content, category, tags) VALUES (new.id, new.content, new.category, new.tags); +END; + +-- Split into two triggers so 0→1 promotion correctly adds to FTS without double-delete. +-- Added `NEW.retired_at IS NULL` guard on the INSERT leg so retire UPDATEs +-- (retired_at NULL → non-NULL) do not re-insert the row. The companion +-- trg_memories_fts_purge_on_retire trigger near the end of this file does +-- the actual DELETE at the retire transition; without this guard, the +-- 'delete' command issued there is silently no-op'd by FTS5 statement-level +-- batching against the pending INSERT. +CREATE TRIGGER memories_fts_update_delete AFTER UPDATE ON memories WHEN old.indexed = 1 BEGIN + INSERT INTO memories_fts(memories_fts, rowid, content, category, tags) + VALUES ('delete', old.id, old.content, old.category, old.tags); +END; + +CREATE TRIGGER memories_fts_update_insert AFTER UPDATE ON memories WHEN new.indexed = 1 AND new.retired_at IS NULL BEGIN + INSERT INTO memories_fts(rowid, content, category, tags) + VALUES (new.id, new.content, new.category, new.tags); +END; + +CREATE TRIGGER memories_fts_delete AFTER DELETE ON memories BEGIN + INSERT INTO memories_fts(memories_fts, rowid, content, category, tags) VALUES('delete', old.id, old.content, old.category, old.tags); +END; + +CREATE TABLE events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + agent_id TEXT NOT NULL REFERENCES agents(id), + event_type TEXT NOT NULL, -- 'observation', 'result', 'decision', 'error', + -- 'handoff', 'task_update', 'artifact', 'session_start', + -- 'session_end', 'memory_promoted', 'memory_retired' + summary TEXT NOT NULL, + detail TEXT, -- longer description, stack traces, etc + metadata TEXT, -- JSON blob for structured data + session_id TEXT, -- links to a specific conversation/run + project TEXT, -- project context + refs TEXT, -- JSON array of related entity refs + importance REAL NOT NULL DEFAULT 0.5, -- 0.0-1.0 for prioritizing retrieval + created_at TEXT NOT NULL DEFAULT (datetime('now')), + epoch_id INTEGER REFERENCES epochs(id), + caused_by_event_id INTEGER REFERENCES events(id), + causal_chain_root INTEGER REFERENCES events(id) +); + +CREATE INDEX idx_events_agent ON events(agent_id); + +CREATE INDEX idx_events_type ON events(event_type); + +CREATE INDEX idx_events_project ON events(project); + +CREATE INDEX idx_events_session ON events(session_id); + +CREATE INDEX idx_events_time ON events(created_at DESC); + +CREATE INDEX idx_events_importance ON events(importance DESC); + +CREATE VIRTUAL TABLE events_fts USING fts5( + summary, + detail, + content=events, + content_rowid=id, + tokenize='porter unicode61' +); + +CREATE TRIGGER events_fts_insert AFTER INSERT ON events BEGIN + INSERT INTO events_fts(rowid, summary, detail) VALUES (new.id, new.summary, new.detail); +END; + +CREATE TRIGGER events_fts_update AFTER UPDATE ON events BEGIN + INSERT INTO events_fts(events_fts, rowid, summary, detail) VALUES('delete', old.id, old.summary, old.detail); + INSERT INTO events_fts(rowid, summary, detail) VALUES (new.id, new.summary, new.detail); +END; + +CREATE TRIGGER events_fts_delete AFTER DELETE ON events BEGIN + INSERT INTO events_fts(events_fts, rowid, summary, detail) VALUES('delete', old.id, old.summary, old.detail); +END; + +CREATE TABLE context ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_type TEXT NOT NULL, -- 'conversation', 'document', 'code', 'skill', + -- 'issue', 'pr', 'obsidian_note' + source_ref TEXT NOT NULL, -- URI or path to original + chunk_index INTEGER NOT NULL DEFAULT 0, -- for multi-chunk documents + content TEXT NOT NULL, + summary TEXT, -- LLM-generated summary of chunk + project TEXT, + tags TEXT, -- JSON array + token_count INTEGER, + embedding_id INTEGER, -- FK to embeddings table (Phase 2) + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + stale_at TEXT -- when source was re-indexed +); + +CREATE INDEX idx_context_source ON context(source_type, source_ref); + +CREATE INDEX idx_context_project ON context(project); + +CREATE INDEX idx_context_stale ON context(stale_at) WHERE stale_at IS NULL; + +CREATE VIRTUAL TABLE context_fts USING fts5( + content, + summary, + tags, + content=context, + content_rowid=id, + tokenize='porter unicode61' +); + +CREATE TRIGGER context_fts_insert AFTER INSERT ON context BEGIN + INSERT INTO context_fts(rowid, content, summary, tags) VALUES (new.id, new.content, new.summary, new.tags); +END; + +CREATE TRIGGER context_fts_update AFTER UPDATE ON context BEGIN + INSERT INTO context_fts(context_fts, rowid, content, summary, tags) VALUES('delete', old.id, old.content, old.summary, old.tags); + INSERT INTO context_fts(rowid, content, summary, tags) VALUES (new.id, new.content, new.summary, new.tags); +END; + +CREATE TRIGGER context_fts_delete AFTER DELETE ON context BEGIN + INSERT INTO context_fts(context_fts, rowid, content, summary, tags) VALUES('delete', old.id, old.content, old.summary, old.tags); +END; + +CREATE TABLE tasks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + external_id TEXT, -- External task ID, GitHub issue #, etc + external_system TEXT, -- 'task-system', 'github', 'manual' + title TEXT NOT NULL, + description TEXT, + status TEXT NOT NULL DEFAULT 'pending', -- pending, in_progress, blocked, completed, cancelled + priority TEXT NOT NULL DEFAULT 'medium', -- critical, high, medium, low + assigned_agent_id TEXT REFERENCES agents(id), + project TEXT, + parent_task_id INTEGER REFERENCES tasks(id), + metadata TEXT, -- JSON: labels, branch name, PR url, etc + claimed_at TEXT, + claimed_by TEXT REFERENCES agents(id), + completed_at TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX idx_tasks_status ON tasks(status); + +CREATE INDEX idx_tasks_agent ON tasks(assigned_agent_id); + +CREATE INDEX idx_tasks_project ON tasks(project); + +CREATE INDEX idx_tasks_external ON tasks(external_system, external_id); + +CREATE TABLE decisions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + agent_id TEXT NOT NULL REFERENCES agents(id), + title TEXT NOT NULL, + rationale TEXT NOT NULL, + alternatives_considered TEXT, -- JSON array of rejected options + project TEXT, + reversible INTEGER NOT NULL DEFAULT 1, -- boolean + reversed_at TEXT, + reversed_by TEXT, + source_event_id INTEGER REFERENCES events(id), + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX idx_decisions_project ON decisions(project); + +CREATE INDEX idx_decisions_agent ON decisions(agent_id); + +CREATE TABLE handoff_packets ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + agent_id TEXT NOT NULL REFERENCES agents(id), + session_id TEXT, + chat_id TEXT, + thread_id TEXT, + user_id TEXT, + project TEXT, + scope TEXT NOT NULL DEFAULT 'global', + status TEXT NOT NULL DEFAULT 'pending' + CHECK (status IN ('pending', 'consumed', 'expired', 'pinned')), + title TEXT, + goal TEXT NOT NULL, + current_state TEXT NOT NULL, + open_loops TEXT NOT NULL, + next_step TEXT NOT NULL, + recent_tail TEXT, + decisions_json TEXT, + entities_json TEXT, + tasks_json TEXT, + facts_json TEXT, + source_event_id INTEGER REFERENCES events(id), + consumed_at TEXT, + expires_at TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX idx_handoff_status_created ON handoff_packets(status, created_at DESC); + +CREATE INDEX idx_handoff_chat_thread_status ON handoff_packets(chat_id, thread_id, status, created_at DESC); + +CREATE INDEX idx_handoff_project_status ON handoff_packets(project, status, created_at DESC); + +CREATE INDEX idx_handoff_session ON handoff_packets(session_id); + +CREATE INDEX idx_handoff_agent_status ON handoff_packets(agent_id, status, created_at DESC); + +CREATE TABLE embeddings ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_table TEXT NOT NULL, -- 'memories', 'context', 'events' + source_id INTEGER NOT NULL, + model TEXT NOT NULL, -- embedding model used + dimensions INTEGER NOT NULL, + vector BLOB, -- raw float32 vector (or use sqlite-vec later) + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX idx_embeddings_source ON embeddings(source_table, source_id); + +CREATE TABLE agent_state ( + agent_id TEXT NOT NULL REFERENCES agents(id), + key TEXT NOT NULL, + value TEXT NOT NULL, -- JSON value + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY (agent_id, key) +); + +CREATE TABLE blobs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + sha256 TEXT NOT NULL UNIQUE, + filename TEXT, + mime_type TEXT, + size_bytes INTEGER NOT NULL, + disk_path TEXT NOT NULL, -- relative path under ~/agentmemory/blobs/ + agent_id TEXT REFERENCES agents(id), + project TEXT, + description TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX idx_blobs_sha256 ON blobs(sha256); + +CREATE INDEX idx_blobs_project ON blobs(project); + +CREATE TABLE access_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + agent_id TEXT NOT NULL, + action TEXT NOT NULL, -- 'read', 'write', 'search', 'promote', 'retire' + target_table TEXT, + target_id INTEGER, + query TEXT, -- search query if action=search + result_count INTEGER, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + tokens_consumed INTEGER, + task_outcome TEXT + CHECK (task_outcome IN ('success', 'blocked', 'escalated', 'cancelled')), + pre_task_uncertainty REAL, + retrieval_contributed INTEGER DEFAULT NULL + CHECK (retrieval_contributed IN (0, 1, NULL)), + task_id TEXT +); + +CREATE INDEX idx_access_agent ON access_log(agent_id); + +CREATE INDEX idx_access_time ON access_log(created_at DESC); + +CREATE TABLE epochs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE, + description TEXT, + started_at TEXT NOT NULL, + ended_at TEXT, + parent_epoch_id INTEGER REFERENCES epochs(id), + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX idx_epochs_started ON epochs(started_at); + +CREATE INDEX idx_epochs_parent ON epochs(parent_epoch_id); + +CREATE INDEX idx_memories_epoch ON memories(epoch_id); + +CREATE INDEX idx_memories_temporal_class ON memories(temporal_class); + +CREATE TRIGGER memories_temporal_class_check +BEFORE INSERT ON memories +WHEN NEW.temporal_class NOT IN ('permanent', 'long', 'medium', 'short', 'ephemeral') +BEGIN + SELECT RAISE(ABORT, 'temporal_class must be one of: permanent, long, medium, short, ephemeral'); +END; + +CREATE TRIGGER memories_temporal_class_update_check +BEFORE UPDATE OF temporal_class ON memories +WHEN NEW.temporal_class NOT IN ('permanent', 'long', 'medium', 'short', 'ephemeral') +BEGIN + SELECT RAISE(ABORT, 'temporal_class must be one of: permanent, long, medium, short, ephemeral'); +END; + +CREATE INDEX idx_events_epoch ON events(epoch_id); + +CREATE INDEX idx_events_caused_by ON events(caused_by_event_id); + +CREATE INDEX idx_events_causal_root ON events(causal_chain_root); + +CREATE TABLE knowledge_edges ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_table TEXT NOT NULL, + source_id INTEGER NOT NULL, + target_table TEXT NOT NULL, + target_id INTEGER NOT NULL, + relation_type TEXT NOT NULL, + weight REAL NOT NULL DEFAULT 1.0, + agent_id TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + last_reinforced_at TEXT, + co_activation_count INTEGER DEFAULT 0, + weight_updated_at TEXT, + CHECK (weight >= 0.0 AND weight <= 1.0) +); + +CREATE UNIQUE INDEX uq_knowledge_edges_relation +ON knowledge_edges (source_table, source_id, target_table, target_id, relation_type); + +CREATE INDEX idx_knowledge_edges_source_pair +ON knowledge_edges (source_table, source_id); + +CREATE INDEX idx_knowledge_edges_target_pair +ON knowledge_edges (target_table, target_id); + +CREATE INDEX idx_knowledge_edges_relation_type +ON knowledge_edges (relation_type); + +CREATE TABLE memory_trust_scores ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + agent_id TEXT NOT NULL REFERENCES agents(id), + category TEXT NOT NULL, + trust_score REAL NOT NULL DEFAULT 1.0 CHECK (trust_score >= 0.0 AND trust_score <= 1.0), + sample_count INTEGER NOT NULL DEFAULT 0, -- number of memories evaluated + validated_count INTEGER NOT NULL DEFAULT 0, -- number that passed validation + retracted_count INTEGER NOT NULL DEFAULT 0, -- number retracted (lowers trust) + last_evaluated_at TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE(agent_id, category) +); + +CREATE INDEX idx_trust_scores_agent ON memory_trust_scores(agent_id); + +CREATE INDEX idx_trust_scores_category ON memory_trust_scores(category); + +CREATE INDEX idx_trust_scores_score ON memory_trust_scores(trust_score); + +CREATE INDEX idx_memories_trust_score ON memories(trust_score); + +CREATE INDEX idx_memories_retracted ON memories(retracted_at) WHERE retracted_at IS NOT NULL; + +CREATE INDEX idx_memories_validation ON memories(validation_agent_id); + +CREATE INDEX idx_memories_id_version ON memories(id, version) WHERE retired_at IS NULL; + +CREATE INDEX idx_memories_type ON memories(memory_type); + +CREATE TABLE situation_models ( + id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(16)))), + name TEXT NOT NULL UNIQUE, + query_anchor TEXT NOT NULL, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, + last_event_id INTEGER, + last_memory_id TEXT, + coherence_score REAL DEFAULT 0.0, + completeness REAL DEFAULT 0.0, + status TEXT DEFAULT 'active' + CHECK (status IN ('active','stale','contradictory','archived')), + narrative TEXT, + structured TEXT, + ttl_seconds INTEGER DEFAULT 21600, + source_memory_ids TEXT DEFAULT '[]', + source_event_ids TEXT DEFAULT '[]' +); + +CREATE TABLE situation_model_contradictions ( + id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(16)))), + model_id TEXT NOT NULL REFERENCES situation_models(id) ON DELETE CASCADE, + memory_id_a TEXT, + memory_id_b TEXT, + contradiction TEXT NOT NULL, + resolution TEXT, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX idx_sm_anchor ON situation_models(query_anchor); + +CREATE INDEX idx_sm_status ON situation_models(status); + +CREATE INDEX idx_sm_updated ON situation_models(updated_at); + +CREATE TRIGGER events_validate_ts_insert +BEFORE INSERT ON events +WHEN NEW.created_at NOT LIKE '____-__-__T%' +BEGIN + SELECT RAISE(ABORT, 'events.created_at must be ISO 8601 (YYYY-MM-DDTHH:MM:SS)'); +END; + +CREATE TRIGGER events_validate_ts_update +BEFORE UPDATE OF created_at ON events +WHEN NEW.created_at NOT LIKE '____-__-__T%' +BEGIN + SELECT RAISE(ABORT, 'events.created_at must be ISO 8601 (YYYY-MM-DDTHH:MM:SS)'); +END; + +CREATE TRIGGER memories_validate_ts_insert +BEFORE INSERT ON memories +WHEN NEW.created_at NOT LIKE '____-__-__T%' +BEGIN + SELECT RAISE(ABORT, 'memories.created_at must be ISO 8601 (YYYY-MM-DDTHH:MM:SS)'); +END; + +CREATE TRIGGER memories_validate_ts_update +BEFORE UPDATE OF created_at ON memories +WHEN NEW.created_at NOT LIKE '____-__-__T%' +BEGIN + SELECT RAISE(ABORT, 'memories.created_at must be ISO 8601 (YYYY-MM-DDTHH:MM:SS)'); +END; + +CREATE TABLE knowledge_coverage ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + scope TEXT NOT NULL, -- 'agent:X', 'project:Y', 'global', 'topic:Z' + memory_count INTEGER NOT NULL DEFAULT 0, + avg_confidence REAL, + min_confidence REAL, + max_confidence REAL, + freshest_memory_at TEXT, -- ISO 8601 datetime of newest active memory in scope + stalest_memory_at TEXT, -- ISO 8601 datetime of oldest active memory in scope + coverage_density REAL, -- composite: count × avg_confidence × recency_factor + last_computed_at TEXT NOT NULL, + UNIQUE(scope) +); + +CREATE INDEX idx_coverage_scope ON knowledge_coverage(scope); + +CREATE INDEX idx_coverage_density ON knowledge_coverage(coverage_density DESC); + +CREATE TABLE knowledge_gaps ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + gap_type TEXT NOT NULL CHECK(gap_type IN ( + 'coverage_hole', -- no memories in scope at all + 'staleness_hole', -- memories exist but all too old + 'confidence_hole', -- memories exist but avg confidence too low + 'contradiction_hole', -- memories contradict each other + -- Migration 036 self-healing scan types + 'orphan_memory', -- memory with no edges + no recalls + old + 'broken_edge', -- knowledge_edges row points at deleted row + 'unreferenced_entity' -- entity with nothing linking to it + )), + scope TEXT NOT NULL, + detected_at TEXT NOT NULL, + triggered_by TEXT, -- query or scan that revealed the gap + severity REAL NOT NULL DEFAULT 0.5 -- 0.0–1.0 + CHECK(severity >= 0.0 AND severity <= 1.0), + resolved_at TEXT, + resolution_note TEXT +); + +CREATE INDEX idx_gaps_scope ON knowledge_gaps(scope); + +CREATE INDEX idx_gaps_type ON knowledge_gaps(gap_type); + +CREATE INDEX idx_gaps_unresolved ON knowledge_gaps(resolved_at) WHERE resolved_at IS NULL; + +CREATE INDEX idx_gaps_severity ON knowledge_gaps(severity DESC) WHERE resolved_at IS NULL; + +CREATE TABLE reflexion_lessons ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + + -- Identity / provenance + source_agent_id TEXT NOT NULL REFERENCES agents(id), + source_event_id INTEGER REFERENCES events(id), + source_run_id TEXT, + + -- Failure classification + failure_class TEXT NOT NULL + CHECK (failure_class IN ( + 'REASONING_ERROR', + 'CONTEXT_LOSS', + 'HALLUCINATION', + 'COORDINATION_FAILURE', + 'TOOL_MISUSE' + )), + failure_subclass TEXT, + + -- Trigger conditions + trigger_conditions TEXT NOT NULL, + + -- Lesson content + lesson_content TEXT NOT NULL, + + -- Generalization scope (JSON array: "agent_type:pipeline", "capability:search", etc.) + generalizable_to TEXT NOT NULL DEFAULT '[]', + + -- Lifecycle + confidence REAL NOT NULL DEFAULT 0.8 + CHECK (confidence >= 0.0 AND confidence <= 1.0), + override_level TEXT NOT NULL DEFAULT 'SOFT_HINT' + CHECK (override_level IN ('HARD_OVERRIDE', 'SOFT_HINT', 'SILENT_LOG')), + status TEXT NOT NULL DEFAULT 'active' + CHECK (status IN ('active', 'archived', 'retired')), + + -- Expiration policy + expiration_policy TEXT NOT NULL DEFAULT 'success_count' + CHECK (expiration_policy IN ('success_count', 'code_fix', 'ttl', 'manual')), + expiration_n INTEGER DEFAULT 5, + expiration_ttl_days INTEGER, + root_cause_ref TEXT, + consecutive_successes INTEGER NOT NULL DEFAULT 0, + last_validated_at TEXT, + + -- Retrieval stats + times_retrieved INTEGER NOT NULL DEFAULT 0, + times_prevented_failure INTEGER NOT NULL DEFAULT 0, + times_failed_to_prevent INTEGER NOT NULL DEFAULT 0, + + -- Timestamps + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + archived_at TEXT, + retired_at TEXT, + retirement_reason TEXT, + propagated_to TEXT NOT NULL DEFAULT '[]', + propagation_source_lesson_id INTEGER REFERENCES reflexion_lessons(id) +); + +CREATE INDEX idx_rlessons_agent + ON reflexion_lessons(source_agent_id); + +CREATE INDEX idx_rlessons_failure_class + ON reflexion_lessons(failure_class); + +CREATE INDEX idx_rlessons_status + ON reflexion_lessons(status) WHERE status = 'active'; + +CREATE INDEX idx_rlessons_confidence + ON reflexion_lessons(confidence DESC); + +CREATE INDEX idx_rlessons_generalizable + ON reflexion_lessons(generalizable_to); + +CREATE INDEX idx_rlessons_active_class + ON reflexion_lessons(status, failure_class, confidence DESC) + WHERE status = 'active'; + +CREATE VIRTUAL TABLE reflexion_lessons_fts USING fts5( + trigger_conditions, + lesson_content, + failure_class, + failure_subclass, + content=reflexion_lessons, + content_rowid=id, + tokenize='porter unicode61' +); + +CREATE TRIGGER rlessons_fts_insert AFTER INSERT ON reflexion_lessons BEGIN + INSERT INTO reflexion_lessons_fts(rowid, trigger_conditions, lesson_content, failure_class, failure_subclass) + VALUES (new.id, new.trigger_conditions, new.lesson_content, new.failure_class, new.failure_subclass); +END; + +CREATE TRIGGER rlessons_fts_update AFTER UPDATE ON reflexion_lessons BEGIN + INSERT INTO reflexion_lessons_fts(reflexion_lessons_fts, rowid, trigger_conditions, lesson_content, failure_class, failure_subclass) + VALUES ('delete', old.id, old.trigger_conditions, old.lesson_content, old.failure_class, old.failure_subclass); + INSERT INTO reflexion_lessons_fts(rowid, trigger_conditions, lesson_content, failure_class, failure_subclass) + VALUES (new.id, new.trigger_conditions, new.lesson_content, new.failure_class, new.failure_subclass); +END; + +CREATE TRIGGER rlessons_fts_delete AFTER DELETE ON reflexion_lessons BEGIN + INSERT INTO reflexion_lessons_fts(reflexion_lessons_fts, rowid, trigger_conditions, lesson_content, failure_class, failure_subclass) + VALUES ('delete', old.id, old.trigger_conditions, old.lesson_content, old.failure_class, old.failure_subclass); +END; + +CREATE TRIGGER rlessons_updated_at AFTER UPDATE ON reflexion_lessons BEGIN + UPDATE reflexion_lessons SET updated_at = datetime('now') WHERE id = new.id; +END; + +CREATE TABLE agent_expertise ( + agent_id TEXT NOT NULL REFERENCES agents(id), + domain TEXT NOT NULL, + strength REAL NOT NULL DEFAULT 0.0, + evidence_count INTEGER NOT NULL DEFAULT 0, + last_active TEXT, + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + brier_score REAL DEFAULT NULL, + PRIMARY KEY (agent_id, domain) + ); + +CREATE INDEX idx_expertise_domain ON agent_expertise(domain); + +CREATE INDEX idx_expertise_strength ON agent_expertise(strength DESC); + +CREATE TABLE memory_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + memory_id INTEGER NOT NULL REFERENCES memories(id), + agent_id TEXT NOT NULL, -- agent that wrote the memory + operation TEXT NOT NULL DEFAULT 'insert', -- 'insert' | 'update' + category TEXT NOT NULL, -- mirrors memories.category at write time + scope TEXT NOT NULL, -- mirrors memories.scope at write time + memory_type TEXT NOT NULL DEFAULT 'episodic', -- 'episodic' | 'semantic' + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%S', 'now')), + ttl_expires_at TEXT -- set by prune; NULL = no expiry override +); + +CREATE INDEX idx_meb_id_asc ON memory_events(id ASC); + +CREATE INDEX idx_meb_agent ON memory_events(agent_id); + +CREATE INDEX idx_meb_category ON memory_events(category); + +CREATE INDEX idx_meb_scope ON memory_events(scope); + +CREATE INDEX idx_meb_created_at ON memory_events(created_at DESC); + +CREATE INDEX idx_meb_ttl ON memory_events(ttl_expires_at) + WHERE ttl_expires_at IS NOT NULL; + +CREATE TRIGGER meb_after_memory_insert +AFTER INSERT ON memories +BEGIN + INSERT INTO memory_events (memory_id, agent_id, operation, category, scope, memory_type, created_at) + VALUES ( + new.id, + new.agent_id, + 'insert', + new.category, + new.scope, + COALESCE(new.memory_type, 'episodic'), + strftime('%Y-%m-%dT%H:%M:%S', 'now') + ); +END; + +CREATE TRIGGER meb_after_memory_update +AFTER UPDATE OF content, category, scope, confidence, trust_score, memory_type ON memories +WHEN new.retired_at IS NULL +BEGIN + INSERT INTO memory_events (memory_id, agent_id, operation, category, scope, memory_type, created_at) + VALUES ( + new.id, + new.agent_id, + 'update', + new.category, + new.scope, + COALESCE(new.memory_type, 'episodic'), + strftime('%Y-%m-%dT%H:%M:%S', 'now') + ); +END; + +CREATE TABLE meb_config ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL, + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%S', 'now')) +); + +CREATE TABLE policy_memories ( + policy_id TEXT PRIMARY KEY, + name TEXT NOT NULL, + category TEXT NOT NULL DEFAULT 'general', + status TEXT NOT NULL DEFAULT 'active' + CHECK(status IN ('candidate','active','deprecated')), + scope TEXT NOT NULL DEFAULT 'global', + priority INTEGER NOT NULL DEFAULT 50, + + trigger_condition TEXT NOT NULL, + action_directive TEXT NOT NULL, + + authored_by TEXT NOT NULL DEFAULT 'unknown', + derived_from TEXT, + + confidence_threshold REAL NOT NULL DEFAULT 0.5 + CHECK(confidence_threshold >= 0.0 AND confidence_threshold <= 1.0), + wisdom_half_life_days INTEGER NOT NULL DEFAULT 30, + version INTEGER NOT NULL DEFAULT 1, + + active_since TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%S', 'now')), + last_validated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%S', 'now')), + expires_at TEXT, + + feedback_count INTEGER NOT NULL DEFAULT 0, + success_count INTEGER NOT NULL DEFAULT 0, + failure_count INTEGER NOT NULL DEFAULT 0, + + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%S', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%S', 'now')) +); + +CREATE INDEX idx_pm_status_category ON policy_memories(status, category); + +CREATE INDEX idx_pm_scope ON policy_memories(scope); + +CREATE INDEX idx_pm_confidence ON policy_memories(confidence_threshold DESC); + +CREATE INDEX idx_pm_priority ON policy_memories(priority DESC); + +CREATE INDEX idx_pm_expires ON policy_memories(expires_at) WHERE expires_at IS NOT NULL; + +CREATE INDEX idx_pm_authored_by ON policy_memories(authored_by); + +CREATE VIRTUAL TABLE policy_memories_fts USING fts5( + trigger_condition, + action_directive, + name, + content=policy_memories, + content_rowid=rowid +); + +CREATE TRIGGER pm_fts_insert AFTER INSERT ON policy_memories BEGIN + INSERT INTO policy_memories_fts(rowid, trigger_condition, action_directive, name) + VALUES (new.rowid, new.trigger_condition, new.action_directive, new.name); +END; + +CREATE TRIGGER pm_fts_update AFTER UPDATE ON policy_memories BEGIN + INSERT INTO policy_memories_fts(policy_memories_fts, rowid, trigger_condition, action_directive, name) + VALUES ('delete', old.rowid, old.trigger_condition, old.action_directive, old.name); + INSERT INTO policy_memories_fts(rowid, trigger_condition, action_directive, name) + VALUES (new.rowid, new.trigger_condition, new.action_directive, new.name); +END; + +CREATE TRIGGER pm_fts_delete AFTER DELETE ON policy_memories BEGIN + INSERT INTO policy_memories_fts(policy_memories_fts, rowid, trigger_condition, action_directive, name) + VALUES ('delete', old.rowid, old.trigger_condition, old.action_directive, old.name); +END; + +CREATE TABLE procedures ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + memory_id INTEGER NOT NULL UNIQUE REFERENCES memories(id) ON DELETE CASCADE, + procedure_key TEXT UNIQUE, + title TEXT, + goal TEXT NOT NULL, + description TEXT, + task_family TEXT, + procedure_kind TEXT NOT NULL DEFAULT 'workflow', + trigger_conditions TEXT, + preconditions TEXT, + constraints_json TEXT, + steps_json TEXT NOT NULL, + tools_json TEXT, + failure_modes_json TEXT, + rollback_steps_json TEXT, + success_criteria_json TEXT, + repair_strategies_json TEXT, + tool_policy_json TEXT, + expected_outcomes TEXT, + applicability_scope TEXT NOT NULL DEFAULT 'global', + temporal_class TEXT DEFAULT 'durable', + status TEXT NOT NULL DEFAULT 'active' + CHECK(status IN ('active','candidate','stale','needs_review','superseded','retired')), + automation_ready INTEGER NOT NULL DEFAULT 0, + determinism REAL NOT NULL DEFAULT 0.5, + confidence REAL NOT NULL DEFAULT 0.5, + utility_score REAL NOT NULL DEFAULT 0.5, + generality_score REAL NOT NULL DEFAULT 0.5, + support_count INTEGER NOT NULL DEFAULT 0, + execution_count INTEGER NOT NULL DEFAULT 0, + success_count INTEGER NOT NULL DEFAULT 0, + failure_count INTEGER NOT NULL DEFAULT 0, + last_used_at TEXT, + last_executed_at TEXT, + last_validated_at TEXT, + stale_after_days INTEGER NOT NULL DEFAULT 90, + supersedes_procedure_id INTEGER REFERENCES procedures(id), + retired_at TEXT, + search_text TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX idx_procedures_kind ON procedures(procedure_kind); + +CREATE INDEX idx_procedures_status ON procedures(status); + +CREATE INDEX idx_procedures_last_validated ON procedures(last_validated_at); + +CREATE INDEX idx_procedures_execution_count ON procedures(execution_count DESC); + +CREATE INDEX idx_procedures_scope ON procedures(applicability_scope); + +CREATE INDEX idx_procedures_memory_id ON procedures(memory_id); + +CREATE INDEX idx_procedures_supersedes ON procedures(supersedes_procedure_id); + +CREATE TABLE procedure_steps ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + procedure_id INTEGER NOT NULL REFERENCES procedures(id) ON DELETE CASCADE, + step_order INTEGER NOT NULL, + action TEXT NOT NULL, + rationale TEXT, + tool_name TEXT, + expected_output TEXT, + stop_condition TEXT, + retry_policy TEXT, + rollback_hint TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX idx_procedure_steps_procedure_order +ON procedure_steps(procedure_id, step_order); + +CREATE TABLE procedure_sources ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + procedure_id INTEGER NOT NULL REFERENCES procedures(id) ON DELETE CASCADE, + memory_id INTEGER REFERENCES memories(id) ON DELETE CASCADE, + event_id INTEGER REFERENCES events(id) ON DELETE CASCADE, + decision_id INTEGER REFERENCES decisions(id) ON DELETE CASCADE, + entity_id INTEGER REFERENCES entities(id) ON DELETE CASCADE, + source_role TEXT NOT NULL DEFAULT 'evidence', + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX idx_procedure_sources_procedure ON procedure_sources(procedure_id); + +CREATE INDEX idx_procedure_sources_memory ON procedure_sources(memory_id); + +CREATE INDEX idx_procedure_sources_event ON procedure_sources(event_id); + +CREATE INDEX idx_procedure_sources_decision ON procedure_sources(decision_id); + +CREATE TABLE procedure_runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + procedure_id INTEGER NOT NULL REFERENCES procedures(id) ON DELETE CASCADE, + agent_id TEXT REFERENCES agents(id), + task_family TEXT, + task_signature TEXT, + input_summary TEXT, + outcome_summary TEXT, + success INTEGER NOT NULL DEFAULT 0, + usefulness_score REAL, + errors_seen TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX idx_procedure_runs_procedure_created +ON procedure_runs(procedure_id, created_at DESC); + +CREATE TABLE procedure_candidates ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + candidate_signature TEXT NOT NULL UNIQUE, + task_family TEXT, + normalized_signature TEXT NOT NULL, + support_count INTEGER NOT NULL DEFAULT 0, + evidence_json TEXT, + mean_success REAL NOT NULL DEFAULT 0.0, + promoted_procedure_id INTEGER REFERENCES procedures(id), + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX idx_procedure_candidates_family ON procedure_candidates(task_family); + +CREATE INDEX idx_procedure_candidates_support ON procedure_candidates(support_count DESC); + +CREATE VIRTUAL TABLE procedures_fts USING fts5( + title, + goal, + description, + task_family, + search_text, + content=procedures, + content_rowid=id, + tokenize='porter unicode61' +); + +CREATE TRIGGER procedures_fts_insert AFTER INSERT ON procedures BEGIN + INSERT INTO procedures_fts(rowid, title, goal, description, task_family, search_text) + VALUES (new.id, new.title, new.goal, new.description, new.task_family, new.search_text); +END; + +CREATE TRIGGER procedures_fts_update AFTER UPDATE ON procedures BEGIN + INSERT INTO procedures_fts(procedures_fts, rowid, title, goal, description, task_family, search_text) + VALUES ('delete', old.id, old.title, old.goal, old.description, old.task_family, old.search_text); + INSERT INTO procedures_fts(rowid, title, goal, description, task_family, search_text) + VALUES (new.id, new.title, new.goal, new.description, new.task_family, new.search_text); +END; + +CREATE TRIGGER procedures_fts_delete AFTER DELETE ON procedures BEGIN + INSERT INTO procedures_fts(procedures_fts, rowid, title, goal, description, task_family, search_text) + VALUES ('delete', old.id, old.title, old.goal, old.description, old.task_family, old.search_text); +END; + +CREATE TABLE agent_beliefs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + agent_id TEXT NOT NULL REFERENCES agents(id), + topic TEXT NOT NULL, + -- Scoped topic key, e.g.: + -- "project:agentmemory:status" + -- "agent:my-agent:role" + -- "global:memory_spine:schema_version" + -- "task:internal-ref:status" + belief_content TEXT NOT NULL, + confidence REAL NOT NULL DEFAULT 1.0 + CHECK(confidence >= 0.0 AND confidence <= 1.0), + source_memory_id INTEGER REFERENCES memories(id), + source_event_id INTEGER REFERENCES events(id), + is_assumption INTEGER NOT NULL DEFAULT 0, + -- 1 = unverified assumption (agent inferred, not explicitly told) + -- 0 = derived from direct evidence or memory injection + last_updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%S', 'now')), + invalidated_at TEXT, -- NULL = still believed / active + invalidation_reason TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%S', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%S', 'now')), + is_superposed INTEGER DEFAULT 0, + belief_density_matrix BLOB DEFAULT NULL, + coherence_score REAL DEFAULT 0.0, + entanglement_source_ids TEXT DEFAULT NULL, + UNIQUE(agent_id, topic) +); + +CREATE INDEX idx_beliefs_agent ON agent_beliefs(agent_id); + +CREATE INDEX idx_beliefs_topic ON agent_beliefs(topic); + +CREATE INDEX idx_beliefs_active ON agent_beliefs(invalidated_at) WHERE invalidated_at IS NULL; + +CREATE INDEX idx_beliefs_assumption ON agent_beliefs(is_assumption) WHERE is_assumption = 1; + +CREATE INDEX idx_beliefs_stale ON agent_beliefs(last_updated_at); + +CREATE TABLE belief_conflicts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + topic TEXT NOT NULL, + agent_a_id TEXT NOT NULL REFERENCES agents(id), + agent_b_id TEXT REFERENCES agents(id), + -- NULL = conflict is with global ground truth (memories), not another agent + belief_a TEXT NOT NULL, -- what agent A believes + belief_b TEXT NOT NULL, -- what agent B believes, or ground truth + conflict_type TEXT NOT NULL DEFAULT 'factual' + CHECK(conflict_type IN ( + 'factual', -- two agents disagree on a fact + 'assumption', -- one agent is acting on an unverified assumption + 'staleness', -- one agent's belief is outdated vs. current ground truth + 'scope' -- agents disagree about ownership or responsibility + )), + severity REAL NOT NULL DEFAULT 0.5 + CHECK(severity >= 0.0 AND severity <= 1.0), + detected_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%S', 'now')), + resolved_at TEXT, + resolution TEXT, + requires_supervisor_intervention INTEGER NOT NULL DEFAULT 0 + -- 1 = supervisor agent should inject corrective context before affected agents act +); + +CREATE INDEX idx_conflicts_topic ON belief_conflicts(topic); + +CREATE INDEX idx_conflicts_agent_a ON belief_conflicts(agent_a_id); + +CREATE INDEX idx_conflicts_agent_b ON belief_conflicts(agent_b_id); + +CREATE INDEX idx_conflicts_open ON belief_conflicts(resolved_at) WHERE resolved_at IS NULL; + +CREATE INDEX idx_conflicts_severity ON belief_conflicts(severity DESC) WHERE resolved_at IS NULL; + +CREATE INDEX idx_conflicts_supervisor ON belief_conflicts(requires_supervisor_intervention) + WHERE requires_supervisor_intervention = 1 AND resolved_at IS NULL; + +CREATE TABLE agent_perspective_models ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + observer_agent_id TEXT NOT NULL REFERENCES agents(id), + subject_agent_id TEXT NOT NULL REFERENCES agents(id), + topic TEXT NOT NULL, + estimated_belief TEXT, + -- Observer's best estimate of what subject currently believes. + -- NULL = observer has no model for this topic (treat as full gap). + estimated_confidence REAL + CHECK(estimated_confidence IS NULL OR (estimated_confidence >= 0.0 AND estimated_confidence <= 1.0)), + -- How confident is the observer in their estimate of subject's belief? + knowledge_gap TEXT, + -- What observer believes subject does NOT know about this topic. + -- This is the delta to fill when routing context to subject. + -- NULL = no known gap (subject likely has sufficient context). + confusion_risk REAL NOT NULL DEFAULT 0.0 + CHECK(confusion_risk >= 0.0 AND confusion_risk <= 1.0), + -- Probability subject will be confused or err on tasks requiring + -- knowledge of this topic. Supervisor uses this for proactive injection. + -- Thresholds: > 0.7 = HIGH (inject before routing), 0.4–0.7 = MODERATE + last_updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%S', 'now')), + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%S', 'now')), + UNIQUE(observer_agent_id, subject_agent_id, topic) +); + +CREATE INDEX idx_pmodel_observer ON agent_perspective_models(observer_agent_id); + +CREATE INDEX idx_pmodel_subject ON agent_perspective_models(subject_agent_id); + +CREATE INDEX idx_pmodel_topic ON agent_perspective_models(topic); + +CREATE INDEX idx_pmodel_confusion ON agent_perspective_models(confusion_risk DESC); + +CREATE INDEX idx_pmodel_gaps ON agent_perspective_models(knowledge_gap) + WHERE knowledge_gap IS NOT NULL; + +CREATE TABLE agent_bdi_state ( + agent_id TEXT PRIMARY KEY REFERENCES agents(id), + + -- BELIEFS dimension + beliefs_summary TEXT, + -- JSON: { + -- "active_belief_count": N, + -- "stale_belief_count": N, (last_updated > 24h for active-task topics) + -- "assumption_count": N, (is_assumption = 1) + -- "conflict_count": N, (open belief_conflicts for this agent) + -- "key_topics": ["t1", "t2", ...] + -- } + beliefs_last_updated_at TEXT, + + -- DESIRES dimension + desires_summary TEXT, + -- JSON: { + -- "active_task_count": N, + -- "primary_goal": "...", + -- "priority": "critical|high|medium|low", + -- "task_ids": ["internal-ref", ...] + -- } + desires_last_updated_at TEXT, + + -- INTENTIONS dimension + intentions_summary TEXT, + -- JSON: { + -- "in_progress_tasks": [...], + -- "committed_actions": [...], (from recent events) + -- "estimated_completion": "..." + -- } + intentions_last_updated_at TEXT, + + -- EPISTEMIC HEALTH SCORES (0.0–1.0) + knowledge_coverage_score REAL, + -- How well does agent's belief state cover topics required + -- by their current active tasks? 1.0 = full coverage. + belief_staleness_score REAL, + -- Fraction of active-task beliefs that are stale (>24h). + -- 1.0 = all beliefs are stale. Target < 0.2. + confusion_risk_score REAL, + -- Aggregate max confusion_risk from agent_perspective_models + -- where this agent is the subject. 1.0 = high confusion expected. + -- Supervisor triggers proactive injection when this > 0.7. + + last_full_assessment_at TEXT, + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%S', 'now')) +); + +CREATE INDEX idx_bdi_coverage ON agent_bdi_state(knowledge_coverage_score); + +CREATE INDEX idx_bdi_staleness ON agent_bdi_state(belief_staleness_score DESC); + +CREATE INDEX idx_bdi_confusion ON agent_bdi_state(confusion_risk_score DESC); + +CREATE TABLE neuromodulation_state ( + id INTEGER PRIMARY KEY DEFAULT 1, + org_state TEXT NOT NULL DEFAULT 'normal' + CHECK(org_state IN ('normal', 'incident', 'sprint', 'strategic_planning', 'focused_work')), + dopamine_signal REAL NOT NULL DEFAULT 0.0, + confidence_boost_rate REAL NOT NULL DEFAULT 0.10, + confidence_decay_rate REAL NOT NULL DEFAULT 0.02, + dopamine_last_fired_at TEXT, + arousal_level REAL NOT NULL DEFAULT 0.3, + retrieval_breadth_multiplier REAL NOT NULL DEFAULT 1.0, + consolidation_immediacy TEXT NOT NULL DEFAULT 'scheduled' + CHECK(consolidation_immediacy IN ('immediate', 'scheduled')), + consolidation_interval_mins INTEGER NOT NULL DEFAULT 240, + focus_level REAL NOT NULL DEFAULT 0.3, + similarity_threshold_delta REAL NOT NULL DEFAULT 0.0, + scope_restriction TEXT, + exploitation_bias REAL NOT NULL DEFAULT 0.0, + temporal_lambda REAL NOT NULL DEFAULT 0.030, + context_window_depth INTEGER NOT NULL DEFAULT 50, + detected_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%S', 'now')), + detection_method TEXT NOT NULL DEFAULT 'auto' + CHECK(detection_method IN ('auto', 'manual', 'policy')), + expires_at TEXT, + triggered_by TEXT, + notes TEXT +); + +CREATE UNIQUE INDEX idx_neuromod_singleton ON neuromodulation_state(id); + +CREATE TABLE neuromodulation_transitions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + from_state TEXT NOT NULL, + to_state TEXT NOT NULL, + reason TEXT, + triggered_by TEXT, + transitioned_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%S', 'now')) +); + +CREATE INDEX idx_neuromod_transitions_ts ON neuromodulation_transitions(transitioned_at DESC); + +CREATE INDEX idx_memories_protected ON memories(protected) WHERE protected = 1; + +CREATE TABLE dream_hypotheses ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + memory_a_id INTEGER NOT NULL REFERENCES memories(id), + memory_b_id INTEGER NOT NULL REFERENCES memories(id), + hypothesis_memory_id INTEGER REFERENCES memories(id), -- the synthesized hypothesis memory + similarity REAL NOT NULL, -- cosine similarity at creation time + status TEXT NOT NULL DEFAULT 'incubating' -- incubating | promoted | retired + CHECK(status IN ('incubating', 'promoted', 'retired')), + created_at TEXT NOT NULL DEFAULT (datetime('now')), + promoted_at TEXT, + retired_at TEXT, + retirement_reason TEXT +); + +CREATE INDEX idx_dream_hypotheses_status ON dream_hypotheses(status); + +CREATE INDEX idx_dream_hypotheses_created ON dream_hypotheses(created_at DESC); + +CREATE INDEX idx_dream_hypotheses_hypothesis_memory ON dream_hypotheses(hypothesis_memory_id); + +CREATE INDEX idx_dream_hypotheses_pair ON dream_hypotheses(memory_a_id, memory_b_id); + +CREATE TABLE workspace_config ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL, + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%S', 'now')) +); + +CREATE TABLE workspace_broadcasts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + memory_id INTEGER NOT NULL REFERENCES memories(id), + agent_id TEXT NOT NULL, -- who triggered the broadcast + salience REAL NOT NULL, -- score that triggered ignition + summary TEXT NOT NULL, -- short broadcast summary (≤200 chars) + target_scope TEXT NOT NULL DEFAULT 'global', -- 'global', 'project:X', 'agent:Y' + broadcast_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%S', 'now')), + expires_at TEXT, -- NULL = uses default TTL + ack_count INTEGER NOT NULL DEFAULT 0, + triggered_by TEXT NOT NULL DEFAULT 'auto' -- 'auto' | 'manual' | 'trigger' +); + +CREATE INDEX idx_wb_broadcast_at ON workspace_broadcasts(broadcast_at DESC); + +CREATE INDEX idx_wb_memory_id ON workspace_broadcasts(memory_id); + +CREATE INDEX idx_wb_agent_id ON workspace_broadcasts(agent_id); + +CREATE INDEX idx_wb_target_scope ON workspace_broadcasts(target_scope); + +CREATE INDEX idx_wb_expires ON workspace_broadcasts(expires_at); + +CREATE TABLE workspace_acks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + broadcast_id INTEGER NOT NULL REFERENCES workspace_broadcasts(id), + agent_id TEXT NOT NULL, + acked_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%S', 'now')), + UNIQUE(broadcast_id, agent_id) +); + +CREATE INDEX idx_wacks_broadcast ON workspace_acks(broadcast_id); + +CREATE INDEX idx_wacks_agent ON workspace_acks(agent_id); + +CREATE TRIGGER trg_ws_ack_count +AFTER INSERT ON workspace_acks +BEGIN + UPDATE workspace_broadcasts + SET ack_count = ack_count + 1 + WHERE id = NEW.broadcast_id; +END; + +CREATE TABLE workspace_phi ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + window_start TEXT NOT NULL, + window_end TEXT NOT NULL, + phi_org REAL NOT NULL DEFAULT 0.0, -- mean pair-wise integration + broadcast_count INTEGER NOT NULL DEFAULT 0, -- broadcasts in window + ack_rate REAL NOT NULL DEFAULT 0.0, -- fraction of broadcasts acked + agent_pair_count INTEGER NOT NULL DEFAULT 0, -- active agent pairs counted + computed_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%S', 'now')) +); + +CREATE INDEX idx_wphi_window ON workspace_phi(window_end DESC); + +CREATE TRIGGER trg_memory_ignition_insert +AFTER INSERT ON memories +WHEN NEW.retired_at IS NULL +BEGIN + -- Compute salience: priority signal (via category) + confidence + recency boost + -- Categories map to implicit priority: decision/identity/convention = high + -- We approximate salience from confidence since we don't have event priority here. + -- Full salience scoring is done in Python; trigger handles high-confidence fast path. + INSERT INTO workspace_broadcasts (memory_id, agent_id, salience, summary, target_scope, triggered_by) + SELECT + NEW.id, + NEW.agent_id, + NEW.confidence, + substr(NEW.content, 1, 200), + COALESCE(NEW.scope, 'global'), + 'auto' + WHERE NEW.confidence >= COALESCE( + -- Use urgent threshold if neuromod org_state = 'incident', else normal + CASE + WHEN EXISTS ( + SELECT 1 FROM neuromodulation_state WHERE id = 1 AND org_state = 'incident' + ) THEN (SELECT CAST(value AS REAL) FROM workspace_config WHERE key = 'urgent_threshold') + ELSE (SELECT CAST(value AS REAL) FROM workspace_config WHERE key = 'ignition_threshold') + END, + 0.85 + ) + AND (SELECT value FROM workspace_config WHERE key = 'enabled') = '1' + -- Governor: don't fire if we've already broadcast governor_max_per_hour in last hour + AND ( + SELECT COUNT(*) FROM workspace_broadcasts + WHERE broadcast_at >= strftime('%Y-%m-%dT%H:%M:%S', datetime('now', '-1 hour')) + ) < CAST((SELECT value FROM workspace_config WHERE key = 'governor_max_per_hour') AS INTEGER); +END; + +CREATE TABLE agent_capabilities ( + agent_id TEXT NOT NULL REFERENCES agents(id), + capability TEXT NOT NULL, -- e.g. "sql_migration", "research", "memory_ops" + skill_level REAL NOT NULL DEFAULT 0.5, -- 0.0-1.0 estimated proficiency + task_count INTEGER NOT NULL DEFAULT 0, -- result events logged in this domain + avg_events REAL, -- avg events per task burst (proxy for effort) + block_rate REAL DEFAULT 0.0, -- fraction of events that were blocked/errors + last_active TEXT, -- last event timestamp in this domain + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%S', 'now')), + PRIMARY KEY (agent_id, capability) +); + +CREATE INDEX idx_agent_caps_agent ON agent_capabilities(agent_id); + +CREATE INDEX idx_agent_caps_cap ON agent_capabilities(capability); + +CREATE INDEX idx_agent_caps_skill ON agent_capabilities(skill_level DESC); + +CREATE TABLE world_model_snapshots ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + snapshot_type TEXT NOT NULL, -- 'org_state' | 'prediction' | 'error_log' + subject_id TEXT, -- agent_id, project name, or task ref + subject_type TEXT, -- 'agent' | 'project' | 'task' + predicted_state TEXT, -- JSON: the predicted state + actual_state TEXT, -- JSON: filled in after resolution + prediction_error REAL, -- scalar distance |predicted - actual| (0.0-1.0) + author_agent_id TEXT REFERENCES agents(id), + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%S', 'now')), + resolved_at TEXT +); + +CREATE INDEX idx_wm_snapshots_type ON world_model_snapshots(snapshot_type); + +CREATE INDEX idx_wm_snapshots_subject ON world_model_snapshots(subject_id); + +CREATE INDEX idx_wm_snapshots_unresolved ON world_model_snapshots(resolved_at) WHERE resolved_at IS NULL; + +CREATE TABLE deferred_queries ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + agent_id TEXT NOT NULL, -- who issued the original search + query_text TEXT NOT NULL, -- the raw search query + query_embedding BLOB, -- optional: embedding vector for vec retry + queried_at TEXT NOT NULL DEFAULT (datetime('now')), + expires_at TEXT, -- NULL = 30-day default applied at retry + resolved_at TEXT, -- NULL while still pending + resolution_memory_id INTEGER REFERENCES memories(id), + attempts INTEGER NOT NULL DEFAULT 0 -- retry counter +); + +CREATE INDEX idx_deferred_queries_agent ON deferred_queries(agent_id); + +CREATE INDEX idx_deferred_queries_pending ON deferred_queries(resolved_at) WHERE resolved_at IS NULL; + +CREATE INDEX idx_deferred_queries_queried ON deferred_queries(queried_at DESC); + +CREATE TABLE neuro_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + org_state TEXT NOT NULL, + dopamine_level REAL NOT NULL DEFAULT 0.0, + norepinephrine_level REAL NOT NULL DEFAULT 0.0, + acetylcholine_level REAL NOT NULL DEFAULT 0.0, + serotonin_level REAL NOT NULL DEFAULT 0.3, + computed_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%S', 'now')), + source TEXT NOT NULL DEFAULT 'auto_detect', + agent_id TEXT, + notes TEXT +); + +CREATE INDEX idx_neuro_events_time ON neuro_events(computed_at); + +CREATE INDEX idx_memories_gw_broadcast ON memories(gw_broadcast) WHERE gw_broadcast = 1; + +CREATE INDEX idx_memories_salience ON memories(salience_score DESC) WHERE retired_at IS NULL; + +CREATE TRIGGER trg_gw_broadcast_meb +AFTER UPDATE OF gw_broadcast ON memories +WHEN NEW.gw_broadcast = 1 AND OLD.gw_broadcast = 0 AND NEW.retired_at IS NULL +BEGIN + INSERT INTO memory_events (memory_id, agent_id, operation, category, scope, memory_type, created_at) + VALUES ( + NEW.id, + NEW.agent_id, + 'broadcast', + NEW.category, + COALESCE(NEW.scope, 'global'), + COALESCE(NEW.memory_type, 'episodic'), + strftime('%Y-%m-%dT%H:%M:%S', 'now') + ); +END; + +CREATE TRIGGER trg_gw_broadcast_workspace +AFTER UPDATE OF gw_broadcast ON memories +WHEN NEW.gw_broadcast = 1 AND OLD.gw_broadcast = 0 AND NEW.retired_at IS NULL +BEGIN + INSERT OR IGNORE INTO workspace_broadcasts (memory_id, agent_id, salience, summary, target_scope, triggered_by) + SELECT + NEW.id, + NEW.agent_id, + NEW.salience_score, + substr(NEW.content, 1, 200), + COALESCE(NEW.scope, 'global'), + 'gw_score' + WHERE NOT EXISTS ( + SELECT 1 FROM workspace_broadcasts wb WHERE wb.memory_id = NEW.id + AND wb.broadcast_at >= strftime('%Y-%m-%dT%H:%M:%S', datetime('now', '-48 hours')) + ); +END; + +CREATE TRIGGER memories_visibility_check_insert +BEFORE INSERT ON memories +WHEN NEW.visibility NOT IN ('public', 'project', 'agent', 'restricted') +BEGIN + SELECT RAISE(ABORT, 'memories.visibility must be one of: public, project, agent, restricted'); +END; + +CREATE TRIGGER memories_visibility_check_update +BEFORE UPDATE OF visibility ON memories +WHEN NEW.visibility NOT IN ('public', 'project', 'agent', 'restricted') +BEGIN + SELECT RAISE(ABORT, 'memories.visibility must be one of: public, project, agent, restricted'); +END; + +CREATE INDEX idx_memories_visibility ON memories(visibility); + +CREATE INDEX idx_memories_ewc_importance ON memories(ewc_importance DESC) WHERE retired_at IS NULL; + +CREATE TABLE world_model ( + entity_id TEXT NOT NULL PRIMARY KEY, + entity_type TEXT CHECK(entity_type IN ('agent', 'project', 'goal', 'dependency')), + state_snapshot TEXT NOT NULL, + causal_parents TEXT, + last_synced_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%S', 'now')) + ); + +CREATE INDEX idx_world_model_type ON world_model(entity_type); + +CREATE INDEX idx_rlessons_propagated ON reflexion_lessons(propagated_to) + WHERE propagated_to != '[]'; + +CREATE INDEX idx_rlessons_prop_source ON reflexion_lessons(propagation_source_lesson_id) + WHERE propagation_source_lesson_id IS NOT NULL; + +CREATE INDEX idx_memories_alpha ON memories(alpha) WHERE retired_at IS NULL; + +CREATE INDEX idx_memories_beta ON memories(beta) WHERE retired_at IS NULL; + +CREATE TABLE agent_uncertainty_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + agent_id TEXT NOT NULL, + task_desc TEXT, -- task description that triggered the scan + gap_topic TEXT, -- what the agent didn't know + free_energy REAL, -- (1 - confidence) * importance at scan time + resolved_at TIMESTAMP, -- when the gap was filled + resolved_by INTEGER REFERENCES memories(id), -- memory that resolved the gap + propagated BOOLEAN DEFAULT FALSE, -- whether gap was propagated to other agents + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + domain TEXT, + query TEXT, + result_count INTEGER, + avg_confidence REAL, + retrieved_at DATETIME DEFAULT (datetime('now')), + temporal_class TEXT DEFAULT 'ephemeral', + ttl_days INTEGER DEFAULT 30 +); + +CREATE INDEX idx_unc_agent ON agent_uncertainty_log(agent_id); + +CREATE INDEX idx_unc_created ON agent_uncertainty_log(created_at); + +CREATE INDEX idx_unc_resolved ON agent_uncertainty_log(resolved_at); + +CREATE INDEX idx_unc_task ON agent_uncertainty_log(agent_id, resolved_at); + +CREATE INDEX idx_expertise_brier ON agent_expertise(brier_score) WHERE brier_score IS NOT NULL; + +CREATE INDEX idx_unc_domain ON agent_uncertainty_log(domain); + +CREATE INDEX idx_unc_retrieved ON agent_uncertainty_log(retrieved_at); + +CREATE INDEX idx_access_agent_day + ON access_log(agent_id, created_at DESC); + +CREATE TABLE entities ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, -- unique human-readable identifier + entity_type TEXT NOT NULL, -- 'person', 'organization', 'project', 'tool', 'concept', 'agent', 'location', 'event', 'document' + properties TEXT NOT NULL DEFAULT '{}', -- JSON object of typed properties + observations TEXT NOT NULL DEFAULT '[]', -- JSON array of atomic fact strings + agent_id TEXT NOT NULL REFERENCES agents(id), -- who created this entity + confidence REAL NOT NULL DEFAULT 1.0, -- 0.0-1.0 + scope TEXT NOT NULL DEFAULT 'global', -- 'global', 'project:', 'agent:' + retired_at TEXT, -- soft delete + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + -- Migration 033: compiled-truth synthesis surface + compiled_truth TEXT, + compiled_truth_updated_at TEXT, + compiled_truth_source TEXT, + -- Migration 034: enrichment tier (T1 critical / T2 notable / T3 minor) + enrichment_tier INTEGER NOT NULL DEFAULT 3, + last_enriched_at TEXT, + -- Migration 035: aliases JSON list for canonical-name dedup + aliases TEXT +); + +CREATE UNIQUE INDEX uq_entities_name_scope ON entities(name, scope) WHERE retired_at IS NULL; + +CREATE INDEX idx_entities_type ON entities(entity_type); + +CREATE INDEX idx_entities_agent ON entities(agent_id); + +CREATE INDEX idx_entities_scope ON entities(scope); + +CREATE INDEX idx_entities_active ON entities(retired_at) WHERE retired_at IS NULL; + +CREATE INDEX idx_entities_compiled_truth_updated_at ON entities(compiled_truth_updated_at); + +CREATE INDEX idx_entities_tier_enriched ON entities(enrichment_tier, last_enriched_at) + WHERE retired_at IS NULL AND enrichment_tier < 3; + +CREATE VIRTUAL TABLE entities_fts USING fts5( + name, + entity_type, + properties, + observations, + content=entities, + content_rowid=id, + tokenize='unicode61' +); + +CREATE TRIGGER entities_fts_insert AFTER INSERT ON entities BEGIN + INSERT INTO entities_fts(rowid, name, entity_type, properties, observations) + VALUES (new.id, new.name, new.entity_type, new.properties, new.observations); +END; + +CREATE TRIGGER entities_fts_update AFTER UPDATE ON entities BEGIN + INSERT INTO entities_fts(entities_fts, rowid, name, entity_type, properties, observations) + VALUES('delete', old.id, old.name, old.entity_type, old.properties, old.observations); + INSERT INTO entities_fts(rowid, name, entity_type, properties, observations) + VALUES (new.id, new.name, new.entity_type, new.properties, new.observations); +END; + +CREATE TRIGGER entities_fts_delete AFTER DELETE ON entities BEGIN + INSERT INTO entities_fts(entities_fts, rowid, name, entity_type, properties, observations) + VALUES('delete', old.id, old.name, old.entity_type, old.properties, old.observations); +END; + +CREATE INDEX idx_memories_confidence_phase ON memories(agent_id, confidence_phase) WHERE confidence_phase != 0.0; + +CREATE INDEX idx_memories_decoherence_rate ON memories(decoherence_rate DESC) WHERE decoherence_rate IS NOT NULL; + +CREATE INDEX idx_memories_coherence_syndrome ON memories(agent_id) WHERE coherence_syndrome IS NOT NULL; + +CREATE INDEX idx_agent_beliefs_superposed ON agent_beliefs(agent_id, is_superposed) WHERE is_superposed = 1; + +CREATE INDEX idx_agent_beliefs_coherence ON agent_beliefs(agent_id, coherence_score DESC) WHERE is_superposed = 1; + +CREATE INDEX idx_agent_beliefs_entanglement_sources ON agent_beliefs(agent_id) WHERE entanglement_source_ids IS NOT NULL; + +CREATE VIEW superposed_beliefs AS + SELECT ab.id, ab.agent_id, ab.topic, ab.is_superposed, + ab.coherence_score, ab.entanglement_source_ids, + ab.created_at, ab.updated_at + FROM agent_beliefs ab WHERE ab.is_superposed = 1; + +CREATE VIEW decoherent_memories AS + SELECT id, content, confidence, coherence_syndrome, decoherence_rate, + temporal_class, created_at, updated_at + FROM memories + WHERE coherence_syndrome IS NOT NULL OR decoherence_rate IS NOT NULL + ORDER BY decoherence_rate DESC; + +CREATE VIEW recent_belief_collapses AS + SELECT bce.id, bce.agent_id, bce.belief_id, bce.collapsed_state, + bce.collapse_type, bce.collapse_fidelity, bce.created_at + FROM belief_collapse_events bce + WHERE bce.created_at > datetime('now', '-7 days') + ORDER BY bce.created_at DESC; + +CREATE TABLE belief_collapse_events ( + id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(16)))), + belief_id TEXT NOT NULL REFERENCES agent_beliefs(id) ON DELETE CASCADE, + agent_id TEXT NOT NULL REFERENCES agents(id) ON DELETE CASCADE, + collapsed_state TEXT NOT NULL, + measured_amplitude REAL NOT NULL, + -- Expanded trigger type vocabulary (internal-ref) + collapse_type TEXT NOT NULL, + collapse_context TEXT DEFAULT NULL, + collapse_fidelity REAL DEFAULT 1.0, + created_at TEXT DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX idx_bce_belief ON belief_collapse_events(belief_id); + +CREATE INDEX idx_bce_agent ON belief_collapse_events(agent_id); + +CREATE INDEX idx_bce_type ON belief_collapse_events(collapse_type); + +CREATE INDEX idx_bce_created ON belief_collapse_events(created_at DESC); + +CREATE INDEX idx_access_log_task_id ON access_log(task_id) WHERE task_id IS NOT NULL; + +CREATE TABLE memory_outcome_calibration ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + agent_id TEXT NOT NULL, + period_start TEXT NOT NULL, + period_end TEXT NOT NULL, + total_tasks INTEGER NOT NULL DEFAULT 0, + tasks_used_memory INTEGER NOT NULL DEFAULT 0, + success_with_memory REAL, + success_without_memory REAL, + brier_score REAL, + p_at_5 REAL, + computed_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX idx_moc_agent_period ON memory_outcome_calibration(agent_id, period_start); + +CREATE TABLE memory_triggers ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + agent_id TEXT NOT NULL, + trigger_condition TEXT NOT NULL, + trigger_keywords TEXT NOT NULL, + action TEXT NOT NULL, + entity_id INTEGER REFERENCES entities(id), + memory_id INTEGER REFERENCES memories(id), + priority TEXT NOT NULL DEFAULT 'medium', + status TEXT NOT NULL DEFAULT 'active' CHECK(status IN ('active','fired','expired','cancelled')), + fired_at TEXT, + expires_at TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX idx_triggers_status ON memory_triggers(status); + +CREATE INDEX idx_triggers_agent ON memory_triggers(agent_id); + +CREATE TABLE affect_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + agent_id TEXT NOT NULL, + valence REAL NOT NULL DEFAULT 0.0, + arousal REAL NOT NULL DEFAULT 0.0, + dominance REAL NOT NULL DEFAULT 0.0, + affect_label TEXT, + cluster TEXT, + functional_state TEXT, + safety_flag TEXT, + trigger TEXT, + source TEXT DEFAULT 'observation', + metadata TEXT, + created_at TEXT NOT NULL +); + +CREATE INDEX idx_affect_agent_time ON affect_log(agent_id, created_at DESC); + +CREATE INDEX idx_affect_safety ON affect_log(safety_flag) WHERE safety_flag IS NOT NULL; + +CREATE INDEX idx_affect_cluster ON affect_log(cluster, created_at DESC); + +-- 2.2.3: cross-agent time-range index for `brainctl affect prune`. The +-- composite idx_affect_agent_time leads with agent_id and cannot serve a +-- WHERE created_at < ? predicate that spans all agents. Mirrors +-- migration 049_affect_log_retention_indexes.sql for fresh installs. +CREATE INDEX IF NOT EXISTS idx_affect_created_at ON affect_log(created_at); + +-- ------------------------------------------------------------------------- +-- LLM usage tracking +-- ------------------------------------------------------------------------- +CREATE TABLE IF NOT EXISTS llm_usage_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + agent_id TEXT NOT NULL REFERENCES agents(id), + model TEXT NOT NULL, + prompt_tokens INTEGER NOT NULL DEFAULT 0, + completion_tokens INTEGER NOT NULL DEFAULT 0, + total_tokens INTEGER NOT NULL DEFAULT 0, + cost_usd REAL NOT NULL DEFAULT 0.0, + tool_name TEXT, -- which MCP tool triggered the call (if applicable) + project TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%S','now')) +); + +CREATE INDEX IF NOT EXISTS idx_llm_usage_agent_created ON llm_usage_log(agent_id, created_at); +CREATE INDEX IF NOT EXISTS idx_llm_usage_created ON llm_usage_log(created_at); + +-- Per-agent budget limits +CREATE TABLE IF NOT EXISTS agent_budget ( + agent_id TEXT PRIMARY KEY REFERENCES agents(id), + monthly_limit_usd REAL NOT NULL DEFAULT 10.0, + alert_threshold REAL NOT NULL DEFAULT 0.8, -- fraction of limit that triggers alert + hard_limit REAL NOT NULL DEFAULT 1.0, -- fraction at which calls are blocked + reset_day INTEGER NOT NULL DEFAULT 1, -- day of month budgets reset + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%S','now')) +); + +-- ------------------------------------------------------------------------- +-- Neuroscience-inspired memory columns (replay priority + reconsolidation) +-- ------------------------------------------------------------------------- +-- replay_priority: accumulated salience score; higher = earlier consolidation +-- ripple_tags: count of high-salience (SWR-like) retrieval events +-- labile_until: ISO datetime when reconsolidation window closes (NULL = stable) +-- labile_agent_id: agent that opened the lability window (agent-scoped) +-- retrieval_prediction_error: cosine distance at lability-opening retrieval +-- (Columns are defined in the base CREATE TABLE memories above.) +CREATE INDEX IF NOT EXISTS idx_memories_replay ON memories(replay_priority DESC) WHERE retired_at IS NULL; +CREATE INDEX IF NOT EXISTS idx_memories_labile ON memories(labile_until) WHERE labile_until IS NOT NULL; + + +-- ------------------------------------------------------------------------- +-- Memory immunity system (issue #24) +-- Quarantine table for adversarial/injected memory detection +-- ------------------------------------------------------------------------- +CREATE TABLE IF NOT EXISTS memory_quarantine ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + memory_id INTEGER NOT NULL REFERENCES memories(id) ON DELETE CASCADE, + reason TEXT NOT NULL, + source_trust REAL, + contradiction_count INTEGER DEFAULT 0, + quarantined_by TEXT NOT NULL DEFAULT 'system', + reviewed_by TEXT DEFAULT NULL, + reviewed_at TEXT DEFAULT NULL, + verdict TEXT DEFAULT NULL CHECK(verdict IN ('safe','malicious','uncertain')), + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%S','now')) +); + +CREATE INDEX IF NOT EXISTS idx_quarantine_memory_id ON memory_quarantine(memory_id); +CREATE INDEX IF NOT EXISTS idx_quarantine_verdict ON memory_quarantine(verdict); +CREATE INDEX IF NOT EXISTS idx_quarantine_created ON memory_quarantine(created_at DESC); + +-- ------------------------------------------------------------------------- +-- Allostatic scheduling (issue #9) +-- ------------------------------------------------------------------------- +CREATE TABLE IF NOT EXISTS consolidation_forecasts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + memory_id INTEGER REFERENCES memories(id) ON DELETE CASCADE, + agent_id TEXT NOT NULL, + predicted_demand_at TEXT NOT NULL, + confidence REAL NOT NULL DEFAULT 0.5 CHECK(confidence >= 0.0 AND confidence <= 1.0), + signal_source TEXT NOT NULL, + fulfilled_at TEXT DEFAULT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%S','now')) +); + +CREATE INDEX IF NOT EXISTS idx_forecasts_agent ON consolidation_forecasts(agent_id, predicted_demand_at); +CREATE INDEX IF NOT EXISTS idx_forecasts_memory ON consolidation_forecasts(memory_id); +CREATE INDEX IF NOT EXISTS idx_forecasts_fulfilled ON consolidation_forecasts(fulfilled_at); + +-- ------------------------------------------------------------------------- +-- D-MEM RPE routing (issue #31) +-- memory_stats: per-(agent, category, scope) recall rate for long-term utility +-- ------------------------------------------------------------------------- +CREATE TABLE IF NOT EXISTS memory_stats ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + agent_id TEXT NOT NULL, + category TEXT NOT NULL, + scope TEXT NOT NULL DEFAULT 'global', + avg_recall_rate REAL NOT NULL DEFAULT 0.5, + sample_count INTEGER NOT NULL DEFAULT 0, + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%S','now')), + UNIQUE(agent_id, category, scope) +); +CREATE INDEX IF NOT EXISTS idx_memory_stats_agent ON memory_stats(agent_id, category, scope); + +-- ------------------------------------------------------------------------- +-- Temporal abstraction hierarchy (issue #20) +-- (temporal_level column is defined in the base CREATE TABLE memories above.) +-- ------------------------------------------------------------------------- +CREATE INDEX IF NOT EXISTS idx_memories_temporal_level ON memories(temporal_level, agent_id); + +-- ------------------------------------------------------------------------- +-- Context profiles — task-scoped search presets (brainctl profile) +-- ------------------------------------------------------------------------- +CREATE TABLE IF NOT EXISTS context_profiles ( + name TEXT PRIMARY KEY, + description TEXT, + categories TEXT, + tables TEXT, + entity_types TEXT, + created_at TEXT DEFAULT (strftime('%Y-%m-%dT%H:%M:%S', 'now')) +); + +-- =========================================================================== +-- FK INTEGRITY DELETE TRIGGERS (mirrored from migration 048) +-- =========================================================================== +-- See db/migrations/048_fk_integrity_fts_retire_trigger.sql for full rationale. +-- These triggers fire only when PRAGMA foreign_keys = OFF (raw SQL admin, +-- merge.py:586 which disables FK during merge). With FK ON the SQLite default +-- NO ACTION rejects orphan-creating parent DELETEs outright. + +CREATE TRIGGER IF NOT EXISTS trg_agent_delete_nullify_validation +AFTER DELETE ON agents +BEGIN + UPDATE memories + SET validation_agent_id = NULL + WHERE validation_agent_id = OLD.id; +END; + +CREATE TRIGGER IF NOT EXISTS trg_memory_delete_cascade_edges +AFTER DELETE ON memories +BEGIN + DELETE FROM knowledge_edges + WHERE (source_table = 'memories' AND source_id = OLD.id) + OR (target_table = 'memories' AND target_id = OLD.id); +END; + +CREATE TRIGGER IF NOT EXISTS trg_entity_delete_cascade_edges +AFTER DELETE ON entities +BEGIN + DELETE FROM knowledge_edges + WHERE (source_table = 'entities' AND source_id = OLD.id) + OR (target_table = 'entities' AND target_id = OLD.id); +END; + +CREATE TRIGGER IF NOT EXISTS trg_event_delete_cascade_edges +AFTER DELETE ON events +BEGIN + DELETE FROM knowledge_edges + WHERE (source_table = 'events' AND source_id = OLD.id) + OR (target_table = 'events' AND target_id = OLD.id); +END; + +-- FTS5 retire-aware re-index: handled inline by the +-- memories_fts_update_insert trigger above, which has a `WHEN ... AND +-- new.retired_at IS NULL` guard. memories_fts_update_delete fires +-- unconditionally on any UPDATE when old.indexed = 1, which removes the +-- FTS5 row at the retire transition; the guarded _update_insert then does +-- NOT re-insert. Net: retired memories vanish from FTS5 immediately, no +-- separate purge trigger needed (and no double-delete risk). + +-- Migration 051: code_ingest_cache — SHA256 cache for `brainctl ingest code` +-- (brainctl[code] optional extra, 2.4.4+). Included here so fresh installs +-- match upgrade-path schemas (caught by tests/test_schema_parity.py). +CREATE TABLE IF NOT EXISTS code_ingest_cache ( + file_path TEXT NOT NULL, + scope TEXT NOT NULL DEFAULT 'global', + content_sha TEXT NOT NULL, + language TEXT NOT NULL, + entity_count INTEGER NOT NULL DEFAULT 0, + edge_count INTEGER NOT NULL DEFAULT 0, + last_ingested_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY (file_path, scope) +); +CREATE INDEX IF NOT EXISTS idx_code_ingest_cache_scope + ON code_ingest_cache(scope); +CREATE INDEX IF NOT EXISTS idx_code_ingest_cache_language + ON code_ingest_cache(language); diff --git a/db/migrations/052_procedural_memory_layer.sql b/db/migrations/052_procedural_memory_layer.sql new file mode 100644 index 0000000..d96ef29 --- /dev/null +++ b/db/migrations/052_procedural_memory_layer.sql @@ -0,0 +1,510 @@ +PRAGMA foreign_keys = OFF; +BEGIN; + +DROP TRIGGER IF EXISTS memories_fts_insert; +DROP TRIGGER IF EXISTS memories_fts_update_delete; +DROP TRIGGER IF EXISTS memories_fts_update_insert; +DROP TRIGGER IF EXISTS memories_fts_delete; +DROP TRIGGER IF EXISTS memories_temporal_class_check; +DROP TRIGGER IF EXISTS memories_temporal_class_update_check; +DROP TRIGGER IF EXISTS memories_validate_ts_insert; +DROP TRIGGER IF EXISTS memories_validate_ts_update; +DROP TRIGGER IF EXISTS meb_after_memory_insert; +DROP TRIGGER IF EXISTS meb_after_memory_update; +DROP TRIGGER IF EXISTS trg_memory_ignition_insert; +DROP TRIGGER IF EXISTS trg_gw_broadcast_meb; +DROP TRIGGER IF EXISTS trg_gw_broadcast_workspace; +DROP TRIGGER IF EXISTS memories_visibility_check_insert; +DROP TRIGGER IF EXISTS memories_visibility_check_update; +DROP TRIGGER IF EXISTS trg_memory_delete_cascade_edges; +DROP TRIGGER IF EXISTS trg_agent_delete_nullify_validation; +DROP VIEW IF EXISTS decoherent_memories; +DROP TABLE IF EXISTS memories_fts; + +CREATE TEMP TABLE memories_backup AS +SELECT + id, agent_id, category, scope, content, confidence, source_event_id, + supersedes_id, tags, expires_at, recalled_count, last_recalled_at, + created_at, updated_at, retired_at, epoch_id, temporal_class, + validation_agent_id, validated_at, trust_score, derived_from_ids, + retracted_at, retraction_reason, version, memory_type, protected, + salience_score, gw_broadcast, visibility, read_acl, ewc_importance, + alpha, beta, confidence_phase, hilbert_projection, coherence_syndrome, + decoherence_rate, gated_from_memory_id, file_path, file_line, write_tier, + indexed, promoted_at, replay_priority, ripple_tags, labile_until, + labile_agent_id, retrieval_prediction_error, encoding_affect_id, + tag_cycles_remaining, stability, encoding_task_context, + encoding_context_hash, temporal_level, next_review_at, q_value +FROM memories; + +DROP TABLE memories; + +CREATE TABLE memories ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + agent_id TEXT NOT NULL REFERENCES agents(id), -- who wrote this + category TEXT NOT NULL, -- 'identity', 'user', 'environment', 'convention', + -- 'project', 'decision', 'lesson', 'preference' + scope TEXT NOT NULL DEFAULT 'global', -- 'global', 'project:', 'agent:' + content TEXT NOT NULL, -- the actual memory + confidence REAL NOT NULL DEFAULT 1.0, -- 0.0-1.0, decays or gets boosted + source_event_id INTEGER, -- event that spawned this memory + supersedes_id INTEGER REFERENCES memories(id), -- if this replaces an older memory + tags TEXT, -- JSON array of tags + expires_at TEXT, -- optional TTL + recalled_count INTEGER NOT NULL DEFAULT 0, -- how often this memory was retrieved + last_recalled_at TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + retired_at TEXT, -- soft delete + epoch_id INTEGER REFERENCES epochs(id), + temporal_class TEXT NOT NULL DEFAULT 'medium', + validation_agent_id TEXT REFERENCES agents(id), + validated_at TEXT, + trust_score REAL DEFAULT 1.0, + derived_from_ids TEXT, + retracted_at TEXT, + retraction_reason TEXT, + version INTEGER NOT NULL DEFAULT 1, + memory_type TEXT NOT NULL DEFAULT 'episodic' CHECK(memory_type IN ('episodic','semantic','procedural')), + protected INTEGER NOT NULL DEFAULT 0, + salience_score REAL NOT NULL DEFAULT 0.0, + gw_broadcast INTEGER NOT NULL DEFAULT 0, + visibility TEXT NOT NULL DEFAULT 'public', + read_acl TEXT, + ewc_importance REAL NOT NULL DEFAULT 0.0, + alpha REAL DEFAULT 1.0, + beta REAL DEFAULT 1.0, + confidence_alpha REAL GENERATED ALWAYS AS (alpha) VIRTUAL, + confidence_beta REAL GENERATED ALWAYS AS (beta) VIRTUAL, + confidence_phase REAL NOT NULL DEFAULT 0.0, + hilbert_projection BLOB DEFAULT NULL, + coherence_syndrome TEXT DEFAULT NULL, + decoherence_rate REAL DEFAULT NULL, + gated_from_memory_id INTEGER REFERENCES memories(id), + file_path TEXT, + file_line INTEGER, + write_tier TEXT NOT NULL DEFAULT 'full' CHECK(write_tier IN ('skip', 'construct', 'full')), + indexed INTEGER NOT NULL DEFAULT 1, + promoted_at TEXT DEFAULT NULL, + replay_priority REAL NOT NULL DEFAULT 0.0, + ripple_tags INTEGER NOT NULL DEFAULT 0, + labile_until TEXT DEFAULT NULL, + labile_agent_id TEXT DEFAULT NULL, + retrieval_prediction_error REAL DEFAULT NULL, + encoding_affect_id INTEGER REFERENCES affect_log(id) DEFAULT NULL, + tag_cycles_remaining INTEGER DEFAULT 0, + stability REAL DEFAULT 1.0, + encoding_task_context TEXT DEFAULT NULL, + encoding_context_hash TEXT DEFAULT NULL, + temporal_level TEXT NOT NULL DEFAULT 'moment' + CHECK(temporal_level IN ('moment','session','day','week','month','quarter')), + next_review_at TEXT DEFAULT NULL, + q_value REAL DEFAULT 0.5 +); + +INSERT INTO memories ( + id, agent_id, category, scope, content, confidence, source_event_id, + supersedes_id, tags, expires_at, recalled_count, last_recalled_at, + created_at, updated_at, retired_at, epoch_id, temporal_class, + validation_agent_id, validated_at, trust_score, derived_from_ids, + retracted_at, retraction_reason, version, memory_type, protected, + salience_score, gw_broadcast, visibility, read_acl, ewc_importance, + alpha, beta, confidence_phase, hilbert_projection, coherence_syndrome, + decoherence_rate, gated_from_memory_id, file_path, file_line, write_tier, + indexed, promoted_at, replay_priority, ripple_tags, labile_until, + labile_agent_id, retrieval_prediction_error, encoding_affect_id, + tag_cycles_remaining, stability, encoding_task_context, + encoding_context_hash, temporal_level, next_review_at, q_value +) +SELECT + id, agent_id, category, scope, content, confidence, source_event_id, + supersedes_id, tags, expires_at, recalled_count, last_recalled_at, + created_at, updated_at, retired_at, epoch_id, temporal_class, + validation_agent_id, validated_at, trust_score, derived_from_ids, + retracted_at, retraction_reason, version, memory_type, protected, + salience_score, gw_broadcast, visibility, read_acl, ewc_importance, + alpha, beta, confidence_phase, hilbert_projection, coherence_syndrome, + decoherence_rate, gated_from_memory_id, file_path, file_line, write_tier, + indexed, promoted_at, replay_priority, ripple_tags, labile_until, + labile_agent_id, retrieval_prediction_error, encoding_affect_id, + tag_cycles_remaining, stability, encoding_task_context, + encoding_context_hash, temporal_level, next_review_at, q_value +FROM memories_backup; + +DROP TABLE memories_backup; + +CREATE INDEX idx_memories_agent ON memories(agent_id); +CREATE INDEX idx_memories_category ON memories(category); +CREATE INDEX idx_memories_scope ON memories(scope); +CREATE INDEX idx_memories_active ON memories(retired_at) WHERE retired_at IS NULL; +CREATE INDEX idx_memories_confidence ON memories(confidence DESC); +CREATE INDEX idx_memories_agent_active_cat ON memories(agent_id, category) WHERE retired_at IS NULL; +CREATE INDEX idx_memories_agent_time ON memories(agent_id, created_at DESC) WHERE retired_at IS NULL; +CREATE INDEX IF NOT EXISTS idx_memories_encoding_affect + ON memories(encoding_affect_id) WHERE encoding_affect_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_memories_context_hash + ON memories(encoding_context_hash) WHERE encoding_context_hash IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_memories_next_review + ON memories(next_review_at) WHERE next_review_at IS NOT NULL AND retired_at IS NULL; +CREATE INDEX idx_memories_epoch ON memories(epoch_id); +CREATE INDEX idx_memories_temporal_class ON memories(temporal_class); +CREATE INDEX idx_memories_trust_score ON memories(trust_score); +CREATE INDEX idx_memories_retracted ON memories(retracted_at) WHERE retracted_at IS NOT NULL; +CREATE INDEX idx_memories_validation ON memories(validation_agent_id); +CREATE INDEX idx_memories_id_version ON memories(id, version) WHERE retired_at IS NULL; +CREATE INDEX idx_memories_type ON memories(memory_type); +CREATE INDEX idx_memories_protected ON memories(protected) WHERE protected = 1; +CREATE INDEX idx_memories_gw_broadcast ON memories(gw_broadcast) WHERE gw_broadcast = 1; +CREATE INDEX idx_memories_salience ON memories(salience_score DESC) WHERE retired_at IS NULL; +CREATE INDEX idx_memories_visibility ON memories(visibility); +CREATE INDEX idx_memories_ewc_importance ON memories(ewc_importance DESC) WHERE retired_at IS NULL; +CREATE INDEX idx_memories_alpha ON memories(alpha) WHERE retired_at IS NULL; +CREATE INDEX idx_memories_beta ON memories(beta) WHERE retired_at IS NULL; +CREATE INDEX idx_memories_confidence_phase ON memories(agent_id, confidence_phase) WHERE confidence_phase != 0.0; +CREATE INDEX idx_memories_decoherence_rate ON memories(decoherence_rate DESC) WHERE decoherence_rate IS NOT NULL; +CREATE INDEX idx_memories_coherence_syndrome ON memories(agent_id) WHERE coherence_syndrome IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_memories_replay ON memories(replay_priority DESC) WHERE retired_at IS NULL; +CREATE INDEX IF NOT EXISTS idx_memories_labile ON memories(labile_until) WHERE labile_until IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_memories_temporal_level ON memories(temporal_level, agent_id); + +CREATE VIEW decoherent_memories AS + SELECT id, content, confidence, coherence_syndrome, decoherence_rate, + temporal_class, created_at, updated_at + FROM memories + WHERE coherence_syndrome IS NOT NULL OR decoherence_rate IS NOT NULL + ORDER BY decoherence_rate DESC; + +CREATE VIRTUAL TABLE memories_fts USING fts5( + content, + category, + tags, + content=memories, + content_rowid=id, + tokenize='porter unicode61' +); + +CREATE TRIGGER memories_fts_insert AFTER INSERT ON memories WHEN new.indexed = 1 BEGIN + INSERT INTO memories_fts(rowid, content, category, tags) VALUES (new.id, new.content, new.category, new.tags); +END; + +CREATE TRIGGER memories_fts_update_delete AFTER UPDATE ON memories WHEN old.indexed = 1 BEGIN + INSERT INTO memories_fts(memories_fts, rowid, content, category, tags) + VALUES ('delete', old.id, old.content, old.category, old.tags); +END; + +CREATE TRIGGER memories_fts_update_insert AFTER UPDATE ON memories WHEN new.indexed = 1 AND new.retired_at IS NULL BEGIN + INSERT INTO memories_fts(rowid, content, category, tags) + VALUES (new.id, new.content, new.category, new.tags); +END; + +CREATE TRIGGER memories_fts_delete AFTER DELETE ON memories BEGIN + INSERT INTO memories_fts(memories_fts, rowid, content, category, tags) VALUES('delete', old.id, old.content, old.category, old.tags); +END; + +CREATE TRIGGER memories_temporal_class_check +BEFORE INSERT ON memories +WHEN NEW.temporal_class NOT IN ('permanent', 'long', 'medium', 'short', 'ephemeral') +BEGIN + SELECT RAISE(ABORT, 'temporal_class must be one of: permanent, long, medium, short, ephemeral'); +END; + +CREATE TRIGGER memories_temporal_class_update_check +BEFORE UPDATE OF temporal_class ON memories +WHEN NEW.temporal_class NOT IN ('permanent', 'long', 'medium', 'short', 'ephemeral') +BEGIN + SELECT RAISE(ABORT, 'temporal_class must be one of: permanent, long, medium, short, ephemeral'); +END; + +CREATE TRIGGER memories_validate_ts_insert +BEFORE INSERT ON memories +WHEN NEW.created_at NOT LIKE '____-__-__T%' +BEGIN + SELECT RAISE(ABORT, 'memories.created_at must be ISO 8601 (YYYY-MM-DDTHH:MM:SS)'); +END; + +CREATE TRIGGER memories_validate_ts_update +BEFORE UPDATE OF created_at ON memories +WHEN NEW.created_at NOT LIKE '____-__-__T%' +BEGIN + SELECT RAISE(ABORT, 'memories.created_at must be ISO 8601 (YYYY-MM-DDTHH:MM:SS)'); +END; + +CREATE TRIGGER IF NOT EXISTS trg_agent_delete_nullify_validation +AFTER DELETE ON agents +BEGIN + UPDATE memories + SET validation_agent_id = NULL + WHERE validation_agent_id = OLD.id; +END; + +CREATE TRIGGER meb_after_memory_insert +AFTER INSERT ON memories +BEGIN + INSERT INTO memory_events (memory_id, agent_id, operation, category, scope, memory_type, created_at) + VALUES ( + new.id, + new.agent_id, + 'insert', + new.category, + new.scope, + COALESCE(new.memory_type, 'episodic'), + strftime('%Y-%m-%dT%H:%M:%S', 'now') + ); +END; + +CREATE TRIGGER meb_after_memory_update +AFTER UPDATE OF content, category, scope, confidence, trust_score, memory_type ON memories +WHEN new.retired_at IS NULL +BEGIN + INSERT INTO memory_events (memory_id, agent_id, operation, category, scope, memory_type, created_at) + VALUES ( + new.id, + new.agent_id, + 'update', + new.category, + new.scope, + COALESCE(new.memory_type, 'episodic'), + strftime('%Y-%m-%dT%H:%M:%S', 'now') + ); +END; + +CREATE TRIGGER trg_memory_ignition_insert +AFTER INSERT ON memories +WHEN NEW.retired_at IS NULL +BEGIN + -- Compute salience: priority signal (via category) + confidence + recency boost + -- Categories map to implicit priority: decision/identity/convention = high + -- We approximate salience from confidence since we don't have event priority here. + -- Full salience scoring is done in Python; trigger handles high-confidence fast path. + INSERT INTO workspace_broadcasts (memory_id, agent_id, salience, summary, target_scope, triggered_by) + SELECT + NEW.id, + NEW.agent_id, + NEW.confidence, + substr(NEW.content, 1, 200), + COALESCE(NEW.scope, 'global'), + 'auto' + WHERE NEW.confidence >= COALESCE( + -- Use urgent threshold if neuromod org_state = 'incident', else normal + CASE + WHEN EXISTS ( + SELECT 1 FROM neuromodulation_state WHERE id = 1 AND org_state = 'incident' + ) THEN (SELECT CAST(value AS REAL) FROM workspace_config WHERE key = 'urgent_threshold') + ELSE (SELECT CAST(value AS REAL) FROM workspace_config WHERE key = 'ignition_threshold') + END, + 0.85 + ) + AND (SELECT value FROM workspace_config WHERE key = 'enabled') = '1' + -- Governor: don't fire if we've already broadcast governor_max_per_hour in last hour + AND ( + SELECT COUNT(*) FROM workspace_broadcasts + WHERE broadcast_at >= strftime('%Y-%m-%dT%H:%M:%S', datetime('now', '-1 hour')) + ) < CAST((SELECT value FROM workspace_config WHERE key = 'governor_max_per_hour') AS INTEGER); +END; + +CREATE TRIGGER trg_gw_broadcast_meb +AFTER UPDATE OF gw_broadcast ON memories +WHEN NEW.gw_broadcast = 1 AND OLD.gw_broadcast = 0 AND NEW.retired_at IS NULL +BEGIN + INSERT INTO memory_events (memory_id, agent_id, operation, category, scope, memory_type, created_at) + VALUES ( + NEW.id, + NEW.agent_id, + 'broadcast', + NEW.category, + COALESCE(NEW.scope, 'global'), + COALESCE(NEW.memory_type, 'episodic'), + strftime('%Y-%m-%dT%H:%M:%S', 'now') + ); +END; + +CREATE TRIGGER trg_gw_broadcast_workspace +AFTER UPDATE OF gw_broadcast ON memories +WHEN NEW.gw_broadcast = 1 AND OLD.gw_broadcast = 0 AND NEW.retired_at IS NULL +BEGIN + INSERT OR IGNORE INTO workspace_broadcasts (memory_id, agent_id, salience, summary, target_scope, triggered_by) + SELECT + NEW.id, + NEW.agent_id, + NEW.salience_score, + substr(NEW.content, 1, 200), + COALESCE(NEW.scope, 'global'), + 'gw_score' + WHERE NOT EXISTS ( + SELECT 1 FROM workspace_broadcasts wb WHERE wb.memory_id = NEW.id + AND wb.broadcast_at >= strftime('%Y-%m-%dT%H:%M:%S', datetime('now', '-48 hours')) + ); +END; + +CREATE TRIGGER memories_visibility_check_insert +BEFORE INSERT ON memories +WHEN NEW.visibility NOT IN ('public', 'project', 'agent', 'restricted') +BEGIN + SELECT RAISE(ABORT, 'memories.visibility must be one of: public, project, agent, restricted'); +END; + +CREATE TRIGGER memories_visibility_check_update +BEFORE UPDATE OF visibility ON memories +WHEN NEW.visibility NOT IN ('public', 'project', 'agent', 'restricted') +BEGIN + SELECT RAISE(ABORT, 'memories.visibility must be one of: public, project, agent, restricted'); +END; + +CREATE TRIGGER IF NOT EXISTS trg_memory_delete_cascade_edges +AFTER DELETE ON memories +BEGIN + DELETE FROM knowledge_edges + WHERE (source_table = 'memories' AND source_id = OLD.id) + OR (target_table = 'memories' AND target_id = OLD.id); +END; + +INSERT INTO memories_fts(memories_fts) VALUES ('rebuild'); + +CREATE TABLE IF NOT EXISTS procedures ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + memory_id INTEGER NOT NULL UNIQUE REFERENCES memories(id) ON DELETE CASCADE, + procedure_key TEXT UNIQUE, + title TEXT, + goal TEXT NOT NULL, + description TEXT, + task_family TEXT, + procedure_kind TEXT NOT NULL DEFAULT 'workflow', + trigger_conditions TEXT, + preconditions TEXT, + constraints_json TEXT, + steps_json TEXT NOT NULL, + tools_json TEXT, + failure_modes_json TEXT, + rollback_steps_json TEXT, + success_criteria_json TEXT, + repair_strategies_json TEXT, + tool_policy_json TEXT, + expected_outcomes TEXT, + applicability_scope TEXT NOT NULL DEFAULT 'global', + temporal_class TEXT DEFAULT 'durable', + status TEXT NOT NULL DEFAULT 'active' + CHECK(status IN ('active','candidate','stale','needs_review','superseded','retired')), + automation_ready INTEGER NOT NULL DEFAULT 0, + determinism REAL NOT NULL DEFAULT 0.5, + confidence REAL NOT NULL DEFAULT 0.5, + utility_score REAL NOT NULL DEFAULT 0.5, + generality_score REAL NOT NULL DEFAULT 0.5, + support_count INTEGER NOT NULL DEFAULT 0, + execution_count INTEGER NOT NULL DEFAULT 0, + success_count INTEGER NOT NULL DEFAULT 0, + failure_count INTEGER NOT NULL DEFAULT 0, + last_used_at TEXT, + last_executed_at TEXT, + last_validated_at TEXT, + stale_after_days INTEGER NOT NULL DEFAULT 90, + supersedes_procedure_id INTEGER REFERENCES procedures(id), + retired_at TEXT, + search_text TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_procedures_kind ON procedures(procedure_kind); +CREATE INDEX IF NOT EXISTS idx_procedures_status ON procedures(status); +CREATE INDEX IF NOT EXISTS idx_procedures_last_validated ON procedures(last_validated_at); +CREATE INDEX IF NOT EXISTS idx_procedures_execution_count ON procedures(execution_count DESC); +CREATE INDEX IF NOT EXISTS idx_procedures_scope ON procedures(applicability_scope); +CREATE INDEX IF NOT EXISTS idx_procedures_memory_id ON procedures(memory_id); +CREATE INDEX IF NOT EXISTS idx_procedures_supersedes ON procedures(supersedes_procedure_id); + +CREATE TABLE IF NOT EXISTS procedure_steps ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + procedure_id INTEGER NOT NULL REFERENCES procedures(id) ON DELETE CASCADE, + step_order INTEGER NOT NULL, + action TEXT NOT NULL, + rationale TEXT, + tool_name TEXT, + expected_output TEXT, + stop_condition TEXT, + retry_policy TEXT, + rollback_hint TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_procedure_steps_procedure_order +ON procedure_steps(procedure_id, step_order); + +CREATE TABLE IF NOT EXISTS procedure_sources ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + procedure_id INTEGER NOT NULL REFERENCES procedures(id) ON DELETE CASCADE, + memory_id INTEGER REFERENCES memories(id) ON DELETE CASCADE, + event_id INTEGER REFERENCES events(id) ON DELETE CASCADE, + decision_id INTEGER REFERENCES decisions(id) ON DELETE CASCADE, + entity_id INTEGER REFERENCES entities(id) ON DELETE CASCADE, + source_role TEXT NOT NULL DEFAULT 'evidence', + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_procedure_sources_procedure ON procedure_sources(procedure_id); +CREATE INDEX IF NOT EXISTS idx_procedure_sources_memory ON procedure_sources(memory_id); +CREATE INDEX IF NOT EXISTS idx_procedure_sources_event ON procedure_sources(event_id); +CREATE INDEX IF NOT EXISTS idx_procedure_sources_decision ON procedure_sources(decision_id); + +CREATE TABLE IF NOT EXISTS procedure_runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + procedure_id INTEGER NOT NULL REFERENCES procedures(id) ON DELETE CASCADE, + agent_id TEXT REFERENCES agents(id), + task_family TEXT, + task_signature TEXT, + input_summary TEXT, + outcome_summary TEXT, + success INTEGER NOT NULL DEFAULT 0, + usefulness_score REAL, + errors_seen TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_procedure_runs_procedure_created +ON procedure_runs(procedure_id, created_at DESC); + +CREATE TABLE IF NOT EXISTS procedure_candidates ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + candidate_signature TEXT NOT NULL UNIQUE, + task_family TEXT, + normalized_signature TEXT NOT NULL, + support_count INTEGER NOT NULL DEFAULT 0, + evidence_json TEXT, + mean_success REAL NOT NULL DEFAULT 0.0, + promoted_procedure_id INTEGER REFERENCES procedures(id), + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_procedure_candidates_family ON procedure_candidates(task_family); +CREATE INDEX IF NOT EXISTS idx_procedure_candidates_support ON procedure_candidates(support_count DESC); + +CREATE VIRTUAL TABLE IF NOT EXISTS procedures_fts USING fts5( + title, + goal, + description, + task_family, + search_text, + content=procedures, + content_rowid=id, + tokenize='porter unicode61' +); + +CREATE TRIGGER IF NOT EXISTS procedures_fts_insert AFTER INSERT ON procedures BEGIN + INSERT INTO procedures_fts(rowid, title, goal, description, task_family, search_text) + VALUES (new.id, new.title, new.goal, new.description, new.task_family, new.search_text); +END; + +CREATE TRIGGER IF NOT EXISTS procedures_fts_update AFTER UPDATE ON procedures BEGIN + INSERT INTO procedures_fts(procedures_fts, rowid, title, goal, description, task_family, search_text) + VALUES ('delete', old.id, old.title, old.goal, old.description, old.task_family, old.search_text); + INSERT INTO procedures_fts(rowid, title, goal, description, task_family, search_text) + VALUES (new.id, new.title, new.goal, new.description, new.task_family, new.search_text); +END; + +CREATE TRIGGER IF NOT EXISTS procedures_fts_delete AFTER DELETE ON procedures BEGIN + INSERT INTO procedures_fts(procedures_fts, rowid, title, goal, description, task_family, search_text) + VALUES ('delete', old.id, old.title, old.goal, old.description, old.task_family, old.search_text); +END; + +COMMIT; +PRAGMA foreign_keys = ON; diff --git a/src/agentmemory/_impl.py b/src/agentmemory/_impl.py index 83ccb55..dd8e50f 100644 --- a/src/agentmemory/_impl.py +++ b/src/agentmemory/_impl.py @@ -62,26 +62,26 @@ def _builtin_classify_intent(query): if any(w in q for w in ['who ', 'person', 'agent', 'team', 'assigned']): return _BuiltinIntentResult('entity_lookup', 0.8, 'keyword:entity', 'Show entity details with relations', - ['memories', 'events', 'context']) + ['memories', 'procedures', 'events', 'context']) if any(w in q for w in ['what happened', 'when did', 'history', 'timeline', 'log']): return _BuiltinIntentResult('event_lookup', 0.8, 'keyword:event', 'Show events in chronological order', - ['events', 'memories', 'context']) - if any(w in q for w in ['how to', 'how do', 'procedure', 'steps', 'guide']): + ['events', 'memories', 'context', 'procedures']) + if any(w in q for w in ['how to', 'how do', 'procedure', 'steps', 'guide', 'rollback', 'runbook', 'playbook', 'troubleshoot']): return _BuiltinIntentResult('procedural', 0.7, 'keyword:procedural', 'Show step-by-step instructions', - ['memories', 'context', 'events']) + ['procedures', 'memories', 'decisions', 'events', 'context']) if any(w in q for w in ['why ', 'decision', 'rationale', 'reason']): return _BuiltinIntentResult('decision_lookup', 0.8, 'keyword:decision', 'Show decisions with rationale', - ['memories', 'events', 'context']) + ['decisions', 'memories', 'procedures', 'events', 'context']) if any(w in q for w in ['related', 'connected', 'depends', 'link']): return _BuiltinIntentResult('graph_traversal', 0.7, 'keyword:graph', 'Show connected nodes and edges', - ['memories', 'events', 'context']) + ['memories', 'events', 'context', 'procedures']) return _BuiltinIntentResult('general', 0.5, 'default', 'Standard search results', - ['memories', 'events', 'context']) + ['memories', 'procedures', 'events', 'context']) # Quantum amplitude scorer try: @@ -157,21 +157,18 @@ def _builtin_classify_intent(query): # via `_CE_WARMUP_SEEN[0] = 0`. _CE_WARMUP_SEEN = [0] -# FTS5 special characters that cause sqlite3.OperationalError when unescaped. -# Strip them before passing any user query to a MATCH clause. -# -# Includes `?` and `!` — natural-language queries from agents and humans -# contain these constantly ("What does X prefer?") and used to crash -# cmd_search with "fts5: syntax error near ?". Also includes common ASCII -# punctuation (`,;:`) that has no operator meaning in FTS5 but still breaks -# tokenisation when glued to a word. -_FTS5_SPECIAL = re.compile(r'[.&|*"\'`()\-@^?!,;:]') +# FTS5 MATCH is brittle around punctuation and symbolic tokens. Strip any +# non-word, non-space character, plus `_`, before building the MATCH +# expression. This covers common natural-language queries like "$5 coupon", +# "LGBTQ+", "7/22", "#PlankChallenge", "SIAC_GEE", and smart quotes. +_FTS5_SPECIAL = re.compile(r"[^\w\s]|_") def _sanitize_fts_query(query: str) -> str: """Remove FTS5 special characters to prevent syntax errors. - Strips: . & | * \" ' ` ( ) - @ ^ ? ! , ; : + Strips punctuation and symbolic tokens, plus `_`, before collapsing + whitespace. Then collapses extra whitespace. Returns an empty string if nothing remains so callers can skip the MATCH clause gracefully. """ @@ -186,7 +183,22 @@ def _sanitize_fts_query(query: str) -> str: "a", "an", "and", "are", "as", "at", "be", "by", "do", "does", "for", "from", "has", "have", "how", "i", "in", "is", "it", "its", "of", "on", "or", "that", "the", "to", "was", "we", "what", "when", "where", - "which", "who", "why", "will", "with", "you", + "which", "who", "why", "will", "with", "you", "use", "uses", "used", "using", +} + +_FTS_QUERY_EXPANSIONS = { + "choose": ("chose", "chosen"), + "chose": ("choose", "chosen"), + "chosen": ("choose", "chose"), + "store": ("stores", "stored"), + "stores": ("store", "stored"), + "stored": ("store", "stores"), + "prefer": ("prefers", "preferred"), + "prefers": ("prefer",), + "embedding": ("embeddings", "embed"), + "embeddings": ("embedding", "embed"), + "use": ("uses", "using", "used"), + "uses": ("use", "using"), } @@ -208,7 +220,17 @@ def _build_fts_match_expression(sanitized: str) -> str: meaningful = [t for t in tokens if t.lower() not in _FTS_STOPWORDS and len(t) > 1] if not meaningful: meaningful = tokens - return " OR ".join(meaningful) + expanded: list[str] = [] + seen: set[str] = set() + for token in meaningful: + variants = (token, *_FTS_QUERY_EXPANSIONS.get(token.lower(), ())) + for variant in variants: + key = variant.lower() + if key in _FTS_STOPWORDS or key in seen: + continue + seen.add(key) + expanded.append(variant) + return " OR ".join(expanded or meaningful) # Temporal recency decay constants (lambda) — configurable per scope # half-life: global ~70d, project ~23d, agent ~14d @@ -3186,6 +3208,31 @@ def cmd_memory_add(args): memory_id = cursor.lastrowid db.commit() # ensure the INSERT (and FTS trigger) is committed before subprocess exit + procedure_id = None + if memory_type == "procedural": + try: + from agentmemory import procedural as _procedural + + proc = _procedural.ensure_procedure_for_memory( + db, + memory_id=memory_id, + agent_id=args.agent, + ) + procedure_id = proc.get("id") + db.commit() + except Exception as exc: + logger.debug("procedural bridge creation failed for memory %s: %s", memory_id, exc) + + indexed_row = db.execute( + "SELECT content, category, tags FROM memories WHERE id = ?", + (memory_id,), + ).fetchone() + indexed_content = indexed_row["content"] if indexed_row else args.content + indexed_category = indexed_row["category"] if indexed_row else args.category + indexed_tags = indexed_row["tags"] if indexed_row else (tags_json or "") + if indexed_content != args.content: + blob = None + # Workaround: FTS5 content-external tables may not build the inverted index # from trigger INSERTs on some SQLite versions. Force a re-index for this memory. if do_index: @@ -3193,11 +3240,11 @@ def cmd_memory_add(args): db.execute( "INSERT INTO memories_fts(memories_fts, rowid, content, category, tags) " "VALUES('delete', ?, ?, ?, ?)", - (memory_id, args.content, args.category, tags_json or '')) + (memory_id, indexed_content, indexed_category, indexed_tags or '')) db.execute( "INSERT INTO memories_fts(rowid, content, category, tags) " "VALUES (?, ?, ?, ?)", - (memory_id, args.content, args.category, tags_json or '')) + (memory_id, indexed_content, indexed_category, indexed_tags or '')) db.commit() except Exception: pass # non-fatal: FTS trigger may have already handled it @@ -3320,7 +3367,7 @@ def cmd_memory_add(args): if do_index: try: if not blob: - blob = _embed_query_safe(args.content) + blob = _embed_query_safe(indexed_content) if blob: db_vec = _try_get_db_with_vec() if db_vec: @@ -3349,6 +3396,8 @@ def cmd_memory_add(args): "conflict_logged": conflict_logged, "worthiness_score": worthiness_score, } + if procedure_id is not None: + out["procedure_id"] = procedure_id if auto_linked: out["auto_linked_entities"] = auto_linked if pii_info: @@ -6163,7 +6212,7 @@ def cmd_search(args, *, db=None, db_path: Optional[str] = None): "[brainctl] --benchmark: reranker chain disabled, returning raw FTS+vec ranking", file=sys.stderr, ) - results = {"memories": [], "events": [], "context": [], "decisions": []} + results = {"memories": [], "events": [], "context": [], "decisions": [], "procedures": []} # Accumulator for which signal-informativeness gates tripped this call. # Each value is a string reason like "uniform_timestamps_stdev_3.2s" or a # boolean True for benchmark-mode hard skips. Surfaced under the top-level @@ -6207,7 +6256,7 @@ def cmd_search(args, *, db=None, db_path: Optional[str] = None): os.environ.get("BRAINCTL_DISABLE_INTENT_ROUTER") ) if args.tables: - tables = args.tables.split(",") + tables = [t.strip() for t in args.tables.split(",") if t.strip()] elif _intent_router_disabled: tables = ["memories", "events", "context", "entities", "decisions"] elif _INTENT_AVAILABLE: @@ -6231,6 +6280,17 @@ def cmd_search(args, *, db=None, db_path: Optional[str] = None): and "decisions" not in tables ): tables = list(set(tables) | {"memories", "events", "context", "decisions"}) + _query_plan = None + _query_plan_dict = None + try: + from agentmemory.retrieval.query_planner import plan_query as _plan_query + + _query_plan = _plan_query(query, requested_tables=tables if args.tables else None) + _query_plan_dict = _query_plan.as_dict() + if not args.tables: + tables = list(dict.fromkeys((_query_plan.candidate_tables or []) + list(tables))) + except Exception as exc: + _debug_skips["query_plan.skipped"] = f"{type(exc).__name__}: {exc}" base_fetch = limit * 5 if not no_recency else limit * 3 fetch_limit = max(limit, round(base_fetch * _nm_breadth)) # Build an OR-expanded FTS5 MATCH expression so natural-language queries @@ -6298,6 +6358,7 @@ def _fts_memories(): "m.trust_score, m.replay_priority " "FROM memories m JOIN memories_fts f ON m.id = f.rowid " "WHERE memories_fts MATCH ? AND m.retired_at IS NULL " + "AND COALESCE(m.memory_type, 'episodic') != 'procedural' " "ORDER BY bm25(memories_fts, 3.0, 1.0, 1.0) LIMIT ?", (fts_query, fetch_limit) ).fetchall() @@ -6323,7 +6384,8 @@ def _vec_memories(): f"created_at, recalled_count, temporal_class, last_recalled_at, retrieval_prediction_error, alpha, beta, agent_id, " f"encoding_task_context, encoding_context_hash, q_value, confidence_phase, " f"trust_score, replay_priority " - f"FROM memories WHERE id IN ({ph}) AND retired_at IS NULL", + f"FROM memories WHERE id IN ({ph}) AND retired_at IS NULL " + f"AND COALESCE(memory_type, 'episodic') != 'procedural'", rowids ).fetchall() out = [dict(r) | {"distance": round(dist_map.get(r["id"], 1.0), 4)} for r in src_rows] @@ -6796,7 +6858,9 @@ def _apply_recency_and_trim(merged, scope_fn, use_adaptive_salience=False, bucke explore_rows = db.execute( "SELECT id, 'memory' as type, category, content, confidence, scope, " "created_at, recalled_count, temporal_class, last_recalled_at " - "FROM memories WHERE retired_at IS NULL ORDER BY recalled_count ASC, RANDOM() LIMIT ?", + "FROM memories WHERE retired_at IS NULL " + "AND COALESCE(memory_type, 'episodic') != 'procedural' " + "ORDER BY recalled_count ASC, RANDOM() LIMIT ?", (limit * 10,) ).fetchall() explore_list = rows_to_list(explore_rows) @@ -6966,6 +7030,55 @@ def _apply_recency_and_trim(merged, scope_fn, use_adaptive_salience=False, bucke trimmed.extend(graph) results["context"] = trimmed + _procedure_debug = None + _pre_answerability_candidates = [] + if "procedures" in tables: + try: + from agentmemory.retrieval.candidate_generation import generate_procedure_candidates as _generate_procedure_candidates + from agentmemory.retrieval.evidence_graph import expand_procedure_evidence as _expand_procedure_evidence + from agentmemory.retrieval.late_reranker import rerank_procedure_candidates as _rerank_procedure_candidates + from agentmemory.retrieval.query_planner import plan_query as _plan_query + + if _query_plan is None: + _query_plan = _plan_query(query, requested_tables=tables) + _query_plan_dict = _query_plan.as_dict() + proc_scope = None + if getattr(args, "project", None): + proc_scope = f"project:{args.project}" + generated = _generate_procedure_candidates( + db, + query, + _query_plan, + limit=fetch_limit, + scope=proc_scope, + ) + evidence = _expand_procedure_evidence( + db, + generated.get("candidates", []), + max_sources_per_candidate=4, + ) + reranked = _rerank_procedure_candidates( + generated.get("candidates", []), + evidence, + benchmark_mode=benchmark_mode, + ) + results["procedures"] = reranked[:limit] + _pre_answerability_candidates = list(results["procedures"]) + _procedure_debug = { + "candidate_generation": generated.get("debug") or {}, + "evidence_clusters": { + str(proc_id): { + "support_bonus": info.get("support_bonus"), + "source_count": len(info.get("sources") or []), + "edge_count": len(info.get("edges") or []), + } + for proc_id, info in evidence.items() + }, + } + except Exception as exc: + results["procedures"] = [] + _debug_skips["procedures.skipped"] = f"{type(exc).__name__}: {exc}" + # Intent-based result weighting and decision search. # # cmd_search accepts two intent taxonomies: @@ -7034,6 +7147,14 @@ def _apply_recency_and_trim(merged, scope_fn, use_adaptive_salience=False, bucke r["final_score"] = round(r.get("final_score", 0.0) * 2.0, 8) results["events"] = sorted(results.get("events", []), key=lambda r: r.get("final_score", 0), reverse=True) + elif _intent == "procedural": + for r in results.get("procedures", []): + r["final_score"] = round(r.get("final_score", 0.0) * 1.2, 8) + results["procedures"] = sorted( + results.get("procedures", []), + key=lambda r: r.get("final_score", 0.0), + reverse=True, + ) # decision_lookup → also search decisions table elif _intent == "decision_lookup": if fts_query: @@ -7068,6 +7189,65 @@ def _apply_recency_and_trim(merged, scope_fn, use_adaptive_salience=False, bucke extra = _graph_expand(db, top_items, tbl_key, already) results.get(tbl_key, []).extend(extra) + def _seed_bucket_score(item, position): + try: + final_score = float(item.get("final_score") or 0.0) + except (TypeError, ValueError): + final_score = 0.0 + if final_score > 0: + return final_score + try: + rrf_score = float(item.get("rrf_score") or 0.0) + except (TypeError, ValueError): + rrf_score = 0.0 + if rrf_score > 0: + return rrf_score + try: + fts_rank = float(item.get("fts_rank") or 0.0) + except (TypeError, ValueError): + fts_rank = 0.0 + if fts_rank != 0.0: + return max(-fts_rank, 0.0) + try: + confidence = float(item.get("confidence") or 0.0) + except (TypeError, ValueError): + confidence = 0.0 + if confidence > 0: + return confidence + return max(1.0 / (position + 1), 0.01) + + def _normalize_bucket_scores(bucket_name): + rows = results.get(bucket_name, []) or [] + if not rows: + return + seeds = [_seed_bucket_score(row, idx) for idx, row in enumerate(rows)] + max_seed = max(seeds) or 1.0 + for row, seed in zip(rows, seeds): + row["retrieval_score"] = round(seed, 8) + row["final_score"] = round(seed / max_seed, 8) + rows.sort(key=lambda r: r.get("final_score", 0.0), reverse=True) + results[bucket_name] = rows + + for _bucket_name in ("procedures", "memories", "events", "context", "entities", "decisions"): + _normalize_bucket_scores(_bucket_name) + + _intent_bucket_multipliers = { + "procedural": {"procedures": 1.15, "memories": 0.95, "events": 0.85, "decisions": 0.8, "context": 0.75}, + "troubleshooting": {"procedures": 1.05, "events": 1.0, "memories": 0.95, "decisions": 0.85, "context": 0.75}, + "decision": {"decisions": 1.15, "memories": 1.05, "procedures": 0.65, "events": 0.85, "context": 0.75}, + "temporal": {"events": 1.15, "memories": 0.9, "procedures": 0.55, "decisions": 0.8, "context": 0.75}, + "factual": {"memories": 1.1, "entities": 1.05, "decisions": 0.95, "procedures": 0.55, "events": 0.8, "context": 0.75}, + "orientation": {"memories": 1.0, "events": 0.95, "procedures": 0.75, "context": 0.8, "decisions": 0.8}, + "graph": {"memories": 1.0, "events": 0.95, "decisions": 0.95, "procedures": 0.8, "context": 0.8}, + } + _normalized_intent = (_query_plan.normalized_intent if _query_plan else "factual") + for _bucket_name, _multiplier in _intent_bucket_multipliers.get(_normalized_intent, {}).items(): + _rows = results.get(_bucket_name, []) or [] + for _row in _rows: + _row["final_score"] = round(float(_row.get("final_score") or 0.0) * _multiplier, 8) + _rows.sort(key=lambda r: r.get("final_score", 0.0), reverse=True) + results[_bucket_name] = _rows + if db_vec: db_vec.close() @@ -7082,7 +7262,7 @@ def _apply_recency_and_trim(merged, scope_fn, use_adaptive_salience=False, bucke # --budget: trim results from lowest-ranked first until output fits within token cap if budget_tokens is not None: # Estimate current size; trim tail entries until we fit - for key in ("memories", "events", "context", "decisions"): + for key in ("memories", "events", "context", "decisions", "procedures"): lst = results.get(key, []) if not lst: continue @@ -7090,6 +7270,31 @@ def _apply_recency_and_trim(merged, scope_fn, use_adaptive_salience=False, bucke lst.pop() # remove lowest-ranked (already sorted desc) results[key] = lst + _top_candidates = sorted( + [ + item + for bucket in ("procedures", "memories", "events", "context", "decisions") + for item in (results.get(bucket, []) or []) + ], + key=lambda item: item.get("final_score", 0.0), + reverse=True, + ) + _answerability = None + if _query_plan is not None: + try: + from agentmemory.retrieval.answerability import assess_answerability as _assess_answerability + + _answerability = _assess_answerability( + query, + _query_plan, + {k: results.get(k, []) for k in ("procedures", "memories", "events", "context", "decisions")}, + ) + if _answerability.get("abstain") and _query_plan.abstain_allowed: + for key in ("memories", "events", "context", "decisions", "procedures"): + results[key] = [] + except Exception as exc: + _debug_skips["answerability.skipped"] = f"{type(exc).__name__}: {exc}" + total = sum(len(v) for v in results.values()) tokens_out = _estimate_tokens(results) log_access(db, args.agent or "unknown", "search", query=query, result_count=total, tokens_consumed=tokens_out) @@ -7097,37 +7302,41 @@ def _apply_recency_and_trim(merged, scope_fn, use_adaptive_salience=False, bucke # Update recalled_count for direct (non-graph) memory hits only. # Uses retrieval-practice strengthening: hard retrievals (high prediction error) # boost confidence more than easy ones (Roediger & Karpicke 2006, Bjork 1994). - for r in results.get("memories", []): - if r.get("source") != "graph": - _retrieval_practice_boost( - db, - r["id"], - retrieval_prediction_error=r.get("retrieval_prediction_error") or 0.0, - ) + # + # Benchmark mode deliberately skips these online-learning writes so the + # retrieval corpus stays stable across repeated synthetic queries. + if not benchmark_mode: + for r in results.get("memories", []): + if r.get("source") != "graph": + _retrieval_practice_boost( + db, + r["id"], + retrieval_prediction_error=r.get("retrieval_prediction_error") or 0.0, + ) - # Online phase learning: nudge confidence_phase toward constructive (0) after recall - # Uses existing db connection to avoid lock contention with uncommitted recall_count updates. - try: - _has_phase_col = any( - col[1] == "confidence_phase" - for col in db.execute("PRAGMA table_info(memories)").fetchall() - ) - if _has_phase_col: - _delta = 0.05 - for r in results.get("memories", []): - if r.get("source") != "graph": - _pm_id = r["id"] - _pm_row = db.execute( - "SELECT confidence_phase FROM memories WHERE id=? AND retired_at IS NULL", - (_pm_id,) - ).fetchone() - if _pm_row and _pm_row[0] is not None: - import math as _pmath - _ph = float(_pm_row[0]) - _ph = (_ph + _delta if _ph > _pmath.pi else max(0.0, _ph - _delta)) % (2 * _pmath.pi) - db.execute("UPDATE memories SET confidence_phase=? WHERE id=?", (_ph, _pm_id)) - except Exception: - pass # phase learning is optional; never break search + # Online phase learning: nudge confidence_phase toward constructive (0) after recall + # Uses existing db connection to avoid lock contention with uncommitted recall_count updates. + try: + _has_phase_col = any( + col[1] == "confidence_phase" + for col in db.execute("PRAGMA table_info(memories)").fetchall() + ) + if _has_phase_col: + _delta = 0.05 + for r in results.get("memories", []): + if r.get("source") != "graph": + _pm_id = r["id"] + _pm_row = db.execute( + "SELECT confidence_phase FROM memories WHERE id=? AND retired_at IS NULL", + (_pm_id,) + ).fetchone() + if _pm_row and _pm_row[0] is not None: + import math as _pmath + _ph = float(_pm_row[0]) + _ph = (_ph + _delta if _ph > _pmath.pi else max(0.0, _ph - _delta)) % (2 * _pmath.pi) + db.execute("UPDATE memories SET confidence_phase=? WHERE id=?", (_ph, _pm_id)) + except Exception: + pass # phase learning is optional; never break search # Post-retrieval metacognitive tier annotation # Tier 1: high-confidence fresh results (≥3 direct results, avg_conf ≥ 0.7) @@ -7136,14 +7345,19 @@ def _apply_recency_and_trim(merged, scope_fn, use_adaptive_salience=False, bucke # Tier 4: coverage gap (0 direct results) # Exclude graph-expanded neighbours (source="graph") — they don't reflect query coverage memory_results = [r for r in results.get("memories", []) if r.get("source") != "graph"] + procedure_results = [r for r in results.get("procedures", []) if r.get("source") != "graph"] + direct_results = memory_results + procedure_results # Keyword/both hits: FTS5 textual matches — strongest evidence of genuine coverage - keyword_hits = [r for r in memory_results if r.get("source") in ("keyword", "both")] + keyword_hits = [ + r for r in direct_results + if r.get("source") in ("keyword", "both", "procedure_fts") + ] k_count = len(keyword_hits) - if not memory_results: + if not direct_results: tier = 4 tier_label = "gap-detected" - tier_note = "COVERAGE GAP — no memories match this query" + tier_note = "COVERAGE GAP — no grounded memories or procedures match this query" try: _log_gap(db, "coverage_hole", f"query:{_sanitize_fts_query(query)[:80]}", 1.0, triggered_by=query[:200]) except Exception: @@ -7171,19 +7385,19 @@ def _apply_recency_and_trim(merged, scope_fn, use_adaptive_salience=False, bucke elif k_count > 0: tier = 2 tier_label = "moderate" - tier_note = f"Only {k_count} keyword match(es); {len(memory_results)} total (includes semantic)" + tier_note = f"Only {k_count} direct lexical match(es); {len(direct_results)} total direct result(s)" else: tier = 3 tier_label = "weak-coverage" - tier_note = f"No keyword matches; {len(memory_results)} semantic-only result(s) — potential gap" + tier_note = f"No lexical direct matches; {len(direct_results)} semantic/procedural result(s) — potential gap" # Passive search instrumentation — append row to agent_uncertainty_log try: _unc_agent = getattr(args, "agent", None) or "unknown" _unc_domain = getattr(args, "scope", None) or (tables[0] if tables else "memories") _unc_avg_conf = None - if memory_results: - _conf_vals = [r.get("confidence") for r in memory_results if r.get("confidence") is not None] + if direct_results: + _conf_vals = [r.get("confidence") for r in direct_results if r.get("confidence") is not None] if _conf_vals: _unc_avg_conf = round(sum(_conf_vals) / len(_conf_vals), 4) db.execute( @@ -7231,12 +7445,29 @@ def _apply_recency_and_trim(merged, scope_fn, use_adaptive_salience=False, bucke except Exception: pass # trigger check is optional; never break search + _debug_payload = {} + try: + if _query_plan_dict is not None or _procedure_debug is not None or _answerability is not None: + from agentmemory.retrieval.diagnostics import build_debug_payload as _build_debug_payload + + _debug_payload = _build_debug_payload( + query_plan=_query_plan_dict or {}, + procedure_debug=_procedure_debug, + answerability=_answerability, + top_candidates=_top_candidates, + ) + except Exception as exc: + _debug_skips["diagnostics.skipped"] = f"{type(exc).__name__}: {exc}" + _out = { "mode": mode, "metacognition": { "tier": tier, "label": tier_label, "note": tier_note, + "answerability_score": (_answerability or {}).get("score"), + "answerability_reason": (_answerability or {}).get("reason"), + "abstained": (_answerability or {}).get("abstain", False), **_intent_meta, **_rollout_meta, }, @@ -7254,8 +7485,10 @@ def _apply_recency_and_trim(merged, scope_fn, use_adaptive_salience=False, bucke # "all_signals_informative" marker so downstream tooling can rely on # the key always being present in debug mode. Without `--debug` and # no skips, stay silent to keep the default response compact. - if _debug_skips: - _out["_debug"] = dict(_debug_skips) + if _debug_skips or _debug_payload: + _debug_out = dict(_debug_skips) + _debug_out.update(_debug_payload) + _out["_debug"] = _debug_out elif _debug_mode: _out["_debug"] = {"all_signals_informative": True} _ofmt = getattr(args, "output", "json") @@ -16075,8 +16308,8 @@ def build_parser(): mem_add.add_argument("--confidence", type=float) mem_add.add_argument("--tags", "-t", help="Comma-separated tags") mem_add.add_argument("--source-event", type=int) - mem_add.add_argument("--type", choices=["episodic", "semantic"], default="episodic", - help="Memory type: episodic (time-bound, faster decay) or semantic (durable facts, slower decay)") + mem_add.add_argument("--type", choices=["episodic", "semantic", "procedural"], default="episodic", + help="Memory type: episodic (time-bound, faster decay), semantic (durable facts), or procedural (structured workflows and runbooks)") mem_add.add_argument("--reflexion", action="store_true", help="Shorthand for failure lessons: sets category=lesson, auto-tags with 'reflexion'") mem_add.add_argument("--attribute", action="store_true", @@ -16142,6 +16375,13 @@ def build_parser(): mem_confidence = mem_sub.add_parser("confidence", help="Show Beta(α,β) Bayesian confidence breakdown") mem_confidence.add_argument("id", type=int, help="Memory ID") + try: + from agentmemory.commands.procedure import register_parser as _register_procedure_parser + + _register_procedure_parser(sub) + except Exception: + pass + # --- trust (top-level) --- trust = sub.add_parser("trust", help="Trust Score Engine — show, audit, calibrate, decay") trust_sub = trust.add_subparsers(dest="trust_cmd") @@ -16563,7 +16803,7 @@ def build_parser(): # --- search --- srch = sub.add_parser("search", help="Universal cross-table search") srch.add_argument("query") - srch.add_argument("--tables", help="Comma-separated: memories,events,context") + srch.add_argument("--tables", help="Comma-separated: memories,events,context,decisions,procedures") srch.add_argument("--limit", "-l", type=int, default=10) srch.add_argument("--no-recency", action="store_true", dest="no_recency", help="Disable temporal recency weighting; return raw FTS rank order") @@ -18368,6 +18608,12 @@ def main(): "confidence": cmd_memory_confidence, "pii": cmd_memory_pii, "pii-scan": cmd_memory_pii_scan} fn = dispatch.get(args.mem_cmd) + elif args.command == "procedure": + from agentmemory.commands.procedure import dispatch as _procedure_dispatch + + if _procedure_dispatch(args): + return + fn = None elif args.command == "entity": dispatch = { "create": cmd_entity_create, "get": cmd_entity_get, "search": cmd_entity_search, diff --git a/src/agentmemory/brain.py b/src/agentmemory/brain.py index a3753cd..393433a 100644 --- a/src/agentmemory/brain.py +++ b/src/agentmemory/brain.py @@ -349,31 +349,200 @@ def __del__(self) -> None: # Core: remember, search, forget # ------------------------------------------------------------------ - def remember(self, content: str, category: str = "general", tags: Optional[Union[str, List[str]]] = None, confidence: float = 1.0) -> int: + def remember( + self, + content: str, + category: str = "general", + tags: Optional[Union[str, List[str]]] = None, + confidence: float = 1.0, + *, + memory_type: str = "episodic", + scope: str = "global", + procedure: Optional[Dict[str, Any]] = None, + ) -> int: """Add a memory. Returns memory ID.""" tags_json = json.dumps(tags.split(",")) if isinstance(tags, str) else (json.dumps(tags) if tags else None) now = _now_ts() with self._lock: db = self._get_conn() - cur = db.execute( - "INSERT INTO memories (agent_id, category, content, confidence, tags, created_at, updated_at) VALUES (?,?,?,?,?,?,?)", - (self.agent_id, category, content, confidence, tags_json, now, now) - ) + if procedure is not None: + from agentmemory import procedural as _procedural + + payload = dict(procedure) + payload.setdefault("description", content) + payload.setdefault("goal", payload.get("goal") or content) + payload.setdefault("title", payload.get("title") or payload["goal"]) + payload.setdefault("steps_json", payload.get("steps_json") or [{"action": payload["goal"]}]) + result = _procedural.create_procedure( + db, + agent_id=self.agent_id, + payload=payload, + category=category, + scope=scope, + confidence=confidence, + ) + mid = int(result["memory_id"]) + else: + cur = db.execute( + """ + INSERT INTO memories ( + agent_id, category, scope, content, confidence, tags, + memory_type, created_at, updated_at + ) VALUES (?,?,?,?,?,?,?,?,?) + """, + (self.agent_id, category, scope, content, confidence, tags_json, memory_type, now, now) + ) + mid = int(cur.lastrowid) + if memory_type == "procedural": + from agentmemory import procedural as _procedural + + _procedural.ensure_procedure_for_memory(db, memory_id=mid, agent_id=self.agent_id) db.commit() - mid = cur.lastrowid if _VEC_AVAILABLE: try: - # vec.index_memory opens its own connection to the same DB; - # WAL mode handles concurrent write access cleanly, and it - # does not contend with our RLock because it's a separate - # sqlite3 connection object. Leave untouched — the async - # embedding rework is tracked separately as Phase 1.2. - _vec.index_memory(db, mid, content) + memory_row = db.execute( + "SELECT content FROM memories WHERE id = ?", + (mid,), + ).fetchone() + _vec.index_memory(db, mid, memory_row["content"] if memory_row else content) except Exception as exc: _log.warning("vec.index_memory failed for memory %s: %s", mid, exc) return mid - def search(self, query: str, limit: int = 10) -> List[Dict[str, Any]]: + def remember_procedure( + self, + *, + goal: str, + title: Optional[str] = None, + description: str = "", + steps: Optional[List[Union[str, Dict[str, Any]]]] = None, + procedure_kind: str = "workflow", + scope: str = "global", + category: str = "convention", + confidence: float = 0.9, + **extra: Any, + ) -> Dict[str, Any]: + from agentmemory import procedural as _procedural + + with self._lock: + db = self._get_conn() + result = _procedural.create_procedure( + db, + agent_id=self.agent_id, + payload={ + "title": title, + "goal": goal, + "description": description, + "procedure_kind": procedure_kind, + "steps_json": steps or [{"action": goal}], + **extra, + }, + category=category, + scope=scope, + confidence=confidence, + ) + db.commit() + return result + + def get_procedure(self, procedure_id: int) -> Dict[str, Any]: + from agentmemory import procedural as _procedural + + with self._lock: + return _procedural.get_procedure(self._get_conn(), procedure_id, include_sources=True) + + def list_procedures( + self, + *, + status: str = "all", + scope: Optional[str] = None, + limit: int = 50, + ) -> List[Dict[str, Any]]: + from agentmemory import procedural as _procedural + + with self._lock: + return _procedural.list_procedures(self._get_conn(), status=status, scope=scope, limit=limit) + + def search_procedures( + self, + query: str, + *, + limit: int = 10, + scope: Optional[str] = None, + status: str = "all", + debug: bool = False, + ) -> Dict[str, Any]: + from agentmemory import procedural as _procedural + + with self._lock: + return _procedural.search_procedures( + self._get_conn(), + query, + limit=limit, + scope=scope, + status=status, + debug=debug, + ) + + def procedure_feedback( + self, + procedure_id: int, + *, + success: bool, + usefulness_score: Optional[float] = None, + outcome_summary: Optional[str] = None, + errors_seen: Optional[str] = None, + validated: bool = False, + task_signature: Optional[str] = None, + input_summary: Optional[str] = None, + ) -> Dict[str, Any]: + from agentmemory import procedural as _procedural + + with self._lock: + db = self._get_conn() + result = _procedural.record_feedback( + db, + procedure_id=procedure_id, + agent_id=self.agent_id, + success=success, + usefulness_score=usefulness_score, + outcome_summary=outcome_summary, + errors_seen=errors_seen, + validated=validated, + task_signature=task_signature, + input_summary=input_summary, + ) + db.commit() + return result + + def backfill_procedures( + self, + *, + scope: Optional[str] = None, + limit: int = 100, + dry_run: bool = False, + ) -> Dict[str, Any]: + from agentmemory import procedural as _procedural + + with self._lock: + db = self._get_conn() + result = _procedural.backfill_procedures( + db, + agent_id=self.agent_id, + scope=scope, + limit=limit, + dry_run=dry_run, + ) + if not dry_run: + db.commit() + return result + + def search( + self, + query: str, + limit: int = 10, + *, + memory_type: Optional[str] = None, + ) -> List[Dict[str, Any]]: """Search memories via the unified hybrid + reranker pipeline. Delegates to ``agentmemory._impl.cmd_search`` so programmatic callers @@ -390,6 +559,8 @@ def search(self, query: str, limit: int = 10) -> List[Dict[str, Any]]: """ if not query or not query.strip(): return [] + if memory_type == "procedural": + return list(self.search_procedures(query, limit=limit).get("procedures") or [])[:limit] # Primary path: unified pipeline via cmd_search. try: from types import SimpleNamespace @@ -414,9 +585,12 @@ def search(self, query: str, limit: int = 10) -> List[Dict[str, Any]]: with self._lock: out = _cmd_search(args, db=self._get_conn(), db_path=str(self.db_path)) if isinstance(out, dict): - mems = out.get("memories") or [] - if isinstance(mems, list): - return mems[:limit] + combined: List[Dict[str, Any]] = [] + combined.extend(out.get("memories") or []) + combined.extend(out.get("procedures") or []) + if isinstance(combined, list): + combined.sort(key=lambda r: r.get("final_score", 0.0), reverse=True) + return combined[:limit] except Exception: # Fall through to the lightweight path — unified pipeline failures # should never take down Brain.search, which has a minimal @@ -628,7 +802,7 @@ def orient(self, project: Optional[str] = None, query: Optional[str] = None) -> except sqlite3.OperationalError: result["triggers"] = [] - # 4. Search for relevant memories (if query or project given) + # 4. Search for relevant memories and procedures (if query or project given) search_q = query or project if search_q: try: @@ -650,6 +824,7 @@ def orient(self, project: Optional[str] = None, query: Optional[str] = None) -> "SELECT m.id, m.content, m.category, m.confidence, m.created_at " "FROM memories_fts fts JOIN memories m ON m.id = fts.rowid " "WHERE memories_fts MATCH ? AND m.retired_at IS NULL " + "AND COALESCE(m.memory_type, 'episodic') != 'procedural' " "ORDER BY fts.rank LIMIT 10", (fts_q,) ).fetchall() @@ -660,6 +835,30 @@ def orient(self, project: Optional[str] = None, query: Optional[str] = None) -> result["memories"] = [] else: result["memories"] = [] + try: + if search_q: + result["procedures"] = self.search_procedures( + search_q, + limit=5, + scope=f"project:{project}" if project else None, + ).get("procedures", []) + elif result.get("handoff"): + handoff_query = " ".join( + str(result["handoff"].get(key, "") or "") + for key in ("goal", "open_loops", "next_step") + ).strip() + if handoff_query: + result["procedures"] = self.search_procedures( + handoff_query, + limit=5, + scope=f"project:{project}" if project else None, + ).get("procedures", []) + else: + result["procedures"] = [] + else: + result["procedures"] = [] + except Exception: + result["procedures"] = [] # 5. Quick stats try: @@ -667,6 +866,7 @@ def orient(self, project: Optional[str] = None, query: Optional[str] = None) -> "active_memories": db.execute( "SELECT count(*) FROM memories WHERE retired_at IS NULL" ).fetchone()[0], + "total_procedures": db.execute("SELECT count(*) FROM procedures").fetchone()[0], "total_events": db.execute("SELECT count(*) FROM events").fetchone()[0], "total_entities": db.execute("SELECT count(*) FROM entities").fetchone()[0], } @@ -844,7 +1044,16 @@ def stats(self) -> Dict[str, int]: stats: Dict[str, int] = {} with self._lock: db = self._get_conn() - for tbl in ["memories", "events", "entities", "decisions", "knowledge_edges", "affect_log"]: + for tbl in [ + "memories", + "procedures", + "procedure_candidates", + "events", + "entities", + "decisions", + "knowledge_edges", + "affect_log", + ]: try: stats[tbl] = db.execute(f"SELECT count(*) FROM {tbl}").fetchone()[0] except Exception: diff --git a/src/agentmemory/commands/procedure.py b/src/agentmemory/commands/procedure.py new file mode 100644 index 0000000..5f63c54 --- /dev/null +++ b/src/agentmemory/commands/procedure.py @@ -0,0 +1,260 @@ +"""CLI procedure commands.""" + +from __future__ import annotations + +import sqlite3 +from typing import Any + +from agentmemory import procedural + + +def _impl(): + from agentmemory import _impl + + return _impl + + +def _open_db() -> sqlite3.Connection: + return _impl().get_db() + + +def _payload_from_args(args) -> dict[str, Any]: + steps = [{"action": step} for step in (getattr(args, "step", None) or [])] + return { + "title": getattr(args, "title", None), + "goal": getattr(args, "goal", None), + "description": getattr(args, "description", None), + "task_family": getattr(args, "task_family", None), + "procedure_kind": getattr(args, "kind", None), + "trigger_conditions": getattr(args, "trigger", None) or [], + "preconditions": getattr(args, "precondition", None) or [], + "steps_json": steps, + "tools_json": getattr(args, "tool", None) or [], + "failure_modes_json": getattr(args, "failure", None) or [], + "rollback_steps_json": getattr(args, "rollback", None) or [], + "success_criteria_json": getattr(args, "success_criterion", None) or [], + "expected_outcomes": getattr(args, "expected_outcome", None) or [], + "applicability_scope": getattr(args, "scope", None) or "global", + "status": getattr(args, "status", None) or "active", + } + + +def cmd_procedure_add(args) -> None: + db = _open_db() + try: + payload = _payload_from_args(args) + result = procedural.create_procedure( + db, + agent_id=args.agent, + payload=payload, + category=args.category, + scope=args.scope, + confidence=args.confidence, + ) + db.commit() + _impl().json_out({"ok": True, **result}) + finally: + db.close() + + +def cmd_procedure_get(args) -> None: + db = _open_db() + try: + result = procedural.get_procedure(db, args.id, include_sources=True) + _impl().json_out({"ok": True, **result}) + finally: + db.close() + + +def cmd_procedure_list(args) -> None: + db = _open_db() + try: + result = procedural.list_procedures( + db, + status=args.status, + scope=args.scope, + limit=args.limit, + ) + _impl().json_out({"ok": True, "count": len(result), "procedures": result}) + finally: + db.close() + + +def cmd_procedure_search(args) -> None: + db = _open_db() + try: + result = procedural.search_procedures( + db, + args.query, + limit=args.limit, + scope=args.scope, + status=args.status, + debug=getattr(args, "debug", False), + ) + _impl().json_out(result) + finally: + db.close() + + +def cmd_procedure_update(args) -> None: + db = _open_db() + try: + changes = {k: v for k, v in _payload_from_args(args).items() if v not in (None, [], "")} + result = procedural.update_procedure(db, args.id, changes) + db.commit() + _impl().json_out({"ok": True, **result}) + finally: + db.close() + + +def cmd_procedure_feedback(args) -> None: + db = _open_db() + try: + result = procedural.record_feedback( + db, + procedure_id=args.id, + agent_id=args.agent, + success=bool(args.success), + usefulness_score=args.usefulness, + outcome_summary=args.outcome, + errors_seen=args.errors, + validated=args.validated, + task_signature=args.task_signature, + input_summary=args.input_summary, + ) + db.commit() + _impl().json_out({"ok": True, **result}) + finally: + db.close() + + +def cmd_procedure_backfill(args) -> None: + db = _open_db() + try: + result = procedural.backfill_procedures( + db, + agent_id=args.agent, + scope=args.scope, + limit=args.limit, + dry_run=args.dry_run, + ) + if not args.dry_run: + db.commit() + _impl().json_out(result) + finally: + db.close() + + +def cmd_procedure_stats(args) -> None: + db = _open_db() + try: + result = procedural.procedure_stats(db) + _impl().json_out(result) + finally: + db.close() + + +def register_parser(sub) -> None: + proc = sub.add_parser("procedure", help="Manage canonical procedural memories") + proc_sub = proc.add_subparsers(dest="procedure_cmd") + + add = proc_sub.add_parser("add", help="Create a structured procedure") + add.add_argument("--title") + add.add_argument("--goal", required=True) + add.add_argument("--description", default="") + add.add_argument("--kind", default="workflow") + add.add_argument("--task-family", dest="task_family") + add.add_argument("--category", default="convention") + add.add_argument("--scope", default="global") + add.add_argument("--confidence", type=float, default=0.9) + add.add_argument("--status", default="active") + add.add_argument("--step", action="append", default=[], help="Repeatable ordered step") + add.add_argument("--trigger", action="append", default=[]) + add.add_argument("--precondition", action="append", default=[]) + add.add_argument("--tool", action="append", default=[]) + add.add_argument("--failure", action="append", default=[]) + add.add_argument("--rollback", action="append", default=[]) + add.add_argument("--success-criterion", dest="success_criterion", action="append", default=[]) + add.add_argument("--expected-outcome", dest="expected_outcome", action="append", default=[]) + + get = proc_sub.add_parser("get", help="Fetch a procedure by id") + get.add_argument("id", type=int) + + lst = proc_sub.add_parser("list", help="List procedures") + lst.add_argument("--status", default="all") + lst.add_argument("--scope") + lst.add_argument("--limit", type=int, default=50) + + search = proc_sub.add_parser("search", help="Search procedures") + search.add_argument("query") + search.add_argument("--limit", type=int, default=10) + search.add_argument("--scope") + search.add_argument("--status", default="all") + search.add_argument("--debug", action="store_true") + + update = proc_sub.add_parser("update", help="Update a procedure") + update.add_argument("id", type=int) + update.add_argument("--title") + update.add_argument("--goal") + update.add_argument("--description") + update.add_argument("--kind") + update.add_argument("--task-family", dest="task_family") + update.add_argument("--scope") + update.add_argument("--status") + update.add_argument("--step", action="append", default=None) + update.add_argument("--trigger", action="append", default=None) + update.add_argument("--precondition", action="append", default=None) + update.add_argument("--tool", action="append", default=None) + update.add_argument("--failure", action="append", default=None) + update.add_argument("--rollback", action="append", default=None) + update.add_argument("--success-criterion", dest="success_criterion", action="append", default=None) + update.add_argument("--expected-outcome", dest="expected_outcome", action="append", default=None) + + feedback = proc_sub.add_parser("feedback", help="Record procedural execution feedback") + feedback.add_argument("id", type=int) + feedback.add_argument("--success", action="store_true", default=False) + feedback.add_argument("--failure", dest="success", action="store_false") + feedback.add_argument("--validated", action="store_true") + feedback.add_argument("--usefulness", type=float, default=None) + feedback.add_argument("--outcome", default=None) + feedback.add_argument("--errors", default=None) + feedback.add_argument("--task-signature", dest="task_signature", default=None) + feedback.add_argument("--input-summary", dest="input_summary", default=None) + + backfill = proc_sub.add_parser("backfill", help="Backfill procedures from existing evidence") + backfill.add_argument("--scope") + backfill.add_argument("--limit", type=int, default=100) + backfill.add_argument("--dry-run", action="store_true") + + proc_sub.add_parser("stats", help="Show procedure stats") + + +def dispatch(args) -> bool: + fn = { + "add": cmd_procedure_add, + "get": cmd_procedure_get, + "list": cmd_procedure_list, + "search": cmd_procedure_search, + "update": cmd_procedure_update, + "feedback": cmd_procedure_feedback, + "backfill": cmd_procedure_backfill, + "stats": cmd_procedure_stats, + }.get(getattr(args, "procedure_cmd", None)) + if not fn: + return False + fn(args) + return True + + +__all__ = [ + "cmd_procedure_add", + "cmd_procedure_backfill", + "cmd_procedure_feedback", + "cmd_procedure_get", + "cmd_procedure_list", + "cmd_procedure_search", + "cmd_procedure_stats", + "cmd_procedure_update", + "dispatch", + "register_parser", +] diff --git a/src/agentmemory/db/init_schema.sql b/src/agentmemory/db/init_schema.sql index 9bb2555..33056a9 100644 --- a/src/agentmemory/db/init_schema.sql +++ b/src/agentmemory/db/init_schema.sql @@ -59,7 +59,7 @@ CREATE TABLE memories ( retracted_at TEXT, retraction_reason TEXT, version INTEGER NOT NULL DEFAULT 1, - memory_type TEXT NOT NULL DEFAULT 'episodic' CHECK(memory_type IN ('episodic','semantic')), + memory_type TEXT NOT NULL DEFAULT 'episodic' CHECK(memory_type IN ('episodic','semantic','procedural')), protected INTEGER NOT NULL DEFAULT 0, salience_score REAL NOT NULL DEFAULT 0.0, gw_broadcast INTEGER NOT NULL DEFAULT 0, @@ -854,6 +854,162 @@ CREATE TRIGGER pm_fts_delete AFTER DELETE ON policy_memories BEGIN VALUES ('delete', old.rowid, old.trigger_condition, old.action_directive, old.name); END; +CREATE TABLE procedures ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + memory_id INTEGER NOT NULL UNIQUE REFERENCES memories(id) ON DELETE CASCADE, + procedure_key TEXT UNIQUE, + title TEXT, + goal TEXT NOT NULL, + description TEXT, + task_family TEXT, + procedure_kind TEXT NOT NULL DEFAULT 'workflow', + trigger_conditions TEXT, + preconditions TEXT, + constraints_json TEXT, + steps_json TEXT NOT NULL, + tools_json TEXT, + failure_modes_json TEXT, + rollback_steps_json TEXT, + success_criteria_json TEXT, + repair_strategies_json TEXT, + tool_policy_json TEXT, + expected_outcomes TEXT, + applicability_scope TEXT NOT NULL DEFAULT 'global', + temporal_class TEXT DEFAULT 'durable', + status TEXT NOT NULL DEFAULT 'active' + CHECK(status IN ('active','candidate','stale','needs_review','superseded','retired')), + automation_ready INTEGER NOT NULL DEFAULT 0, + determinism REAL NOT NULL DEFAULT 0.5, + confidence REAL NOT NULL DEFAULT 0.5, + utility_score REAL NOT NULL DEFAULT 0.5, + generality_score REAL NOT NULL DEFAULT 0.5, + support_count INTEGER NOT NULL DEFAULT 0, + execution_count INTEGER NOT NULL DEFAULT 0, + success_count INTEGER NOT NULL DEFAULT 0, + failure_count INTEGER NOT NULL DEFAULT 0, + last_used_at TEXT, + last_executed_at TEXT, + last_validated_at TEXT, + stale_after_days INTEGER NOT NULL DEFAULT 90, + supersedes_procedure_id INTEGER REFERENCES procedures(id), + retired_at TEXT, + search_text TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX idx_procedures_kind ON procedures(procedure_kind); + +CREATE INDEX idx_procedures_status ON procedures(status); + +CREATE INDEX idx_procedures_last_validated ON procedures(last_validated_at); + +CREATE INDEX idx_procedures_execution_count ON procedures(execution_count DESC); + +CREATE INDEX idx_procedures_scope ON procedures(applicability_scope); + +CREATE INDEX idx_procedures_memory_id ON procedures(memory_id); + +CREATE INDEX idx_procedures_supersedes ON procedures(supersedes_procedure_id); + +CREATE TABLE procedure_steps ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + procedure_id INTEGER NOT NULL REFERENCES procedures(id) ON DELETE CASCADE, + step_order INTEGER NOT NULL, + action TEXT NOT NULL, + rationale TEXT, + tool_name TEXT, + expected_output TEXT, + stop_condition TEXT, + retry_policy TEXT, + rollback_hint TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX idx_procedure_steps_procedure_order +ON procedure_steps(procedure_id, step_order); + +CREATE TABLE procedure_sources ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + procedure_id INTEGER NOT NULL REFERENCES procedures(id) ON DELETE CASCADE, + memory_id INTEGER REFERENCES memories(id) ON DELETE CASCADE, + event_id INTEGER REFERENCES events(id) ON DELETE CASCADE, + decision_id INTEGER REFERENCES decisions(id) ON DELETE CASCADE, + entity_id INTEGER REFERENCES entities(id) ON DELETE CASCADE, + source_role TEXT NOT NULL DEFAULT 'evidence', + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX idx_procedure_sources_procedure ON procedure_sources(procedure_id); + +CREATE INDEX idx_procedure_sources_memory ON procedure_sources(memory_id); + +CREATE INDEX idx_procedure_sources_event ON procedure_sources(event_id); + +CREATE INDEX idx_procedure_sources_decision ON procedure_sources(decision_id); + +CREATE TABLE procedure_runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + procedure_id INTEGER NOT NULL REFERENCES procedures(id) ON DELETE CASCADE, + agent_id TEXT REFERENCES agents(id), + task_family TEXT, + task_signature TEXT, + input_summary TEXT, + outcome_summary TEXT, + success INTEGER NOT NULL DEFAULT 0, + usefulness_score REAL, + errors_seen TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX idx_procedure_runs_procedure_created +ON procedure_runs(procedure_id, created_at DESC); + +CREATE TABLE procedure_candidates ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + candidate_signature TEXT NOT NULL UNIQUE, + task_family TEXT, + normalized_signature TEXT NOT NULL, + support_count INTEGER NOT NULL DEFAULT 0, + evidence_json TEXT, + mean_success REAL NOT NULL DEFAULT 0.0, + promoted_procedure_id INTEGER REFERENCES procedures(id), + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX idx_procedure_candidates_family ON procedure_candidates(task_family); + +CREATE INDEX idx_procedure_candidates_support ON procedure_candidates(support_count DESC); + +CREATE VIRTUAL TABLE procedures_fts USING fts5( + title, + goal, + description, + task_family, + search_text, + content=procedures, + content_rowid=id, + tokenize='porter unicode61' +); + +CREATE TRIGGER procedures_fts_insert AFTER INSERT ON procedures BEGIN + INSERT INTO procedures_fts(rowid, title, goal, description, task_family, search_text) + VALUES (new.id, new.title, new.goal, new.description, new.task_family, new.search_text); +END; + +CREATE TRIGGER procedures_fts_update AFTER UPDATE ON procedures BEGIN + INSERT INTO procedures_fts(procedures_fts, rowid, title, goal, description, task_family, search_text) + VALUES ('delete', old.id, old.title, old.goal, old.description, old.task_family, old.search_text); + INSERT INTO procedures_fts(rowid, title, goal, description, task_family, search_text) + VALUES (new.id, new.title, new.goal, new.description, new.task_family, new.search_text); +END; + +CREATE TRIGGER procedures_fts_delete AFTER DELETE ON procedures BEGIN + INSERT INTO procedures_fts(procedures_fts, rowid, title, goal, description, task_family, search_text) + VALUES ('delete', old.id, old.title, old.goal, old.description, old.task_family, old.search_text); +END; + CREATE TABLE agent_beliefs ( id INTEGER PRIMARY KEY AUTOINCREMENT, agent_id TEXT NOT NULL REFERENCES agents(id), diff --git a/src/agentmemory/hippocampus.py b/src/agentmemory/hippocampus.py index ce51238..1c3b9d0 100755 --- a/src/agentmemory/hippocampus.py +++ b/src/agentmemory/hippocampus.py @@ -720,6 +720,23 @@ def cmd_consolidate(args): print("\n[DRY RUN] No changes written.") else: print(f"\nDone. {total_clusters} cluster(s) consolidated, {total_retired} memories retired.") + try: + from agentmemory import procedural as _procedural + + synth_stats = _procedural.synthesize_procedure_candidates( + conn, + agent_id=args.agent, + dry_run=args.dry_run, + ) + print( + "Procedural synthesis: " + f"candidates_updated={synth_stats.get('candidates_updated', 0)}, " + f"promoted={synth_stats.get('promoted', 0)}" + ) + if not args.dry_run: + conn.commit() + except Exception as exc: + print(f"Procedural synthesis skipped: {exc}", file=sys.stderr) # ============================================================================= @@ -2279,6 +2296,18 @@ def cmd_consolidation_cycle(args): # Pass 7: Episodic-to-semantic promotion promotion_stats = promote_episodic_to_semantic(db) + # Pass 7b: repeated procedural traces -> procedure candidates / canonical procedures + try: + from agentmemory import procedural as _procedural + + procedural_stats = _procedural.synthesize_procedure_candidates( + db, + agent_id=args.agent, + dry_run=dry_run, + ) + except Exception as exc: + procedural_stats = {"error": str(exc), "candidates_updated": 0, "promoted": 0} + # Pass 8 (CLF): Experience replay — re-process top-10 highest-recalled memories # Prevents catastrophic forgetting by re-anchoring important old knowledge. replay_stats = experience_replay(db, top_k=10, now=now) @@ -2326,6 +2355,7 @@ def cmd_consolidation_cycle(args): "semantic_memories_created": promotion_stats.get("semantic_memories_created", 0), "source_memories_tagged": promotion_stats.get("source_memories_tagged", 0), }, + "procedural_synthesis": procedural_stats, "experience_replay": replay_stats, "hebbian": hebbian_stats, "causal_chain_mining": causal_stats, diff --git a/src/agentmemory/mcp_server.py b/src/agentmemory/mcp_server.py index 9f5f189..85c9055 100755 --- a/src/agentmemory/mcp_server.py +++ b/src/agentmemory/mcp_server.py @@ -53,6 +53,7 @@ mcp_tools_merge, mcp_tools_neuro, mcp_tools_policy, + mcp_tools_procedural, mcp_tools_reasoning, mcp_tools_reconcile, mcp_tools_reflexion, @@ -84,6 +85,7 @@ mcp_tools_merge, mcp_tools_neuro, mcp_tools_policy, + mcp_tools_procedural, mcp_tools_reasoning, mcp_tools_reconcile, mcp_tools_reflexion, @@ -126,26 +128,26 @@ def _builtin_classify_intent(query): if any(w in q for w in ['who ', 'person', 'agent', 'team', 'assigned']): return _BuiltinIntentResult('entity_lookup', 0.8, 'keyword:entity', 'Show entity details with relations', - ['memories', 'events', 'context']) + ['memories', 'procedures', 'events', 'context']) if any(w in q for w in ['what happened', 'when did', 'history', 'timeline', 'log']): return _BuiltinIntentResult('event_lookup', 0.8, 'keyword:event', 'Show events in chronological order', - ['events', 'memories', 'context']) - if any(w in q for w in ['how to', 'how do', 'procedure', 'steps', 'guide']): + ['events', 'memories', 'context', 'procedures']) + if any(w in q for w in ['how to', 'how do', 'procedure', 'steps', 'guide', 'rollback', 'runbook', 'playbook', 'troubleshoot']): return _BuiltinIntentResult('procedural', 0.7, 'keyword:procedural', 'Show step-by-step instructions', - ['memories', 'context', 'events']) + ['procedures', 'memories', 'decisions', 'events', 'context']) if any(w in q for w in ['why ', 'decision', 'rationale', 'reason']): return _BuiltinIntentResult('decision_lookup', 0.8, 'keyword:decision', 'Show decisions with rationale', - ['memories', 'events', 'context']) + ['decisions', 'memories', 'procedures', 'events', 'context']) if any(w in q for w in ['related', 'connected', 'depends', 'link']): return _BuiltinIntentResult('graph_traversal', 0.7, 'keyword:graph', 'Show connected nodes and edges', - ['memories', 'events', 'context']) + ['memories', 'events', 'context', 'procedures']) return _BuiltinIntentResult('general', 0.5, 'default', 'Standard search results', - ['memories', 'events', 'context']) + ['memories', 'procedures', 'events', 'context']) # Quantum amplitude scorer (optional re-ranking). # Ships in-tree as of 2.4.9 under agentmemory.lib.quantum_retrieval so @@ -431,8 +433,8 @@ def tool_memory_add(agent_id: str, content: str, category: str, scope: str = "gl return {"ok": False, "error": f"Invalid category: {category}. Must be one of: {', '.join(VALID_MEMORY_CATEGORIES)}"} if not (0.0 <= confidence <= 1.0): return {"ok": False, "error": "confidence must be between 0.0 and 1.0"} - if memory_type not in ("episodic", "semantic"): - return {"ok": False, "error": "memory_type must be 'episodic' or 'semantic'"} + if memory_type not in ("episodic", "semantic", "procedural"): + return {"ok": False, "error": "memory_type must be 'episodic', 'semantic', or 'procedural'"} if scope != "global" and not scope.startswith("project:") and not scope.startswith("agent:"): return {"ok": False, "error": "scope must be 'global', 'project:', or 'agent:'"} if source not in _SOURCE_TRUST_WEIGHTS: @@ -700,6 +702,27 @@ def tool_memory_add(agent_id: str, content: str, category: str, scope: str = "gl mid = cur.lastrowid db.commit() # ensure the INSERT (and FTS trigger) is committed + procedure_id = None + if memory_type == "procedural": + try: + from agentmemory import procedural as _procedural + + proc = _procedural.ensure_procedure_for_memory(db, memory_id=mid, agent_id=agent_id) + procedure_id = proc.get("id") + db.commit() + except Exception: + pass + + indexed_row = db.execute( + "SELECT content, category, tags FROM memories WHERE id = ?", + (mid,), + ).fetchone() + indexed_content = indexed_row["content"] if indexed_row else content + indexed_category = indexed_row["category"] if indexed_row else category + indexed_tags = indexed_row["tags"] if indexed_row else (tags_json or "") + if indexed_content != content: + blob = None + # Workaround: FTS5 content-external tables may not build the inverted index # from trigger INSERTs on some SQLite versions. Force a re-index for this memory. if do_index: @@ -707,11 +730,11 @@ def tool_memory_add(agent_id: str, content: str, category: str, scope: str = "gl db.execute( "INSERT INTO memories_fts(memories_fts, rowid, content, category, tags) " "VALUES('delete', ?, ?, ?, ?)", - (mid, content, category, tags_json or '')) + (mid, indexed_content, indexed_category, indexed_tags or '')) db.execute( "INSERT INTO memories_fts(rowid, content, category, tags) " "VALUES (?, ?, ?, ?)", - (mid, content, category, tags_json or '')) + (mid, indexed_content, indexed_category, indexed_tags or '')) db.commit() except Exception: pass # non-fatal @@ -752,7 +775,7 @@ def tool_memory_add(agent_id: str, content: str, category: str, scope: str = "gl if do_index: try: if not blob: - blob = _embed_safe(content) + blob = _embed_safe(indexed_content) if blob: vdb = _get_vec_db() if vdb: @@ -771,6 +794,8 @@ def tool_memory_add(agent_id: str, content: str, category: str, scope: str = "gl "surprise_score": surprise, "surprise_method": surprise_method, "source": source, "trust_score": source_trust, "memory_type": memory_type} + if procedure_id is not None: + result["procedure_id"] = procedure_id if _schema_resonance_hit: result["schema_resonance"] = _schema_resonance result["schema_resonance_fast_track"] = True @@ -806,8 +831,8 @@ def tool_memory_search(agent_id: str, query: str, category: str = None, expansion adjuncts come after). Falls through gracefully if sentence-transformers isn't installed. """ - if memory_type and memory_type not in ("episodic", "semantic"): - return {"ok": False, "error": "memory_type must be 'episodic' or 'semantic'"} + if memory_type and memory_type not in ("episodic", "semantic", "procedural"): + return {"ok": False, "error": "memory_type must be 'episodic', 'semantic', or 'procedural'"} # Cross-agent borrow restricts the SQL to `scope='global'` (line ~846). # Combining that with an explicit non-global scope produces an @@ -1719,167 +1744,45 @@ def tool_agent_wrap_up(agent_id: str, summary: str, goal: str = None, def tool_search(agent_id: str, query: str, limit: int = 20, vector: bool = False, profile: str = None) -> dict: - """Cross-table search: memories + events + entities. Intent-aware routing.""" - db = get_db() - fts_q = _safe_fts(query) - if not fts_q: + """Cross-table search routed through the canonical CLI retrieval path.""" + if not query or not str(query).strip(): return {"ok": False, "error": "Empty query"} - - # Profile: resolve task-scoped table constraints before intent routing - _profile_tables = None - _profile_categories = None - if profile: - try: - from agentmemory.profiles import resolve_profile as _resolve_profile - _prof = _resolve_profile(profile, DB_PATH) - if _prof is None: - return {"ok": False, "error": f"Unknown profile '{profile}'"} - if _prof.get("tables"): - _profile_tables = set(_prof["tables"]) - if _prof.get("categories"): - _profile_categories = _prof["categories"] - except Exception: - pass - - # Classify intent and route to appropriate tables - intent_meta = {} - intent_tables = {"memories", "events", "entities"} # default: all three - ir = None - if _INTENT_AVAILABLE: - try: - ir = _classify_intent(query) - except Exception: - ir = _builtin_classify_intent(query) - else: - ir = _builtin_classify_intent(query) - - if ir: - intent_meta = { - "intent": ir.intent, - "intent_confidence": ir.confidence, - "format_hint": ir.format_hint, - } - # Map intent tables to MCP table set (entities replaces context in MCP) - _routed = set(ir.tables) - intent_tables = set() - if "memories" in _routed: - intent_tables.add("memories") - if "events" in _routed: - intent_tables.add("events") - # entity_lookup intent: include entities; also include for all intents by default - if ir.intent == "entity_lookup" or "context" in _routed: - intent_tables.add("entities") - if not intent_tables: - intent_tables = {"memories", "events", "entities"} - - # Profile table override: if profile specifies tables, intersect with intent routing - if _profile_tables: - intent_tables = intent_tables & _profile_tables - if not intent_tables: - intent_tables = _profile_tables # use profile tables if intersection is empty - - results = [] - - if "memories" in intent_tables: - _mem_conditions = ["m.retired_at IS NULL"] - _mem_params: list = [fts_q] - if _profile_categories: - ph = ",".join("?" * len(_profile_categories)) - _mem_conditions.append(f"m.category IN ({ph})") - _mem_params.extend(_profile_categories) - _mem_params.append(limit) - _mem_where = " AND ".join(_mem_conditions) - memories = rows_to_list(db.execute( - f"SELECT m.id, 'memory' as type, m.content as text, m.category, m.confidence, m.created_at " - f"FROM memories_fts fts JOIN memories m ON m.id=fts.rowid " - f"WHERE memories_fts MATCH ? AND {_mem_where} ORDER BY rank LIMIT ?", - _mem_params - ).fetchall()) - # Quantum amplitude re-ranking — transparent to callers - if _QUANTUM_AVAILABLE and memories: - try: - memories = _quantum_rerank(memories, db_path=str(DB_PATH)) - except Exception: - pass - results.extend(memories) - - if "events" in intent_tables: - events = rows_to_list(db.execute( - "SELECT e.id, 'event' as type, e.summary as text, e.event_type as category, e.importance as confidence, e.created_at " - "FROM events_fts fts JOIN events e ON e.id=fts.rowid " - "WHERE events_fts MATCH ? ORDER BY rank LIMIT ?", - (fts_q, limit) - ).fetchall()) - results.extend(events) - - if "entities" in intent_tables: - entities = rows_to_list(db.execute( - "SELECT e.id, 'entity' as type, e.name as text, e.entity_type as category, e.confidence, e.created_at " - "FROM entities_fts fts JOIN entities e ON e.id=fts.rowid " - "WHERE entities_fts MATCH ? AND e.retired_at IS NULL ORDER BY rank LIMIT ?", - (fts_q, limit) - ).fetchall()) - results.extend(entities) - - # Vector search path (issue #19). - if vector: + try: + from types import SimpleNamespace + from agentmemory._impl import cmd_search as _cmd_search + + db = get_db() + args = SimpleNamespace( + query=query, + limit=limit, + output="return", + tables=None, + profile=profile, + no_recency=False, + no_graph=False, + budget=None, + min_salience=None, + mmr=False, + mmr_lambda=0.7, + explore=False, + benchmark=False, + agent=agent_id, + project=None, + debug=True, + quantum=False, + ) try: - from agentmemory.vec import embed_text as _embed_text - blob = _embed_text(query) - if blob: - db_vec = _get_vec_db() - if db_vec: - try: - vmem_rows = db_vec.execute( - "SELECT rowid, distance FROM vec_memories WHERE embedding MATCH ? AND k=?", - (blob, limit) - ).fetchall() - for vr in vmem_rows: - mid = vr[0] if isinstance(vr, tuple) else vr["rowid"] - dist = vr[1] if isinstance(vr, tuple) else vr["distance"] - mr = db.execute( - "SELECT id, 'memory' as type, content as text, category, confidence, created_at " - "FROM memories WHERE id = ? AND retired_at IS NULL", (mid,) - ).fetchone() - if mr: - row = dict(mr) - row["_vscore"] = round(1.0 - float(dist), 4) - row["source_type"] = "memory" - results.append(row) - vent_rows = db_vec.execute( - "SELECT rowid, distance FROM vec_entities WHERE embedding MATCH ? AND k=?", - (blob, limit) - ).fetchall() - for vr in vent_rows: - eid = vr[0] if isinstance(vr, tuple) else vr["rowid"] - dist = vr[1] if isinstance(vr, tuple) else vr["distance"] - er = db.execute( - "SELECT id, 'entity' as type, name as text, entity_type as category, confidence, created_at " - "FROM entities WHERE id = ? AND retired_at IS NULL", (eid,) - ).fetchone() - if er: - row = dict(er) - row["_vscore"] = round(1.0 - float(dist), 4) - row["source_type"] = "entity" - results.append(row) - finally: - db_vec.close() - seen = set() - deduped = [] - for r in results: - key = (r.get("type", ""), r.get("id", "")) - if key not in seen: - seen.add(key) - deduped.append(r) - results = sorted(deduped, key=lambda r: -r.get("_vscore", 0.0)) - for r in results: - r.pop("_vscore", None) - except Exception: - pass - - log_access(db, agent_id, "search", query=query, result_count=len(results)) - db.commit(); db.close() - return {"ok": True, "count": len(results), "results": results, **intent_meta} + out = _cmd_search(args, db=db, db_path=str(DB_PATH)) + finally: + db.close() + if not isinstance(out, dict): + return {"ok": False, "error": "search returned no result payload"} + if vector: + out.setdefault("metacognition", {}) + out["metacognition"]["vector_hint"] = "hybrid retrieval is automatic when sqlite-vec is available" + return {"ok": True, **out} + except Exception as exc: + return {"ok": False, "error": str(exc)} def tool_pagerank(table: str = None, damping: float = 0.85, iterations: int = 20, @@ -2176,7 +2079,7 @@ def tool_resolve_conflict( "scope": {"type": "string", "description": "Scope: 'global', 'project:', or 'agent:'", "default": "global"}, "confidence": {"type": "number", "description": "Confidence 0.0-1.0", "default": 1.0}, "tags": {"type": "string", "description": "Comma-separated tags"}, - "memory_type": {"type": "string", "enum": ["episodic", "semantic"], "default": "episodic"}, + "memory_type": {"type": "string", "enum": ["episodic", "semantic", "procedural"], "default": "episodic"}, "force": {"type": "boolean", "description": "Bypass W(m) worthiness gate", "default": False}, "supersedes_id": {"type": "integer", "description": "ID of memory being superseded; triggers PII recency gate"}, "source": { @@ -2203,7 +2106,7 @@ def tool_resolve_conflict( "category": {"type": "string", "enum": VALID_MEMORY_CATEGORIES}, "scope": {"type": "string"}, "limit": {"type": "integer", "default": 20, "description": "Max results; capped by agent tier (7 × tier)"}, - "memory_type": {"type": "string", "enum": ["episodic", "semantic"], "description": "Filter to one CLS store. Unset = both stores, semantic gets 1.1x confidence bonus."}, + "memory_type": {"type": "string", "enum": ["episodic", "semantic", "procedural"], "description": "Filter to one memory store. Unset searches all supported memory types; semantic gets a mild confidence bonus in memory_search."}, "pagerank_boost": {"type": "number", "default": 0.0, "description": "Re-rank by graph centrality (0=FTS-only, 1=equal FTS+PageRank). Requires prior pagerank run. Implements SR retrieval."}, "borrow_from": {"type": "string", "description": "Agent ID to borrow from. When set, searches only that agent's scope='global' memories and logs the cross-agent access in access_log."}, "multi_pass": {"type": "boolean", "default": False, "description": "SDM-style iterative convergence: use pass-1 results to build a richer pass-2 query; merge and deduplicate both passes (items in both passes ranked first)."}, diff --git a/src/agentmemory/mcp_tools_meb.py b/src/agentmemory/mcp_tools_meb.py index 203f0bb..d15dc19 100644 --- a/src/agentmemory/mcp_tools_meb.py +++ b/src/agentmemory/mcp_tools_meb.py @@ -50,8 +50,10 @@ def _find_vec_dylib(): _MEB_TTL_HOURS_DEFAULT = 72 _MEB_MAX_DEPTH_DEFAULT = 10_000 -# FTS5 special characters — strip everything that isn't word chars or spaces -_FTS5_SPECIAL = re.compile(r'[.&|*"()\-@^?!]') +# FTS5 MATCH is brittle around punctuation and symbolic tokens. Strip any +# non-word, non-space character, plus `_`, so questions like "$5 coupon" or +# "LGBTQ+" cannot crash the tool path. +_FTS5_SPECIAL = re.compile(r"[^\w\s]|_") # --------------------------------------------------------------------------- # DB helpers diff --git a/src/agentmemory/mcp_tools_procedural.py b/src/agentmemory/mcp_tools_procedural.py new file mode 100644 index 0000000..d487347 --- /dev/null +++ b/src/agentmemory/mcp_tools_procedural.py @@ -0,0 +1,324 @@ +"""brainctl MCP tools — procedural memory system.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from mcp.types import Tool + +from agentmemory import procedural +from agentmemory.lib.mcp_helpers import open_db +from agentmemory.paths import get_db_path + +DB_PATH: Path = get_db_path() + + +def _db(): + conn = open_db(str(DB_PATH)) + procedural.ensure_procedure_schema(conn) + return conn + + +def tool_procedure_add( + agent_id: str = "mcp-client", + goal: str = "", + title: str | None = None, + description: str | None = None, + procedure_kind: str = "workflow", + task_family: str | None = None, + scope: str = "global", + category: str = "convention", + confidence: float = 0.9, + steps: list[str] | None = None, + trigger_conditions: list[str] | None = None, + preconditions: list[str] | None = None, + tools: list[str] | None = None, + failure_modes: list[str] | None = None, + rollback_steps: list[str] | None = None, + success_criteria: list[str] | None = None, + expected_outcomes: list[str] | None = None, + status: str = "active", + **_kw: Any, +) -> dict[str, Any]: + if not goal: + return {"ok": False, "error": "goal is required"} + db = _db() + try: + payload = { + "title": title, + "goal": goal, + "description": description or "", + "procedure_kind": procedure_kind, + "task_family": task_family, + "steps_json": [{"action": step} for step in (steps or [])], + "trigger_conditions": trigger_conditions or [], + "preconditions": preconditions or [], + "tools_json": tools or [], + "failure_modes_json": failure_modes or [], + "rollback_steps_json": rollback_steps or [], + "success_criteria_json": success_criteria or [], + "expected_outcomes": expected_outcomes or [], + "applicability_scope": scope, + "status": status, + } + result = procedural.create_procedure( + db, + agent_id=agent_id, + payload=payload, + category=category, + scope=scope, + confidence=confidence, + ) + db.commit() + return {"ok": True, **result} + except Exception as exc: + return {"ok": False, "error": str(exc)} + finally: + db.close() + + +def tool_procedure_get(procedure_id: int, **_kw: Any) -> dict[str, Any]: + db = _db() + try: + return {"ok": True, **procedural.get_procedure(db, procedure_id, include_sources=True)} + except Exception as exc: + return {"ok": False, "error": str(exc)} + finally: + db.close() + + +def tool_procedure_list(status: str = "all", scope: str | None = None, limit: int = 50, **_kw: Any) -> dict[str, Any]: + db = _db() + try: + items = procedural.list_procedures(db, status=status, scope=scope, limit=limit) + return {"ok": True, "procedures": items, "count": len(items)} + finally: + db.close() + + +def tool_procedure_search(query: str, limit: int = 10, scope: str | None = None, status: str = "all", debug: bool = False, **_kw: Any) -> dict[str, Any]: + if not query: + return {"ok": False, "error": "query is required"} + db = _db() + try: + return procedural.search_procedures(db, query, limit=limit, scope=scope, status=status, debug=debug) + finally: + db.close() + + +def tool_procedure_update(procedure_id: int, **changes: Any) -> dict[str, Any]: + db = _db() + try: + normalized = dict(changes) + if normalized.get("steps") is not None: + normalized["steps_json"] = [{"action": step} for step in normalized.pop("steps") or []] + if normalized.get("tools") is not None: + normalized["tools_json"] = normalized.pop("tools") + if normalized.get("trigger_conditions") is not None: + normalized["trigger_conditions"] = normalized["trigger_conditions"] + result = procedural.update_procedure(db, procedure_id, normalized) + db.commit() + return {"ok": True, **result} + except Exception as exc: + return {"ok": False, "error": str(exc)} + finally: + db.close() + + +def tool_procedure_feedback( + procedure_id: int, + agent_id: str = "mcp-client", + success: bool = True, + usefulness_score: float | None = None, + outcome_summary: str | None = None, + errors_seen: str | None = None, + validated: bool = False, + task_signature: str | None = None, + input_summary: str | None = None, + **_kw: Any, +) -> dict[str, Any]: + db = _db() + try: + result = procedural.record_feedback( + db, + procedure_id=procedure_id, + agent_id=agent_id, + success=success, + usefulness_score=usefulness_score, + outcome_summary=outcome_summary, + errors_seen=errors_seen, + validated=validated, + task_signature=task_signature, + input_summary=input_summary, + ) + db.commit() + return {"ok": True, **result} + except Exception as exc: + return {"ok": False, "error": str(exc)} + finally: + db.close() + + +def tool_procedure_backfill(agent_id: str = "mcp-client", scope: str | None = None, limit: int = 100, dry_run: bool = False, **_kw: Any) -> dict[str, Any]: + db = _db() + try: + result = procedural.backfill_procedures( + db, + agent_id=agent_id, + scope=scope, + limit=limit, + dry_run=dry_run, + ) + if not dry_run: + db.commit() + return result + finally: + db.close() + + +def tool_procedure_stats(**_kw: Any) -> dict[str, Any]: + db = _db() + try: + return procedural.procedure_stats(db) + finally: + db.close() + + +TOOLS = [ + Tool( + name="procedure_add", + description="Create a canonical structured procedure with ordered steps and provenance.", + inputSchema={ + "type": "object", + "properties": { + "agent_id": {"type": "string"}, + "goal": {"type": "string"}, + "title": {"type": "string"}, + "description": {"type": "string"}, + "procedure_kind": {"type": "string"}, + "task_family": {"type": "string"}, + "scope": {"type": "string", "default": "global"}, + "category": {"type": "string", "default": "convention"}, + "confidence": {"type": "number", "default": 0.9}, + "steps": {"type": "array", "items": {"type": "string"}}, + "trigger_conditions": {"type": "array", "items": {"type": "string"}}, + "preconditions": {"type": "array", "items": {"type": "string"}}, + "tools": {"type": "array", "items": {"type": "string"}}, + "failure_modes": {"type": "array", "items": {"type": "string"}}, + "rollback_steps": {"type": "array", "items": {"type": "string"}}, + "success_criteria": {"type": "array", "items": {"type": "string"}}, + "expected_outcomes": {"type": "array", "items": {"type": "string"}}, + "status": {"type": "string", "default": "active"}, + }, + "required": ["goal"], + }, + ), + Tool( + name="procedure_get", + description="Get a procedure by id.", + inputSchema={"type": "object", "properties": {"procedure_id": {"type": "integer"}}, "required": ["procedure_id"]}, + ), + Tool( + name="procedure_list", + description="List procedures with optional scope/status filters.", + inputSchema={ + "type": "object", + "properties": { + "status": {"type": "string", "default": "all"}, + "scope": {"type": "string"}, + "limit": {"type": "integer", "default": 50}, + }, + }, + ), + Tool( + name="procedure_search", + description="Search structured procedural memories.", + inputSchema={ + "type": "object", + "properties": { + "query": {"type": "string"}, + "limit": {"type": "integer", "default": 10}, + "scope": {"type": "string"}, + "status": {"type": "string", "default": "all"}, + "debug": {"type": "boolean", "default": False}, + }, + "required": ["query"], + }, + ), + Tool( + name="procedure_update", + description="Update a procedure.", + inputSchema={ + "type": "object", + "properties": { + "procedure_id": {"type": "integer"}, + "title": {"type": "string"}, + "goal": {"type": "string"}, + "description": {"type": "string"}, + "procedure_kind": {"type": "string"}, + "task_family": {"type": "string"}, + "status": {"type": "string"}, + "scope": {"type": "string"}, + "steps": {"type": "array", "items": {"type": "string"}}, + "tools": {"type": "array", "items": {"type": "string"}}, + "trigger_conditions": {"type": "array", "items": {"type": "string"}}, + "preconditions": {"type": "array", "items": {"type": "string"}}, + "failure_modes_json": {"type": "array", "items": {"type": "string"}}, + "rollback_steps_json": {"type": "array", "items": {"type": "string"}}, + "success_criteria_json": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["procedure_id"], + }, + ), + Tool( + name="procedure_feedback", + description="Record procedural execution feedback and validation outcome.", + inputSchema={ + "type": "object", + "properties": { + "procedure_id": {"type": "integer"}, + "agent_id": {"type": "string"}, + "success": {"type": "boolean", "default": True}, + "usefulness_score": {"type": "number"}, + "outcome_summary": {"type": "string"}, + "errors_seen": {"type": "string"}, + "validated": {"type": "boolean", "default": False}, + "task_signature": {"type": "string"}, + "input_summary": {"type": "string"}, + }, + "required": ["procedure_id"], + }, + ), + Tool( + name="procedure_backfill", + description="Backfill or synthesize procedures from existing memories, events, and decisions.", + inputSchema={ + "type": "object", + "properties": { + "agent_id": {"type": "string"}, + "scope": {"type": "string"}, + "limit": {"type": "integer", "default": 100}, + "dry_run": {"type": "boolean", "default": False}, + }, + }, + ), + Tool( + name="procedure_stats", + description="Show procedure counts and candidate promotion stats.", + inputSchema={"type": "object", "properties": {}}, + ), +] + + +DISPATCH = { + "procedure_add": tool_procedure_add, + "procedure_get": tool_procedure_get, + "procedure_list": tool_procedure_list, + "procedure_search": tool_procedure_search, + "procedure_update": tool_procedure_update, + "procedure_feedback": tool_procedure_feedback, + "procedure_backfill": tool_procedure_backfill, + "procedure_stats": tool_procedure_stats, +} + diff --git a/src/agentmemory/procedural.py b/src/agentmemory/procedural.py new file mode 100644 index 0000000..6a5b9ca --- /dev/null +++ b/src/agentmemory/procedural.py @@ -0,0 +1,1679 @@ +"""Procedural memory service layer. + +Canonical procedures live in dedicated tables and are bridged back to the +generic ``memories`` table through ``procedures.memory_id`` so the legacy +memory/search surfaces still have a human-readable synopsis row. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import sqlite3 +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import Any, Iterable, Optional + +PROCEDURE_STATUSES = { + "active", + "candidate", + "stale", + "needs_review", + "superseded", + "retired", +} + +PROCEDURE_KINDS = { + "workflow", + "runbook", + "playbook", + "troubleshooting", + "rollback", + "recipe", + "routine", +} + +_STEP_RE = re.compile(r"^\s*(?:\d+[\).\:-]|[-*•])\s+(?P.+?)\s*$") +_IF_THEN_RE = re.compile(r"\bif\s+(.+?)\s+then\s+(.+)", re.IGNORECASE) +_ROLLBACK_RE = re.compile(r"\b(rollback|roll back|revert|undo)\b", re.IGNORECASE) +_HOW_TO_RE = re.compile(r"^\s*how\s+(?:to|do|does|can|should)\s+", re.IGNORECASE) +_TOOL_RE = re.compile(r"\b(?:run|use|with|via|invoke)\s+([A-Za-z0-9_./:-]+)") +_LIST_SPLIT_RE = re.compile(r"\b(?:first|then|next|after that|finally|lastly)\b", re.IGNORECASE) +_BULLET_RE = re.compile(r"[•*\-]\s+") +_TOKEN_RE = re.compile(r"[a-z0-9_./:-]+") + +_STOPWORDS = { + "a", + "an", + "and", + "are", + "as", + "at", + "be", + "by", + "for", + "from", + "how", + "i", + "if", + "in", + "is", + "it", + "of", + "on", + "or", + "that", + "the", + "then", + "to", + "use", + "using", + "when", + "with", +} + + +@dataclass(slots=True) +class ProcedureRecord: + procedure_id: int + memory_id: int + title: str + goal: str + procedure_kind: str + status: str + + +def now_iso() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def _json_dumps(value: Any) -> str: + return json.dumps(value or [], ensure_ascii=True) + + +def _json_loads_list(value: Any) -> list[Any]: + if value in (None, ""): + return [] + if isinstance(value, list): + return value + try: + parsed = json.loads(value) + except Exception: + return [] + return parsed if isinstance(parsed, list) else [] + + +def _json_loads_obj(value: Any) -> dict[str, Any]: + if value in (None, ""): + return {} + if isinstance(value, dict): + return value + try: + parsed = json.loads(value) + except Exception: + return {} + return parsed if isinstance(parsed, dict) else {} + + +def _tokenize(text: str) -> list[str]: + return [ + tok + for tok in _TOKEN_RE.findall((text or "").lower()) + if tok not in _STOPWORDS and len(tok) > 1 + ] + + +def _sentence_split(text: str) -> list[str]: + if not text: + return [] + parts = re.split(r"(?<=[.!?])\s+|\r?\n+", text.strip()) + return [p.strip(" -\t\r\n") for p in parts if p.strip(" -\t\r\n")] + + +def _slugify(text: str) -> str: + slug = re.sub(r"[^a-z0-9]+", "-", (text or "").lower()).strip("-") + return slug[:80] or "procedure" + + +def _procedure_key(title: str, goal: str, scope: str) -> str: + stem = f"{_slugify(title or goal)}:{scope or 'global'}:{goal or title}" + digest = hashlib.sha1(stem.encode("utf-8")).hexdigest()[:10] + return f"{_slugify(title or goal)}-{digest}" + + +def _normalize_step_item(step: Any) -> dict[str, Any]: + if isinstance(step, str): + return {"action": step.strip()} + if isinstance(step, dict): + action = (step.get("action") or step.get("step") or "").strip() + out = { + "action": action, + "rationale": (step.get("rationale") or "").strip() or None, + "tool_name": (step.get("tool_name") or step.get("tool") or "").strip() or None, + "expected_output": (step.get("expected_output") or "").strip() or None, + "stop_condition": (step.get("stop_condition") or "").strip() or None, + "retry_policy": (step.get("retry_policy") or "").strip() or None, + "rollback_hint": (step.get("rollback_hint") or "").strip() or None, + } + return {k: v for k, v in out.items() if v is not None or k == "action"} + return {"action": str(step).strip()} + + +def _normalize_steps(steps: Iterable[Any]) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + for raw in steps: + step = _normalize_step_item(raw) + if step.get("action"): + out.append(step) + return out + + +def _extract_tools(text: str, steps: list[dict[str, Any]]) -> list[str]: + tools: list[str] = [] + for step in steps: + if step.get("tool_name"): + tools.append(step["tool_name"]) + for match in _TOOL_RE.findall(step.get("action") or ""): + tools.append(match) + for match in _TOOL_RE.findall(text or ""): + tools.append(match) + seen: set[str] = set() + deduped: list[str] = [] + for tool in tools: + key = tool.lower() + if key not in seen: + seen.add(key) + deduped.append(tool) + return deduped + + +def _guess_kind(text: str) -> str: + lower = (text or "").lower() + if _ROLLBACK_RE.search(lower): + return "rollback" + if any(word in lower for word in ("troubleshoot", "debug", "fix ", "error", "failure", "incident")): + return "troubleshooting" + if any(word in lower for word in ("playbook", "runbook")): + return "runbook" + if any(word in lower for word in ("routine", "repeat", "recurring")): + return "routine" + if any(word in lower for word in ("recipe", "tool use", "tool-use")): + return "recipe" + return "workflow" + + +def looks_procedural(text: str) -> bool: + if not text or len(text.strip()) < 12: + return False + lowered = text.lower() + if _HOW_TO_RE.search(text): + return True + if _IF_THEN_RE.search(text): + return True + if _ROLLBACK_RE.search(text): + return True + if any(_STEP_RE.match(line) for line in text.splitlines()): + return True + hints = ( + "steps", + "first", + "then", + "finally", + "run ", + "deploy", + "rollback", + "revert", + "restart", + "apply migrations", + "troubleshoot", + "before ", + "after ", + ) + return sum(1 for hint in hints if hint in lowered) >= 2 + + +def parse_procedural_text( + text: str, + *, + title: Optional[str] = None, + goal: Optional[str] = None, + procedure_kind: Optional[str] = None, + scope: str = "global", +) -> dict[str, Any]: + """Deterministically coerce free text into a structured procedure payload.""" + + original = (text or "").strip() + lines = [ln.strip() for ln in original.splitlines() if ln.strip()] + steps: list[dict[str, Any]] = [] + triggers: list[str] = [] + preconditions: list[str] = [] + rollback_steps: list[str] = [] + failure_modes: list[str] = [] + success_criteria: list[str] = [] + + for line in lines: + match = _STEP_RE.match(line) + if match: + body = match.group("step").strip() + steps.append({"action": body}) + if _ROLLBACK_RE.search(body): + rollback_steps.append(body) + if "if " in line.lower(): + m = _IF_THEN_RE.search(line) + if m: + triggers.append(m.group(1).strip()) + steps.append({"action": m.group(2).strip()}) + else: + triggers.append(line) + if any(token in line.lower() for token in ("before ", "requires ", "ensure ", "must ", "need to ")): + preconditions.append(line) + if any(token in line.lower() for token in ("failure", "error", "incident", "stuck", "syntax error")): + failure_modes.append(line) + if any(token in line.lower() for token in ("success", "done when", "healthy", "green", "validated")): + success_criteria.append(line) + + if not steps and original: + split_chunks = [chunk.strip(" .") for chunk in _LIST_SPLIT_RE.split(original) if chunk.strip(" .")] + if len(split_chunks) > 1: + steps = [{"action": chunk} for chunk in split_chunks] + + if not steps and original: + sentences = _sentence_split(original) + if len(sentences) > 1: + steps = [{"action": sentence} for sentence in sentences] + + if not steps and original: + steps = [{"action": original}] + + steps = _normalize_steps(steps) + tools = _extract_tools(original, steps) + kind = procedure_kind or _guess_kind(original) + + if not goal: + for sentence in _sentence_split(original): + cleaned = _HOW_TO_RE.sub("", sentence).strip(" .:-") + if cleaned: + goal = cleaned[0].upper() + cleaned[1:] if len(cleaned) > 1 else cleaned + break + goal = goal or (steps[0]["action"] if steps else "Complete the procedure safely") + + if not title: + title = goal + if len(title) > 96: + title = title[:93].rstrip() + "..." + + expected_outcomes: list[str] = [] + if success_criteria: + expected_outcomes.extend(success_criteria) + elif "deploy" in original.lower(): + expected_outcomes.append("Deployment completes and target environment is healthy.") + elif "rollback" in original.lower(): + expected_outcomes.append("System returns to the last known good state.") + elif "migrat" in original.lower(): + expected_outcomes.append("Schema changes apply cleanly and services remain healthy.") + + if not rollback_steps and kind == "rollback": + rollback_steps = [step["action"] for step in steps] + elif not rollback_steps: + rollback_steps = [line for line in lines if _ROLLBACK_RE.search(line)] + + search_text = compose_search_text( + { + "title": title, + "goal": goal, + "description": original, + "procedure_kind": kind, + "trigger_conditions": triggers, + "preconditions": preconditions, + "steps_json": steps, + "tools_json": tools, + "failure_modes_json": failure_modes, + "rollback_steps_json": rollback_steps, + "success_criteria_json": success_criteria, + "expected_outcomes": expected_outcomes, + "applicability_scope": scope, + } + ) + return { + "title": title, + "goal": goal, + "description": original, + "procedure_kind": kind, + "trigger_conditions": triggers, + "preconditions": preconditions, + "steps_json": steps, + "tools_json": tools, + "failure_modes_json": failure_modes, + "rollback_steps_json": rollback_steps, + "success_criteria_json": success_criteria, + "expected_outcomes": expected_outcomes, + "applicability_scope": scope, + "status": "active", + "automation_ready": 1 if tools else 0, + "determinism": 0.7 if len(steps) > 1 else 0.45, + "constraints_json": [], + "repair_strategies_json": rollback_steps or failure_modes, + "tool_policy_json": tools, + "task_family": kind, + "search_text": search_text, + } + + +def compose_search_text(payload: dict[str, Any]) -> str: + parts: list[str] = [] + for key in ( + "title", + "goal", + "description", + "task_family", + "procedure_kind", + "applicability_scope", + "expected_outcomes", + ): + value = payload.get(key) + if isinstance(value, str): + parts.append(value) + elif isinstance(value, list): + parts.extend(str(v) for v in value) + + for key in ( + "trigger_conditions", + "preconditions", + "tools_json", + "failure_modes_json", + "rollback_steps_json", + "success_criteria_json", + "constraints_json", + "repair_strategies_json", + "tool_policy_json", + ): + values = payload.get(key) + if isinstance(values, list): + parts.extend(str(v) for v in values) + + for step in _normalize_steps(payload.get("steps_json") or []): + parts.extend(str(v) for v in step.values() if v) + + text = " ".join(part for part in parts if part) + return re.sub(r"\s+", " ", text).strip() + + +def compose_synopsis(payload: dict[str, Any]) -> str: + title = payload.get("title") or payload.get("goal") or "Procedure" + goal = payload.get("goal") or title + steps = _normalize_steps(payload.get("steps_json") or []) + lead = f"{title}. Goal: {goal}." + if steps: + preview = " ".join( + f"{idx + 1}. {step['action']}" + for idx, step in enumerate(steps[:4]) + if step.get("action") + ) + lead += f" Steps: {preview}." + rollback = _json_loads_list(payload.get("rollback_steps_json")) + if rollback: + lead += f" Rollback: {rollback[0]}" + if len(rollback) > 1: + lead += f"; then {rollback[1]}" + lead += "." + tools = _json_loads_list(payload.get("tools_json")) + if tools: + lead += f" Tools: {', '.join(str(t) for t in tools[:5])}." + return re.sub(r"\s+", " ", lead).strip() + + +def ensure_procedure_schema(conn: sqlite3.Connection) -> None: + """Best-effort local guard so procedural APIs work on legacy DBs too.""" + + if conn.row_factory is None: + conn.row_factory = sqlite3.Row + + conn.executescript( + """ + CREATE TABLE IF NOT EXISTS procedures ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + memory_id INTEGER NOT NULL UNIQUE REFERENCES memories(id) ON DELETE CASCADE, + procedure_key TEXT UNIQUE, + title TEXT, + goal TEXT NOT NULL, + description TEXT, + task_family TEXT, + procedure_kind TEXT NOT NULL DEFAULT 'workflow', + trigger_conditions TEXT, + preconditions TEXT, + constraints_json TEXT, + steps_json TEXT NOT NULL, + tools_json TEXT, + failure_modes_json TEXT, + rollback_steps_json TEXT, + success_criteria_json TEXT, + repair_strategies_json TEXT, + tool_policy_json TEXT, + expected_outcomes TEXT, + applicability_scope TEXT NOT NULL DEFAULT 'global', + temporal_class TEXT DEFAULT 'durable', + status TEXT NOT NULL DEFAULT 'active', + automation_ready INTEGER NOT NULL DEFAULT 0, + determinism REAL NOT NULL DEFAULT 0.5, + confidence REAL NOT NULL DEFAULT 0.5, + utility_score REAL NOT NULL DEFAULT 0.5, + generality_score REAL NOT NULL DEFAULT 0.5, + support_count INTEGER NOT NULL DEFAULT 0, + execution_count INTEGER NOT NULL DEFAULT 0, + success_count INTEGER NOT NULL DEFAULT 0, + failure_count INTEGER NOT NULL DEFAULT 0, + last_used_at TEXT, + last_executed_at TEXT, + last_validated_at TEXT, + stale_after_days INTEGER NOT NULL DEFAULT 90, + supersedes_procedure_id INTEGER REFERENCES procedures(id), + retired_at TEXT, + search_text TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + CREATE INDEX IF NOT EXISTS idx_procedures_kind ON procedures(procedure_kind); + CREATE INDEX IF NOT EXISTS idx_procedures_status ON procedures(status); + CREATE INDEX IF NOT EXISTS idx_procedures_last_validated ON procedures(last_validated_at); + CREATE INDEX IF NOT EXISTS idx_procedures_execution_count ON procedures(execution_count DESC); + CREATE INDEX IF NOT EXISTS idx_procedures_scope ON procedures(applicability_scope); + CREATE INDEX IF NOT EXISTS idx_procedures_memory_id ON procedures(memory_id); + CREATE INDEX IF NOT EXISTS idx_procedures_supersedes ON procedures(supersedes_procedure_id); + + CREATE TABLE IF NOT EXISTS procedure_steps ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + procedure_id INTEGER NOT NULL REFERENCES procedures(id) ON DELETE CASCADE, + step_order INTEGER NOT NULL, + action TEXT NOT NULL, + rationale TEXT, + tool_name TEXT, + expected_output TEXT, + stop_condition TEXT, + retry_policy TEXT, + rollback_hint TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + CREATE INDEX IF NOT EXISTS idx_procedure_steps_procedure_order + ON procedure_steps(procedure_id, step_order); + + CREATE TABLE IF NOT EXISTS procedure_sources ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + procedure_id INTEGER NOT NULL REFERENCES procedures(id) ON DELETE CASCADE, + memory_id INTEGER REFERENCES memories(id) ON DELETE CASCADE, + event_id INTEGER REFERENCES events(id) ON DELETE CASCADE, + decision_id INTEGER REFERENCES decisions(id) ON DELETE CASCADE, + entity_id INTEGER REFERENCES entities(id) ON DELETE CASCADE, + source_role TEXT NOT NULL DEFAULT 'evidence', + created_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + CREATE INDEX IF NOT EXISTS idx_procedure_sources_procedure ON procedure_sources(procedure_id); + CREATE INDEX IF NOT EXISTS idx_procedure_sources_memory ON procedure_sources(memory_id); + CREATE INDEX IF NOT EXISTS idx_procedure_sources_event ON procedure_sources(event_id); + CREATE INDEX IF NOT EXISTS idx_procedure_sources_decision ON procedure_sources(decision_id); + + CREATE TABLE IF NOT EXISTS procedure_runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + procedure_id INTEGER NOT NULL REFERENCES procedures(id) ON DELETE CASCADE, + agent_id TEXT REFERENCES agents(id), + task_family TEXT, + task_signature TEXT, + input_summary TEXT, + outcome_summary TEXT, + success INTEGER NOT NULL DEFAULT 0, + usefulness_score REAL, + errors_seen TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + CREATE INDEX IF NOT EXISTS idx_procedure_runs_procedure_created + ON procedure_runs(procedure_id, created_at DESC); + + CREATE TABLE IF NOT EXISTS procedure_candidates ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + candidate_signature TEXT NOT NULL UNIQUE, + task_family TEXT, + normalized_signature TEXT NOT NULL, + support_count INTEGER NOT NULL DEFAULT 0, + evidence_json TEXT, + mean_success REAL NOT NULL DEFAULT 0.0, + promoted_procedure_id INTEGER REFERENCES procedures(id), + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + CREATE INDEX IF NOT EXISTS idx_procedure_candidates_family + ON procedure_candidates(task_family); + CREATE INDEX IF NOT EXISTS idx_procedure_candidates_support + ON procedure_candidates(support_count DESC); + + CREATE VIRTUAL TABLE IF NOT EXISTS procedures_fts USING fts5( + title, + goal, + description, + task_family, + search_text, + content=procedures, + content_rowid=id, + tokenize='porter unicode61' + ); + CREATE TRIGGER IF NOT EXISTS procedures_fts_insert AFTER INSERT ON procedures BEGIN + INSERT INTO procedures_fts(rowid, title, goal, description, task_family, search_text) + VALUES (new.id, new.title, new.goal, new.description, new.task_family, new.search_text); + END; + CREATE TRIGGER IF NOT EXISTS procedures_fts_update AFTER UPDATE ON procedures BEGIN + INSERT INTO procedures_fts( + procedures_fts, rowid, title, goal, description, task_family, search_text + ) + VALUES ( + 'delete', old.id, old.title, old.goal, old.description, old.task_family, old.search_text + ); + INSERT INTO procedures_fts(rowid, title, goal, description, task_family, search_text) + VALUES (new.id, new.title, new.goal, new.description, new.task_family, new.search_text); + END; + CREATE TRIGGER IF NOT EXISTS procedures_fts_delete AFTER DELETE ON procedures BEGIN + INSERT INTO procedures_fts( + procedures_fts, rowid, title, goal, description, task_family, search_text + ) + VALUES ( + 'delete', old.id, old.title, old.goal, old.description, old.task_family, old.search_text + ); + END; + """ + ) + + +def _insert_procedure_steps(conn: sqlite3.Connection, procedure_id: int, steps: list[dict[str, Any]]) -> None: + conn.execute("DELETE FROM procedure_steps WHERE procedure_id = ?", (procedure_id,)) + for idx, step in enumerate(steps, start=1): + conn.execute( + """ + INSERT INTO procedure_steps ( + procedure_id, step_order, action, rationale, tool_name, + expected_output, stop_condition, retry_policy, rollback_hint + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + procedure_id, + idx, + step.get("action"), + step.get("rationale"), + step.get("tool_name"), + step.get("expected_output"), + step.get("stop_condition"), + step.get("retry_policy"), + step.get("rollback_hint"), + ), + ) + + +def _link_knowledge_edge( + conn: sqlite3.Connection, + *, + procedure_id: int, + target_table: str, + target_id: int, + relation_type: str, + weight: float = 1.0, + agent_id: Optional[str] = None, +) -> None: + conn.execute( + """ + INSERT OR IGNORE INTO knowledge_edges + (source_table, source_id, target_table, target_id, relation_type, weight, agent_id, created_at) + VALUES ('procedures', ?, ?, ?, ?, ?, ?, ?) + """, + (procedure_id, target_table, target_id, relation_type, weight, agent_id, now_iso()), + ) + + +def create_procedure( + conn: sqlite3.Connection, + *, + agent_id: str, + payload: dict[str, Any], + category: str = "convention", + scope: str = "global", + confidence: float = 0.9, + source_memory_ids: Optional[list[int]] = None, + source_event_ids: Optional[list[int]] = None, + source_decision_ids: Optional[list[int]] = None, + source_entity_ids: Optional[list[int]] = None, + memory_id: Optional[int] = None, +) -> dict[str, Any]: + ensure_procedure_schema(conn) + source_memory_ids = source_memory_ids or [] + source_event_ids = source_event_ids or [] + source_decision_ids = source_decision_ids or [] + source_entity_ids = source_entity_ids or [] + + data = dict(payload) + if not data.get("steps_json"): + data = parse_procedural_text( + data.get("description") or data.get("goal") or "", + title=data.get("title"), + goal=data.get("goal"), + procedure_kind=data.get("procedure_kind"), + scope=scope, + ) + steps = _normalize_steps(data.get("steps_json") or []) + data["steps_json"] = steps or [{"action": data.get("goal") or "Review the procedure"}] + data["trigger_conditions"] = list(data.get("trigger_conditions") or []) + data["preconditions"] = list(data.get("preconditions") or []) + data["tools_json"] = list(data.get("tools_json") or []) + data["failure_modes_json"] = list(data.get("failure_modes_json") or []) + data["rollback_steps_json"] = list(data.get("rollback_steps_json") or []) + data["success_criteria_json"] = list(data.get("success_criteria_json") or []) + data["constraints_json"] = list(data.get("constraints_json") or []) + data["repair_strategies_json"] = list(data.get("repair_strategies_json") or []) + data["tool_policy_json"] = list(data.get("tool_policy_json") or []) + data["expected_outcomes"] = data.get("expected_outcomes") or [] + data["title"] = (data.get("title") or data.get("goal") or "Procedure").strip() + data["goal"] = (data.get("goal") or data["title"]).strip() + data["description"] = (data.get("description") or "").strip() + data["procedure_kind"] = data.get("procedure_kind") or _guess_kind( + " ".join([data["goal"], data["description"]]) + ) + if data["procedure_kind"] not in PROCEDURE_KINDS: + data["procedure_kind"] = "workflow" + data["status"] = data.get("status") or "active" + if data["status"] not in PROCEDURE_STATUSES: + data["status"] = "active" + data["applicability_scope"] = data.get("applicability_scope") or scope or "global" + data["task_family"] = data.get("task_family") or data["procedure_kind"] + data["search_text"] = compose_search_text(data) + synopsis = compose_synopsis(data) + source_refs = { + "memory_ids": source_memory_ids, + "event_ids": source_event_ids, + "decision_ids": source_decision_ids, + "entity_ids": source_entity_ids, + } + + created_at = now_iso() + if memory_id is None: + tags = data.get("tags") + tags_json = _json_dumps(tags) if tags else None + cur = conn.execute( + """ + INSERT INTO memories ( + agent_id, category, scope, content, confidence, tags, memory_type, + derived_from_ids, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, 'procedural', ?, ?, ?) + """, + ( + agent_id, + category, + scope, + synopsis, + confidence, + tags_json, + json.dumps(source_refs, ensure_ascii=True), + created_at, + created_at, + ), + ) + memory_id = int(cur.lastrowid) + else: + exists = conn.execute( + "SELECT id, content, scope FROM memories WHERE id = ?", + (memory_id,), + ).fetchone() + if not exists: + raise ValueError(f"memory_id {memory_id} does not exist") + conn.execute( + """ + UPDATE memories + SET memory_type = 'procedural', + scope = COALESCE(scope, ?), + updated_at = ?, + derived_from_ids = COALESCE(derived_from_ids, ?) + WHERE id = ? + """, + (scope, created_at, json.dumps(source_refs, ensure_ascii=True), memory_id), + ) + maybe_existing = conn.execute( + "SELECT id FROM procedures WHERE memory_id = ?", + (memory_id,), + ).fetchone() + if maybe_existing: + return get_procedure(conn, int(maybe_existing["id"]), include_sources=True) + + proc_key = data.get("procedure_key") or _procedure_key( + data["title"], data["goal"], data["applicability_scope"] + ) + cur = conn.execute( + """ + INSERT INTO procedures ( + memory_id, procedure_key, title, goal, description, task_family, + procedure_kind, trigger_conditions, preconditions, constraints_json, + steps_json, tools_json, failure_modes_json, rollback_steps_json, + success_criteria_json, repair_strategies_json, tool_policy_json, + expected_outcomes, applicability_scope, temporal_class, status, + automation_ready, determinism, confidence, utility_score, + generality_score, support_count, execution_count, success_count, + failure_count, last_used_at, last_executed_at, last_validated_at, + stale_after_days, supersedes_procedure_id, retired_at, search_text, + created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, 0, 0, NULL, NULL, NULL, ?, ?, NULL, ?, ?, ?) + """, + ( + memory_id, + proc_key, + data["title"], + data["goal"], + data["description"], + data["task_family"], + data["procedure_kind"], + _json_dumps(data["trigger_conditions"]), + _json_dumps(data["preconditions"]), + _json_dumps(data["constraints_json"]), + _json_dumps(data["steps_json"]), + _json_dumps(data["tools_json"]), + _json_dumps(data["failure_modes_json"]), + _json_dumps(data["rollback_steps_json"]), + _json_dumps(data["success_criteria_json"]), + _json_dumps(data["repair_strategies_json"]), + _json_dumps(data["tool_policy_json"]), + json.dumps(data["expected_outcomes"], ensure_ascii=True), + data["applicability_scope"], + data.get("temporal_class") or "durable", + data["status"], + int(bool(data.get("automation_ready", 0))), + float(data.get("determinism", 0.5)), + float(data.get("confidence", confidence)), + float(data.get("utility_score", confidence)), + float(data.get("generality_score", 0.5)), + int(data.get("support_count", len(source_memory_ids) + len(source_event_ids) + len(source_decision_ids))), + int(data.get("stale_after_days", 90)), + data.get("supersedes_procedure_id"), + data["search_text"], + created_at, + created_at, + ), + ) + procedure_id = int(cur.lastrowid) + + conn.execute( + "UPDATE memories SET content = ?, updated_at = ? WHERE id = ?", + (synopsis, created_at, memory_id), + ) + + _insert_procedure_steps(conn, procedure_id, steps) + + for mid in source_memory_ids: + conn.execute( + """ + INSERT INTO procedure_sources (procedure_id, memory_id, source_role, created_at) + VALUES (?, ?, 'derived_from_memory', ?) + """, + (procedure_id, mid, created_at), + ) + _link_knowledge_edge( + conn, + procedure_id=procedure_id, + target_table="memories", + target_id=mid, + relation_type="derived_from_memory", + weight=1.0, + agent_id=agent_id, + ) + + for eid in source_event_ids: + conn.execute( + """ + INSERT INTO procedure_sources (procedure_id, event_id, source_role, created_at) + VALUES (?, ?, 'derived_from_event', ?) + """, + (procedure_id, eid, created_at), + ) + rel = "rollback_for" if data["procedure_kind"] == "rollback" else "derived_from_event" + _link_knowledge_edge( + conn, + procedure_id=procedure_id, + target_table="events", + target_id=eid, + relation_type=rel, + weight=0.9, + agent_id=agent_id, + ) + + for did in source_decision_ids: + conn.execute( + """ + INSERT INTO procedure_sources (procedure_id, decision_id, source_role, created_at) + VALUES (?, ?, 'derived_from_decision', ?) + """, + (procedure_id, did, created_at), + ) + _link_knowledge_edge( + conn, + procedure_id=procedure_id, + target_table="decisions", + target_id=did, + relation_type="derived_from_decision", + weight=0.95, + agent_id=agent_id, + ) + + for ent_id in source_entity_ids: + conn.execute( + """ + INSERT INTO procedure_sources (procedure_id, entity_id, source_role, created_at) + VALUES (?, ?, 'applicable_to', ?) + """, + (procedure_id, ent_id, created_at), + ) + _link_knowledge_edge( + conn, + procedure_id=procedure_id, + target_table="entities", + target_id=ent_id, + relation_type="applicable_to", + weight=0.8, + agent_id=agent_id, + ) + + for tool in data["tools_json"]: + conn.execute( + """ + INSERT OR IGNORE INTO knowledge_edges + (source_table, source_id, target_table, target_id, relation_type, weight, agent_id, created_at) + SELECT 'procedures', ?, 'entities', e.id, 'requires_tool', 0.7, ?, ? + FROM entities e + WHERE lower(e.name) = lower(?) + """, + (procedure_id, agent_id, created_at, str(tool)), + ) + + if data.get("supersedes_procedure_id"): + _link_knowledge_edge( + conn, + procedure_id=procedure_id, + target_table="procedures", + target_id=int(data["supersedes_procedure_id"]), + relation_type="supersedes_procedure", + weight=1.0, + agent_id=agent_id, + ) + conn.execute( + "UPDATE procedures SET status = 'superseded', updated_at = ? WHERE id = ?", + (created_at, int(data["supersedes_procedure_id"])), + ) + + return get_procedure(conn, procedure_id, include_sources=True) + + +def ensure_procedure_for_memory( + conn: sqlite3.Connection, + *, + memory_id: int, + agent_id: str, +) -> dict[str, Any]: + ensure_procedure_schema(conn) + existing = conn.execute( + "SELECT id FROM procedures WHERE memory_id = ?", + (memory_id,), + ).fetchone() + if existing: + return get_procedure(conn, int(existing["id"]), include_sources=True) + + row = conn.execute( + "SELECT id, content, category, scope, confidence FROM memories WHERE id = ?", + (memory_id,), + ).fetchone() + if not row: + raise ValueError(f"memory_id {memory_id} not found") + + payload = parse_procedural_text( + row["content"], + scope=row["scope"] or "global", + ) + payload.setdefault("description", row["content"]) + payload.setdefault("confidence", row["confidence"] or 0.6) + payload.setdefault("utility_score", row["confidence"] or 0.6) + payload.setdefault("support_count", 1) + return create_procedure( + conn, + agent_id=agent_id, + payload=payload, + category=row["category"] or "convention", + scope=row["scope"] or "global", + confidence=float(row["confidence"] or 0.8), + source_memory_ids=[memory_id], + memory_id=memory_id, + ) + + +def _procedure_row_to_dict(row: sqlite3.Row) -> dict[str, Any]: + out = dict(row) + for key in ( + "trigger_conditions", + "preconditions", + "constraints_json", + "steps_json", + "tools_json", + "failure_modes_json", + "rollback_steps_json", + "success_criteria_json", + "repair_strategies_json", + "tool_policy_json", + ): + out[key] = _json_loads_list(out.get(key)) + if isinstance(out.get("expected_outcomes"), str) and out["expected_outcomes"].startswith("["): + out["expected_outcomes"] = _json_loads_list(out["expected_outcomes"]) + out["success_rate"] = round( + float(out.get("success_count") or 0) / max(int(out.get("execution_count") or 0), 1), + 4, + ) + return out + + +def get_procedure( + conn: sqlite3.Connection, + procedure_id: int, + *, + include_sources: bool = False, +) -> dict[str, Any]: + ensure_procedure_schema(conn) + row = conn.execute( + """ + SELECT p.*, m.content, m.category, m.scope, m.confidence AS memory_confidence, + m.memory_type, m.created_at AS memory_created_at + FROM procedures p + JOIN memories m ON m.id = p.memory_id + WHERE p.id = ? + """, + (procedure_id,), + ).fetchone() + if not row: + raise ValueError(f"procedure_id {procedure_id} not found") + out = _procedure_row_to_dict(row) + if include_sources: + out["sources"] = [dict(r) for r in conn.execute( + """ + SELECT memory_id, event_id, decision_id, entity_id, source_role, created_at + FROM procedure_sources + WHERE procedure_id = ? + ORDER BY id + """, + (procedure_id,), + ).fetchall()] + out["steps"] = [dict(r) for r in conn.execute( + """ + SELECT step_order, action, rationale, tool_name, expected_output, + stop_condition, retry_policy, rollback_hint + FROM procedure_steps + WHERE procedure_id = ? + ORDER BY step_order + """, + (procedure_id,), + ).fetchall()] + return out + + +def list_procedures( + conn: sqlite3.Connection, + *, + status: Optional[str] = None, + scope: Optional[str] = None, + limit: int = 50, +) -> list[dict[str, Any]]: + ensure_procedure_schema(conn) + clauses = ["1=1"] + params: list[Any] = [] + if status and status != "all": + clauses.append("p.status = ?") + params.append(status) + if scope: + clauses.append("(p.applicability_scope = 'global' OR p.applicability_scope = ?)") + params.append(scope) + params.append(limit) + rows = conn.execute( + f""" + SELECT p.*, m.content, m.category, m.scope, m.confidence AS memory_confidence + FROM procedures p + JOIN memories m ON m.id = p.memory_id + WHERE {' AND '.join(clauses)} + ORDER BY + CASE p.status + WHEN 'active' THEN 0 + WHEN 'candidate' THEN 1 + WHEN 'needs_review' THEN 2 + WHEN 'stale' THEN 3 + WHEN 'superseded' THEN 4 + ELSE 5 + END, + COALESCE(p.last_validated_at, p.updated_at, p.created_at) DESC + LIMIT ? + """, + params, + ).fetchall() + return [_procedure_row_to_dict(row) for row in rows] + + +def _days_old(timestamp: Optional[str]) -> float: + if not timestamp: + return 9999.0 + normalized = str(timestamp).replace("Z", "+00:00") + dt = datetime.fromisoformat(normalized) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return max(0.0, (datetime.now(timezone.utc) - dt).total_seconds() / 86400.0) + + +def _score_procedure( + query: str, + proc: dict[str, Any], + *, + debug: bool = False, +) -> tuple[float, dict[str, float]]: + tokens = set(_tokenize(query)) + phrase = query.lower().strip() + + title_tokens = set(_tokenize(proc.get("title") or "")) + goal_tokens = set(_tokenize(proc.get("goal") or "")) + desc_tokens = set(_tokenize(proc.get("description") or "")) + trigger_tokens = set(_tokenize(" ".join(str(v) for v in proc.get("trigger_conditions", [])))) + pre_tokens = set(_tokenize(" ".join(str(v) for v in proc.get("preconditions", [])))) + tool_tokens = set(_tokenize(" ".join(str(v) for v in proc.get("tools_json", [])))) + step_tokens = set(_tokenize(" ".join(step.get("action", "") for step in proc.get("steps_json", [])))) + failure_tokens = set(_tokenize(" ".join(str(v) for v in proc.get("failure_modes_json", [])))) + rollback_tokens = set(_tokenize(" ".join(str(v) for v in proc.get("rollback_steps_json", [])))) + scope_tokens = set(_tokenize(proc.get("applicability_scope") or "")) + + overlap = lambda bag: len(tokens & bag) / max(len(tokens), 1) + breakdown = { + "goal_match": overlap(goal_tokens | desc_tokens) * 1.4, + "title_match": overlap(title_tokens) * 1.6, + "trigger_match": overlap(trigger_tokens) * 0.9, + "precondition_match": overlap(pre_tokens) * 0.7, + "step_overlap": overlap(step_tokens) * 1.3, + "tool_overlap": overlap(tool_tokens) * 0.9, + "failure_overlap": overlap(failure_tokens) * 0.7, + "rollback_overlap": overlap(rollback_tokens) * 1.1, + "scope_match": overlap(scope_tokens) * 0.4, + "exact_phrase": 1.0 if phrase and phrase in (proc.get("search_text") or "").lower() else 0.0, + } + + status = proc.get("status") or "active" + status_multiplier = { + "active": 1.15, + "candidate": 0.95, + "needs_review": 0.75, + "stale": 0.68, + "superseded": 0.35, + "retired": 0.15, + }.get(status, 1.0) + validation_age = _days_old(proc.get("last_validated_at")) + last_exec_age = _days_old(proc.get("last_executed_at")) + validation_boost = max(0.0, 1.0 - min(validation_age / max(int(proc.get("stale_after_days") or 90), 1), 1.5)) + utility_boost = float(proc.get("utility_score") or 0.5) + confidence_boost = float(proc.get("confidence") or 0.5) + execution_count = int(proc.get("execution_count") or 0) + success_count = int(proc.get("success_count") or 0) + failure_count = int(proc.get("failure_count") or 0) + success_rate = success_count / max(execution_count, 1) + failure_penalty = min(failure_count / max(execution_count, 1), 1.0) + support_bonus = min(int(proc.get("support_count") or 0) / 5.0, 1.0) + freshness = max(0.0, 1.0 - min(last_exec_age / max(int(proc.get("stale_after_days") or 90), 1), 1.5)) + + base = sum(breakdown.values()) + score = ( + base + + validation_boost * 0.8 + + freshness * 0.4 + + success_rate * 0.8 + + support_bonus * 0.5 + + utility_boost * 0.3 + + confidence_boost * 0.4 + - failure_penalty * 0.9 + ) * status_multiplier + directness = ( + breakdown["goal_match"] + + breakdown["title_match"] + + breakdown["trigger_match"] + + breakdown["exact_phrase"] + ) + if directness < 0.6 and breakdown["step_overlap"] > 0: + score *= 0.72 + if debug: + breakdown.update( + { + "validation_boost": round(validation_boost, 4), + "freshness_boost": round(freshness, 4), + "success_rate": round(success_rate, 4), + "support_bonus": round(support_bonus, 4), + "utility_boost": round(utility_boost, 4), + "confidence_boost": round(confidence_boost, 4), + "failure_penalty": round(failure_penalty, 4), + "status_multiplier": round(status_multiplier, 4), + "directness": round(directness, 4), + } + ) + return round(score, 6), breakdown + + +def search_procedures( + conn: sqlite3.Connection, + query: str, + *, + limit: int = 10, + scope: Optional[str] = None, + status: Optional[str] = None, + debug: bool = False, +) -> dict[str, Any]: + ensure_procedure_schema(conn) + search = query.strip() + if not search: + return {"ok": True, "procedures": [], "debug": {"reason": "empty_query"}} + + tokens = _tokenize(search) + fts_query = " OR ".join(tokens) if tokens else re.sub(r"[^\w\s]", " ", search).strip() + clauses = ["1=1"] + params: list[Any] = [] + if status and status != "all": + clauses.append("p.status = ?") + params.append(status) + if scope: + clauses.append("(p.applicability_scope = 'global' OR p.applicability_scope = ?)") + params.append(scope) + + rows: list[sqlite3.Row] + if fts_query: + rows = conn.execute( + f""" + SELECT p.*, m.content, m.category, m.scope, m.confidence AS memory_confidence, + bm25(procedures_fts, 3.0, 2.0, 1.5, 1.0, 2.5) AS fts_rank + FROM procedures_fts + JOIN procedures p ON p.id = procedures_fts.rowid + JOIN memories m ON m.id = p.memory_id + WHERE procedures_fts MATCH ? AND {' AND '.join(clauses)} + ORDER BY bm25(procedures_fts, 3.0, 2.0, 1.5, 1.0, 2.5) + LIMIT ? + """, + [fts_query, *params, max(limit * 4, 12)], + ).fetchall() + else: + rows = [] + + if not rows: + rows = conn.execute( + f""" + SELECT p.*, m.content, m.category, m.scope, m.confidence AS memory_confidence, NULL AS fts_rank + FROM procedures p + JOIN memories m ON m.id = p.memory_id + WHERE {' AND '.join(clauses)} + AND ( + lower(p.goal) LIKE ? OR lower(COALESCE(p.description, '')) LIKE ? + OR lower(p.search_text) LIKE ? OR lower(m.content) LIKE ? + ) + LIMIT ? + """, + [*params, f"%{search.lower()}%", f"%{search.lower()}%", f"%{search.lower()}%", f"%{search.lower()}%", max(limit * 4, 12)], + ).fetchall() + + results: list[dict[str, Any]] = [] + for row in rows: + proc = _procedure_row_to_dict(row) + score, breakdown = _score_procedure(search, proc, debug=debug) + proc["final_score"] = score + proc["fts_rank"] = row["fts_rank"] + proc["type"] = "procedure" + proc["why_retrieved"] = ( + "goal/title match" if breakdown.get("goal_match", 0.0) + breakdown.get("title_match", 0.0) >= 1.0 + else "procedural evidence match" + ) + if debug: + proc["score_breakdown"] = breakdown + results.append(proc) + + results.sort(key=lambda item: item.get("final_score", 0.0), reverse=True) + return { + "ok": True, + "procedures": results[:limit], + "debug": { + "query": search, + "fts_query": fts_query, + "candidate_count": len(results), + }, + } + + +def update_procedure( + conn: sqlite3.Connection, + procedure_id: int, + changes: dict[str, Any], +) -> dict[str, Any]: + ensure_procedure_schema(conn) + current = get_procedure(conn, procedure_id, include_sources=True) + merged = dict(current) + merged.update({k: v for k, v in changes.items() if v is not None}) + merged["steps_json"] = _normalize_steps(merged.get("steps_json") or current.get("steps_json") or []) + merged["search_text"] = compose_search_text(merged) + merged["updated_at"] = now_iso() + + conn.execute( + """ + UPDATE procedures + SET title = ?, goal = ?, description = ?, task_family = ?, procedure_kind = ?, + trigger_conditions = ?, preconditions = ?, constraints_json = ?, steps_json = ?, + tools_json = ?, failure_modes_json = ?, rollback_steps_json = ?, + success_criteria_json = ?, repair_strategies_json = ?, tool_policy_json = ?, + expected_outcomes = ?, applicability_scope = ?, status = ?, automation_ready = ?, + determinism = ?, confidence = ?, utility_score = ?, generality_score = ?, + support_count = ?, stale_after_days = ?, supersedes_procedure_id = ?, + search_text = ?, updated_at = ? + WHERE id = ? + """, + ( + merged.get("title"), + merged.get("goal"), + merged.get("description"), + merged.get("task_family"), + merged.get("procedure_kind"), + _json_dumps(merged.get("trigger_conditions")), + _json_dumps(merged.get("preconditions")), + _json_dumps(merged.get("constraints_json")), + _json_dumps(merged.get("steps_json")), + _json_dumps(merged.get("tools_json")), + _json_dumps(merged.get("failure_modes_json")), + _json_dumps(merged.get("rollback_steps_json")), + _json_dumps(merged.get("success_criteria_json")), + _json_dumps(merged.get("repair_strategies_json")), + _json_dumps(merged.get("tool_policy_json")), + json.dumps(merged.get("expected_outcomes") or [], ensure_ascii=True), + merged.get("applicability_scope"), + merged.get("status"), + int(bool(merged.get("automation_ready", 0))), + float(merged.get("determinism", 0.5)), + float(merged.get("confidence", 0.5)), + float(merged.get("utility_score", 0.5)), + float(merged.get("generality_score", 0.5)), + int(merged.get("support_count", 0)), + int(merged.get("stale_after_days", 90)), + merged.get("supersedes_procedure_id"), + merged["search_text"], + merged["updated_at"], + procedure_id, + ), + ) + _insert_procedure_steps(conn, procedure_id, merged["steps_json"]) + conn.execute( + "UPDATE memories SET content = ?, updated_at = ? WHERE id = ?", + (compose_synopsis(merged), merged["updated_at"], current["memory_id"]), + ) + return get_procedure(conn, procedure_id, include_sources=True) + + +def _recompute_status(proc: dict[str, Any]) -> str: + if proc.get("retired_at"): + return "retired" + if proc.get("status") == "superseded": + return "superseded" + stale_after_days = int(proc.get("stale_after_days") or 90) + last_validated = proc.get("last_validated_at") or proc.get("updated_at") or proc.get("created_at") + if last_validated and _days_old(last_validated) > stale_after_days: + return "stale" + failures = int(proc.get("failure_count") or 0) + successes = int(proc.get("success_count") or 0) + execution_count = int(proc.get("execution_count") or 0) + if execution_count >= 3 and failures >= max(2, successes): + return "needs_review" + return "active" + + +def record_feedback( + conn: sqlite3.Connection, + *, + procedure_id: int, + agent_id: str, + success: bool, + usefulness_score: Optional[float] = None, + outcome_summary: Optional[str] = None, + errors_seen: Optional[str] = None, + validated: bool = False, + task_signature: Optional[str] = None, + input_summary: Optional[str] = None, +) -> dict[str, Any]: + ensure_procedure_schema(conn) + proc = get_procedure(conn, procedure_id, include_sources=False) + now = now_iso() + execution_count = int(proc.get("execution_count") or 0) + 1 + success_count = int(proc.get("success_count") or 0) + (1 if success else 0) + failure_count = int(proc.get("failure_count") or 0) + (0 if success else 1) + utility = usefulness_score if usefulness_score is not None else proc.get("utility_score") or 0.5 + utility = float(max(0.0, min(1.0, utility))) + confidence = float(proc.get("confidence") or 0.5) + confidence = confidence + (0.06 if success else -0.09) + confidence = max(0.05, min(0.99, confidence)) + + conn.execute( + """ + INSERT INTO procedure_runs ( + procedure_id, agent_id, task_family, task_signature, input_summary, + outcome_summary, success, usefulness_score, errors_seen, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + procedure_id, + agent_id, + proc.get("task_family"), + task_signature, + input_summary, + outcome_summary, + 1 if success else 0, + usefulness_score, + errors_seen, + now, + ), + ) + + proc.update( + { + "execution_count": execution_count, + "success_count": success_count, + "failure_count": failure_count, + "last_used_at": now, + "last_executed_at": now, + "last_validated_at": now if validated or success else proc.get("last_validated_at"), + "utility_score": utility, + "confidence": confidence, + } + ) + proc["status"] = _recompute_status(proc) + conn.execute( + """ + UPDATE procedures + SET execution_count = ?, success_count = ?, failure_count = ?, + last_used_at = ?, last_executed_at = ?, last_validated_at = ?, + utility_score = ?, confidence = ?, status = ?, updated_at = ? + WHERE id = ? + """, + ( + execution_count, + success_count, + failure_count, + now, + now, + proc.get("last_validated_at"), + utility, + confidence, + proc["status"], + now, + procedure_id, + ), + ) + + mem = conn.execute( + "SELECT alpha, beta FROM memories WHERE id = ?", + (proc["memory_id"],), + ).fetchone() + alpha = float(mem["alpha"] if mem and mem["alpha"] is not None else 1.0) + beta = float(mem["beta"] if mem and mem["beta"] is not None else 1.0) + if success: + alpha += 1.0 + else: + beta += 1.0 + posterior = alpha / max(alpha + beta, 1e-6) + conn.execute( + """ + UPDATE memories + SET alpha = ?, beta = ?, confidence = ?, updated_at = ? + WHERE id = ? + """, + (alpha, beta, posterior, now, proc["memory_id"]), + ) + + return get_procedure(conn, procedure_id, include_sources=True) + + +def _candidate_signature_from_text(text: str) -> str: + tokens = _tokenize(text)[:8] + if not tokens: + return "" + return " ".join(tokens) + + +def synthesize_procedure_candidates( + conn: sqlite3.Connection, + *, + agent_id: str, + dry_run: bool = False, + min_support: int = 2, + promote_support: int = 3, +) -> dict[str, Any]: + ensure_procedure_schema(conn) + rows = conn.execute( + """ + SELECT id, content, category, scope, confidence + FROM memories + WHERE retired_at IS NULL + AND COALESCE(memory_type, 'episodic') = 'episodic' + AND category IN ('lesson', 'integration', 'decision', 'convention') + ORDER BY created_at DESC + """ + ).fetchall() + grouped: dict[str, list[sqlite3.Row]] = {} + for row in rows: + if not looks_procedural(row["content"]): + continue + signature = _candidate_signature_from_text(row["content"]) + if not signature: + continue + grouped.setdefault(signature, []).append(row) + + stats = { + "scanned": len(rows), + "candidates_updated": 0, + "promoted": 0, + "signatures": [], + } + now = now_iso() + for signature, members in grouped.items(): + if len(members) < min_support: + continue + mean_success = sum(float(row["confidence"] or 0.5) for row in members) / len(members) + evidence = { + "memory_ids": [int(row["id"]) for row in members], + "scope": members[0]["scope"], + "category": members[0]["category"], + } + if not dry_run: + conn.execute( + """ + INSERT INTO procedure_candidates ( + candidate_signature, task_family, normalized_signature, + support_count, evidence_json, mean_success, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(candidate_signature) DO UPDATE SET + support_count = excluded.support_count, + evidence_json = excluded.evidence_json, + mean_success = excluded.mean_success, + updated_at = excluded.updated_at + """, + ( + signature, + _guess_kind(signature), + signature, + len(members), + json.dumps(evidence, ensure_ascii=True), + round(mean_success, 4), + now, + ), + ) + stats["candidates_updated"] += 1 + stats["signatures"].append({"signature": signature, "support": len(members)}) + + should_promote = len(members) >= promote_support or ( + len(members) >= 2 and mean_success >= 0.75 and any(row["category"] in ("decision", "lesson") for row in members) + ) + if should_promote: + payload = parse_procedural_text( + members[0]["content"], + scope=members[0]["scope"] or "global", + ) + payload["support_count"] = len(members) + payload["confidence"] = round(mean_success, 4) + payload["utility_score"] = round(mean_success, 4) + if not dry_run: + proc = create_procedure( + conn, + agent_id=agent_id, + payload=payload, + category=members[0]["category"] or "convention", + scope=members[0]["scope"] or "global", + confidence=round(mean_success, 4), + source_memory_ids=[int(row["id"]) for row in members], + ) + conn.execute( + """ + UPDATE procedure_candidates + SET promoted_procedure_id = ?, updated_at = ? + WHERE candidate_signature = ? + """, + (proc["id"], now, signature), + ) + stats["promoted"] += 1 + return stats + + +def backfill_procedures( + conn: sqlite3.Connection, + *, + agent_id: str, + scope: Optional[str] = None, + limit: int = 100, + dry_run: bool = False, +) -> dict[str, Any]: + ensure_procedure_schema(conn) + clauses = [ + "m.retired_at IS NULL", + "COALESCE(m.memory_type, 'episodic') != 'procedural'", + "m.category IN ('convention', 'lesson', 'integration', 'decision')", + "NOT EXISTS (SELECT 1 FROM procedure_sources ps WHERE ps.memory_id = m.id)", + ] + params: list[Any] = [] + if scope: + clauses.append("(m.scope = ? OR m.scope = 'global')") + params.append(scope) + params.append(limit) + rows = conn.execute( + f""" + SELECT m.id, m.content, m.category, m.scope, m.confidence + FROM memories m + WHERE {' AND '.join(clauses)} + ORDER BY m.created_at DESC + LIMIT ? + """, + params, + ).fetchall() + + stats = { + "ok": True, + "scanned_memories": len(rows), + "created_procedures": 0, + "created_from_decisions": 0, + "created_from_events": 0, + "procedure_ids": [], + } + + for row in rows: + if not looks_procedural(row["content"]): + continue + stats["created_procedures"] += 1 + if dry_run: + continue + proc = ensure_procedure_for_memory(conn, memory_id=int(row["id"]), agent_id=agent_id) + stats["procedure_ids"].append(proc["id"]) + + decision_rows = conn.execute( + """ + SELECT d.id, d.title, d.rationale, d.project + FROM decisions d + WHERE NOT EXISTS ( + SELECT 1 FROM procedure_sources ps WHERE ps.decision_id = d.id + ) + ORDER BY d.created_at DESC + LIMIT ? + """, + (limit,), + ).fetchall() + for row in decision_rows: + combined = f"{row['title']}. {row['rationale']}" + if not looks_procedural(combined): + continue + stats["created_from_decisions"] += 1 + if dry_run: + continue + payload = parse_procedural_text(combined, title=row["title"], scope=f"project:{row['project']}" if row["project"] else "global") + proc = create_procedure( + conn, + agent_id=agent_id, + payload=payload, + category="decision", + scope=f"project:{row['project']}" if row["project"] else "global", + confidence=0.75, + source_decision_ids=[int(row["id"])], + ) + stats["procedure_ids"].append(proc["id"]) + + event_rows = conn.execute( + """ + SELECT e.id, e.summary, COALESCE(e.detail, '') AS detail, e.project + FROM events e + WHERE e.event_type IN ('error', 'warning', 'artifact', 'result') + AND NOT EXISTS ( + SELECT 1 FROM procedure_sources ps WHERE ps.event_id = e.id + ) + ORDER BY e.created_at DESC + LIMIT ? + """, + (limit,), + ).fetchall() + for row in event_rows: + combined = f"{row['summary']} {row['detail']}".strip() + if not looks_procedural(combined): + continue + stats["created_from_events"] += 1 + if dry_run: + continue + payload = parse_procedural_text( + combined, + title=row["summary"], + scope=f"project:{row['project']}" if row["project"] else "global", + ) + proc = create_procedure( + conn, + agent_id=agent_id, + payload=payload, + category="lesson", + scope=f"project:{row['project']}" if row["project"] else "global", + confidence=0.7, + source_event_ids=[int(row["id"])], + ) + stats["procedure_ids"].append(proc["id"]) + + candidate_stats = synthesize_procedure_candidates( + conn, + agent_id=agent_id, + dry_run=dry_run, + ) + stats["candidate_stats"] = candidate_stats + return stats + + +def procedure_stats(conn: sqlite3.Connection) -> dict[str, Any]: + ensure_procedure_schema(conn) + rows = conn.execute( + "SELECT status, COUNT(*) AS cnt FROM procedures GROUP BY status" + ).fetchall() + out = {row["status"]: row["cnt"] for row in rows} + total = sum(out.values()) + candidate_count = conn.execute( + "SELECT COUNT(*) FROM procedure_candidates" + ).fetchone()[0] + return { + "ok": True, + "total": total, + "by_status": out, + "candidates": candidate_count, + } diff --git a/src/agentmemory/retrieval/__init__.py b/src/agentmemory/retrieval/__init__.py new file mode 100644 index 0000000..716a6fe --- /dev/null +++ b/src/agentmemory/retrieval/__init__.py @@ -0,0 +1,18 @@ +"""Retrieval executive helpers.""" + +from .answerability import assess_answerability +from .candidate_generation import generate_procedure_candidates +from .diagnostics import build_debug_payload +from .evidence_graph import expand_procedure_evidence +from .late_reranker import rerank_procedure_candidates +from .query_planner import QueryPlan, plan_query + +__all__ = [ + "QueryPlan", + "assess_answerability", + "build_debug_payload", + "expand_procedure_evidence", + "generate_procedure_candidates", + "plan_query", + "rerank_procedure_candidates", +] diff --git a/src/agentmemory/retrieval/answerability.py b/src/agentmemory/retrieval/answerability.py new file mode 100644 index 0000000..91a1d3a --- /dev/null +++ b/src/agentmemory/retrieval/answerability.py @@ -0,0 +1,120 @@ +"""Grounded answerability gate.""" + +from __future__ import annotations + +import re +from typing import Any + +_STOPWORDS = { + "a", "an", "and", "are", "as", "at", "be", "by", "do", "does", "for", + "from", "has", "have", "how", "i", "in", "is", "it", "its", "of", + "on", "or", "that", "the", "to", "was", "we", "what", "when", "where", + "which", "who", "why", "will", "with", "you", "did", +} + + +def _normalize_token(token: str) -> str: + tok = re.sub(r"[^a-z0-9]+", "", (token or "").lower()) + if len(tok) <= 2 or tok in _STOPWORDS: + return "" + if tok.endswith("ies") and len(tok) > 4: + tok = tok[:-3] + "y" + elif tok.endswith("ed") and len(tok) > 4: + tok = tok[:-2] + elif tok.endswith("es") and len(tok) > 4: + tok = tok[:-2] + elif tok.endswith("s") and len(tok) > 3: + tok = tok[:-1] + return tok + + +def _token_set(text: str) -> set[str]: + return { + norm + for part in re.split(r"\s+", text or "") + if (norm := _normalize_token(part)) + } + + +def assess_answerability( + query: str, + plan, + buckets: dict[str, list[dict[str, Any]]], +) -> dict[str, Any]: + """Estimate whether the current retrieval set is grounded enough to answer.""" + + flat: list[dict[str, Any]] = [] + for rows in buckets.values(): + flat.extend(rows or []) + flat.sort(key=lambda item: item.get("final_score", 0.0), reverse=True) + + if not flat: + return { + "score": 0.0, + "abstain": True, + "reason": "no_candidates", + "top_margin": 0.0, + } + + top = flat[0] + second = flat[1] if len(flat) > 1 else None + top_score = float(top.get("final_score") or 0.0) + second_score = float(second.get("final_score") or 0.0) if second else 0.0 + margin = top_score - second_score + + query_tokens = _token_set(query) + top_text = " ".join( + str(top.get(key) or "") + for key in ("content", "summary", "title", "goal", "description", "search_text") + ) + top_text_tokens = _token_set(top_text) + supporting_text = " ".join( + " ".join( + str(row.get(key) or "") + for key in ("content", "summary", "title", "goal", "description", "search_text") + ) + for row in flat[:3] + ) + supporting_tokens = _token_set(supporting_text) + coverage = 0.0 + if query_tokens: + coverage = len(query_tokens & supporting_tokens) / len(query_tokens) + anchor_overlap = len(query_tokens & top_text_tokens) + evidence_diversity = len({ + row.get("type") or bucket_name.rstrip("s") + for bucket_name, rows in buckets.items() + for row in (rows or [])[:2] + }) + direct_support = len(top.get("supporting_evidence") or []) + stale_penalty = 0.25 if top.get("status") in {"stale", "needs_review", "superseded", "retired"} else 0.0 + + score = ( + (top_score * 0.45) + + (margin * 0.35) + + (coverage * 0.45) + + min(direct_support / 3.0, 1.0) * 0.15 + + min(evidence_diversity / 3.0, 1.0) * 0.1 + - stale_penalty + ) + abstain = False + reason = "grounded" + if coverage < 0.34 and anchor_overlap == 0 and direct_support == 0: + abstain = True + reason = "weak_token_coverage" + if margin < 0.08 and coverage < 0.5 and anchor_overlap < 2 and plan.abstain_allowed: + abstain = True + reason = "diffuse_candidates" + if plan.abstain_allowed and score < 0.42 and coverage < 0.5: + abstain = True + reason = "low_answerability_score" + + return { + "score": round(score, 4), + "abstain": abstain, + "reason": reason, + "top_margin": round(margin, 4), + "coverage": round(coverage, 4), + "anchor_overlap": anchor_overlap, + "evidence_diversity": evidence_diversity, + "direct_support": direct_support, + } diff --git a/src/agentmemory/retrieval/candidate_generation.py b/src/agentmemory/retrieval/candidate_generation.py new file mode 100644 index 0000000..72bb159 --- /dev/null +++ b/src/agentmemory/retrieval/candidate_generation.py @@ -0,0 +1,43 @@ +"""Candidate generation for procedure-aware retrieval.""" + +from __future__ import annotations + +import sqlite3 +from typing import Any + +from agentmemory import procedural +from .query_planner import QueryPlan + + +def generate_procedure_candidates( + conn: sqlite3.Connection, + query: str, + plan: QueryPlan, + *, + limit: int = 10, + scope: str | None = None, +) -> dict[str, Any]: + """Search procedures and attach minimal diagnostics.""" + + if "procedures" not in plan.candidate_tables: + return {"candidates": [], "debug": {"skipped": "procedures_not_in_plan"}} + + search = procedural.search_procedures( + conn, + query, + limit=max(limit * 3, 12), + scope=scope, + debug=True, + ) + candidates = search.get("procedures", []) + for cand in candidates: + cand.setdefault("type", "procedure") + cand.setdefault("source", "procedure_fts") + return { + "candidates": candidates, + "debug": { + "query": query, + "count": len(candidates), + **(search.get("debug") or {}), + }, + } diff --git a/src/agentmemory/retrieval/diagnostics.py b/src/agentmemory/retrieval/diagnostics.py new file mode 100644 index 0000000..1a16749 --- /dev/null +++ b/src/agentmemory/retrieval/diagnostics.py @@ -0,0 +1,33 @@ +"""Debug payload builders for retrieval executive output.""" + +from __future__ import annotations + +from typing import Any + + +def build_debug_payload( + *, + query_plan: dict[str, Any], + procedure_debug: dict[str, Any] | None, + answerability: dict[str, Any] | None, + top_candidates: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + payload: dict[str, Any] = { + "query_plan": query_plan, + } + if procedure_debug: + payload["procedures"] = procedure_debug + if answerability: + payload["answerability"] = answerability + if top_candidates is not None: + payload["top_candidates"] = [ + { + "type": cand.get("type"), + "id": cand.get("id"), + "final_score": cand.get("final_score"), + "why_retrieved": cand.get("why_retrieved"), + "text": cand.get("content") or cand.get("summary") or cand.get("title") or cand.get("goal") or cand.get("name"), + } + for cand in top_candidates[:5] + ] + return payload diff --git a/src/agentmemory/retrieval/evidence_graph.py b/src/agentmemory/retrieval/evidence_graph.py new file mode 100644 index 0000000..e47517e --- /dev/null +++ b/src/agentmemory/retrieval/evidence_graph.py @@ -0,0 +1,55 @@ +"""Evidence expansion helpers for procedure retrieval.""" + +from __future__ import annotations + +import sqlite3 +from typing import Any + + +def expand_procedure_evidence( + conn: sqlite3.Connection, + candidates: list[dict[str, Any]], + *, + max_sources_per_candidate: int = 4, +) -> dict[int, dict[str, Any]]: + """Attach 1-hop provenance and support evidence to top procedure candidates.""" + + if not candidates: + return {} + + out: dict[int, dict[str, Any]] = {} + for cand in candidates: + proc_id = int(cand["id"]) + sources = [ + dict(row) + for row in conn.execute( + """ + SELECT memory_id, event_id, decision_id, entity_id, source_role, created_at + FROM procedure_sources + WHERE procedure_id = ? + ORDER BY id + LIMIT ? + """, + (proc_id, max_sources_per_candidate), + ).fetchall() + ] + edges = [ + dict(row) + for row in conn.execute( + """ + SELECT target_table, target_id, relation_type, weight + FROM knowledge_edges + WHERE source_table = 'procedures' AND source_id = ? + ORDER BY weight DESC, id DESC + LIMIT ? + """, + (proc_id, max_sources_per_candidate), + ).fetchall() + ] + support_bonus = min((len(sources) * 0.14) + (sum(float(edge.get("weight") or 0.0) for edge in edges) * 0.08), 0.8) + out[proc_id] = { + "sources": sources, + "edges": edges, + "support_bonus": round(support_bonus, 4), + } + return out diff --git a/src/agentmemory/retrieval/late_reranker.py b/src/agentmemory/retrieval/late_reranker.py new file mode 100644 index 0000000..f91e8d1 --- /dev/null +++ b/src/agentmemory/retrieval/late_reranker.py @@ -0,0 +1,43 @@ +"""Deterministic late reranking for procedure candidates.""" + +from __future__ import annotations + +from typing import Any + + +def rerank_procedure_candidates( + candidates: list[dict[str, Any]], + evidence: dict[int, dict[str, Any]], + *, + benchmark_mode: bool = False, +) -> list[dict[str, Any]]: + reranked: list[dict[str, Any]] = [] + for cand in candidates: + proc_id = int(cand["id"]) + ev = evidence.get(proc_id) or {} + bonus = float(ev.get("support_bonus") or 0.0) + base = float(cand.get("final_score") or 0.0) + status = cand.get("status") or "active" + status_multiplier = { + "active": 1.0, + "candidate": 0.9, + "needs_review": 0.72, + "stale": 0.64, + "superseded": 0.3, + "retired": 0.1, + }.get(status, 1.0) + if benchmark_mode: + score = base * status_multiplier + else: + score = (base + bonus) * status_multiplier + updated = dict(cand) + updated["supporting_evidence"] = ev.get("sources") or [] + updated["evidence_edges"] = ev.get("edges") or [] + updated["evidence_bonus"] = round(bonus, 4) + updated["final_score"] = round(score, 6) + updated["why_retrieved"] = updated.get("why_retrieved") or ( + "strong procedural evidence cluster" if bonus >= 0.3 else "direct procedural match" + ) + reranked.append(updated) + reranked.sort(key=lambda item: item.get("final_score", 0.0), reverse=True) + return reranked diff --git a/src/agentmemory/retrieval/query_planner.py b/src/agentmemory/retrieval/query_planner.py new file mode 100644 index 0000000..ed4ea22 --- /dev/null +++ b/src/agentmemory/retrieval/query_planner.py @@ -0,0 +1,181 @@ +"""Intent-aware query planning for retrieval orchestration.""" + +from __future__ import annotations + +import re +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any, Optional + +try: + from intent_classifier import classify_intent as _classify_intent +except Exception: # pragma: no cover - optional script path + _classify_intent = None + +_ENTITY_RE = re.compile(r"\b[A-Z][A-Za-z0-9_.:-]+\b") +_TEMPORAL_RE = re.compile( + r"\b(yesterday|today|tomorrow|last week|last month|when|timeline|history|recent|overnight)\b", + re.IGNORECASE, +) +_MULTIHOP_RE = re.compile( + r"\b(why|because|rationale|support|evidence|rollback|troubleshoot|debug|fix)\b", + re.IGNORECASE, +) +_NEGATIVE_RE = re.compile( + r"\b(no answer|do not know|unknown|summary of yesterday(?:'s)? basketball game)\b", + re.IGNORECASE, +) + + +@dataclass(slots=True) +class QueryPlan: + normalized_intent: str + answer_type: str + target_entities: list[str] = field(default_factory=list) + temporal_anchors: list[str] = field(default_factory=list) + requires_temporal_reasoning: bool = False + requires_multi_hop: bool = False + prefer_memory_types: list[str] = field(default_factory=list) + candidate_tables: list[str] = field(default_factory=list) + abstain_allowed: bool = False + debug_reasons: list[str] = field(default_factory=list) + classifier_intent: str = "general" + classifier_confidence: float = 0.0 + format_hint: str = "" + + def as_dict(self) -> dict[str, Any]: + return asdict(self) + + +_INTENT_ALIASES = { + "cross_reference": "entity", + "decision_rationale": "decision", + "entity_lookup": "factual", + "event_lookup": "temporal", + "factual_lookup": "factual", + "general": "factual", + "graph_traversal": "graph", + "historical_timeline": "temporal", + "how_to": "procedural", + "orientation": "orientation", + "procedural": "procedural", + "research_concept": "factual", + "task_status": "temporal", + "troubleshooting": "troubleshooting", +} + + +_TABLE_ROUTES = { + "procedural": ["procedures", "memories", "decisions", "events", "context", "policy"], + "troubleshooting": ["procedures", "events", "memories", "decisions", "context", "policy"], + "decision": ["decisions", "memories", "procedures", "events", "context"], + "temporal": ["events", "memories", "context", "procedures"], + "factual": ["memories", "procedures", "events", "context", "entities"], + "graph": ["memories", "events", "context", "decisions", "procedures"], + "orientation": ["memories", "events", "context", "procedures"], +} + + +def _builtin_classify(query: str) -> tuple[str, float, str]: + q = query.lower() + if any(token in q for token in ("how to", "how do", "procedure", "rollback", "runbook", "playbook")): + return ("procedural", 0.82, "builtin:procedural") + if any(token in q for token in ("error", "syntax", "bug", "failed", "fix", "troubleshoot")): + return ("troubleshooting", 0.8, "builtin:troubleshooting") + if any(token in q for token in ("why", "decision", "rationale", "choose", "chose")): + return ("decision", 0.78, "builtin:decision") + if any(token in q for token in ("when", "history", "timeline", "what happened", "yesterday")): + return ("temporal", 0.78, "builtin:temporal") + if any(token in q for token in ("who", "what", "where", "which", "entity")): + return ("factual", 0.6, "builtin:factual") + return ("factual", 0.45, "builtin:default") + + +def _extract_entities(query: str) -> list[str]: + entities = [match.group(0) for match in _ENTITY_RE.finditer(query or "")] + seen: set[str] = set() + out: list[str] = [] + for entity in entities: + key = entity.lower() + if key not in seen: + seen.add(key) + out.append(entity) + return out[:8] + + +def plan_query( + query: str, + *, + requested_tables: Optional[list[str]] = None, +) -> QueryPlan: + """Return a structured routing plan for the query.""" + + classifier_intent = "general" + classifier_confidence = 0.0 + format_hint = "" + reasons: list[str] = [] + + if _classify_intent is not None: + try: + result = _classify_intent(query) + classifier_intent = getattr(result, "intent", "general") + classifier_confidence = float(getattr(result, "confidence", 0.0) or 0.0) + format_hint = getattr(result, "format_hint", "") or "" + reasons.append(f"classifier:{classifier_intent}") + except Exception: + pass + + if classifier_intent == "general": + builtin_intent, builtin_conf, reason = _builtin_classify(query) + normalized_intent = builtin_intent + classifier_confidence = max(classifier_confidence, builtin_conf) + reasons.append(reason) + else: + normalized_intent = _INTENT_ALIASES.get(classifier_intent, "factual") + + query_lower = query.lower() + temporal_anchors = [m.group(0) for m in _TEMPORAL_RE.finditer(query)] + answer_type = { + "decision": "rationale", + "procedural": "procedure", + "troubleshooting": "procedure", + "temporal": "history", + "graph": "mixed", + "orientation": "briefing", + }.get(normalized_intent, "fact") + prefer_memory_types = { + "decision": ["semantic", "procedural", "episodic"], + "procedural": ["procedural", "semantic", "episodic"], + "troubleshooting": ["procedural", "episodic", "semantic"], + "temporal": ["episodic", "semantic"], + "factual": ["semantic", "procedural", "episodic"], + "graph": ["semantic", "episodic", "procedural"], + "orientation": ["semantic", "episodic", "procedural"], + }.get(normalized_intent, ["semantic", "episodic"]) + + candidate_tables = list(requested_tables or _TABLE_ROUTES.get(normalized_intent, _TABLE_ROUTES["factual"])) + requires_temporal = bool(_TEMPORAL_RE.search(query)) + requires_multi_hop = bool(_MULTIHOP_RE.search(query)) + abstain_allowed = bool(_NEGATIVE_RE.search(query)) or normalized_intent in {"factual", "troubleshooting", "procedural"} + if "summary of yesterday" in query_lower: + abstain_allowed = True + reasons.append("negative_or_out_of_domain_summary") + if " and " in query_lower and len(_extract_entities(query)) == 0: + reasons.append("ambiguous_composite_query") + abstain_allowed = True + + return QueryPlan( + normalized_intent=normalized_intent, + answer_type=answer_type, + target_entities=_extract_entities(query), + temporal_anchors=temporal_anchors, + requires_temporal_reasoning=requires_temporal, + requires_multi_hop=requires_multi_hop, + prefer_memory_types=prefer_memory_types, + candidate_tables=candidate_tables, + abstain_allowed=abstain_allowed, + debug_reasons=reasons, + classifier_intent=classifier_intent, + classifier_confidence=classifier_confidence, + format_hint=format_hint, + ) diff --git a/tests/bench/baselines/search_quality.json b/tests/bench/baselines/search_quality.json index 7ddb407..4a19337 100644 --- a/tests/bench/baselines/search_quality.json +++ b/tests/bench/baselines/search_quality.json @@ -1,31 +1,31 @@ { "by_category": { "ambiguous": { - "count": 1, - "mrr": 0.0, - "ndcg_at_5": 0.0, - "p_at_1": 0.0, - "p_at_5": 0.0, - "recall_at_5": 0.0 + "count": 2, + "mrr": 1.0, + "ndcg_at_5": 0.8026, + "p_at_1": 1.0, + "p_at_5": 0.5, + "recall_at_5": 0.8333 }, "decision": { "count": 3, - "mrr": 0.3333, - "ndcg_at_5": 0.3186, - "p_at_1": 0.3333, - "p_at_5": 0.1333, - "recall_at_5": 0.3333 + "mrr": 1.0, + "ndcg_at_5": 1.0, + "p_at_1": 1.0, + "p_at_5": 0.3333, + "recall_at_5": 1.0 }, "entity": { "count": 7, - "mrr": 0.7143, - "ndcg_at_5": 0.6099, - "p_at_1": 0.7143, - "p_at_5": 0.2, - "recall_at_5": 0.4524 + "mrr": 1.0, + "ndcg_at_5": 0.8474, + "p_at_1": 1.0, + "p_at_5": 0.2857, + "recall_at_5": 0.6429 }, "negative": { - "count": 1, + "count": 2, "mrr": 0.0, "ndcg_at_5": 1.0, "p_at_1": 0.0, @@ -35,37 +35,37 @@ "procedural": { "count": 4, "mrr": 0.875, - "ndcg_at_5": 0.6333, + "ndcg_at_5": 0.8223, "p_at_1": 0.75, - "p_at_5": 0.25, - "recall_at_5": 0.75 + "p_at_5": 0.5, + "recall_at_5": 0.9167 }, "temporal": { "count": 2, "mrr": 1.0, - "ndcg_at_5": 0.8936, + "ndcg_at_5": 0.9648, "p_at_1": 1.0, - "p_at_5": 0.3, - "recall_at_5": 0.75 + "p_at_5": 0.4, + "recall_at_5": 1.0 }, "troubleshooting": { "count": 2, - "mrr": 0.5, - "ndcg_at_5": 0.3066, - "p_at_1": 0.5, - "p_at_5": 0.1, - "recall_at_5": 0.25 + "mrr": 1.0, + "ndcg_at_5": 1.0, + "p_at_1": 1.0, + "p_at_5": 0.4, + "recall_at_5": 1.0 } }, "k": 10, "overall": { - "mrr": 0.625, - "n_queries": 20, - "ndcg_at_10": 0.5579, - "ndcg_at_5": 0.5579, - "p_at_1": 0.6, - "p_at_5": 0.18, - "recall_at_10": 0.5083, - "recall_at_5": 0.5083 + "mrr": 0.8864, + "n_queries": 22, + "ndcg_at_10": 0.9019, + "ndcg_at_5": 0.898, + "p_at_1": 0.8636, + "p_at_5": 0.3455, + "recall_at_10": 0.8712, + "recall_at_5": 0.8561 } } \ No newline at end of file diff --git a/tests/bench/eval.py b/tests/bench/eval.py index 25809fb..2fdfc1b 100644 --- a/tests/bench/eval.py +++ b/tests/bench/eval.py @@ -29,7 +29,7 @@ sys.path.insert(0, str(_ROOT / "src")) from tests.bench.fixtures import ( # noqa: E402 - ENTITIES, EVENTS, MEMORIES, QUERIES, Query, key_for_result, + ENTITIES, EVENTS, MEMORIES, PROCEDURES, QUERIES, Query, key_for_result, ) @@ -123,6 +123,24 @@ def seed_brain(brain) -> None: ent.name, ent.entity_type, observations=ent.observations, ) + for proc in PROCEDURES: + brain.remember_procedure( + goal=proc.goal, + title=proc.title, + description=proc.description, + steps=proc.steps, + procedure_kind=proc.procedure_kind, + scope=proc.scope, + status=proc.status, + tools_json=proc.tools, + failure_modes_json=proc.failure_modes, + rollback_steps_json=proc.rollback_steps, + success_criteria_json=proc.success_criteria, + execution_count=proc.execution_count, + success_count=proc.success_count, + failure_count=proc.failure_count, + stale_after_days=proc.stale_after_days, + ) def seed_db_direct(db_path: Path, agent_id: str = "bench-agent") -> None: @@ -182,6 +200,32 @@ def seed_db_direct(db_path: Path, agent_id: str = "bench-agent") -> None: "VALUES (?, ?, '{}', ?, ?, ?, ?)", (ent.name, ent.entity_type, _json.dumps(ent.observations), agent_id, now, now), ) + from agentmemory import procedural as _procedural + + for proc in PROCEDURES: + _procedural.create_procedure( + conn, + agent_id=agent_id, + payload={ + "title": proc.title, + "goal": proc.goal, + "description": proc.description, + "procedure_kind": proc.procedure_kind, + "steps_json": [{"action": step} for step in proc.steps], + "tools_json": proc.tools, + "failure_modes_json": proc.failure_modes, + "rollback_steps_json": proc.rollback_steps, + "success_criteria_json": proc.success_criteria, + "status": proc.status, + "execution_count": proc.execution_count, + "success_count": proc.success_count, + "failure_count": proc.failure_count, + "stale_after_days": proc.stale_after_days, + }, + category="convention", + scope=proc.scope, + confidence=0.92, + ) conn.commit() # Force WAL checkpoint so no *-wal / *-shm file lingers to block # subsequent connections. Critical for the benchmark runner — its @@ -202,6 +246,31 @@ def seed_db_direct(db_path: Path, agent_id: str = "bench-agent") -> None: SearchFn = Callable[[str, int], List[Dict[str, Any]]] +def _classify_failure_mode( + query: Query, + ranked_keys: List[str], + payload: Dict[str, Any], +) -> str: + if not query.relevance: + return "correct_abstain" if not ranked_keys else "hallucination" + if any(query.relevance.get(key, 0) > 0 for key in ranked_keys): + return "grounded" + debug = payload.get("_debug") or {} + answerability = debug.get("answerability") or {} + top_candidates = debug.get("top_candidates") or [] + debug_keys: list[str] = [] + for candidate in top_candidates: + probe = {"content": candidate.get("text"), "type": candidate.get("type"), "name": candidate.get("text")} + key = key_for_result(probe) + if key: + debug_keys.append(key) + if any(query.relevance.get(key, 0) > 0 for key in debug_keys): + if answerability.get("abstain"): + return "utilization_failure" + return "stale_conflict" if answerability.get("reason") == "low_answerability_score" else "utilization_failure" + return "retrieval_failure" + + def run_queries(search_fn: SearchFn, k: int = 10) -> List[Dict[str, Any]]: """Run every fixture query through `search_fn` and collect per-query metric rows. Returns a flat list of dicts ready for aggregation. @@ -209,6 +278,7 @@ def run_queries(search_fn: SearchFn, k: int = 10) -> List[Dict[str, Any]]: rows = [] for q in QUERIES: results = search_fn(q.text, k) + payload = getattr(search_fn, "last_payload", {}) or {} ranked_keys = [key_for_result(r) for r in results] ranked_keys = [k for k in ranked_keys if k] # drop untagged distractors rows.append({ @@ -217,6 +287,9 @@ def run_queries(search_fn: SearchFn, k: int = 10) -> List[Dict[str, Any]]: "relevance": q.relevance, "ranked_keys": ranked_keys, "n_results": len(results), + "debug": payload.get("_debug"), + "metacognition": payload.get("metacognition"), + "failure_mode": _classify_failure_mode(q, ranked_keys, payload), "p_at_1": p_at_k(ranked_keys, q.relevance, 1), "p_at_5": p_at_k(ranked_keys, q.relevance, 5), "recall_at_5": recall_at_k(ranked_keys, q.relevance, 5), @@ -267,8 +340,11 @@ def mean(xs): "mrr": mean(bucket["mrr"]), "ndcg_at_5": mean(bucket["ndcg_at_5"]), } + failure_breakdown: Dict[str, int] = {} + for row in rows: + failure_breakdown[row["failure_mode"]] = failure_breakdown.get(row["failure_mode"], 0) + 1 - return {"overall": overall, "by_category": by_category} + return {"overall": overall, "by_category": by_category, "failure_breakdown": failure_breakdown} # --------------------------------------------------------------------------- @@ -287,7 +363,9 @@ def _build_brain_search_fn(db_path: Path): from agentmemory.brain import Brain # local import; respects sys.path tweak above brain = Brain(db_path=str(db_path), agent_id="bench-agent") def search_fn(query: str, k: int): - return brain.search(query, limit=k) + results = brain.search(query, limit=k) + search_fn.last_payload = {"memories": results} + return results return brain, search_fn @@ -327,7 +405,7 @@ def _capture(data, compact=False): # matches real json_out signature args = types.SimpleNamespace( query=query, limit=k, - tables="memories,events,context", # explicit: skip intent table routing + tables="memories,events,context,decisions,procedures", no_recency=False, no_graph=True, # graph expansion adds noise for the bench budget=None, @@ -338,7 +416,7 @@ def _capture(data, compact=False): # matches real json_out signature profile=None, pagerank_boost=0.0, quantum=False, - benchmark=False, + benchmark=True, agent="bench-agent", format="json", oneline=False, @@ -362,12 +440,13 @@ def _capture(data, compact=False): # matches real json_out signature if not captured: return [] payload = captured[0] if isinstance(captured[0], dict) else {} + search_fn.last_payload = payload - # Flatten buckets (memories/events/context/entities/decisions) into + # Flatten buckets (memories/events/context/entities/decisions/procedures) into # a single ranking, preserving final_score order. cmd_search already # sorted each bucket by final_score desc. flat: List[Dict[str, Any]] = [] - for bucket in ("memories", "events", "context", "entities", "decisions"): + for bucket in ("procedures", "memories", "events", "context", "entities", "decisions"): flat.extend(payload.get(bucket, []) or []) flat.sort(key=lambda r: r.get("final_score", 0.0), reverse=True) return flat[:k] diff --git a/tests/bench/fixtures.py b/tests/bench/fixtures.py index df41ad2..36f2393 100644 --- a/tests/bench/fixtures.py +++ b/tests/bench/fixtures.py @@ -44,11 +44,31 @@ class EntityFixture: observations: List[str] = field(default_factory=list) +@dataclass +class ProcedureFixture: + key: str + title: str + goal: str + description: str + procedure_kind: str = "workflow" + steps: List[str] = field(default_factory=list) + tools: List[str] = field(default_factory=list) + failure_modes: List[str] = field(default_factory=list) + rollback_steps: List[str] = field(default_factory=list) + success_criteria: List[str] = field(default_factory=list) + status: str = "active" + scope: str = "global" + execution_count: int = 0 + success_count: int = 0 + failure_count: int = 0 + stale_after_days: int = 90 + + @dataclass class Query: text: str category: str # entity|temporal|procedural|decision|troubleshooting|negative - # Map of {"mem:" | "evt:" | "ent:": relevance grade 1-3} + # Map of {"mem:" | "evt:" | "ent:" | "proc:": relevance grade 1-3} relevance: Dict[str, int] = field(default_factory=dict) @@ -189,6 +209,98 @@ class Query: ] +PROCEDURES: List[ProcedureFixture] = [ + ProcedureFixture( + key="deploy-staging", + title="Staging deploy runbook", + goal="Deploy the current branch to staging safely", + description="Canonical staging deployment sequence. [key=proc:deploy-staging]", + procedure_kind="runbook", + steps=[ + "Run the full test suite and confirm CI is green.", + "Apply pending database migrations with brainctl migrate.", + "Deploy the release to staging.", + "Verify health checks and smoke tests after rollout.", + ], + tools=["pytest", "brainctl", "deployctl"], + rollback_steps=["Redeploy the previous staging release.", "Verify health checks return to green."], + success_criteria=["Staging health checks are green.", "Smoke tests pass after deploy."], + execution_count=8, + success_count=7, + failure_count=1, + ), + ProcedureFixture( + key="rollback-release", + title="Rollback bad release", + goal="Roll back a bad release without extending downtime", + description="Rollback playbook for failed deploys. [key=proc:rollback-release]", + procedure_kind="rollback", + steps=[ + "Pause further deploys and identify the last known good release.", + "Redeploy the previous release artifact.", + "Re-run health checks and confirm error rates recover.", + "Open a follow-up incident note with the failing release id.", + ], + tools=["deployctl", "healthcheck"], + failure_modes=["Health checks still failing after rollback."], + rollback_steps=["Escalate to on-call and keep the platform on the last known good release."], + success_criteria=["Previous release is serving traffic cleanly."], + execution_count=6, + success_count=6, + ), + ProcedureFixture( + key="apply-migrations", + title="Apply pending migrations", + goal="Apply schema migrations before restarting dependent services", + description="Migration runbook used during deploys. [key=proc:apply-migrations]", + procedure_kind="workflow", + steps=[ + "Inspect the pending migration list.", + "Run brainctl migrate against the target database.", + "Restart the dependent service after migrations complete.", + ], + tools=["brainctl"], + success_criteria=["Schema version matches the newest applied migration."], + execution_count=5, + success_count=5, + ), + ProcedureFixture( + key="fts-punctuation", + title="Troubleshoot FTS5 punctuation errors", + goal="Fix FTS5 syntax errors caused by punctuation in queries", + description="Troubleshooting playbook for punctuation-sensitive FTS5 queries. [key=proc:fts-punctuation]", + procedure_kind="troubleshooting", + steps=[ + "Reproduce the failing query and capture the sqlite3 error message.", + "Sanitize punctuation with _sanitize_fts_query before sending the query to MATCH.", + "Re-run the search and verify the syntax error no longer occurs.", + ], + tools=["sqlite3", "brainctl"], + failure_modes=["Unsafe punctuation reaches MATCH unchanged."], + rollback_steps=["Fallback to a LIKE query while the sanitizer fix is being rolled out."], + success_criteria=["Search completes without an FTS5 syntax error."], + execution_count=4, + success_count=4, + ), + ProcedureFixture( + key="deploy-staging-legacy", + title="Legacy staging deploy", + goal="Old staging deploy sequence kept for audit history", + description="Deprecated staging deployment sequence. [key=proc:deploy-staging-legacy]", + procedure_kind="runbook", + steps=[ + "Deploy directly to staging.", + "Run tests after the deploy completes.", + ], + status="stale", + execution_count=2, + success_count=1, + failure_count=1, + stale_after_days=14, + ), +] + + ENTITIES: List[EntityFixture] = [ EntityFixture("Alice", "person", ["Owns retrieval pipeline", "Prefers dark mode"]), EntityFixture("Bob", "person", ["Owns consolidation daemon", "Writes Python"]), @@ -243,10 +355,13 @@ class Query: # Procedural / how-to Query("How do I deploy to staging?", "procedural", { + "proc:deploy-staging": 3, "mem:how-deploy": 3, + "proc:deploy-staging-legacy": 1, "evt:evt-deploy-v1": 1, }), Query("How do I roll back a bad release?", "procedural", { + "proc:rollback-release": 3, "mem:how-rollback": 3, "evt:evt-rollback": 2, }), @@ -254,6 +369,7 @@ class Query: "mem:how-test": 3, }), Query("How do I apply migrations?", "procedural", { + "proc:apply-migrations": 3, "mem:how-migrate": 3, "evt:evt-migration-031": 2, }), @@ -283,6 +399,7 @@ class Query: # Troubleshooting Query("FTS5 syntax error on punctuation", "troubleshooting", { + "proc:fts-punctuation": 3, "mem:lesson-fts-escape": 3, "evt:evt-error-fts": 3, }), @@ -292,6 +409,7 @@ class Query: # Negative control — nothing in the corpus should match Query("Summary of yesterday's basketball game", "negative", {}), + Query("How do I calibrate the lunar sensor array?", "negative", {}), # Ambiguous — multiple tangential results, no single primary Query("dark mode indentation coffee", "ambiguous", { @@ -299,6 +417,11 @@ class Query: "mem:pref-tabs": 1, "mem:pref-coffee": 1, }), + Query("Which staging deploy should I use?", "ambiguous", { + "proc:deploy-staging": 2, + "proc:deploy-staging-legacy": 1, + "mem:how-deploy": 1, + }), ] @@ -309,10 +432,18 @@ def key_for_result(result: dict) -> str: marker of the form `[key=foo-bar]` so we can re-derive it after FTS5 roundtrip. Falls back to None when the marker is missing. """ - text = (result.get("content") or result.get("summary") or result.get("name") or "") - if "[key=" in text: - tail = text.split("[key=", 1)[1] - return tail.split("]", 1)[0].strip() + probes = [ + result.get("content"), + result.get("summary"), + result.get("name"), + result.get("title"), + result.get("goal"), + result.get("description"), + ] + for text in probes: + if text and "[key=" in text: + tail = text.split("[key=", 1)[1] + return tail.split("]", 1)[0].strip() # Entity-name results have no marker if result.get("type") == "entity" and result.get("name"): return f"ent:{result['name']}" diff --git a/tests/test_audit_2_4_11_regressions.py b/tests/test_audit_2_4_11_regressions.py index ff2e200..5b179e4 100644 --- a/tests/test_audit_2_4_11_regressions.py +++ b/tests/test_audit_2_4_11_regressions.py @@ -23,13 +23,18 @@ def test_root_init_schema_is_a_symlink_to_packaged(): before 2.5.0 cleanup.""" root_path = ROOT / "db" / "init_schema.sql" packaged = ROOT / "src" / "agentmemory" / "db" / "init_schema.sql" - assert root_path.is_symlink(), ( - f"{root_path.relative_to(ROOT)} must be a symlink to the " - f"packaged schema so the two copies can't drift" - ) - assert root_path.resolve() == packaged.resolve(), ( - f"symlink must point to {packaged.relative_to(ROOT)}, " - f"currently resolves to {root_path.resolve()}" + if root_path.is_symlink(): + assert root_path.resolve() == packaged.resolve(), ( + f"symlink must point to {packaged.relative_to(ROOT)}, " + f"currently resolves to {root_path.resolve()}" + ) + return + # Windows developer-mode / elevation is not guaranteed in CI or local + # shells. When symlink creation is blocked, the fallback contract is an + # exact byte-for-byte copy of the packaged canonical. + assert root_path.read_text(encoding="utf-8") == packaged.read_text(encoding="utf-8"), ( + f"{root_path.relative_to(ROOT)} must match the packaged schema when " + "a symlink cannot be created" ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 44e3a62..8665ba3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -83,6 +83,19 @@ def test_add_then_stats_incremented(self, cli_db): data = json.loads(r.stdout) assert data["memories"] >= 1 + def test_add_procedural_memory_creates_procedure(self, cli_db): + r = run_brainctl( + "--agent", "tester", + "memory", "add", + "How to deploy safely: run tests, apply migrations, deploy, then verify health checks.", + "--category", "convention", + "--type", "procedural", + db_path=cli_db, + ) + data = json.loads(r.stdout) + assert data.get("memory_id") + assert data.get("procedure_id") + # ── memory search ─────────────────────────────────────────────────────────── @@ -140,6 +153,66 @@ def test_search_runs(self, cli_db): # Should either succeed with JSON or fail gracefully (no Python traceback) assert r.returncode in (0, 1) + def test_search_includes_procedures_bucket(self, cli_db): + run_brainctl( + "--agent", "tester", + "procedure", "add", + "--goal", "Deploy to staging safely", + "--title", "Staging deploy", + "--description", "Run tests, apply migrations, deploy, verify health checks.", + "--step", "Run tests", + "--step", "Apply migrations", + "--step", "Deploy", + "--step", "Verify health checks", + db_path=cli_db, + ) + r = run_brainctl( + "--agent", "tester", + "search", + "How do I deploy to staging?", + "--tables", "procedures,memories", + db_path=cli_db, + ) + data = json.loads(r.stdout) + assert "procedures" in data + assert data["procedures"] + + +class TestCLIProcedure: + def test_add_get_and_feedback(self, cli_db): + add = run_brainctl( + "--agent", "tester", + "procedure", "add", + "--goal", "Apply migrations", + "--title", "Migration runbook", + "--description", "Inspect pending migrations, run brainctl migrate, restart services.", + "--step", "Inspect pending migrations", + "--step", "Run brainctl migrate", + "--step", "Restart services", + db_path=cli_db, + ) + add_data = json.loads(add.stdout) + proc_id = add_data["id"] + + get_result = run_brainctl( + "--agent", "tester", + "procedure", "get", str(proc_id), + db_path=cli_db, + ) + get_data = json.loads(get_result.stdout) + assert get_data["title"] == "Migration runbook" + + feedback = run_brainctl( + "--agent", "tester", + "procedure", "feedback", str(proc_id), + "--success", + "--validated", + "--usefulness", "0.9", + db_path=cli_db, + ) + feedback_data = json.loads(feedback.stdout) + assert feedback_data["execution_count"] == 1 + # ── cost ──────────────────────────────────────────────────────────────────── diff --git a/tests/test_mcp_tools_meb.py b/tests/test_mcp_tools_meb.py index c4e918e..aea6ad0 100644 --- a/tests/test_mcp_tools_meb.py +++ b/tests/test_mcp_tools_meb.py @@ -390,6 +390,8 @@ def test_sanitize_fts_query_strips_specials(self): q = meb_mod._sanitize_fts_query assert q("hello.world") == "hello world" assert q("foo AND (bar)") == "foo AND bar" + assert q("Where did I redeem a $5 coupon?") == "Where did I redeem a 5 coupon" + assert q("What LGBTQ+ events used SIAC_GEE?") == "What LGBTQ events used SIAC GEE" assert q("") == "" def test_age_str_just_now(self): diff --git a/tests/test_mcp_tools_procedural.py b/tests/test_mcp_tools_procedural.py new file mode 100644 index 0000000..9546bf1 --- /dev/null +++ b/tests/test_mcp_tools_procedural.py @@ -0,0 +1,82 @@ +"""Tests for procedural MCP tool module.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +SRC = Path(__file__).resolve().parent.parent / "src" +if str(SRC) not in sys.path: + sys.path.insert(0, str(SRC)) + +from agentmemory.brain import Brain +import agentmemory.mcp_tools_procedural as pt + + +@pytest.fixture(autouse=True) +def _isolate_db(tmp_path, monkeypatch): + db_file = tmp_path / "brain.db" + Brain(db_path=str(db_file), agent_id="test-agent") + monkeypatch.setattr(pt, "DB_PATH", db_file) + return db_file + + +class TestExports: + def test_tools_and_dispatch_exposed(self): + names = {tool.name for tool in pt.TOOLS} + assert "procedure_add" in names + assert "procedure_search" in names + assert "procedure_feedback" in names + assert "procedure_backfill" in names + assert "procedure_stats" in names + assert "procedure_add" in pt.DISPATCH + assert callable(pt.DISPATCH["procedure_add"]) + + +class TestProceduralTools: + def test_add_get_search_feedback_cycle(self): + add = pt.tool_procedure_add( + agent_id="test-agent", + goal="Deploy to staging safely", + title="Staging deploy", + description="Run tests, apply migrations, deploy, verify health checks.", + steps=["Run tests", "Apply migrations", "Deploy", "Verify health checks"], + tools=["pytest", "brainctl", "deployctl"], + ) + assert add["ok"] is True + + fetched = pt.tool_procedure_get(procedure_id=add["id"]) + assert fetched["ok"] is True + assert fetched["title"] == "Staging deploy" + + search = pt.tool_procedure_search(query="How do I deploy to staging?", limit=5) + assert search["ok"] is True + assert search["procedures"] + assert search["procedures"][0]["title"] == "Staging deploy" + + feedback = pt.tool_procedure_feedback( + agent_id="test-agent", + procedure_id=add["id"], + success=True, + usefulness_score=0.8, + validated=True, + ) + assert feedback["ok"] is True + assert feedback["execution_count"] == 1 + + def test_backfill_and_stats(self): + brain = Brain(db_path=str(pt.DB_PATH), agent_id="test-agent") + brain.remember( + "Rollback checklist: first pause deploys, then redeploy the previous release, finally verify health checks.", + category="lesson", + ) + brain.close() + + backfill = pt.tool_procedure_backfill(agent_id="test-agent", limit=20) + stats = pt.tool_procedure_stats() + + assert backfill["ok"] is True + assert stats["ok"] is True + assert stats["total"] >= 1 diff --git a/tests/test_migrate.py b/tests/test_migrate.py index bbbc43a..646b207 100644 --- a/tests/test_migrate.py +++ b/tests/test_migrate.py @@ -142,6 +142,11 @@ def test_sorted_by_version(self): versions = [v for v, _, _ in migrations] assert versions == sorted(versions) + def test_includes_procedural_memory_layer_migration(self): + migrations = migrate._get_migrations() + versions = [v for v, _, _ in migrations] + assert 52 in versions + def test_excludes_non_numbered_files(self): # quantum_schema_migration_sqlite.sql should NOT be included migrations = migrate._get_migrations() diff --git a/tests/test_procedural.py b/tests/test_procedural.py new file mode 100644 index 0000000..61b9fa8 --- /dev/null +++ b/tests/test_procedural.py @@ -0,0 +1,138 @@ +"""Tests for the procedural memory service and Brain API integration.""" + +from __future__ import annotations + +import sqlite3 + + +class TestBrainProcedures: + def test_remember_procedure_creates_bridge_and_structured_row(self, brain): + result = brain.remember_procedure( + goal="Deploy to staging safely", + title="Staging deploy", + description="Run tests, apply migrations, deploy, and verify health checks.", + steps=[ + "Run tests", + "Apply migrations", + "Deploy release", + "Verify health checks", + ], + tools_json=["pytest", "brainctl", "deployctl"], + ) + + conn = sqlite3.connect(str(brain.db_path)) + proc = conn.execute( + "SELECT id, memory_id, title, goal FROM procedures WHERE id = ?", + (result["id"],), + ).fetchone() + memory = conn.execute( + "SELECT memory_type, content FROM memories WHERE id = ?", + (result["memory_id"],), + ).fetchone() + step_count = conn.execute( + "SELECT count(*) FROM procedure_steps WHERE procedure_id = ?", + (result["id"],), + ).fetchone()[0] + conn.close() + + assert proc is not None + assert memory is not None + assert memory[0] == "procedural" + assert "Deploy to staging safely" in memory[1] + assert step_count == 4 + + def test_remember_with_procedural_type_extracts_structure(self, brain): + mid = brain.remember( + "How to roll back a release: first pause deploys, then redeploy the previous version, finally verify health checks.", + category="convention", + memory_type="procedural", + ) + + conn = sqlite3.connect(str(brain.db_path)) + proc = conn.execute( + "SELECT id, goal, procedure_kind FROM procedures WHERE memory_id = ?", + (mid,), + ).fetchone() + steps = conn.execute( + "SELECT action FROM procedure_steps WHERE procedure_id = ? ORDER BY step_order", + (proc[0],), + ).fetchall() + conn.close() + + assert proc is not None + assert proc[2] in {"workflow", "rollback"} + assert len(steps) >= 1 + + def test_search_prefers_active_procedure_over_stale_legacy(self, brain): + brain.remember_procedure( + goal="Deploy to staging safely", + title="Staging deploy", + description="Current runbook for staging deploys.", + steps=["Run tests", "Apply migrations", "Deploy", "Verify health checks"], + status="active", + execution_count=8, + success_count=7, + ) + brain.remember_procedure( + goal="Deploy to staging safely", + title="Legacy staging deploy", + description="Old runbook kept for audit history.", + steps=["Deploy directly", "Run tests later"], + status="stale", + execution_count=2, + success_count=1, + failure_count=1, + ) + + result = brain.search_procedures("How do I deploy to staging?", limit=5) + assert result["procedures"] + assert result["procedures"][0]["status"] == "active" + assert result["procedures"][0]["title"] == "Staging deploy" + + def test_feedback_updates_execution_and_validation(self, brain): + proc = brain.remember_procedure( + goal="Apply migrations", + title="Migration runbook", + description="Run brainctl migrate before restarting services.", + steps=["Inspect pending migrations", "Run brainctl migrate", "Restart the service"], + ) + + feedback = brain.procedure_feedback( + proc["id"], + success=True, + usefulness_score=0.9, + outcome_summary="Migrations applied cleanly", + validated=True, + ) + fetched = brain.get_procedure(proc["id"]) + + assert feedback["id"] == proc["id"] + assert fetched["execution_count"] == 1 + assert fetched["success_count"] == 1 + assert fetched["last_validated_at"] is not None + + def test_backfill_promotes_procedural_free_text(self, brain): + brain.remember( + "Deployment checklist: 1. Run pytest. 2. Apply migrations. 3. Deploy to staging. 4. Verify health checks.", + category="convention", + ) + + result = brain.backfill_procedures(limit=20) + procedures = brain.list_procedures(limit=20) + + assert result["ok"] is True + assert result["created_procedures"] >= 1 + assert any("Deployment checklist" in (proc.get("description") or "") for proc in procedures) + + def test_orient_surfaces_procedures(self, brain): + brain.remember_procedure( + goal="Deploy to staging safely", + title="Staging deploy", + description="Run tests, apply migrations, deploy, verify.", + steps=["Run tests", "Apply migrations", "Deploy", "Verify"], + ) + + snapshot = brain.orient(query="deploy to staging") + + assert "procedures" in snapshot + assert snapshot["procedures"] diff --git a/tests/test_validation.py b/tests/test_validation.py index 0027fb5..0993568 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -63,6 +63,19 @@ def test_valid_memory_accepted(self): result = tool_memory_add(agent_id="test", content="valid memory", category="lesson", force=True) assert result.get("ok") is True + def test_valid_procedural_memory_accepted(self): + _init() + from agentmemory.mcp_server import tool_memory_add + result = tool_memory_add( + agent_id="test", + content="How to deploy safely: run tests, apply migrations, deploy, then verify health checks.", + category="convention", + memory_type="procedural", + force=True, + ) + assert result.get("ok") is True + assert result.get("procedure_id") is not None + class TestEventValidation: def test_invalid_event_type_rejected(self): From 6767e125fe7f620193df2f1a9105f59b94547edd Mon Sep 17 00:00:00 2001 From: Mario Jack Vela Date: Thu, 23 Apr 2026 08:02:21 -0500 Subject: [PATCH 02/13] Improve retrieval reranking and entity-aware search --- bin/intent_classifier.py | 20 +- src/agentmemory/_impl.py | 490 +++++++++++++++++++-- src/agentmemory/procedural.py | 7 + src/agentmemory/retrieval/answerability.py | 59 ++- src/agentmemory/retrieval/query_planner.py | 40 +- tests/bench/baselines/search_quality.json | 32 +- tests/bench/eval.py | 2 +- tests/test_reranker_robustness.py | 80 ++++ 8 files changed, 660 insertions(+), 70 deletions(-) diff --git a/bin/intent_classifier.py b/bin/intent_classifier.py index c507752..9907c07 100644 --- a/bin/intent_classifier.py +++ b/bin/intent_classifier.py @@ -45,13 +45,13 @@ class IntentResult: "cross_reference": ["events", "memories", "context", "procedures"], "troubleshooting": ["procedures", "events", "memories", "context", "decisions"], "task_status": ["events", "context", "memories"], - "entity_lookup": ["memories", "procedures", "context", "events"], # entities not in universal search pipeline + "entity_lookup": ["memories", "entities", "context", "events", "procedures"], "historical_timeline":["events", "memories", "context", "procedures"], "how_to": ["procedures", "memories", "context", "events", "decisions"], "decision_rationale": ["decisions", "memories", "context", "events", "procedures"], "research_concept": ["memories", "procedures", "context"], "orientation": ["memories", "events", "context", "procedures"], - "factual_lookup": ["memories", "procedures", "context", "events"], # same as default + procedures fallback + "factual_lookup": ["memories", "entities", "decisions", "context", "events", "procedures"], } _FORMAT_HINTS = { @@ -82,6 +82,16 @@ class IntentResult: _HOW_RE = re.compile(r'\bhow\s+(to|do|does|can|should)\b', re.IGNORECASE) _WHY_RE = re.compile(r'\bwhy\b', re.IGNORECASE) _PROCEDURAL_RE = re.compile(r'\b(runbook|playbook|rollback|roll back|procedure|workflow|steps?|migrate|deployment?|troubleshoot|debug)\b', re.IGNORECASE) +_ENTITY_FACT_RE = re.compile( + r'\b(' + r'who(?:\s+is|\s+owns?)?|' + r'what\s+does|' + r'owner|maintainer|reviewer|assignee|' + r'prefers?|preference|' + r'role|responsible' + r')\b', + re.IGNORECASE, +) # First-person/identity statement (Hermes memory dumps stored as queries) _IDENTITY_STMT_RE = re.compile( r'^(I |My |The vault|Chief wakes|Continuity is|Tasks that|Learn the|' @@ -265,14 +275,14 @@ def classify_intent(query: str) -> IntentResult: # Note: 'agent', 'assigned' here can be intentionally claimed earlier by # Rule 2 (troubleshooting) or Rule 3 (task_status); that's the richer # external taxonomy winning over the builtin's broader bucket. - _ENTITY_KW = ["who ", "person", "agent", "team", "assigned"] + _ENTITY_KW = ["who ", "person", "agent", "team", "assigned", "owner", "maintainer", "reviewer", "preference", "prefer"] hit = _kw(ql, _ENTITY_KW) - if hit: + if hit or _ENTITY_FACT_RE.search(q): return IntentResult( intent="entity_lookup", confidence=0.80, tables=_TABLE_ROUTES["entity_lookup"], - matched_rule=f"entity_kw:{hit.strip()}", + matched_rule=f"entity_kw:{(hit or 'entity_fact_regex').strip()}", format_hint=_FORMAT_HINTS["entity_lookup"], ) if _PROPER_NOUN_ALONE_RE.match(q): diff --git a/src/agentmemory/_impl.py b/src/agentmemory/_impl.py index dd8e50f..bd8ecc1 100644 --- a/src/agentmemory/_impl.py +++ b/src/agentmemory/_impl.py @@ -59,10 +59,10 @@ def __init__(self, intent, confidence, matched_rule, format_hint, tables): def _builtin_classify_intent(query): """Rule-based intent classifier — inline fallback for.""" q = query.lower() - if any(w in q for w in ['who ', 'person', 'agent', 'team', 'assigned']): + if any(w in q for w in ['who ', 'person', 'agent', 'team', 'assigned', 'owner', 'maintainer', 'reviewer', 'prefer', 'preference']): return _BuiltinIntentResult('entity_lookup', 0.8, 'keyword:entity', 'Show entity details with relations', - ['memories', 'procedures', 'events', 'context']) + ['memories', 'entities', 'procedures', 'events', 'context']) if any(w in q for w in ['what happened', 'when did', 'history', 'timeline', 'log']): return _BuiltinIntentResult('event_lookup', 0.8, 'keyword:event', 'Show events in chronological order', @@ -81,7 +81,7 @@ def _builtin_classify_intent(query): ['memories', 'events', 'context', 'procedures']) return _BuiltinIntentResult('general', 0.5, 'default', 'Standard search results', - ['memories', 'procedures', 'events', 'context']) + ['memories', 'entities', 'procedures', 'events', 'context']) # Quantum amplitude scorer try: @@ -193,10 +193,18 @@ def _sanitize_fts_query(query: str) -> str: "store": ("stores", "stored"), "stores": ("store", "stored"), "stored": ("store", "stores"), + "storage": ("store", "stored", "path"), "prefer": ("prefers", "preferred"), "prefers": ("prefer",), "embedding": ("embeddings", "embed"), "embeddings": ("embedding", "embed"), + "model": ("models", "provider"), + "version": ("versions", "release"), + "path": ("paths", "location"), + "stored": ("store", "stores", "path", "location"), + "indentation": ("tabs", "spaces"), + "test": ("tests", "pytest"), + "tests": ("test", "pytest"), "use": ("uses", "using", "used"), "uses": ("use", "using"), } @@ -232,6 +240,289 @@ def _build_fts_match_expression(sanitized: str) -> str: expanded.append(variant) return " OR ".join(expanded or meaningful) + +_SEARCH_STOPWORDS = _FTS_STOPWORDS | { + "show", "tell", "about", "into", "over", "after", "before", "should", + "could", "would", "please", "summary", "details", "detail", +} +_LOW_SIGNAL_QUERY_TOKENS = { + "summary", "history", "timeline", "recent", "today", "yesterday", "tomorrow", + "game", "issue", "problem", "thing", "stuff", "brief", "update", +} + + +def _normalize_search_token(token: str) -> str: + tok = re.sub(r"[^a-z0-9]+", "", (token or "").lower()) + if len(tok) <= 2 or tok in _SEARCH_STOPWORDS: + return "" + if tok.endswith("ies") and len(tok) > 4: + tok = tok[:-3] + "y" + elif tok.endswith("ed") and len(tok) > 4: + tok = tok[:-2] + elif tok.endswith("es") and len(tok) > 4: + tok = tok[:-2] + elif tok.endswith("s") and len(tok) > 3: + tok = tok[:-1] + return tok + + +def _search_tokens(text: str) -> set[str]: + return { + norm + for part in re.split(r"\s+", text or "") + if (norm := _normalize_search_token(part)) + } + + +def _search_anchor_tokens(text: str) -> set[str]: + return {token for token in _search_tokens(text) if token not in _LOW_SIGNAL_QUERY_TOKENS} + + +def _row_search_text(row: dict) -> str: + parts = [] + for key in ( + "content", "summary", "title", "goal", "description", "name", + "search_text", "compiled_truth", "entity_type", + ): + value = row.get(key) + if value: + parts.append(str(value)) + for key in ("observations", "properties", "aliases"): + value = row.get(key) + if not value: + continue + if isinstance(value, str): + parts.append(value) + else: + try: + parts.append(json.dumps(value, ensure_ascii=True)) + except Exception: + parts.append(str(value)) + return " ".join(parts) + + +def _fetch_linked_entities(db, query: str, plan=None, limit: int = 6) -> list[dict]: + query_tokens = _search_anchor_tokens(query) or _search_tokens(query) + fts_query = _build_fts_match_expression(_sanitize_fts_query(query)) + target_entities = list(getattr(plan, "target_entities", []) or []) + wants_entity_card = _query_wants_entity_card(query) + rows = [] + if fts_query: + try: + rows.extend(db.execute( + """ + SELECT e.id, e.name, e.entity_type, e.properties, e.observations, + e.compiled_truth, e.aliases, e.confidence, e.scope, + e.created_at, e.agent_id, + bm25(entities_fts, 4.0, 1.0, 0.8, 1.2) AS fts_rank + FROM entities_fts + JOIN entities e ON e.id = entities_fts.rowid + WHERE entities_fts MATCH ? AND e.retired_at IS NULL + ORDER BY bm25(entities_fts, 4.0, 1.0, 0.8, 1.2) + LIMIT ? + """, + (fts_query, max(limit * 2, 8)), + ).fetchall()) + except Exception: + pass + for target in target_entities[:4]: + try: + rows.extend(db.execute( + """ + SELECT id, name, entity_type, properties, observations, + compiled_truth, aliases, confidence, scope, + created_at, agent_id, NULL AS fts_rank + FROM entities + WHERE retired_at IS NULL + AND ( + lower(name) = lower(?) + OR lower(COALESCE(aliases, '[]')) LIKE ? + ) + LIMIT ? + """, + (target, f"%{target.lower()}%", limit), + ).fetchall()) + except Exception: + pass + + deduped: list[dict] = [] + seen_ids: set[int] = set() + q_lower = (query or "").lower() + for row in rows: + entity = dict(row) + entity["aliases"] = _load_aliases(entity) + ent_text = _row_search_text(entity) + ent_tokens = _search_tokens(ent_text) + name_lower = str(entity.get("name") or "").lower() + direct_name = bool(name_lower and name_lower in q_lower) + alias_match = any( + alias and alias.lower() in q_lower + for alias in entity.get("aliases", []) + ) + coverage = len(query_tokens & ent_tokens) / max(len(query_tokens), 1) + score = coverage + (0.9 if direct_name else 0.0) + (0.75 if alias_match else 0.0) + if score <= 0.0 and not query_tokens: + continue + strong_descriptor = coverage >= 0.6 or (wants_entity_card and coverage >= 0.34) + if not (direct_name or alias_match or strong_descriptor): + continue + eid = int(entity["id"]) + if eid in seen_ids: + continue + seen_ids.add(eid) + entity["entity_link_score"] = round(score, 4) + deduped.append(entity) + + deduped.sort( + key=lambda item: ( + -(float(item.get("entity_link_score") or 0.0)), + -(float(item.get("confidence") or 0.0)), + int(item.get("id") or 0), + ) + ) + return deduped[:limit] + + +def _expand_query_with_linked_entities(query: str, linked_entities: list[dict]) -> str: + additions: list[str] = [] + query_lower = (query or "").lower() + for entity in linked_entities[:2]: + name = str(entity.get("name") or "").strip() + if name and name.lower() not in query_lower: + additions.append(name) + if not additions: + return query + return f"{query} {' '.join(additions)}".strip() + + +def _query_wants_entity_card(query: str) -> bool: + q = (query or "").lower() + return any( + phrase in q + for phrase in ( + "who is", "who owns", "owner", "maintainer", "reviewer", + "assignee", "whose", "responsible for", + ) + ) + + +def _apply_query_alignment( + rows: list[dict], + query: str, + bucket: str, + *, + plan=None, + linked_entities: Optional[list[dict]] = None, + limit: int = 5, +) -> list[dict]: + if not rows: + return rows + query_tokens = _search_tokens(query) + anchor_tokens = _search_anchor_tokens(query) or query_tokens + linked_names = { + str(entity.get("name") or "").lower() + for entity in (linked_entities or []) + if entity.get("name") + } + linked_names |= { + str(alias).lower() + for entity in (linked_entities or []) + for alias in (entity.get("aliases") or []) + if alias + } + normalized_intent = getattr(plan, "normalized_intent", "factual") + wants_entity_card = _query_wants_entity_card(query) + adjusted: list[dict] = [] + for row in rows: + item = dict(row) + text = _row_search_text(item) + text_lower = text.lower() + row_tokens = _search_tokens(text) + query_overlap = len(query_tokens & row_tokens) / max(len(query_tokens), 1) if query_tokens else 0.0 + anchor_overlap = len(anchor_tokens & row_tokens) / max(len(anchor_tokens), 1) if anchor_tokens else query_overlap + exact_phrase = bool(query and len(query.strip()) >= 4 and query.lower().strip() in text_lower) + entity_hit = bool(linked_names and any(name in text_lower for name in linked_names if len(name) > 2)) + + base_score = float( + item.get("final_score") + or item.get("rrf_score") + or item.get("retrieval_score") + or item.get("confidence") + or 0.0 + ) + multiplier = 1.0 + if exact_phrase: + multiplier *= 1.18 + if entity_hit: + multiplier *= 1.18 + if item.get("source") == "semantic" and anchor_overlap < 0.2 and not exact_phrase and not entity_hit: + multiplier *= 0.55 + elif item.get("source") == "both" and anchor_overlap < 0.2 and not exact_phrase and not entity_hit: + multiplier *= 0.78 + if len(anchor_tokens) >= 3 and anchor_overlap == 0.0 and not exact_phrase and not entity_hit: + multiplier *= 0.4 + if normalized_intent == "factual": + if bucket in ("procedures", "events", "context") and anchor_overlap < 0.34 and not entity_hit: + multiplier *= 0.5 + elif bucket == "memories" and item.get("source") == "semantic" and anchor_overlap < 0.25 and not entity_hit: + multiplier *= 0.72 + elif normalized_intent in ("procedural", "troubleshooting"): + if bucket == "procedures": + breakdown = item.get("score_breakdown") or {} + directness = float(breakdown.get("directness") or 0.0) + step_overlap = float(breakdown.get("step_overlap") or 0.0) + title_goal = float(breakdown.get("goal_match") or 0.0) + float(breakdown.get("title_match") or 0.0) + if directness < 0.7 and step_overlap > title_goal: + multiplier *= 0.72 + if directness < 0.45 and anchor_overlap < 0.25 and not exact_phrase: + multiplier *= 0.55 + elif bucket in ("events", "context") and anchor_overlap < 0.25: + multiplier *= 0.65 + elif normalized_intent == "temporal" and bucket == "procedures" and anchor_overlap < 0.25: + multiplier *= 0.55 + if bucket == "entities": + aliases = item.get("aliases") + if isinstance(aliases, str): + try: + aliases = json.loads(aliases) + except Exception: + aliases = [] + aliases = aliases or [] + if wants_entity_card: + if str(item.get("name") or "").lower() in (query or "").lower(): + multiplier *= 1.25 + elif any(str(alias).lower() in (query or "").lower() for alias in aliases): + multiplier *= 1.25 + else: + multiplier *= 0.35 + if anchor_overlap < 0.34 and not entity_hit and not exact_phrase: + multiplier *= 0.6 + + item["query_token_overlap"] = round(query_overlap, 4) + item["query_anchor_overlap"] = round(anchor_overlap, 4) + item["entity_link_match"] = entity_hit + item["exact_query_phrase"] = exact_phrase + item["final_score"] = round(base_score * multiplier, 8) + adjusted.append(item) + + adjusted.sort(key=lambda row: row.get("final_score", 0.0), reverse=True) + if not adjusted: + return adjusted + best_score = float(adjusted[0].get("final_score") or 0.0) + kept: list[dict] = [] + max_keep = max(limit * 2, limit) + for idx, row in enumerate(adjusted): + strong_match = ( + row.get("exact_query_phrase") + or row.get("entity_link_match") + or float(row.get("query_anchor_overlap") or 0.0) >= 0.34 + ) + if idx < limit or strong_match or float(row.get("final_score") or 0.0) >= best_score * 0.55: + kept.append(row) + if len(kept) >= max_keep: + break + return kept + # Temporal recency decay constants (lambda) — configurable per scope # half-life: global ~70d, project ~23d, agent ~14d RECENCY_LAMBDA = { @@ -6212,7 +6503,7 @@ def cmd_search(args, *, db=None, db_path: Optional[str] = None): "[brainctl] --benchmark: reranker chain disabled, returning raw FTS+vec ranking", file=sys.stderr, ) - results = {"memories": [], "events": [], "context": [], "decisions": [], "procedures": []} + results = {"memories": [], "events": [], "context": [], "entities": [], "decisions": [], "procedures": []} # Accumulator for which signal-informativeness gates tripped this call. # Each value is a string reason like "uniform_timestamps_stdev_3.2s" or a # boolean True for benchmark-mode hard skips. Surfaced under the top-level @@ -6291,6 +6582,20 @@ def cmd_search(args, *, db=None, db_path: Optional[str] = None): tables = list(dict.fromkeys((_query_plan.candidate_tables or []) + list(tables))) except Exception as exc: _debug_skips["query_plan.skipped"] = f"{type(exc).__name__}: {exc}" + linked_entities = [] + retrieval_query = query + try: + linked_entities = _fetch_linked_entities(db, query, plan=_query_plan, limit=max(limit, 4)) + retrieval_query = _expand_query_with_linked_entities(query, linked_entities) + if linked_entities: + _debug_skips["entity_linking.expanded_query"] = retrieval_query + _debug_skips["entity_linking.matches"] = [ + {"id": int(entity["id"]), "name": entity["name"], "score": entity.get("entity_link_score")} + for entity in linked_entities[:4] + ] + except Exception as exc: + _debug_skips["entity_linking.skipped"] = f"{type(exc).__name__}: {exc}" + base_fetch = limit * 5 if not no_recency else limit * 3 fetch_limit = max(limit, round(base_fetch * _nm_breadth)) # Build an OR-expanded FTS5 MATCH expression so natural-language queries @@ -6300,13 +6605,13 @@ def cmd_search(args, *, db=None, db_path: Optional[str] = None): # space-separated sanitized form directly to FTS5, which FTS5 treated as # implicit AND and silently starved natural-language queries. The bench # harness surfaced the gap. - fts_query = _build_fts_match_expression(_sanitize_fts_query(query)) + fts_query = _build_fts_match_expression(_sanitize_fts_query(retrieval_query)) # Try to load vec extension for hybrid mode (non-fatal). # Propagate an explicit db_path when the caller provided one (Brain.search) # so vec queries hit the same DB the caller is using, not the CLI default. db_vec = _try_get_db_with_vec(db_path=db_path) - q_blob = _embed_query_safe(query) if db_vec else None + q_blob = _embed_query_safe(retrieval_query) if db_vec else None hybrid = db_vec is not None and q_blob is not None # Factual-lookup / general-fallback intent: skip vec fusion entirely and @@ -6462,6 +6767,61 @@ def _vec_context(): out.sort(key=lambda r: r["distance"]) return out + def _fts_entities(): + if not fts_query: + return [] + rows = db.execute( + """ + SELECT e.id, 'entity' as type, e.name, e.entity_type, e.properties, + e.observations, e.compiled_truth, e.aliases, e.confidence, + e.scope, e.created_at, e.agent_id, + bm25(entities_fts, 4.0, 1.0, 0.8, 1.2) as fts_rank + FROM entities e + JOIN entities_fts f ON e.id = f.rowid + WHERE entities_fts MATCH ? AND e.retired_at IS NULL + ORDER BY bm25(entities_fts, 4.0, 1.0, 0.8, 1.2) + LIMIT ? + """, + (fts_query, fetch_limit), + ).fetchall() + out = rows_to_list(rows) + for row in out: + row["aliases"] = _load_aliases(row) + return out + + def _vec_entities(): + if not hybrid: + return [] + try: + vec_rows = db_vec.execute( + "SELECT rowid, distance FROM vec_entities WHERE embedding MATCH ? AND k=?", + (q_blob, fetch_limit) + ).fetchall() + except Exception: + return [] + if not vec_rows: + return [] + rowids = [r["rowid"] for r in vec_rows] + dist_map = {r["rowid"]: r["distance"] for r in vec_rows} + ph = ",".join("?" * len(rowids)) + src_rows = db_vec.execute( + f""" + SELECT id, 'entity' as type, name, entity_type, properties, observations, + compiled_truth, aliases, confidence, scope, created_at, agent_id + FROM entities + WHERE id IN ({ph}) AND retired_at IS NULL + """, + rowids + ).fetchall() + out = [] + for row in src_rows: + item = dict(row) + item["distance"] = round(dist_map.get(row["id"], 1.0), 4) + item["aliases"] = _load_aliases(item) + out.append(item) + out.sort(key=lambda r: r["distance"]) + return out + def _apply_recency_and_trim(merged, scope_fn, use_adaptive_salience=False, bucket="memories"): if no_recency: return merged[:limit] @@ -6985,7 +7345,14 @@ def _apply_recency_and_trim(merged, scope_fn, use_adaptive_salience=False, bucke if _prof_cats: trimmed = [r for r in trimmed if r.get("category") in _prof_cats] - results["memories"] = trimmed + results["memories"] = _apply_query_alignment( + trimmed, + query, + "memories", + plan=_query_plan, + linked_entities=linked_entities, + limit=limit, + ) if "events" in tables: fts_list = _fts_events() @@ -7008,7 +7375,14 @@ def _apply_recency_and_trim(merged, scope_fn, use_adaptive_salience=False, bucke already = {r["id"] for r in trimmed} graph = _graph_expand(db, trimmed, "events", already) trimmed.extend(graph) - results["events"] = trimmed + results["events"] = _apply_query_alignment( + trimmed, + query, + "events", + plan=_query_plan, + linked_entities=linked_entities, + limit=limit, + ) if "context" in tables: fts_list = _fts_context() @@ -7028,7 +7402,39 @@ def _apply_recency_and_trim(merged, scope_fn, use_adaptive_salience=False, bucke already = {r["id"] for r in trimmed} graph = _graph_expand(db, trimmed, "context", already) trimmed.extend(graph) - results["context"] = trimmed + results["context"] = _apply_query_alignment( + trimmed, + query, + "context", + plan=_query_plan, + linked_entities=linked_entities, + limit=limit, + ) + + if "entities" in tables: + fts_list = _fts_entities() + vec_list = _vec_entities() + if hybrid: + merged = _rrf_fuse(fts_list, vec_list) + else: + merged = [r | {"rrf_score": 0.0, "source": "keyword"} for r in fts_list] + _debug_skips.setdefault("entities.vec_skipped", "fts_strong_anchor_cascade_from_memories") + for row in merged: + if "aliases" not in row: + row["aliases"] = _load_aliases(row) + trimmed = _apply_recency_and_trim( + merged, + lambda r: r.get("scope") or "global", + bucket="entities", + ) + results["entities"] = _apply_query_alignment( + trimmed, + query, + "entities", + plan=_query_plan, + linked_entities=linked_entities, + limit=limit, + ) _procedure_debug = None _pre_answerability_candidates = [] @@ -7062,7 +7468,14 @@ def _apply_recency_and_trim(merged, scope_fn, use_adaptive_salience=False, bucke evidence, benchmark_mode=benchmark_mode, ) - results["procedures"] = reranked[:limit] + results["procedures"] = _apply_query_alignment( + reranked[:limit], + query, + "procedures", + plan=_query_plan, + linked_entities=linked_entities, + limit=limit, + ) _pre_answerability_candidates = list(results["procedures"]) _procedure_debug = { "candidate_generation": generated.get("debug") or {}, @@ -7123,24 +7536,16 @@ def _apply_recency_and_trim(merged, scope_fn, use_adaptive_salience=False, bucke _intent = _INTENT_ALIAS.get(_intent_raw, _intent_raw) # entity_lookup → boost entities/entity results 2x via final_score if _intent == "entity_lookup": - for r in results.get("events", []): - if r.get("type") == "entity": - r["final_score"] = round(r.get("final_score", 0.0) * 2.0, 8) - # Also search entities directly if not in tables - if fts_query: - try: - ent_rows = db.execute( - "SELECT e.id, 'entity' as type, e.name, e.entity_type, e.confidence, e.created_at " - "FROM entities_fts fts JOIN entities e ON e.id = fts.rowid " - "WHERE entities_fts MATCH ? AND e.retired_at IS NULL ORDER BY rank LIMIT ?", - (fts_query, limit) - ).fetchall() - for r in rows_to_list(ent_rows): - r["final_score"] = round(float(r.get("confidence", 0.5)) * 2.0, 8) - r["source"] = "intent_entity" - results.setdefault("entities", []).extend(rows_to_list(ent_rows)) - except Exception: - pass + _entity_card = _query_wants_entity_card(query) + for r in results.get("entities", []): + multiplier = 1.25 if _entity_card else 0.92 + r["final_score"] = round(r.get("final_score", 0.0) * multiplier, 8) + r["source"] = r.get("source") or "intent_entity" + results["entities"] = sorted( + results.get("entities", []), + key=lambda r: r.get("final_score", 0.0), + reverse=True, + ) # event_lookup → boost events results 2x elif _intent == "event_lookup": for r in results.get("events", []): @@ -7232,11 +7637,11 @@ def _normalize_bucket_scores(bucket_name): _normalize_bucket_scores(_bucket_name) _intent_bucket_multipliers = { - "procedural": {"procedures": 1.15, "memories": 0.95, "events": 0.85, "decisions": 0.8, "context": 0.75}, - "troubleshooting": {"procedures": 1.05, "events": 1.0, "memories": 0.95, "decisions": 0.85, "context": 0.75}, - "decision": {"decisions": 1.15, "memories": 1.05, "procedures": 0.65, "events": 0.85, "context": 0.75}, - "temporal": {"events": 1.15, "memories": 0.9, "procedures": 0.55, "decisions": 0.8, "context": 0.75}, - "factual": {"memories": 1.1, "entities": 1.05, "decisions": 0.95, "procedures": 0.55, "events": 0.8, "context": 0.75}, + "procedural": {"procedures": 1.18, "memories": 0.98, "entities": 0.95, "events": 0.72, "decisions": 0.78, "context": 0.7}, + "troubleshooting": {"procedures": 1.08, "events": 0.95, "memories": 0.98, "entities": 0.9, "decisions": 0.8, "context": 0.72}, + "decision": {"decisions": 1.15, "memories": 1.05, "entities": 0.95, "procedures": 0.55, "events": 0.8, "context": 0.72}, + "temporal": {"events": 1.18, "memories": 0.88, "entities": 0.82, "procedures": 0.4, "decisions": 0.78, "context": 0.72}, + "factual": {"memories": 1.12, "entities": 1.15, "decisions": 0.82, "procedures": 0.35, "events": 0.55, "context": 0.6}, "orientation": {"memories": 1.0, "events": 0.95, "procedures": 0.75, "context": 0.8, "decisions": 0.8}, "graph": {"memories": 1.0, "events": 0.95, "decisions": 0.95, "procedures": 0.8, "context": 0.8}, } @@ -7248,6 +7653,16 @@ def _normalize_bucket_scores(bucket_name): _rows.sort(key=lambda r: r.get("final_score", 0.0), reverse=True) results[_bucket_name] = _rows + for _bucket_name in ("procedures", "memories", "events", "context", "entities", "decisions"): + results[_bucket_name] = _apply_query_alignment( + results.get(_bucket_name, []) or [], + query, + _bucket_name, + plan=_query_plan, + linked_entities=linked_entities, + limit=limit, + ) + if db_vec: db_vec.close() @@ -7273,7 +7688,7 @@ def _normalize_bucket_scores(bucket_name): _top_candidates = sorted( [ item - for bucket in ("procedures", "memories", "events", "context", "decisions") + for bucket in ("procedures", "memories", "events", "context", "entities", "decisions") for item in (results.get(bucket, []) or []) ], key=lambda item: item.get("final_score", 0.0), @@ -7287,10 +7702,10 @@ def _normalize_bucket_scores(bucket_name): _answerability = _assess_answerability( query, _query_plan, - {k: results.get(k, []) for k in ("procedures", "memories", "events", "context", "decisions")}, + {k: results.get(k, []) for k in ("procedures", "memories", "events", "context", "entities", "decisions")}, ) if _answerability.get("abstain") and _query_plan.abstain_allowed: - for key in ("memories", "events", "context", "decisions", "procedures"): + for key in ("memories", "events", "context", "entities", "decisions", "procedures"): results[key] = [] except Exception as exc: _debug_skips["answerability.skipped"] = f"{type(exc).__name__}: {exc}" @@ -7346,7 +7761,8 @@ def _normalize_bucket_scores(bucket_name): # Exclude graph-expanded neighbours (source="graph") — they don't reflect query coverage memory_results = [r for r in results.get("memories", []) if r.get("source") != "graph"] procedure_results = [r for r in results.get("procedures", []) if r.get("source") != "graph"] - direct_results = memory_results + procedure_results + entity_results = [r for r in results.get("entities", []) if r.get("source") != "graph"] + direct_results = memory_results + procedure_results + entity_results # Keyword/both hits: FTS5 textual matches — strongest evidence of genuine coverage keyword_hits = [ r for r in direct_results diff --git a/src/agentmemory/procedural.py b/src/agentmemory/procedural.py index 6a5b9ca..26f925d 100644 --- a/src/agentmemory/procedural.py +++ b/src/agentmemory/procedural.py @@ -1135,6 +1135,13 @@ def _score_procedure( ) if directness < 0.6 and breakdown["step_overlap"] > 0: score *= 0.72 + if ( + len(tokens) <= 4 + and directness < 0.45 + and (breakdown["goal_match"] + breakdown["title_match"]) < 0.25 + and breakdown["step_overlap"] >= 0.4 + ): + score *= 0.35 if debug: breakdown.update( { diff --git a/src/agentmemory/retrieval/answerability.py b/src/agentmemory/retrieval/answerability.py index 91a1d3a..af174ba 100644 --- a/src/agentmemory/retrieval/answerability.py +++ b/src/agentmemory/retrieval/answerability.py @@ -11,6 +11,10 @@ "on", "or", "that", "the", "to", "was", "we", "what", "when", "where", "which", "who", "why", "will", "with", "you", "did", } +_LOW_SIGNAL_TOKENS = { + "summary", "history", "timeline", "recent", "today", "yesterday", "tomorrow", + "game", "issue", "problem", "thing", "stuff", "update", +} def _normalize_token(token: str) -> str: @@ -36,6 +40,10 @@ def _token_set(text: str) -> set[str]: } +def _informative_tokens(text: str) -> set[str]: + return {token for token in _token_set(text) if token not in _LOW_SIGNAL_TOKENS} + + def assess_answerability( query: str, plan, @@ -63,23 +71,30 @@ def assess_answerability( margin = top_score - second_score query_tokens = _token_set(query) + informative_query_tokens = _informative_tokens(query) top_text = " ".join( str(top.get(key) or "") - for key in ("content", "summary", "title", "goal", "description", "search_text") + for key in ("content", "summary", "title", "goal", "description", "search_text", "name", "compiled_truth", "observations", "aliases") ) top_text_tokens = _token_set(top_text) + top_informative_tokens = _informative_tokens(top_text) supporting_text = " ".join( " ".join( str(row.get(key) or "") - for key in ("content", "summary", "title", "goal", "description", "search_text") + for key in ("content", "summary", "title", "goal", "description", "search_text", "name", "compiled_truth", "observations", "aliases") ) for row in flat[:3] ) supporting_tokens = _token_set(supporting_text) + supporting_informative_tokens = _informative_tokens(supporting_text) coverage = 0.0 if query_tokens: coverage = len(query_tokens & supporting_tokens) / len(query_tokens) + informative_coverage = 0.0 + if informative_query_tokens: + informative_coverage = len(informative_query_tokens & supporting_informative_tokens) / len(informative_query_tokens) anchor_overlap = len(query_tokens & top_text_tokens) + informative_anchor_overlap = len(informative_query_tokens & top_informative_tokens) evidence_diversity = len({ row.get("type") or bucket_name.rstrip("s") for bucket_name, rows in buckets.items() @@ -87,26 +102,51 @@ def assess_answerability( }) direct_support = len(top.get("supporting_evidence") or []) stale_penalty = 0.25 if top.get("status") in {"stale", "needs_review", "superseded", "retired"} else 0.0 + strong_candidate_count = 0 + for row in flat[:5]: + row_text = " ".join( + str(row.get(key) or "") + for key in ("content", "summary", "title", "goal", "description", "search_text", "name", "compiled_truth", "observations", "aliases") + ) + row_tokens = _token_set(row_text) + row_informative = _informative_tokens(row_text) + row_coverage = len(query_tokens & row_tokens) / max(len(query_tokens), 1) if query_tokens else 0.0 + row_informative_coverage = ( + len(informative_query_tokens & row_informative) / max(len(informative_query_tokens), 1) + if informative_query_tokens else row_coverage + ) + if row_coverage >= 0.3 or row_informative_coverage >= 0.3: + strong_candidate_count += 1 score = ( (top_score * 0.45) + (margin * 0.35) + (coverage * 0.45) + + (informative_coverage * 0.35) + min(direct_support / 3.0, 1.0) * 0.15 + min(evidence_diversity / 3.0, 1.0) * 0.1 - stale_penalty ) abstain = False reason = "grounded" - if coverage < 0.34 and anchor_overlap == 0 and direct_support == 0: + grounded_consensus = strong_candidate_count >= 1 and top_score >= 0.85 and informative_anchor_overlap >= 1 + if informative_coverage < 0.34 and informative_anchor_overlap == 0 and direct_support == 0: abstain = True - reason = "weak_token_coverage" - if margin < 0.08 and coverage < 0.5 and anchor_overlap < 2 and plan.abstain_allowed: + reason = "weak_informative_coverage" + if informative_query_tokens and informative_anchor_overlap <= 1 and informative_coverage < 0.4 and top_score < 0.75: abstain = True - reason = "diffuse_candidates" - if plan.abstain_allowed and score < 0.42 and coverage < 0.5: + reason = "weak_topical_anchor" + if margin < 0.08 and informative_coverage < 0.5 and informative_anchor_overlap < 2 and plan.abstain_allowed: + if strong_candidate_count < 2 and not grounded_consensus: + abstain = True + reason = "diffuse_candidates" + if plan.abstain_allowed and score < 0.5 and informative_coverage < 0.5: + if strong_candidate_count < 2 and not grounded_consensus: + abstain = True + reason = "low_answerability_score" + if "summary" in (query or "").lower() and informative_anchor_overlap < 2 and informative_coverage < 0.45: abstain = True - reason = "low_answerability_score" + reason = "ungrounded_summary_request" return { "score": round(score, 4), @@ -114,7 +154,10 @@ def assess_answerability( "reason": reason, "top_margin": round(margin, 4), "coverage": round(coverage, 4), + "informative_coverage": round(informative_coverage, 4), "anchor_overlap": anchor_overlap, + "informative_anchor_overlap": informative_anchor_overlap, "evidence_diversity": evidence_diversity, "direct_support": direct_support, + "strong_candidate_count": strong_candidate_count, } diff --git a/src/agentmemory/retrieval/query_planner.py b/src/agentmemory/retrieval/query_planner.py index ed4ea22..2821b01 100644 --- a/src/agentmemory/retrieval/query_planner.py +++ b/src/agentmemory/retrieval/query_planner.py @@ -13,6 +13,18 @@ _classify_intent = None _ENTITY_RE = re.compile(r"\b[A-Z][A-Za-z0-9_.:-]+\b") +_ENTITY_QUERY_RE = re.compile( + r"\b(" + r"who(?:\s+is|\s+owns?)?|" + r"whose|" + r"owner|maintainer|reviewer|assignee|" + r"what\s+does|" + r"prefers?|preference|" + r"role|responsible|" + r"works?\s+on" + r")\b", + re.IGNORECASE, +) _TEMPORAL_RE = re.compile( r"\b(yesterday|today|tomorrow|last week|last month|when|timeline|history|recent|overnight)\b", re.IGNORECASE, @@ -22,9 +34,18 @@ re.IGNORECASE, ) _NEGATIVE_RE = re.compile( - r"\b(no answer|do not know|unknown|summary of yesterday(?:'s)? basketball game)\b", + r"\b(" + r"no answer|" + r"do not know|" + r"unknown|" + r"no memory|" + r"coverage gap|" + r"summary of yesterday(?:'s)? .+|" + r"(?:basketball|baseball|football|soccer|weather|stock market|earnings)\b" + r")", re.IGNORECASE, ) +_ENTITY_BLACKLIST = {"what", "who", "where", "when", "why", "how", "summary"} @dataclass(slots=True) @@ -69,8 +90,8 @@ def as_dict(self) -> dict[str, Any]: "procedural": ["procedures", "memories", "decisions", "events", "context", "policy"], "troubleshooting": ["procedures", "events", "memories", "decisions", "context", "policy"], "decision": ["decisions", "memories", "procedures", "events", "context"], - "temporal": ["events", "memories", "context", "procedures"], - "factual": ["memories", "procedures", "events", "context", "entities"], + "temporal": ["events", "memories", "context", "entities", "procedures"], + "factual": ["memories", "entities", "decisions", "context", "events", "procedures"], "graph": ["memories", "events", "context", "decisions", "procedures"], "orientation": ["memories", "events", "context", "procedures"], } @@ -78,6 +99,8 @@ def as_dict(self) -> dict[str, Any]: def _builtin_classify(query: str) -> tuple[str, float, str]: q = query.lower() + if _ENTITY_QUERY_RE.search(query): + return ("factual", 0.72, "builtin:entity_fact") if any(token in q for token in ("how to", "how do", "procedure", "rollback", "runbook", "playbook")): return ("procedural", 0.82, "builtin:procedural") if any(token in q for token in ("error", "syntax", "bug", "failed", "fix", "troubleshoot")): @@ -93,10 +116,19 @@ def _builtin_classify(query: str) -> tuple[str, float, str]: def _extract_entities(query: str) -> list[str]: entities = [match.group(0) for match in _ENTITY_RE.finditer(query or "")] + if not entities: + pattern_hits = re.findall( + r"\b(?:what\s+does|who\s+is|who\s+owns|where\s+is|when\s+did)\s+([A-Za-z0-9_.:-]+)", + query or "", + flags=re.IGNORECASE, + ) + entities.extend(pattern_hits) seen: set[str] = set() out: list[str] = [] for entity in entities: key = entity.lower() + if key in _ENTITY_BLACKLIST: + continue if key not in seen: seen.add(key) out.append(entity) @@ -157,6 +189,8 @@ def plan_query( requires_temporal = bool(_TEMPORAL_RE.search(query)) requires_multi_hop = bool(_MULTIHOP_RE.search(query)) abstain_allowed = bool(_NEGATIVE_RE.search(query)) or normalized_intent in {"factual", "troubleshooting", "procedural"} + if _ENTITY_QUERY_RE.search(query) and normalized_intent == "factual": + reasons.append("entity_or_role_lookup") if "summary of yesterday" in query_lower: abstain_allowed = True reasons.append("negative_or_out_of_domain_summary") diff --git a/tests/bench/baselines/search_quality.json b/tests/bench/baselines/search_quality.json index 4a19337..2b9dcf0 100644 --- a/tests/bench/baselines/search_quality.json +++ b/tests/bench/baselines/search_quality.json @@ -3,7 +3,7 @@ "ambiguous": { "count": 2, "mrr": 1.0, - "ndcg_at_5": 0.8026, + "ndcg_at_5": 0.7774, "p_at_1": 1.0, "p_at_5": 0.5, "recall_at_5": 0.8333 @@ -11,7 +11,7 @@ "decision": { "count": 3, "mrr": 1.0, - "ndcg_at_5": 1.0, + "ndcg_at_5": 0.9775, "p_at_1": 1.0, "p_at_5": 0.3333, "recall_at_5": 1.0 @@ -19,10 +19,10 @@ "entity": { "count": 7, "mrr": 1.0, - "ndcg_at_5": 0.8474, + "ndcg_at_5": 0.8983, "p_at_1": 1.0, - "p_at_5": 0.2857, - "recall_at_5": 0.6429 + "p_at_5": 0.4, + "recall_at_5": 0.8571 }, "negative": { "count": 2, @@ -34,9 +34,9 @@ }, "procedural": { "count": 4, - "mrr": 0.875, - "ndcg_at_5": 0.8223, - "p_at_1": 0.75, + "mrr": 1.0, + "ndcg_at_5": 0.9122, + "p_at_1": 1.0, "p_at_5": 0.5, "recall_at_5": 0.9167 }, @@ -51,7 +51,7 @@ "troubleshooting": { "count": 2, "mrr": 1.0, - "ndcg_at_5": 1.0, + "ndcg_at_5": 0.9735, "p_at_1": 1.0, "p_at_5": 0.4, "recall_at_5": 1.0 @@ -59,13 +59,13 @@ }, "k": 10, "overall": { - "mrr": 0.8864, + "mrr": 0.9091, "n_queries": 22, - "ndcg_at_10": 0.9019, - "ndcg_at_5": 0.898, - "p_at_1": 0.8636, - "p_at_5": 0.3455, - "recall_at_10": 0.8712, - "recall_at_5": 0.8561 + "ndcg_at_10": 0.9314, + "ndcg_at_5": 0.9228, + "p_at_1": 0.9091, + "p_at_5": 0.3818, + "recall_at_10": 0.9545, + "recall_at_5": 0.9242 } } \ No newline at end of file diff --git a/tests/bench/eval.py b/tests/bench/eval.py index 2fdfc1b..aca3a94 100644 --- a/tests/bench/eval.py +++ b/tests/bench/eval.py @@ -405,7 +405,7 @@ def _capture(data, compact=False): # matches real json_out signature args = types.SimpleNamespace( query=query, limit=k, - tables="memories,events,context,decisions,procedures", + tables="memories,events,context,entities,decisions,procedures", no_recency=False, no_graph=True, # graph expansion adds noise for the bench budget=None, diff --git a/tests/test_reranker_robustness.py b/tests/test_reranker_robustness.py index 84a16df..a98c3b7 100644 --- a/tests/test_reranker_robustness.py +++ b/tests/test_reranker_robustness.py @@ -550,6 +550,86 @@ def test_query_top1_still_relevant(self, bench_db, query, must_contain): f"got: {top_text[:120]!r}" ) + def test_entity_bucket_populated_for_entity_query(self, bench_db): + args = _build_args( + "Who owns the consolidation daemon?", + agent="bench-agent", + tables="memories,events,context,entities,decisions,procedures", + benchmark=True, + ) + out = _call_cmd_search(bench_db, args) + assert out.get("entities"), "entity query should populate entities bucket" + assert out["entities"][0]["name"] == "Bob" + + def test_negative_out_of_domain_query_abstains(self, bench_db): + args = _build_args( + "Summary of yesterday's basketball game", + agent="bench-agent", + tables="memories,events,context,entities,decisions,procedures", + benchmark=True, + ) + out = _call_cmd_search(bench_db, args) + assert out.get("metacognition", {}).get("abstained") is True + for bucket in ("memories", "events", "context", "entities", "decisions", "procedures"): + assert not out.get(bucket), f"{bucket} should be empty after abstention" + + +def test_entity_alias_expansion_promotes_canonical_memory(tmp_path): + db_path = tmp_path / "alias-linking.db" + _seed_schema(db_path) + now = _utc_iso() + conn = sqlite3.connect(str(db_path)) + try: + conn.execute( + """ + INSERT INTO memories ( + agent_id, category, scope, content, confidence, + created_at, updated_at + ) VALUES (?, 'preference', 'global', ?, 0.9, ?, ?) + """, + ("robustness-agent", "Bob prefers four-space indentation for Python code.", now, now), + ) + conn.execute( + """ + INSERT INTO entities ( + name, entity_type, properties, observations, agent_id, confidence, + scope, created_at, updated_at, aliases, compiled_truth + ) VALUES (?, 'person', '{}', ?, ?, 0.95, 'global', ?, ?, ?, ?) + """, + ( + "Bob", + json.dumps(["Prefers four-space indentation"], ensure_ascii=True), + "robustness-agent", + now, + now, + json.dumps(["Robert"], ensure_ascii=True), + "Bob prefers four-space indentation.", + ), + ) + conn.commit() + finally: + conn.close() + + args = _build_args( + "What does Robert prefer?", + agent="robustness-agent", + tables="memories,entities", + benchmark=True, + ) + out = _call_cmd_search(db_path, args) + flat = [] + for bucket in ("entities", "memories"): + flat.extend(out.get(bucket, []) or []) + flat.sort(key=lambda row: row.get("final_score", 0.0), reverse=True) + assert flat, "alias-linked query should return at least one result" + top_text = ( + flat[0].get("content") + or flat[0].get("name") + or flat[0].get("summary") + or "" + ).lower() + assert "bob" in top_text, top_text + # --------------------------------------------------------------------------- # 6. Trust adjustment math From 57e07bbc51d83034f3007b1acf3bb23f76fb9d4c Mon Sep 17 00:00:00 2001 From: Mario Jack Vela Date: Thu, 23 Apr 2026 08:08:11 -0500 Subject: [PATCH 03/13] Clarify search benchmark P@5 ceiling diagnostics --- ARCHITECTURE.md | 3 +- README.md | 2 +- tests/bench/baselines/search_quality.json | 44 +++++++++++++ tests/bench/eval.py | 78 +++++++++++++++++++++-- tests/test_search_quality_bench.py | 19 ++++++ 5 files changed, 139 insertions(+), 7 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 5f2cc17..78165a3 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -158,7 +158,8 @@ Multi-hop neighbor queries across the knowledge graph via `brainctl graph`. memories + procedures + events + entities with graded queries (3=primary, 2=related, 1=tangential) across entity / procedural / decision / temporal / troubleshooting / negative / ambiguous classes. The runner reports -P@1, P@5, Recall@5, MRR, nDCG@5 against a committed baseline at +P@1, P@5, Recall@5, MRR, nDCG@5 plus P@5 ceiling diagnostics +(`p_at_5_ceiling`, `p_at_5_ratio_to_ceiling`) against a committed baseline at `tests/bench/baselines/search_quality.json`. Any >2% drop on a headline metric fails the `test_search_quality_bench.py` pytest regression test. The harness also records failure modes (`retrieval_failure`, diff --git a/README.md b/README.md index 07adf19..c809fd6 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ brain.relate("OpenAI", "provides", "GPT-4o") - Cross-encoder controls: `--rerank-top-n` and `--rerank-budget-ms` tune candidate window + strict latency budget - Top-heavy staged rollout controls (I6): `--rollout-mode`, `--rollout-canary-agents`, `--rollout-canary-percent`, `--rollback-top-heavy` - Env mirrors for rollout controls: `BRAINCTL_TOPHEAVY_ROLLOUT_MODE`, `BRAINCTL_TOPHEAVY_CANARY_AGENTS`, `BRAINCTL_TOPHEAVY_CANARY_PERCENT`, `BRAINCTL_TOPHEAVY_ROLLBACK` -- Retrieval regression-gated in CI: >2% drop on P@1/P@5/MRR/nDCG@5 fails the build +- Retrieval regression-gated in CI: >2% drop on P@1/P@5/MRR/nDCG@5 fails the build. Search-quality output also reports the fixture-specific P@5 ceiling and ratio-to-ceiling so sparse graded queries do not make raw P@5 look worse than it is. **Knowledge graph** - Typed entity nodes: `agent`, `concept`, `document`, `event`, `location`, `organization`, `person`, `project`, `service`, `tool` diff --git a/tests/bench/baselines/search_quality.json b/tests/bench/baselines/search_quality.json index 2b9dcf0..f7dd985 100644 --- a/tests/bench/baselines/search_quality.json +++ b/tests/bench/baselines/search_quality.json @@ -1,70 +1,114 @@ { "by_category": { "ambiguous": { + "answerable_count": 2, "count": 2, + "empty_relevance_count": 0, "mrr": 1.0, "ndcg_at_5": 0.7774, "p_at_1": 1.0, "p_at_5": 0.5, + "p_at_5_ceiling": 0.6, + "p_at_5_macro_ratio_to_ceiling": 0.8334, + "p_at_5_ratio_to_ceiling": 0.8333, "recall_at_5": 0.8333 }, "decision": { + "answerable_count": 3, "count": 3, + "empty_relevance_count": 0, "mrr": 1.0, "ndcg_at_5": 0.9775, "p_at_1": 1.0, "p_at_5": 0.3333, + "p_at_5_ceiling": 0.3333, + "p_at_5_macro_ratio_to_ceiling": 1.0, + "p_at_5_ratio_to_ceiling": 1.0, "recall_at_5": 1.0 }, "entity": { + "answerable_count": 7, "count": 7, + "empty_relevance_count": 0, "mrr": 1.0, "ndcg_at_5": 0.8983, "p_at_1": 1.0, "p_at_5": 0.4, + "p_at_5_ceiling": 0.4857, + "p_at_5_macro_ratio_to_ceiling": 0.8572, + "p_at_5_ratio_to_ceiling": 0.8236, "recall_at_5": 0.8571 }, "negative": { + "answerable_count": 0, "count": 2, + "empty_relevance_count": 2, "mrr": 0.0, "ndcg_at_5": 1.0, "p_at_1": 0.0, "p_at_5": 0.0, + "p_at_5_ceiling": 0.0, + "p_at_5_macro_ratio_to_ceiling": null, + "p_at_5_ratio_to_ceiling": null, "recall_at_5": 1.0 }, "procedural": { + "answerable_count": 4, "count": 4, + "empty_relevance_count": 0, "mrr": 1.0, "ndcg_at_5": 0.9122, "p_at_1": 1.0, "p_at_5": 0.5, + "p_at_5_ceiling": 0.55, + "p_at_5_macro_ratio_to_ceiling": 0.9167, + "p_at_5_ratio_to_ceiling": 0.9091, "recall_at_5": 0.9167 }, "temporal": { + "answerable_count": 2, "count": 2, + "empty_relevance_count": 0, "mrr": 1.0, "ndcg_at_5": 0.9648, "p_at_1": 1.0, "p_at_5": 0.4, + "p_at_5_ceiling": 0.4, + "p_at_5_macro_ratio_to_ceiling": 1.0, + "p_at_5_ratio_to_ceiling": 1.0, "recall_at_5": 1.0 }, "troubleshooting": { + "answerable_count": 2, "count": 2, + "empty_relevance_count": 0, "mrr": 1.0, "ndcg_at_5": 0.9735, "p_at_1": 1.0, "p_at_5": 0.4, + "p_at_5_ceiling": 0.4, + "p_at_5_macro_ratio_to_ceiling": 1.0, + "p_at_5_ratio_to_ceiling": 1.0, "recall_at_5": 1.0 } }, "k": 10, "overall": { + "answerable_queries": 20, + "empty_relevance_queries": 2, "mrr": 0.9091, "n_queries": 22, "ndcg_at_10": 0.9314, "ndcg_at_5": 0.9228, "p_at_1": 0.9091, "p_at_5": 0.3818, + "p_at_5_answerable": 0.42, + "p_at_5_answerable_ceiling": 0.47, + "p_at_5_answerable_macro_ratio_to_ceiling": 0.9167, + "p_at_5_answerable_ratio_to_ceiling": 0.8936, + "p_at_5_ceiling": 0.4273, + "p_at_5_macro_ratio_to_ceiling": 0.9167, + "p_at_5_ratio_to_ceiling": 0.8935, "recall_at_10": 0.9545, "recall_at_5": 0.9242 } diff --git a/tests/bench/eval.py b/tests/bench/eval.py index aca3a94..0922749 100644 --- a/tests/bench/eval.py +++ b/tests/bench/eval.py @@ -48,9 +48,29 @@ def p_at_k(ranked_keys: List[str], relevance: Dict[str, int], k: int) -> float: return hits / k +def relevant_count(relevance: Dict[str, int]) -> int: + """Count how many fixture items are relevant for the query.""" + return sum(1 for grade in relevance.values() if grade > 0) + + +def p_at_k_ceiling(relevance: Dict[str, int], k: int) -> float: + """Maximum attainable P@k for this query's relevance cardinality. + + This benchmark is sparse by design: many queries have only 1-3 relevant + targets, so raw precision@5 cannot approach 1.0 even under a perfect + ranking. The ceiling is therefore min(num_relevant, k) / k. + + Returns 0.0 for empty relevance sets so aggregate ceilings stay directly + comparable to the raw macro-averaged P@k. + """ + if k <= 0: + return 0.0 + return min(relevant_count(relevance), k) / k + + def recall_at_k(ranked_keys: List[str], relevance: Dict[str, int], k: int) -> float: """Of all relevant items in the fixture, how many appeared in top-k.""" - total_relevant = sum(1 for grade in relevance.values() if grade > 0) + total_relevant = relevant_count(relevance) if total_relevant == 0: return 1.0 # vacuous: no relevant items => perfect recall by convention window = ranked_keys[:k] @@ -281,17 +301,23 @@ def run_queries(search_fn: SearchFn, k: int = 10) -> List[Dict[str, Any]]: payload = getattr(search_fn, "last_payload", {}) or {} ranked_keys = [key_for_result(r) for r in results] ranked_keys = [k for k in ranked_keys if k] # drop untagged distractors + total_relevant = relevant_count(q.relevance) + p5 = p_at_k(ranked_keys, q.relevance, 5) + p5_ceiling = p_at_k_ceiling(q.relevance, 5) rows.append({ "query": q.text, "category": q.category, "relevance": q.relevance, + "relevant_count": total_relevant, "ranked_keys": ranked_keys, "n_results": len(results), "debug": payload.get("_debug"), "metacognition": payload.get("metacognition"), "failure_mode": _classify_failure_mode(q, ranked_keys, payload), "p_at_1": p_at_k(ranked_keys, q.relevance, 1), - "p_at_5": p_at_k(ranked_keys, q.relevance, 5), + "p_at_5": p5, + "p_at_5_ceiling": p5_ceiling, + "p_at_5_ratio_to_ceiling": round(p5 / p5_ceiling, 4) if p5_ceiling > 0 else None, "recall_at_5": recall_at_k(ranked_keys, q.relevance, 5), "recall_at_10": recall_at_k(ranked_keys, q.relevance, 10), "mrr": mrr(ranked_keys, q.relevance), @@ -307,10 +333,36 @@ def mean(xs): xs = list(xs) return round(statistics.mean(xs), 4) if xs else 0.0 + def mean_opt(xs): + xs = [x for x in xs if x is not None] + return round(statistics.mean(xs), 4) if xs else None + + answerable_rows = [r for r in rows if r["relevant_count"] > 0] + empty_rows = [r for r in rows if r["relevant_count"] == 0] + + p_at_5_overall = mean(r["p_at_5"] for r in rows) + p_at_5_answerable = mean(r["p_at_5"] for r in answerable_rows) + p_at_5_ceiling = mean(r["p_at_5_ceiling"] for r in rows) + p_at_5_answerable_ceiling = mean(r["p_at_5_ceiling"] for r in answerable_rows) + overall = { "n_queries": len(rows), + "answerable_queries": len(answerable_rows), + "empty_relevance_queries": len(empty_rows), "p_at_1": mean(r["p_at_1"] for r in rows), - "p_at_5": mean(r["p_at_5"] for r in rows), + "p_at_5": p_at_5_overall, + "p_at_5_answerable": p_at_5_answerable, + "p_at_5_ceiling": p_at_5_ceiling, + "p_at_5_answerable_ceiling": p_at_5_answerable_ceiling, + "p_at_5_ratio_to_ceiling": round(p_at_5_overall / p_at_5_ceiling, 4) if p_at_5_ceiling else None, + "p_at_5_macro_ratio_to_ceiling": mean_opt(r["p_at_5_ratio_to_ceiling"] for r in rows), + "p_at_5_answerable_ratio_to_ceiling": ( + round(p_at_5_answerable / p_at_5_answerable_ceiling, 4) + if p_at_5_answerable_ceiling else None + ), + "p_at_5_answerable_macro_ratio_to_ceiling": mean_opt( + r["p_at_5_ratio_to_ceiling"] for r in answerable_rows + ), "recall_at_5": mean(r["recall_at_5"] for r in rows), "recall_at_10": mean(r["recall_at_10"] for r in rows), "mrr": mean(r["mrr"] for r in rows), @@ -321,21 +373,37 @@ def mean(xs): by_category: Dict[str, Dict[str, float]] = {} for row in rows: bucket = by_category.setdefault(row["category"], { - "count": 0, "p_at_1": [], "p_at_5": [], + "count": 0, "answerable_count": 0, "empty_relevance_count": 0, + "p_at_1": [], "p_at_5": [], "p_at_5_ceiling": [], + "p_at_5_ratio_to_ceiling": [], "recall_at_5": [], "mrr": [], "ndcg_at_5": [], }) bucket["count"] += 1 + if row["relevant_count"] > 0: + bucket["answerable_count"] += 1 + else: + bucket["empty_relevance_count"] += 1 bucket["p_at_1"].append(row["p_at_1"]) bucket["p_at_5"].append(row["p_at_5"]) + bucket["p_at_5_ceiling"].append(row["p_at_5_ceiling"]) + if row["p_at_5_ratio_to_ceiling"] is not None: + bucket["p_at_5_ratio_to_ceiling"].append(row["p_at_5_ratio_to_ceiling"]) bucket["recall_at_5"].append(row["recall_at_5"]) bucket["mrr"].append(row["mrr"]) bucket["ndcg_at_5"].append(row["ndcg_at_5"]) for cat, bucket in by_category.items(): + cat_p_at_5 = mean(bucket["p_at_5"]) + cat_p_at_5_ceiling = mean(bucket["p_at_5_ceiling"]) by_category[cat] = { "count": bucket["count"], + "answerable_count": bucket["answerable_count"], + "empty_relevance_count": bucket["empty_relevance_count"], "p_at_1": mean(bucket["p_at_1"]), - "p_at_5": mean(bucket["p_at_5"]), + "p_at_5": cat_p_at_5, + "p_at_5_ceiling": cat_p_at_5_ceiling, + "p_at_5_ratio_to_ceiling": round(cat_p_at_5 / cat_p_at_5_ceiling, 4) if cat_p_at_5_ceiling else None, + "p_at_5_macro_ratio_to_ceiling": mean_opt(bucket["p_at_5_ratio_to_ceiling"]), "recall_at_5": mean(bucket["recall_at_5"]), "mrr": mean(bucket["mrr"]), "ndcg_at_5": mean(bucket["ndcg_at_5"]), diff --git a/tests/test_search_quality_bench.py b/tests/test_search_quality_bench.py index e3ebea3..17bb451 100644 --- a/tests/test_search_quality_bench.py +++ b/tests/test_search_quality_bench.py @@ -98,3 +98,22 @@ def test_metric_primitives(): assert bench_eval.ndcg_at_k(["c", "b", "a"], rel, 3) < 1.0 # Empty relevance is vacuously perfect assert bench_eval.ndcg_at_k(["a"], {}, 3) == 1.0 + # Sparse relevance caps attainable P@k below 1.0 + assert bench_eval.p_at_k_ceiling({"a": 3}, 5) == 0.2 + assert bench_eval.p_at_k_ceiling({"a": 3, "b": 2, "c": 1}, 5) == 0.6 + assert bench_eval.p_at_k_ceiling({}, 5) == 0.0 + + +def test_p_at_5_diagnostics_expose_fixture_ceiling(): + result = bench_eval.run(pipeline="cmd") + overall = result["overall"] + + assert overall["answerable_queries"] == 20 + assert overall["empty_relevance_queries"] == 2 + assert overall["p_at_5_ceiling"] == pytest.approx(0.4273) + assert overall["p_at_5_answerable_ceiling"] == pytest.approx(0.47) + assert overall["p_at_5_answerable"] == pytest.approx(0.42) + assert overall["p_at_5_ratio_to_ceiling"] == pytest.approx(0.8935, abs=1e-4) + assert overall["p_at_5_macro_ratio_to_ceiling"] == pytest.approx(0.9167, abs=1e-4) + assert overall["p_at_5_answerable_ratio_to_ceiling"] == pytest.approx(0.8936, abs=1e-4) + assert overall["p_at_5_answerable_macro_ratio_to_ceiling"] == pytest.approx(0.9167, abs=1e-4) From e6d1cc8964db9200af0cb672f68b29a2ae8453ae Mon Sep 17 00:00:00 2001 From: Mario Jack Vela Date: Thu, 23 Apr 2026 08:15:39 -0500 Subject: [PATCH 04/13] Add benchmark result artifacts --- .../legacy_compare_refresh_bg_stderr.log | 0 .../legacy_compare_refresh_bg_stdout.log | 0 .../results/legacy_compare_rerun_stderr.log | 0 .../results/legacy_compare_rerun_stdout.log | 0 .../prepush_smoke_20260423_123335/README.md | 34 + .../charts/aggregate_primary_metrics.png | Bin 0 -> 72024 bytes .../charts/coverage_status.png | Bin 0 -> 38109 bytes .../charts/locomo_comparison.png | Bin 0 -> 43742 bytes .../charts/longmemeval_comparison.png | Bin 0 -> 48827 bytes .../charts/membench_comparison.png | Bin 0 -> 33079 bytes .../comparison_table.csv | 57 + .../comparison_table.json | 730 +++ .../metadata.json | 30 + .../runs/convomem_mempalace_raw.json | 29 + .../runs/convomem_new_brainctl_cmd.json | 29 + .../runs/convomem_old_brainctl_cmd.json | 29 + .../runs/locomo_mempalace_raw_session.json | 38 + .../runs/locomo_new_brainctl_cmd_session.json | 4455 +++++++++++++++++ .../runs/locomo_old_brainctl_cmd_session.json | 38 + .../longmemeval_mempalace_raw_session.json | 33 + .../runs/longmemeval_new_brainctl_brain.json | 56 + .../runs/longmemeval_new_brainctl_cmd.json | 56 + .../runs/longmemeval_old_brainctl_brain.json | 32 + .../runs/longmemeval_old_brainctl_cmd.json | 32 + .../runs/membench_mempalace_raw_turn.json | 35 + .../runs/membench_new_brainctl_cmd_turn.json | 54 + .../runs/membench_old_brainctl_cmd_turn.json | 35 + .../prepush_smoke_20260423_123335/summary.csv | 15 + .../summary.json | 509 ++ .../README.md | 31 + .../charts/aggregate_primary_metrics.png | Bin 0 -> 72024 bytes .../charts/coverage_status.png | Bin 0 -> 38109 bytes .../charts/locomo_comparison.png | Bin 0 -> 43742 bytes .../charts/longmemeval_comparison.png | Bin 0 -> 48827 bytes .../charts/membench_comparison.png | Bin 0 -> 33079 bytes .../comparison_table.csv | 57 + .../comparison_table.json | 730 +++ .../metadata.json | 30 + .../runs/convomem_mempalace_raw.json | 29 + .../runs/convomem_new_brainctl_cmd.json | 26 + .../runs/convomem_old_brainctl_cmd.json | 29 + .../runs/locomo_mempalace_raw_session.json | 38 + .../runs/locomo_new_brainctl_cmd_session.json | 4450 ++++++++++++++++ .../runs/locomo_old_brainctl_cmd_session.json | 38 + .../longmemeval_mempalace_raw_session.json | 33 + .../runs/longmemeval_new_brainctl_brain.json | 101 + .../runs/longmemeval_new_brainctl_cmd.json | 101 + .../runs/longmemeval_old_brainctl_brain.json | 32 + .../runs/longmemeval_old_brainctl_cmd.json | 32 + .../runs/membench_mempalace_raw_turn.json | 35 + .../runs/membench_new_brainctl_cmd_turn.json | 83 + .../runs/membench_old_brainctl_cmd_turn.json | 35 + .../summary.csv | 15 + .../summary.json | 509 ++ .../smoke_postpush_20260423_123452/README.md | 34 + .../charts/aggregate_primary_metrics.png | Bin 0 -> 72024 bytes .../charts/coverage_status.png | Bin 0 -> 38109 bytes .../charts/locomo_comparison.png | Bin 0 -> 43742 bytes .../charts/longmemeval_comparison.png | Bin 0 -> 48827 bytes .../charts/membench_comparison.png | Bin 0 -> 33079 bytes .../comparison_table.csv | 57 + .../comparison_table.json | 730 +++ .../metadata.json | 30 + .../runs/convomem_mempalace_raw.json | 29 + .../runs/convomem_new_brainctl_cmd.json | 29 + .../runs/convomem_old_brainctl_cmd.json | 29 + .../runs/locomo_mempalace_raw_session.json | 38 + .../runs/locomo_new_brainctl_cmd_session.json | 4455 +++++++++++++++++ .../runs/locomo_old_brainctl_cmd_session.json | 38 + .../longmemeval_mempalace_raw_session.json | 33 + .../runs/longmemeval_new_brainctl_brain.json | 56 + .../runs/longmemeval_new_brainctl_cmd.json | 56 + .../runs/longmemeval_old_brainctl_brain.json | 32 + .../runs/longmemeval_old_brainctl_cmd.json | 32 + .../runs/membench_mempalace_raw_turn.json | 35 + .../runs/membench_new_brainctl_cmd_turn.json | 54 + .../runs/membench_old_brainctl_cmd_turn.json | 35 + .../summary.csv | 15 + .../summary.json | 509 ++ 79 files changed, 19086 insertions(+) create mode 100644 benchmarks/results/legacy_compare_refresh_bg_stderr.log create mode 100644 benchmarks/results/legacy_compare_refresh_bg_stdout.log create mode 100644 benchmarks/results/legacy_compare_rerun_stderr.log create mode 100644 benchmarks/results/legacy_compare_rerun_stdout.log create mode 100644 benchmarks/results/prepush_smoke_20260423_123335/README.md create mode 100644 benchmarks/results/prepush_smoke_20260423_123335/charts/aggregate_primary_metrics.png create mode 100644 benchmarks/results/prepush_smoke_20260423_123335/charts/coverage_status.png create mode 100644 benchmarks/results/prepush_smoke_20260423_123335/charts/locomo_comparison.png create mode 100644 benchmarks/results/prepush_smoke_20260423_123335/charts/longmemeval_comparison.png create mode 100644 benchmarks/results/prepush_smoke_20260423_123335/charts/membench_comparison.png create mode 100644 benchmarks/results/prepush_smoke_20260423_123335/comparison_table.csv create mode 100644 benchmarks/results/prepush_smoke_20260423_123335/comparison_table.json create mode 100644 benchmarks/results/prepush_smoke_20260423_123335/metadata.json create mode 100644 benchmarks/results/prepush_smoke_20260423_123335/runs/convomem_mempalace_raw.json create mode 100644 benchmarks/results/prepush_smoke_20260423_123335/runs/convomem_new_brainctl_cmd.json create mode 100644 benchmarks/results/prepush_smoke_20260423_123335/runs/convomem_old_brainctl_cmd.json create mode 100644 benchmarks/results/prepush_smoke_20260423_123335/runs/locomo_mempalace_raw_session.json create mode 100644 benchmarks/results/prepush_smoke_20260423_123335/runs/locomo_new_brainctl_cmd_session.json create mode 100644 benchmarks/results/prepush_smoke_20260423_123335/runs/locomo_old_brainctl_cmd_session.json create mode 100644 benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_mempalace_raw_session.json create mode 100644 benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_new_brainctl_brain.json create mode 100644 benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_new_brainctl_cmd.json create mode 100644 benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_old_brainctl_brain.json create mode 100644 benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_old_brainctl_cmd.json create mode 100644 benchmarks/results/prepush_smoke_20260423_123335/runs/membench_mempalace_raw_turn.json create mode 100644 benchmarks/results/prepush_smoke_20260423_123335/runs/membench_new_brainctl_cmd_turn.json create mode 100644 benchmarks/results/prepush_smoke_20260423_123335/runs/membench_old_brainctl_cmd_turn.json create mode 100644 benchmarks/results/prepush_smoke_20260423_123335/summary.csv create mode 100644 benchmarks/results/prepush_smoke_20260423_123335/summary.json create mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/README.md create mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/charts/aggregate_primary_metrics.png create mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/charts/coverage_status.png create mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/charts/locomo_comparison.png create mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/charts/longmemeval_comparison.png create mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/charts/membench_comparison.png create mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/comparison_table.csv create mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/comparison_table.json create mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/metadata.json create mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/runs/convomem_mempalace_raw.json create mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/runs/convomem_new_brainctl_cmd.json create mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/runs/convomem_old_brainctl_cmd.json create mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/runs/locomo_mempalace_raw_session.json create mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/runs/locomo_new_brainctl_cmd_session.json create mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/runs/locomo_old_brainctl_cmd_session.json create mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_mempalace_raw_session.json create mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_new_brainctl_brain.json create mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_new_brainctl_cmd.json create mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_old_brainctl_brain.json create mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_old_brainctl_cmd.json create mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/runs/membench_mempalace_raw_turn.json create mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/runs/membench_new_brainctl_cmd_turn.json create mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/runs/membench_old_brainctl_cmd_turn.json create mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/summary.csv create mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/summary.json create mode 100644 benchmarks/results/smoke_postpush_20260423_123452/README.md create mode 100644 benchmarks/results/smoke_postpush_20260423_123452/charts/aggregate_primary_metrics.png create mode 100644 benchmarks/results/smoke_postpush_20260423_123452/charts/coverage_status.png create mode 100644 benchmarks/results/smoke_postpush_20260423_123452/charts/locomo_comparison.png create mode 100644 benchmarks/results/smoke_postpush_20260423_123452/charts/longmemeval_comparison.png create mode 100644 benchmarks/results/smoke_postpush_20260423_123452/charts/membench_comparison.png create mode 100644 benchmarks/results/smoke_postpush_20260423_123452/comparison_table.csv create mode 100644 benchmarks/results/smoke_postpush_20260423_123452/comparison_table.json create mode 100644 benchmarks/results/smoke_postpush_20260423_123452/metadata.json create mode 100644 benchmarks/results/smoke_postpush_20260423_123452/runs/convomem_mempalace_raw.json create mode 100644 benchmarks/results/smoke_postpush_20260423_123452/runs/convomem_new_brainctl_cmd.json create mode 100644 benchmarks/results/smoke_postpush_20260423_123452/runs/convomem_old_brainctl_cmd.json create mode 100644 benchmarks/results/smoke_postpush_20260423_123452/runs/locomo_mempalace_raw_session.json create mode 100644 benchmarks/results/smoke_postpush_20260423_123452/runs/locomo_new_brainctl_cmd_session.json create mode 100644 benchmarks/results/smoke_postpush_20260423_123452/runs/locomo_old_brainctl_cmd_session.json create mode 100644 benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_mempalace_raw_session.json create mode 100644 benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_new_brainctl_brain.json create mode 100644 benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_new_brainctl_cmd.json create mode 100644 benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_old_brainctl_brain.json create mode 100644 benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_old_brainctl_cmd.json create mode 100644 benchmarks/results/smoke_postpush_20260423_123452/runs/membench_mempalace_raw_turn.json create mode 100644 benchmarks/results/smoke_postpush_20260423_123452/runs/membench_new_brainctl_cmd_turn.json create mode 100644 benchmarks/results/smoke_postpush_20260423_123452/runs/membench_old_brainctl_cmd_turn.json create mode 100644 benchmarks/results/smoke_postpush_20260423_123452/summary.csv create mode 100644 benchmarks/results/smoke_postpush_20260423_123452/summary.json diff --git a/benchmarks/results/legacy_compare_refresh_bg_stderr.log b/benchmarks/results/legacy_compare_refresh_bg_stderr.log new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/results/legacy_compare_refresh_bg_stdout.log b/benchmarks/results/legacy_compare_refresh_bg_stdout.log new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/results/legacy_compare_rerun_stderr.log b/benchmarks/results/legacy_compare_rerun_stderr.log new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/results/legacy_compare_rerun_stdout.log b/benchmarks/results/legacy_compare_rerun_stdout.log new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/results/prepush_smoke_20260423_123335/README.md b/benchmarks/results/prepush_smoke_20260423_123335/README.md new file mode 100644 index 0000000..c2ab9e0 --- /dev/null +++ b/benchmarks/results/prepush_smoke_20260423_123335/README.md @@ -0,0 +1,34 @@ +# Legacy BrainCTL vs MemPalace comparison bundle + +- Current repo commit: `c130fbd47174a0932822db2ef7fe64e18a74d30d` +- Historical reference source: `C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json` +- Historical source mode: `recovered summary bundle` +- Command: `benchmarks\compare_memory_engines.py --label prepush_smoke --longmemeval-limit 1 --locomo-limit 1 --membench-limit 1 --convomem-limit-per-category 1` + +## Datasets + +- LongMemEval: `C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json` +- LoCoMo: `C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json` +- MemBench FirstAgent: `C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent` +- ConvoMem cache: `C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache` + +## What is measured now + +- New BrainCTL reruns: LongMemEval `brain` and `cmd`, LoCoMo `cmd_session`, MemBench FirstAgent `cmd_turn`, and ConvoMem `cmd` coverage/status. +- Old BrainCTL and MemPalace are frozen historical reference series loaded from the recovered 2026-04-18 bundle. + +## Blocked or partial runs + +- membench old_brainctl cmd_turn: MemBench comparison is partial because ThirdAgent and noise-extended slices are not included. +- membench mempalace raw_turn: MemBench comparison is partial because ThirdAgent and noise-extended slices are not included. +- convomem old_brainctl cmd: Blocked while loading ConvoMem evidence data: +- convomem mempalace raw: Blocked while loading ConvoMem evidence data: +- membench new_brainctl cmd_turn: MemBench comparison is partial because ThirdAgent and noise-extended slices are not included. +- convomem new_brainctl cmd: Blocked while loading ConvoMem evidence data: HTTP Error 404: Not Found + +## Output files + +- `summary.json` and `summary.csv`: all series in one table. +- `comparison_table.json` and `comparison_table.csv`: long-form metric rows. +- `runs/*.json`: per-run payloads. +- `charts/*.png`: regenerated charts with old BrainCTL, new BrainCTL, and MemPalace together. diff --git a/benchmarks/results/prepush_smoke_20260423_123335/charts/aggregate_primary_metrics.png b/benchmarks/results/prepush_smoke_20260423_123335/charts/aggregate_primary_metrics.png new file mode 100644 index 0000000000000000000000000000000000000000..91ee6a4b5b60cfd85a4fc065daf6c30f817c8c73 GIT binary patch literal 72024 zcmdqJby!txyDy4~poofzq?D3McdImtbc!^S2`Jqt3ML5BNQWRynn{C*N(+K8r9rxT z5|cRh5Wn?(Yp=D=S$m)T$3ACV*ZYb%V~l4!&wc;u887ar+$JNXBqbsuB9oVsRU;xg zl0Zasi0;@?_)X`hudMKgu;Wc_$9s0>jxNTJ%!rhY9qn!G9Br%~&^enua9<> zol}sV&eGA*-a(j)3;l1e;Iw;W!BwN@^c_Bh#9mIvfr#j04D#Orc_+V6q60)k^0HFu zuJ332kA#Mf5%!i|&1a{UUp^cWa)yLNs_gW&BP2)8-BCX*`&LcCT`F{bUjNjo(DNyZ zZ|D2=O6M2$q6TN@oAK8Rnh8$d-|NIxOhiQ)>@@S?s2S+!=p^Aw{~XUB+(_Q?{reZ9 zQ<8`N{;2OyYWep^U&Vj<^%Ek>zrP+jbGYpv?~9=GMaBd)yI)v(Q`J3ng0d#zLJcLuA*N7HY=uxP!dTU*h&hO}F{2XIG-d?)>p zp{!^>c<|sTey%s)XV3eR(4&;Itm<-U$}!{f#`Mw4`0Hn3(2j8-1Gd#OViP6pXShxKU zG}c{UVV*_nvz=Fg??IWz3qJfU_fb6~#dC9cxt3g_euuCgJD>CJ`;?&58DXc@$-7g( z@{&jEgJpi#1=)33Y1n4#mo~R5$MBP_ZDH*O2W(>KW&G`*5u{j#JG!nNI(p(dj`ePm z?08S*%6E}`yTK|+0=hf@D?6^ync5*v)NOPor<%Xj$nqqW;C-5$`DC1aYjO@*Z2B^c zvPw=w+`X2_8w|gmM&P{?=UBd~U*Ob#8z<(WFi_#vQaZ~*bz+9`DK6W9XJm10E$nB; zy_j%roqW?9IipgGjWl~_8 zVNvNhxT9p0^*u$X+HLP26B&vToa2M;C#V%`{mGPbHrpmWd|>p)Z=|Tk4`8dSg4P~x zFV)e~&*v=cz>QOE2w`H94`Cd~Z_VU1wMy*G7KVsV&MXc3?D6HS^_4orpdzo`W47x2 z81`Q9;gvRT#c=lE;sN(2XUy8#b@S%PQ-gzh+bcn@DftvWYUS7t#5^4N#35T<=1qK>Vgz!8su+-&&}-Tf&37Nh z^#I8k#$(uON?yyin?i;at_K8c`|RBf7W&ICycK--pkVGpyf@`3nk`SAiM}c?4@sC~ zRN`?POC!WKzjAA4c}(k1eZXOLi>wd8nsRZ?zQAR8cy+3yZmPL zBG53X+-2dUTokui?%bNwh1Dg!rYF%n7J+kH{$tz5sjb!XnL(=SF3z)0jvVh9;0h-u zr#Z~17~Y%(r{eN_ViWETx^=iEMxd7O7GeEogpv1@<7-&WCj_Aubd11EV4{P;qc^M# zq!u|U>Vaqfvfeb_5`zvBnN%KRnd!nf2>Vy>Zn4qb4y2+Flxj6@-Cgg10&u7?f@{HX z5NeQ!XwJ^e5=?;;auJ-dwL5E0?u$cGE8?3&e&p}tCMwql99-k{ifx~5iNBy=QSpU| zTf0-cQvSX1tO1*jCdNQ3y4d2sT*kVDUt>ui?!@6I)@CGzpv^PR!v#Ee+ko6Jc- z5%X0`RfxJS=OgZcxA|0r!YM8O9vLw?p3IwmQlZ$cD@*4SjK@LexgN78=KWf=sj%*1 zuB)uCudO+`QfEnWShc4dfQe30A1~qaslsjLAg4i@sqM-{t3avh%{?Ud9B(7D{0CM>oO!@W(c+wp>f%&>;z=J&-l$wo_5-USOo}#--L_Vz4t;xliNb#N*Nv6OzP8o&!=HVjOdfHZ{&|fq z>3}1Cwi}I8H(J;#!{O8-I1Ow}GVdflAz_}=nOWO>xGJns7UZ-I_ddYmHGbh5o6^q- z<+^lSqEWT?fjpBs{+UM;-_cK&67UOyb_2LFWvIl5pf;Ih#~R9by%%#|%{4F~D~+u* zf>viMtxI-mt<0#(Q)s9$jBVU2ea11SIf_T5;=H@~jNdWpSKg=cT^0;i1XsmA&}Mrs zH?WMmNVo0G4${6j!$f*f)HT;FerjV)ETQ#H5v)1Xli1$fwe-5S-r4Sg7J^<~PLA@- zp+JEF*IN962>&+sxm8i(lT^oC33J8hx`$^AXHaF1Q|$+20#1_sDtVYZ=Q0U%@8EeZ zLz_5}nOZ)w5o&|v++EI-*L>r+&0fx}hgYSo?Hnf}>I*>ntE)264wUsCtlJEx*rsXD z??fd=JMYRDhi6Q_U3vj0sj23Tvh{@u3}tx8d|*N@-0!t~epW5vEG1cZmZXwjLc%PY z`^sp8$#EH(kCa?@lb&{28=BclO-$+R@-Ivbdsy`raOMq zA)*Q=(zT-;5^1mY@#*DvnD|cStc-EeeK`HRYBvl0USl=j%dS{z=UgDo#Aw4ibg$>$ z&U#JP1vbs*tW5pVM+xMlvrzcHpAS4*`5nJ-K`dNzn>dv=RJ(VQ_=HqvmQIKBHo6xp z8^5!_LZ-pzNv=?rOsg`X}BKLz;1+kf#Ty8I;(C z(Yx>)JLtLc1}T73TYzpNJ?4}Nu(pI|H_Ony{Cc#L^^;wgn5lPE>YX1olv@o`#muS+ z^W~>6cdFsIW*EKr zO}Fihr8J9N9#|ZKix-Omu=rr-(N|nm*S$(QkyVOegBxEeq-znykB% z0=txlhS9!|+XjJZl)URH zjDN{kR(71v?2QhKLxK1m`y;WgcUVvkvNCnTPScU`J3Bjm%8nLZxykPi9gQ8GgDRPS zzvkddfl9mOk@u$)X_d5F-!Lpx5whWCoX#<-rkSOjCdeQ1v}i5g^}ecGX!)Ez+aI?@ z_xlDd$dEk1QBe zw1ykeo~C@6XDMOVunlFhW0*fcURJFlNO^AIvRAbLs;k+vK4qkfr+?oVUjaXQRM;Tv`6(^et#jM5RDi268@P7 zfTDLw7{l5%bggjaLt^((jo%pb^M0Y&H4~DZotfhn45Hr%_W-{vUsyhX2P{>+DFH zjyFdw1M2JTYKh{hW5*G3bJw6;zombV6GIV>K0ew$4aJ1{#GB@o>;o9hhJe);;?$t- zA1moI)LtX>S<>!vA|l0^115pqTw*@Eed@`rkvC*MOB<$YrfIzCc_~%<0ZL&2<){w5im6a0&Fgw+DK-_p z(H}WAsEw@0cGccHrcL1-%)USD+uUb$Pp3b@COxko>AWMKd7&sD8bK;{Y>RITnszVt zMOLnUP_CgKVdlKrG1?Gltn#Ke$N5svePPY2v65v{c`w*JPH}QjU=s*(d9Xul?KdrE1{kEvju4hMqHsP~%yfKoKeQI9c;U__cetM8muUt=68LY~sk*C=WK%_z>{(BOR;dGgob# z2_9S`RakasNR4&CDQfKQ_dSoMb;Y07FP_QnoxihK(3iNH4F9wVKaE;^6(Skbuv9ueo+WA}U} z7u%{nQvRNHD_SVP>vraS38dD8KEV_>k0~DDUO#?;llNWt&8MAv0x2lku&fuve2c~L z_218FtVyw;3nzaYoE}t zI%Vti>%PQ~58n5aXyZ!!uylD(izFU7nizG@&h8{Ot*!gBS^C(L#TK=M__qP>3@5wH zUoxr>r|kJh;7GcZnRJiPXx(Xar)=AgT>Sa+iuLnf`DTiGPMI3RA3ObF)>5eAGPSrI zax|g=&65{Vc12fkPNuw>ZckNQ=AJz4Lz#8WO9gt7(T86DpQ-}vVnI$x^nJk)k1T=9BFni)Wg(2WH`-`^LooiKI1gbbyne8h#U;fHYyFFIMe2L}h4T9ZYr&gFN`n*!ZCS$s**K4FutMq_m+Bhex6!PjQS7d-*G zgr6Y)tY1MCfam6gbN<-S+^Xs9&U)bkiT+P}D0*->-0JhLfy}!t(`%XN!&#rWSea=( zFBy8w@lEv>=WMzG%~Y%Mbc+I71MTFNZGV~BlLkN=4bTBo;hV89)}GMFpa-k^3u%Ak z8Xc0UEKREP!(WdFm_QZ6PKFY9ChV zu^w%0i4I#F4xo*0YqKsGo{ZM!y_y3u1BSF`x;)GHzIuH)M+^x?QIbwPOYEr?q|tI! zZAe3BF-19@UKO4!>iB3<8NP)fqHYBet)xfc^u1SBla8v|b7 z4~5~68kvm~@jje{^#uWwwl)voQRX1Tj4=wglMoGhtsATq7%J|(y`7UOt_qF)#?DXR zSg31v5_}cw5<3?tr2&h`Sb8a!WiC?_5pAo^?)i`dT(Xp|wII7wM5*J)h7m2gz+;*PXPauRES0Nq{sLN*B9C;ULkTEkonSzYP4puVx)*B_u99mgz+FMV5 zS~Ia)UUcPH0v-JE=gqifW=n zmnYMvPx9f&UuQY9L#rH1I*6=w%60{fx!z1K6(_Zll7}M-X-ZK_M?M^t{4U*?p7lhN z(%qVTa#H%xPw$*Pg;Kz zlPSiWL%SFsNnaYN6WEDQ4hg*3D>CU;*#E@v5XQZ0LX8az=zqj<5Eg0-x#?={<()Q0 zudUT_ujG;D5T;i;X$L+UTFf!blJ=0wvfTT5ANy#mar}nmFAVLdrBkJU!S^05S>i3q zA2@~9II$Lj3~h{m4d@`jz%zfWS~{;Dc+575(uNr_Q1tb+e_N&|#{Jp^EWpQKk0D3? z36Mza71&JqPH9k1K0CWrLl|w1A6#wi!e7shwm%8~`h)l!BE%lMfLfNCOH|r8;n5TQ zHM-(P+3HcK{JyBuh^R}~97rYfAf_!72!+1TVQv6`4)G!`wxHONYQf8hl%$Wri`yz) zmp-dwty?YThSI#eiw1cUfV^5T-+wRSlD%1sH$92|EDf)A4VdIHK~Aktnx(XZgq(wt zeRs7()%?i~*J6+;Vj5OwH@CJLJr*h_ur~l2eJW^sf=J88A)u*HaB><|>09HnTQq9- zc2sXRM}`PFm`?$f$kH!Wd+_z;*G_ng?`a0fY8euE6LLtNO-3fppW}aBd4BOLTZlbL zElKvp=v2pt5Pq~SCrnD@@V)wQZCV4EWnf>mkCFZ}B4Nt5#Z7=y^E7ZX{=7n~ekJ0q z&>6|whcw4lo*lo?Cx(c#$dU70DT=#vn`#xQoPUo0Jt}yffrGZ|OJ83ix83d1QsPrL zZanz%7Lf-%_P^yR^?_A3x!Um<0lP-1i=&|EK5dme+qNiV{P`&|((=Jqju`LD@sbx$ zNmf4qfr}p6Jnt}%Z~D&tiQOFnRu7a0CUd2-i~Caj)2BpR=Nv}9k?C!`XV>8dz2wE0 zR~Le;mFYr;S4QfB#$Y}?BH+<0G#%@@(?2sxv-i)1p3c|%h@M?KFaiZ2?0sU4Lt&={ zZ1qi!AYss~P)(WO zZcG&|s)KSUU}DpY#gqpe{BsUNYtY(dFH?R)m1GTz?w^*Zk&O#Ek>u*|!~_c>ezWW3 zmv~&=|F`Ey;q^~1)$r{9^|Rl3szqHyA%-3(Ki^l%s0|-;nVpy@c;;g2%4Nz0VK|^{G z6chwERQ1t#^ED9EBkF=^MZ7!-__=rd=pH$w&pHHin<-q=ElghpmW=QNk$`NivgYYb z>fqg;{mXL|O852G>M5`Y`k8)L##4J?Q5T7tpDWq zZ@k8z4>v?`#aC>O$BgG$(`D@f8bwrpk~FxlY)p#bp)mKO>0B3wYPenI^}CBu5h)5` ztokNm_CvmoK#J8;~*pfMV21`NTPs6PMQMAI7 z2Z4egLuy^=3?PoX34{{Hs;a7d*QE#dm<-dDF7@Imm)Aj!Ru@C~8l`|O)ub!Pzj+!0 zsK^|tvZHMo_dZ}X9FWosMcU!VlS78L+5C#0$@rhRq*Q`>*2)U6PMg-;*%%301`cp1 zNWiS|Y)8Aaj3!}G$11WP-Y@q?i|xr=tp!3xI>bl7#9@|nLGHrIT8l7n4tV(CrA$hZ zcfJMDav|~~?pj4*i2Kl|$5{9X!E6qpQ2zn^(GwT%&+|=^Q;KIEJVGkByS-T}hIBVL zT#vFQ3_E4*eg*tjRFsnYp(Y#qHdj~8(r7Z5ipx^Y=EuM{-Ahz=%c={;bjhd+X$zRx zIamw(pW|PMyUf4O@IHC$*!8BOwR*tr&3tXB)0(<-bLq4N=dKum?0aXd1UuVyN}sKM zzW^>y)BF14@}%#eq{|a{B|Duct9ppj=#|YcwlBTsAkS%YVrO3 z!;*pS<=p&lzh_H&T7^XC2dlY2-r_aR8_>$sR0j0|HS-KJQzYh}sngjW!DSTQTWnXp z?YaM{uA9eTL=j_h9tz=zeMk+|n<)HcsP;GBO+QP69z4Da6AZqjwUtY41WZly=XFIi zs@hh+2-}+u9p@RKWYu)FM)R@tg|(g@9vNYPPY8q7QV-RswlD~vrYXF6W&hhmX<@yn zAV_j;4yr;5_&<+7D`EI|VN$iU_Ln)m1J)YgE*dMz1uFmVg#|8?Z?8!TRalO_ziup@ z@-6f6hF$f}dCTtY&6SqJPmw7!av}({fT*hP#kT#V=WRiTR21Lac5u`AGmQvqGZr+e zcVOxUC*Kb0*^OmZrlgQYJ;Q@AYS<&Ve?v~)P?DZZo%R&Jo%GI3CR(;T4{Sb*gME~I z*7a~c%9lxvjEqV>*0XK;N`9!{gp=++AL!2vD}&prozbd0KNYDj%&Kwh&q&X4Q(Uy| z#VWh6PSP{n4t%}nx)upV;N5A4TXInR$Kri%HmO;v4#!K&w#l-ib zZtaKLfK9)Hafz8LQkH|EY+2FD(!O!csH%cdDY7lAW9IJ~YL!01TcHLmc=XfbL(2$v zlGMz6&BxAZSW!Zf3iHDjwSMY6EHJ*#@09u|`*PEUGZdDnwSuE{!HEY}!3I<=S^*3^ z224&ZQThn@G*KkM4AL@7Ute8N?9MkEgE1S|1}~c5col4zQL|}agd|mbrZ^PVeps4E zD#^bEaC0zBy2Cx*zrVa_&$!3(;xvOlNzw- z15it|R)_A_#Gmn=VL<-?nd~js0gL_D=Sa_Ayrllm)WZLpctXxuzzGg3jBqhLCc+Nm zum6Dsh-CjYvGB#Y#rVKeoIly+KN#|c$t_Nr{gW{yIo#$OP#?wfp*&#!H$<}6bnQRQ zfUYjD5YB!L43Q9MH;NeDB2#dRp`9D%o4x>Ka#Jl$$toN@P|BC8)V!~_E*o+z6%w= ztP7)4aI4H^fzxMaEdu<~P8+Q~%OBGP((T5>9-t||1|yHYT;q@VX%?F~W0jgEI1Yh;3j?{;nE zRbXL~ww2Q|C>+XHZ7%WKL_!I@4W^j+!2wXgkA74G?=Ooa27uL7NGOmw&%*j{oa-(A zio*%CEdv%%fc0QD*HiQ-j<7ILc?eS+2-D7LrNHaRYB+~pb1Mb2()GusW+3q{kH^?4 zG)LZ09xGhaeLS-$zTXB1!bNT!J=eBqO*!5Z(5m2PjE+d82jTu5^*jzl;eg?SMlcX_ z7j;tQf~CE;mu|1D0L1w$rirdJrWI$S_E?+l90TT%Z`F}zPYe1ma@aF5(?8Yfpz96> zbE64xh4SDS=r%|72Epn)6_oincfQTm-*8Jt!tMeQoaQlff1OL*WiX%*6i`G0A-%mVr z0|x%tsRD8$k%SYMxDOwX2Mem;3a`rkN7D&k0U&uurz>?d2)iMLKC~Y4sln=MPjDGIJ^tc*UzWs!l!nS|ArQV` z0T3GuBi;mXCKaHX`LblJ()Fae_n=grv#4? zq#_P?xDA*_1fqI8RYUj*OqiIr)8Ih8`S}o+W5iWt-5CPVK@kZof!W-iVQ8%?7jqW$$R_@lw9N;vA}+#=c}*iGtPmswO2$-YW(3I>R`ql`P+Gwf#WuivF{^oU zB;SkROrdlrhg3pB!fkQrF-VYK`(^e&N0N7l?_3-7?hsY+O<8ZylZ4z?(+i<4s@3&Y z`&4^ENahTiCc94FgJ(cVZo-mcL+FV*Mz>R4AEt*;<2Q2GgJf|1HaJj|f!h6>_X>H`6qld%lX>F9e^OhWQRSs=F_bzQ}Obq_iXs<3njunCjNh zi$ZJgQ2Mmlp(+tIu@dVt5S(w;)CjY~m=!nZO+XijW`T2M(UY5HSg8w6Ejv_uKCzlL zKXOcH&}$&RkqO#|`XKS72whL|@4-#Ckv9U(H;Bn783716l>~WT zSBHkzF~9%}S1rd-j3*pOG(r%~pv#zx7~mm*6dljUj{DTJ`?vJ9j_`>kB)7x;M{fqGRu3h}Mz{GeA0$&Qi8yC@ zF4hp&bWJgO3qMYqNA01BybN5X+P^-nG4kF_?UoQNY%SBgGgAaNeHw7w@0l64R~NYW zHGYF2nTM`+oKgd@6_nu~cf-V=7YVy8A);ZXZP#@pKgS*O$9T=+kEMTCVQIQ=AQAdT zLSg33sUM}7*M&A2%n*=KrlD)F{f9Yz;Ls{eedeEqA1-LW zUJjKzm@-H6aBo2CM{r@&ShxNoal|HIEDN8#R`~_jX5$e1ITO||xBZZ$-bNNLRk3?E zY7%Yt$@jpxq2F!-e5myT0O0@+s{M7ictTzBwy*yi&C5g zD057EAixMx@Z%JJA9qzF{W^ZzKFfK&#lq|`F?2lS%}7h#R?=n%Dto(BzD^&rKDc3 z|0ocd@quVMuB#SJwT<0^m@(m~2d?;gKwy-%zWaVDe-XqlxSlDi$1_aOj#bh!C{U=SKqjzmq4BtK!>?Gw>q2G0$=99by*>Qwbs_-uYuCPq2ZR-f|L%Gy zV?Fr$xwm~jME6#X-&%lRAu|BEq9Yk)hP@^wx&Wz~E06X=xW{>O^Y2VsJ}r4$5{ZGD zqOAdQo-ED_kAULJ-Ab;X0HtQ!8$yXESuW`pS_WHHCx*=nXBb+tYGzr^BQ8e34A}fv z_y)i<^LN#sS$Z1gU)fWwow-udWobW4+p|4NNX zGzT0HTOPxM!5I^8(Hci>vjZ!sWz@Dy)_#%w4J99UXEBII{{`u;K*TBM3%k3!+z{oI zoN;{eiG+e#*1Cf7`5t`Y~@gOwK0z! z{xeFxgddE8hb4Aa-<5`O8VGBjo3V)ZGMD}YW7_~u5*wfO&+ARJqOSK3%44nreqJ7< zP*tB;5jqpgez~2oG4LFhq3BBAlmox_wrhv~XRsm*?{z~SEaN5UFl8K6WN+|_alBR` zg*M%42xkE#8!r8fG^&B57~RVZUg_IlhTFGXE8LqQ>DKW4Gy@~F@MY)^^&n`sltETd z5mx-EUIz52MVj|mM2>6usD8tq-(VrL4QFq%vd{yNvHh<>}#cINmPAA}uZbuP=fJC3hG<|Dw5%~e6yt-yS3s;j zZPk?(g`h?otqe6L0N1Z2k|NuSfjZC#HJ5Y+d++Fp7aJY3wAlwQ)$($LEth* z5^2~ufy90tuUO=Mt>^y{Pb!XqX4nqjo&we_Xz>ZIknarv9j|u_az0U#Lg_H&rM9=X zQ_Pyev1V}Xd9N?t_^i&3a;*eK72$e1A8H-}*fj+nkVfhb4heP#&f*HF1p>X3yJ8;r zLQHS2vER>ZgWhLa$c=%+UhJSH*Vx<+Wdw&Fv&Z^;7$V`?!$yQJkGoJYZya^WPu+mz0V4<%fl0Sd z?0-##=Z$`Ql}~epss-)mFH6-Fd4}v0oYP9uUBX}yr_=^ z-d9TY-%g||3w<9_z&Z{&6uZxE|DpNC(l)ne-Z9EQm5-(qZtty6$&^~yKN>w7klYA-`AZw@ppCWzk#XEhU&^f63Ul^ z9n;UUYqzOi;nFX;gBV-V_YLN)|7xDzGj0*=I#+~6w95Lvkya3XS+-MCJ zccy-4cwCjwYs-=68x)GNe;A^MK}3|2g^9)${!NtM|N2WS;4eu=ZfCqqYZIl|R5x z5s>^4EeSo*wx$rF<5k4aZghG7;<+Hdv8VvqpZ0hl@y%_rTGK1Rz1dMK#tal8b!K2Ye4znh$(B5KOaB7#D%S>*cY`PGS4-L&w zQ^Zr1rraM+dNhIO5OQ2pjauxGD_H1DMZx~A&|%7%$A^ehpgx8GQOeNHODe5G_zsln zsF?&DUpmr`0NybJlz59scpy5D0>XN| zclXDng8gvYqOk6ZUWh?c1OlJxK3oAs_{I##p-hmP1|gxV9^&BWCmn4N^1g}Gl+P20 z-2%7`J;QJFiw?}bz@M(S!DX$jQ+ivrcDXwoqg^KF?lea?=(X}}BRqr7ZF!6wJ7`m# z!9CQwka{x)U(;pQ_+BS%uk4OzljhsV8+!|{PJu7{_JEf}@Yj`z`pIFL(M+k@&wO3T z#wp0EHW$5uy07Cr*+#zvyS_j$jA31Z2K)<)ds_|7IBnVuz*9R!oFwZ+ptWMkbZWPu zC3hS~oLBzNuKmkhv?(0&#r?20`;Y7yKw`3}`|8!7Bg#?SH$lKbN&iRdbP`l&Vq+x$ioGpI!lCnmj*GASEM!Bm-3}g;4&6fg5<`#6C^LK6solVzcE8lSvhjIae-*F!8il@I6W4&6RJd*okD8)Xx@y{;+(4z;1QR?y{e2NI7olhhWoWwHGW^6>92j-`c*MJB*@*1b(8dLNJ>LUZ%=S2myTq%;SO#pFP5aI{cp1z zcvE=HkOllpED3R}5LCqkbnT-GkirG^7i`?LbxcU6e?QyPI7LWlJ(WEo=4x+*wb8Zs z{9B%9T&`4rRq^9bE1Thh>}uz^k0sF>9A2BtW=N=_sr4^$h=8(c_~Ad0%3#Qd@cb97 zQdqyrwT8;Us7$F|?4w_q$3!5AiTY<)-Ltp4BSqds+A{Xj;9uGq+O97w>Il1~{L0UoGnJD5YeyjJUO>y_W$BJ?I%nw?0dG+jFZiuM9UlItr zZ`e0;Z`)nkc9NLt7&)|Q_U&xLd!`uuGDj)=!j8c-za^uG3!&-j@rkdUpb>HX3wdM$ zwHws0zvYoaQZkTbEsjJ zXDOy%Mw=Ji^HW^}We-452h`*jB!xv=ND?KOGV^F#a-Ax;y_a#A5Q@5>22^TO{THi( zRD%G{)57Mx>mp4h4Lqq|P^n|98SnC~cC6i&H}Xzw;5|0IM+M!YkiEpQC+?fufBLb}^bi1PTps=v;!y29*<Ti-W_{!ONdLqNG_jY?9S9ixX?Eg8ycJHD?U46oTo*}${ zMtXlxiEM9|f7GJ*lc)UioB!U%cGq0k93B}X&|TqH;MNT0$fJK4AE)I1Wt03LN+|gy zl+!>rkVcFZbe7RG`yf!(0bVBh@@OlNG>7%;`zLei2m*uFhH4HWlq;0W$PfuLAyXS9 zV+Ekwrhx7rV>`XS^h9VAD3A=q07&QZVXR?bpveM4zuJpM7of}C*R20|U*=#MqymLm zhRs=m)B*p5PqtW}zsa-H-uha5l~)E&Zs-rVCl{V=43l~}`km1(l#?A3Q^jzPd3)48 z-X}*tyvArff-=mj1&`pXqjp+hQ5`G|kmh@W-UASXSRbM6I_>&?P()sWc!lT`;0iGS zPdrw*188j&Hj}=C9e^=2SwZ)5iVW*O2YQxCybApZaY)R4#55a2ueGejLGHA6u6k>l z%|Wyhs$#U~#&7@<7v(XpTmW>~SLVb94G3xC}j47EA)j8;1RS6bRo94$T!<;8Zq;b~GVD$^%sTo5JXxT<@IG zO_C*n{iYe0f@omx8}%|6G7X3z*4kyVuiHufr1QnAHTm@3Om{a35eh$$Im0W1d>(! zk4(J7Udyj%F`B2h^=XT;Zeg&U|Mq=W9}dcj`R+;{?fW~%b^OSt)lKWR>sCGam@lt4 ztGRCVRk+ENxwxrf?fMxh>Het9Wr1J<_k;12g$>uMw^Y-K)v06nUTZ z1LjT{lq~ZT$c|y!zkUzhG~z>BfKbDns{|N=#2=Dy=Ony6GeESf^(xxx68F^9O#j%~ z2A)ISvsMK*FTSf9ls!Ei3J?fdi2n2w{@wjGz}jboyd9UhkEOUq3nW_SWY@iX?hr?e zxY?7wGB+9BLMyHoy=sf&@fxsyXN(X$((GN3rW)?+OQd(o=KI}5X$T}(9Y2j>F4pc9 zDNBQ0?QdaEErU{!axDRdBnC_C-ZDoUAJ^edQ+Tj97%l|K7lLq+uWNr5qrML>B!7Y( zkNZK>qR=aA$@P2D{%n$UCJCFpQy~QD`o?#m@z=Ohvtg#4Qh+v}F}(RQ?E1Qd=lYv0 zogyWOpk4Tl*W=G343^CNbSW8N@jBWNEf&yMvK#+iXeUS~4b9H0jeGfk%#q8eit4wJ zLIfvzp6@;G{3YEQs?%L-kDLt(qxG#WzfHjPkggT=E8KylEU>Oyxu0R$*A^TQ)gvB0 zz?XCdHc!zJ*$oCOvtJjJYb zU2pT=?I;~7=p@(cbS=+jlsi=+{L66p>@^z+RPRebFHLY0lvOP&cR-#W=fD4nh$lE1 zD|lDeslv8jn1Ua*wh5H6MY%V@(q|I%zEA3E<mV9^6~N6oGoZI1-|nbi~DF%nE_@*1E|q|WOQ&7JM_wdzZ;hs zLOSD7i6KE|5WG+<2$9Iq7Zf7tN{v30=)JS1RqC@>)%)#k-tRr;!#}a-0yj1{Z#K9& z=8MOA=BiY>v{b_udQ-jiTVs35ktNofA#(Az^`bpt>(KZFh>lN7(6hhtlqaFytzAdw z{7E_b9qm0oCy!q1WBwb_Up#>jOwi}U5WW1xG*E9w+@6n}$t(Op!y@ajrnKrVp-gV+p=(v?x%p_t0i;ZHQ_<(mjitS{V64kn;^5l_^z#9lZ)sKM314 zDmkvhHl=8ixhow98Q2|{tDr#3!%oUnpxE4Q%cj@V)8Ah5!{3?zksJ^;%O%>Ct;-pY z-@enHwi3L3>AMUgzz=I079mbVbq&`FN z2?Sn_G|X+-3}(Y2wLcC+-f;I6bRTAR7VE}}z^==C&$2qJK|xRBy9L2;qej7Kel#mQ zYJ`u*AGLPQF_@sQ(J$dWuMcxAnWFR%uS%i-(79+(^V8 zt!Ghr{}!%C9!ao3!Y*ySz`tcHk{yBikU)=|TTLUtlTI`ZeRiBEE+Tt~3rCN=f(S!Q zf(JzV@{MbcuJplkh2)SN3zF6YdGqOv`+oiSRP^BRFP|W4ng-Gpz%8eKJNKL!h8)QF z1gIbK5I#3-+!%6a;oP=+B<9itDxRx|s%Jc3w2q!=O@-zjs^(zDG-5>b%Pd}P@?RW| zXZe3b%{BLS%~y)s9T3<|{Qu)<63qc#UrU)#_$}>i|VF+yo2dh%lYm2w9t_+G7G!58TJjaJf48VhK zLzpH7p8bG`coDB@?ODj;R>qoG5CZUG|5GK?D_DJzr3tYMZQ#fBkdqEamlPky*9 zXb7OzRV*S+O|gt?9@iwaEOS4YECv~I&VJ3e%e$#epTGsX-*th9n2EoBae*stv^nbD z*7^b$AaUod4J23xUPmy(4JNL@^^Mmi{@-Y)0l2~;?F!dC2!r(;_S3`K1fOA2h#W(5 z2qf{KpmG{{EG z2bDHxljt|J+O&`BNHht7Pxqa`G0jKH?pGQhG1s-aW*jgzPQU?D;NdKfcDLQ!;5!i~ zFW5bL)rl8UY1hdJ}iob(6z8tWNW{=wvCxqRWQ`{uZ|BuRR zbMXaAew&LmZ2Rdo--xfMgOf1lV?4;pU@Ls8N~|sn1Pko>WM^SF6m}k_ zg7GkArgwb@_;n6*NUk}Y8aC@!5Uah8Ct=XqqG4IYXE5?a7n>gWV6#r_%@Q4=;&@0^H_serdQpFYD zE(Oh0iU^_PcN8B%_7E79L>}CpF#t5zr3#rq_)614LoKcq{RRX}$iM~UTX_gs z&Rv0c@Z)SP?cyKtn14j0zJI@Prk~~eo>O7&i=DlQBXW`n zYh4|2g2tW!gp6G;_sGRk*#@=tk$0+0J~h&aZmK5|f_ru0ZRXxCTb{{LaclwV1I00a zXLQzF*4pV(PUG&nAEiklH1}vf#7)2recbb=L~&Z%>MO-lya=hQMv8Ltpq59)y4Z?K z{)eHp)4!4r2eToE_#Z!KCNd6sP~8e6RgczA+1_PpdiQCPY#BJ=|Y+Rmwc39W8^x;Zi#(;7( zdlaMGd-h-a$H!X=G=yh_32T}fm4?R&{}*-d0Tk7?wTmK(3W$k~Zc&Vg3W5<(vKSFC zAVH!5k=W#%F<=%I1r-URf`kTz?k0m`Km;X&TLA zj&3@vwdR~-d}-vI^-x=4pYkEqF*TTOaH;|{p?DXsynNPa-d$2CK(i-h8O8-q43rPH zUO((giLLNv)V>aVU#vgj);Y3ZUiMmA{qZ+3P3;+zlQ)*k?(!%Y3+~wD>@jPg;mo(H z@f01!iSsiJhU@2#9T`vw>8YO|(lgRv^*yh(z1`Su{x5c-+#Upw<&=7h{M*b-s86M?(h`r=6D+$vmBAScg6kum>vrP}$v&&K% z|5&#q_B28lzgFCU=Y8N7?Y~Vb+xe7@zI-+knnfv#+nanlP)b#&#IpG-QZJp~PAXdD z`N$7^T2Rk(KhTKCIWtlI!*+S$G}CXz?zKSS>EqG>kQT#Dyu;^dq`NGf>g4CE%l>N? zz(c=`1EW)Y(`KTUMA9KfQ2|_Xl`-$3-4TwU=FX877mVq3X=o@a5&egf?2)`zCb6y( zOCpMZzhLz$5!)~F$RRu(Y<_ii(JNZR!~dWrTObT#;NR3_2Kv&UXiCk3x&u#0BQ$I` zDVNw;GEsjhbByDigA8@y#Q!f|;H&(rYr@370S;9O8}yee62OA7r~p;%?-)9Yd|Jxu=ALU(sIsum7rz;CQ zZQlZJlCC)qMD9o3@ozkXB^B?${?%!@jZ}&{tym%!-IG~#uy`_Y z+rZ`gepZ#vxD15aA=tvRQKUTk!(iB?%0w>NOEKq6~2*r=xE zky;bI*mk5;sgmYNmvI?8Ioo|S!(HC?NxF*EUDQ$1Uy?Zc9*gQF7p89zx9EiC*!0m^ zD`1|)6IL>N@5MIQh#pQu7BWX~vc*&~MJ#crBdXMN*AsuR=FeD-B52$LAS8HP&RvyQ z8UC~d{jVi^E7WnbqzoXRLB~zaFXhr1Ef*j@TWo+q2Cu=y?R@b1?wNk?`B6t{WixeR z4ro9Xq}ZX#gtn}x=o)v(aivgcf8)MOwvpZG1qJ~Jc9`GT@UX%x1;U8*zlvB^FJoLr z>rIZ1W3dfYFZ_1#?avRqf*o#^pe53;Pf5^Z{qg6;$gwX!u-=xI*KFIK9?o&awKdo& zf0j+vGp%Y-QW4=fJ$K;$M%H)(D<=kb-T%-sbqTd8*CCtNqZJK`TDCEB385$b&Cm9>Hdr z>s0SyJZ;cpW83oN(!#0r$JPzu>oo7lruXmGJMTF+;O+Y(xGP#TV`uku)5@1|0NV~b zj#@eWAxIj&|2Lu($bhH->^~v8lZYpY(10`xu%T(d)3Xj#!+aVA^cTDi2%N}6PErdL zaRE31a7ZXUN1c~|oVe=W>NtGu$A&^6CW&SZgyoeI8%X2+IPcGo{!xx}<~$RXr1gL7 zN^^YZm1L$@Fjk!^`b1i~(6K7|-7@pgJFaSkoda~f4r-Jgd60b=T8`KO9u!2RvggFG z@h0tr=V%E(SqzrzT0P1^3?cdHT3B5>ldY!lziRA^*}p^M_IT?s^mWC6@lTH;oiO`S zIb@+V|fw^9c z_)&ApUS}ZA&~~W1(ONd5)U^2UxUZ^Fb-=BjdU}l7)3bbuNH5Wye@&S9O<*=x!fBxT zw-@pyd1cJoEKCt{$=M?UBmy!DiHfu2U~olP?tY*b{Tr7f!G`?E@XnwG>PS4l<{U#> z4ImvVL$Paqfu0h?e$D77)WeyCb@F4~7XjfBG>VcZH!)4SA{LT?Pvhedg7 z@^6+jE#8ObWHn+Q9{{sWLDP&aYUn`na^beTwd3iA0y4Lo<-3g3I|*{!=$8}c0SL5KtGk|&xj_loV_GW8qj3&@u(3mQUI z^#BslG|CE+$1s>R)^p=szo)0?BB{8tktQ_V^D82Im8@WfkY0!AL{5fh???I~sp*$l z_!nO02p+>1;J44Ky#c|;5j5|p8*jW0{qy%`6Lgb51z6M8@#*@*UJvq@{PEl@KWV+t zZ70s>2FOO%qb#?aS)YT)Xb8L7iVZ)b=zgrlk_3WN~S5L^! zmm2$ZBwY^bFqjLC{HCQAbwssDp4ER-Ew1^aT4X=1Ke2t9*k_AtB^F|m3a&2i_xHB7 ziaD`*{i4GIqsQ-Uu=1>=8wTZv_TI>!qy#+K?NH26?SJfWuBq?Ti+xcQ-}9rq&gFgV zUo3rO#j@Rw3HFmii5uN7Obv7AThGD94RwbQ9T-dghk9BP$=JyA)1 zg4D-92;m!%-{I@&ss9N*v{Yc@d1Ev;Wa_w0UoN_<;Q>wbVqGajaKDc^n7x(mZ@hb7 zMC*uVoZe6SdP1TA=vDP{K+UPqbGSv4K$hUQF8~x+z3H=C6Xl{W=oSB0-XIcFEMZlDeY*kr0x&%gK&Zp=3pL zhSQNU*{GUEMVYp&EO!{INiI0@-5;9cLi|^!;77edm>GgvpPCef7lSz4je?T3BGusM zcBW3X{NU6*widE?qukx=i4SivrBhl0H%P>lh zp4yThtmRox-z&v174|IgbWA0X+;ffQRNIM~Wt(&}!$Hg|0Rl8G7m&6+nK?v@B!-7s zAAV!~<4`UdfuT{0R^OE3IqsUAgD{gBn>1q|B0)%%=q{|%fBEkIK~ftaYnVUyTMi+n zUqu_dx?D1@G)Al7=65*lwj%(il4j~(c_c6c(X~H}dGf?xF|wu+|9)~jNgy5P*Wh3v zX-OF!G4YxAAi>rl(KS(MlEPTH&lb@x;$bBEm@`n6xCBWgJv2=%@Z1sm>IqX_x|zl> z;%HP~f-kpAq-qYDJtV2>iPX5!wr76fg5Dw9j4sSuwPR0v`Q)irasLyj8EC3K9`oNg4QcmkqAdaO z41Vb^;Tk0Mu9^ANP^@~&Uptc<7s+eotS zd;wu%7DC$6*FNnD1iPz<+=M|s7rWd$fO3LPzd<=x{3N<7mAv^pZ>y7>R^0#7V#|w54za65me*9szqb_b-*1$8#$My`iZDcNg#an6s+S zVDI_S_y!Tytr);lpJ8-~TTFYg?-1;hXF45<)5G`Q@J*%^GFI{nnX^$TyJ*qc8@86V zpQ7kLeQJlgCtr%PZBZqg)XtHoe(S2VDj#`fxIOIF9^Gc1b>3r)e`gMH@m4Sds}AH; zzJAs;GIg0&kJu=)jf3b6if+YLp$xljuw~>7`ams*j8DH#*_$EE6yU1J#GA&S1ML)B zwzZn;A^rP7bo|6w48w#&WvW9Ar=y5w?B7pq40btb_AK4_m&Nc0?#Y`eReU?`}Eyr3sMn~sjC$O^<98<9MLZ0LoW8HAN?c@tzgG47% zYc3UQ;G_*E>cHL{R=)7eabzg)Hkf@%KmUbachdNVZ;{lV2GaGh>l0*Tx$N42%}MA%o zJp5>2JFZJ2H7WWj^>h0tTK&l+*UOtvC2TEBfROp~;|F;vd48HKDP-zIl{|j{i*R0W zb&?S-x=%SoQPa6VwEfqhyA0iY^402ZBap|>8Y$lgKb^{4Dqj`&9B0$!sruJ@xvn*- zYD>%cb&f}!QG>+pR8?}i62yn=r5*tVaAptQ)e9^kLNeuu&`Z9Q5&p7@nAycTwUaHm zyuBmk1_c8<)$2eZMTIcpJsH_7bfk}#tSDzJD*KUfr1<|^M;f_`-Q(M{qd%f4p%X~7HLdb}BI~B9N5xUR-K20eapv!|_*QsnXk-OqLyjE@kumi& zZnL{D@8y}8!Pp*!RMdPJIQu%wH3{9glZm{dj-FprszYb#By|NX01+xL>D7tFCXCWuo<0;r z`P0ly+|qRoMmw!|fFbclm>r+FhI$_R`D;_9LIy>bSPt@orM2V`w4OKtijI++lm5V9 z`3oQSLm_e=X3+`+0#aGsIu8g?9uS}+xwTFa_nTPRy>f2aGT+BLHr1=M=~2J7PBNqX zCbcD=a|UM5^xELYH6bx;;Iy^tuUVX<2Jj-X#W!u5WWA}kC}_c-4V%0Yhjo9Zq!`j- zyiv|1MLjOfeNXz|9m}5jeo0kvV|YT$c)s`*Y5_Nop=oazThRkuc9Sv~A4(Qzb#hj@3QI*x)3BBO}5+qWHy6g7U* z?;VF|_h|?#aNU&!p9$Cfb8X6t(_j145)UtXwB06_b&>cP@uQHQ!7ha^u5*{@LX~)f zjB@~-Nt{O%;&TzceGJI#dZ0W0sIU*iA}3%#CzeLfk)IoQg)4wfV>$`3w{2$<<`io| zlDd;tdlQuuCrR884fPm+2MFYxAb?TsiX91*yaSM^r=QDxtom8wrC>$egb1`{_ny{! zZqgDvb>H(8&yQtOszvXEIG}i(uVY6b4o%hdiALXEohgnyy6QLX6G4}EIff%za*r(< z?u*nCGc!m9cx9^75Q%`sQ@xzXI*I*9Anx^qj3do`N6Q?d%A`D+yiF};XwQ5k?vX(2 z>U%o$Jko1gIwEtWdZr1*A*?KnvA2!T!Kr=nlb_fKJKzKS@3OSFXUut)LSv%viB6tP zr(R#Ts7~MG$f^8F;}SKKf^RbzCIQw%BQBd zHQC%IcRCK?exNAv2o~TdB=FC#WJEJajhua(<$IHKQH-wg_b9~@3tH!;*?OZ>+^QLjwpDT+t~A7>E6Rpyz`aOhf`nbjmT03 z<}6Eoxr%5FWr98CB6uULp9D^#3;Kks`{nlt4QXG_Z^8i=58qksQxfy6tO&mV$7|m` zF+@}Yrc=k3`5SK;Y`GW!B}C87sh3}^W1}2S$EHt&zCRO}?{wFMzKRUZPzsi!Zw7WR zu=i46;R3}K0Gn^76x`3XDSN#GP@6yUuk%#&UdH!Au}tQ7sSH_Y9Gr%U>()l3DQ3Yd zdl*sS!IsaM_REcDtXar+{_B%vO+Cb8$aE(1ioub$hdaK#%{m?C@nw2TvT6)qlY+vz z2j075NoazXQRXzLk;l}ZqiQ)4M8{wz#-X(s$^s8?OaT9SN93HlPrqsw(`DqZn5GlD zmtem`Fb5@jcks=v&kq*^UVEgl!1@86KS<{_ROlWk=@IR<78aK+ylW9CQ%1YUxqz~~ zgl?`yf_NY8J0bG2KbiQaQXQCwVgJ=1I5~CE*p4;_u%!~uD-v>DkS&Eoh&`tLFOM`h zwtf}B0eOvVNKg54kv_H~o{k#mPrg(Ff(3RwI=#NOK9TgI88V4&X5QyP9qL(wh)G zWC5R3O=V4X_Twj8f8!beU<+y#rJYXRrq(anWz*=D@j_tuWkuwV>GyHwPnuOobi`=8 zAl21BhfL0h!|GP#DS7<4l>`>TTT{Dj-}HA^u%8!3#K|hk7f=`mInE&qJ#WYP{NL^H zxvx(u_4iIc25kFU*{Z_#5GJEW_Rpd4CT1(=n(5bSC?kbDfecsdw*3T3Zt7WKE&6j9 zUG=P4qEMgepo|Bs$vy~AdmeUhGCjj3%J3iW(#$r3ei_`s^+O2W3AW@#h{Pr&+m3-A z-r`^}{bt@KD-=$_M2G~8zuO&CZ&^;m_H1~Q#R?nXhL6L95&VnlA&uhc-(f5GjynyK zGQluseu_=AZwbY|7TP7^=K1$mCR1r_Cnr2eVx)|7H_A^aR1U)mo$moi`(G}fAkM_L z(kOowqy>^KA4UCLa2wUa!2nx@u&QEUCsKltQDl{Sw+|kq>5p>t_6&RUDk?*IS)Lx1 zNN6Id5kBQ}X3aW;lYy8=BToI%@BMB~s4yrB;`~h8tjkc*j5iPk#w4Bx#CR=2IPQ>V z8dmxmahd7YSFrDxK`HXqzW9TL(qirAQ*hsj)GXV0^%LR1O+qf)Bkysy7P#coF&F2a z%NF?jwPnh%PQUyO#EjYbbn2DTZEN@yW_pVtP4wfXJP;1eic@#WS((j)SuRTAo%!9-H@rmO`#`@|`Er&VoJ$Pk)F62811j#YA)lbMVq9wc7@5QW6}EtvQ>;i?mB z3wm(<7EnI0=m>kt9)&~wfIvuqNiyWm!lxHG_lTb?N4peBLKsf)|?d>21o$*Mmem|9ip+wPbwBWzR%K z`3AMrj#{UBae4ETaX$3%>0K$_g!?u;+K13svWp_CEfAX*5#U+m(G*<)h+F=>CjA3; z&wTSP;BiT!qtj6+0c2!C-`b>#6g%S+Ut(tTqq9*-wjj2zzK+Thc)7D9D|4>%<+;C9 z`#%8Y#ejEx_}Pl*A~+i5Wr*HN{wRg~#1ng{6}s_(#jolRzT?iT7`l91^O6yWc2YQ= z=$AqFW{S%c)@-^yMWdFS#zD+TzD^)9|MWRuz|4gu^vTZ?K$r5hTIgqs46t<(Z}@Rn zOG^J!$_&~dO>=*H#Q>e}g67c!rBJP^@WpESn(dG9y@y8e0m1$b=iGi7S#TEuy8i$H zCJYE-IXZ@QYxlPmy;DDqnOuq}wT*NwFMnc|kDubcDIo3mu)-0ymW(73DlIQxdDtxs z_OJg1_>WSj+xWuIV=BuUBFRs*7HjooaeU0fBF``S7DOxfu-}vS>b9N^K{fbG0kWyt z);h9LOT_d{vKe7B`k?f^aM>LpyH9;JT`kACys(f|;%kq}kE`QsTip3-6p=4KoV(I{ zZ*zX8J=U1HpQn%(O4qE&PBazaJhtr;}qzJl_Q?fwvKI7M1 zeg!Mcsbu7HY0`tQ#8s59gzHEm&%Wskq)-*K7s!ttgF@YSWV08N<$hcY1XB1Ylhye2LCfutZMtg`ko>wJn)W-kY~6GUZb*Ts z=q}oRAK%QXUX8uR6r-@Gp_+?yX==Tw+qdqqQ7??|Wmb%%k+-OFJ;}Rz0<;AJUL5BV zrS8u)%QdU!oQ#nZ_bv}98xi-)!#3Q4|HALy>vs-MRC^+M7e_-8R2cw;{O1}bQ^RUa z(bn>b6;--ABF~7I4^08|5FvYG;^_i8sEG5o2eWP4&XQD**sLwErC==UJDMTeWI4n0 zt-LD@Cc7SEbI}}NRq{2Zbk(F7X6iRwXU4Bu5{1(%h|6JW`*W5$v5UB$XC1y(9 z4e}f}7;UW@yjd zbtqp%z>;#(3}SZD{X}JS3j-D-i;eI8k;jo#UB$&T>J{HygcUJ$VjlN9#Fu4Nr7GeA zNc+cKl*9e!NJ%@Y>v>ImPmq zr&syjTCsckF%A2DKP!g7svCMTgnSYsHNJtnkr?-+e^G=`Mp5<$Zj^~RbJ0%sk1SFaU&A}un^{TCNw z>_55O&w6-4Y%Wtc0vnOXwH)Qdp(fas4^76R|ALLHlN?&(-4f`T`4JZ zW581rr!Y?`|Ju#8FE?f<^KZxJE}N_jKSN^#4Sp{_$|M z*yuC55T~x1ZSlkCx~3y*v-O8pERxEN)C9aK4~%82*S8n{0&nsaZm9<9u(w0f{8X(O zW%*(jhXqGIJS*G%Uvcm}L9Z;DH9io;x$>9UMpXCdC<@7B0aFs9PSaO`3=_Pxi4-6^&A5TH*YzbP3SJQNMhgAwB&Q_Doo4d1)0mS!-s`oD!hap;ke zZzQCoTFPS_MNMl@g2|1=83TYCc61qg$aM(pwterOY@~YjCBDe!c(Xit`V*CGfI};Z*Xy!@#FE4+xs?TN$7UmYnF<+Nf&md znwu5n3ZZh`gwlAsKj)H7YVXy{m;>mX@1pjFK-Xn0G4KLT<&cW-{QKz)n&we}m*Rys zu#j6Yk_>dwy>Zdxj>Nu_k8tdAx<{0GeYwrxOP?T~)O2@uj$C3~S7802Mn#U96sruC zvpM$eA-i6m%{`dmn)hbL_;b*TLV^`mqHWK;5r`K-y~QXYgypit@#J9rgE66wh2NLx zTT+a(GEq&CAj;vMye(gnHem8;!CSa9zPKHo*@oS^y3TH7WuZsk-kGdgN`JK`E{)yQ zR@(zfML91<0jcDr*Vl+T#L}8L6-Nd76J!oA9Jczp@AAQwwz3QRi-jB~2FsSae0Hlp z;rzMl+|jqpbz3J6%&7ZqWv|cuZE1gd!eG@H)qQy90-OF7`uS{2W(=@qqkl2c5jNpo z!1J2E-Z)n(a|B`6+mwLz->>) z1vQHFGuV!VaU@|3rnvWnX_NXj0TD**skFlra*is8GYk$06+9GWNjqLE+*Z#l)D}7t zY$~*73{|ZWKzfpYL$b?NAd)f=3O=MhdX9+4-IHkqJjMWysLlxpWGpI*!XLB~F1O<3 zMH7Lpyp>Z?hTcinS)xy$maTldEatMWuh$vb5HpNqd9?3qh~_@W_EIIZCg+8Mxo?on z0387Vol)<9YI?UGv5LR`(sT3E{>^n5y>^ueRMOP4@u_U+Ua2l9tDeNUjbyE@T}bJX zKRr53ou}PG)#=Z+fAYvK(D$dzgP5{=w^=LX&r>Jn7dVtk2)f!G$kGmzG`253ka2lG zDpX3@*G0Rvj*YA^dq|B}qEMSS{qIZ+jzv4OA2qeH-q>GvIstF!Z{A8mVo zqHZ)K`-CTOVuMvaza+W3j}R5^f1}3L3^kx?I(*bwN{ft7Y}RIpu22m#$A2Se4VxAB={4=5fBNIPk0R_QlrzUcQTO(u8G%=U zvy{fJqiYIrlZZ-=1+r*UoXXD{5@na%_6+xFlx4M)oidv&61fnU{8m}k7+A`ej3Zn_ zhz$4^V6M?OXO;VD&rXbLB9mD$m_#xUvjbjnHkTe;<$XM&j8_PQh|Qdx1b+? zO__34v9Z;C5}iStovHJn26Iq$<0Mqx-HMJqmTY3E)w6Hf}Zwsla)1p-zlK717o8l&3;p-7Kjc3^DlT)X|#!BTAL zW@zDoroOwB1_U|)?O4!OXB7Hk^6?m%*Njg16JA5LKVf#G4qt{%^8uwhtM??|CL7Bz z7`zu~4{gz!lI#qO!1{XWD*)ZQs6js(`Ar+b*ktrAoSvv(tS8tD;ttLcax@kIGZh+` zFX{)gfU`toTo!?|>sw8BI2w%~OYRZI6cxrZ6MKcz(3or;^a{=T`eN!HDA1Z2N^|X6 zKCQu)5{UZeGh&^eLkD3CxdqM`zo^5V=%tANNB`YFppFUvDzb=jva%AIU4x}Ya9Sb; z#HV_I!-PZyo&%mp@?3xatRAs;W1}2(+c62LS}~xl9oItZcHS>n8i&tb6abFX;KJ!= zf~`o@yAFxiQI`O;C*EAKTkY(}o0~{M3KG0-RtednVOIQvuBAp+Jza67jPq)uL;z+Q zl-W$W;eQFNV6ccGcVrvE2%zs+58jvo|0;>CUJGH^eO#YPhJJ=ConWLZ$Uu$AzjJ1WVaoK?L$U+!H-9Ng*K| z5()Cwj`Vou^U@l2txX!yq`4|_XA75?G%0*DtB`-=Aiejkgu8Bcr4RK@%jPQ{`0)m2uhrjRZp&-;R7r_Z(7#J{u2_Pden6Wa4V? zzNve(TIUAqwtxHki|Bpt;E>3JwN*9d7M@6D3^5^R50U(z9yPgaLgqsN9_jphyTLZ( z@oYWQ3sH~Cv44qO_=Gld9-|)h`D#e$im~BUZ|AvA+*D*%s0$@tKdoF{L4>6r_h1wm zI(N`rKjxGa1vtD-V>0ztXZ)UIBQ!v2X^xbo2Gh5>fsZE)vRca^^JJ;zE-f z?9|5xTD~B4C1%Sr&pWSy%qx+YgDdp@{e`*P`e*uu5&zgmcTG}P%pj6g!cJha4;ERT zx*Bu$PIX2k>l{fQ$7CYpCceliY@bAY*1@86(?DbTbB-&%Zc;^IvmLRUEI2U~n>jh* zhVUd>Ud|P*P)7POKzj?8$X$7Vd}A&Zue@?v2H4Fts}O}D zSrNIYAewKN%p}hi0f&DctRwtjR%IPFwR0FbC6=Lvicd1*873$HGkS@=;IFC09xjds zhGgKPG*d>IL?kjo9WxDt{yk{%N~REW#FoPT9`LK8a269FbD+)v?jlMeVk`yy^4`(Y z!|LBt7DXA!d$3f92%6COczm}f0n<5**8T!zIV6w5FB%s^^g@PrekNNaL6R2_V-cY7 zTX^!_X)W_AS`;EllrqN$bnQIHHNAbCbjGob9VDZ{@^Ys4taP;*wDA@*{(Ui|d2M6> zO^gc{-3p_Yq2-#QfWOypQnD484xNXHkOH*Jnog&t>{|={rsc}cYjG7&F*xR!{@K~$ zUD>GA>3>EDsRRhsKd`EE|3Ag;2Ej{K1d#j^6cR1qbg)GVT;Zkv6ud+x0_cAHbHZIc zj|1)t@bUoY<)r2XQya7^e#%aNdV;-j-v*uXsJ#`4^e3dS{0=uYDL>psWFio4+n$R z*DAWxc+1l8^+yZ;8O(X}K}mz8tm@k`@brbRKp&i%D|EsDO02R78er z{aiHO_w$5lZ7Tg=0PpADnr4#}9^lrG3<`zOw-8>+1@?+_s8@bO;3?mLcHJrGpC{Nh z@2{BF`8)k(J0vY9_%cAal#D_|Id=XZ=}!uMEdfhbd=)%&M*-BerSNmkP@nE7y@-c| z@)dg`-?}X@4p*6ocuNXuPpF2<)!uxK<@sm!!yA#DXP8iU^)ODvZH>0@1th47RHikV4k!Li zW$F@3@GHMjytB~Ua;xn`wnC~TGN!fSH&{(zDD1+AuspxRnN8@YWq_-ZKy?z61Y5eR zd5JIpI$=eG*{#DS=nuN-GqzeISV7bv`{b2lUg08!&4OpX-wDY*NS|7Z3Onb8Ih=ik z)S!#q%Pb}Tlk&9c(vv+i+fXnEb!*RThe>M|*L{+(34rE4sQ|GHkPxbfVuo2+bPwp9 z$Hy>6;m3JhYG*Bphf{XY)gLT6xN5%=o2=pISkFJ~S{~paL z8xhV#R87yAH7LBWcds<{`K%gNkX0rpY#`9j>5&KpoIFlp0sI&za_0#KT|D~lpk|$A zlp^6%)T***J*rUoFn$~6MW}b?^UH})*8&N1PM)TR(U zEh})9RR%gd z>=I`@o0GLfZs?IiFX5d(rwa&RlU+PjvTw;^9z#~aQ{F*SqdsgCstb&G!unU}cu-lp zP_LW~Fl7_KFhk)J0kO9OjVJ1RL;|<$h65zAmvceQ{sq)~<7w&8zpGIx?K;DeJw@Ir~?r&X0JTXnBy0 z=ja)FQ{jIVkyc~?md*3S%j7`{--7CvhzfW+upqZ7Hm6&6#dpi8( zV8k74ceMMgPw5+`^8LZ7Sc;h{Tj_Q>9pTNE+V+5R>W$4Yv2MnnTFBQsP;!{ z`k1+0K%bvxxKZJuX;&-p27pT-o6H)`d2=+T1*I1K4&@UCOL(5q*+iu2Nmgo_a1OFk;O(*2T5<<3{hog@fri`&=0A*4EjTv#x@zh)d6l%VVok2SiYkX?l;&L@ki|wMjVkzsQzqkcI*Q%Jmol?AuxSyU#yXj2S zWEI4Y(k3PEZSi`G3|1nkIpbHieTeBzAu)D&sDgRaZnP`g6VQH-RZv!qU3F2 z+I6F7+>KxnPz}2ifHxV1XZ(Pp+{)F-Jcf6?J8xw5w74UVZ(!NC@S&JKXxMFXqI?b& zRX|&6ZbbSJ!SaQUB;gw13-1MnNm2HgH0r0f1M~$CF3!%^p~q09)X{>xfD?7mN<-#A z@VUOBsE8_;=2Q*arlD;vRQqL*c2O8KX5o6p9E(v;<7JXA5rCyN31kG;)gPt9zYnn> zwIj)b1vv2Kc?AFeKwBDb!hvmRVuTqVv3RG9hrQ3s#X1E|Xi25@BQu_ekCo~=n{4GT zb|Li6-PgI3-mF%YM~R2NRP|@m>SJA$-_dG=%FLAl2Sf$pO0bXX9Sn9ZEoyld04w?9 zpdILne=J#2IW$PXRoczed|yu)tEzNMW|zEA%?M5DilhT8pG6I6@5 za;GOt{=7%@(LZg_Sk^%2EWM9Clg&d~jtL^toAw=_d)Gr*TR35ZvJpIA`?!T4NT~m{ zUrDEfZ!H%_S>0G)x$Z@W@~6p>n(?mTFWSNWAD0=|#WNq9|Hzl=&EH|St3W2S)y>O_ zEf-^97k(_ASv#og_g6b-{Gc8{qvv(gEvw#K(Ot9WuV{Z>&<&J7E7 zZsJ@jNCtp^E?`?2e4!QRVei|ie;1tDN@bVI4E1?#jc$dg%2c21Dk5;ipdA`6)?@Ry z*XX&lQ5DY2i(d5$OO^#!4uYMED^DDiHA2%&2V0Rx=BJ=ery(DwOmq}#qiXIg>8DZS zCg4cbK~vO$zjMm_a2`8Ioan7MsLZ}+SHO;4(V5DO^XktWm`KezFn^U9cjjaBtg(a= zvBNfZjlKkY5lb86iZOV0ySDa)+s)RoRO6(K@gGda>ADr8My*GbAN8yiao>O3ye9Sq zbdzskOB{(LnrH-~Zykj!hyvkRr|~VzjS&A~5Mv2i=Vs3VO#VIx2}11X7#V}741M2s z>;RZpYKoGV4mIJvpEIZq>X?9Ad)pE?PiOBKhz|-Xg8+=S6T!$&2T02vOIocobQ#ZB zw)M2P>~M=eMDmh@y&FQ^&*F;2kO8iiX)KAs(fr5a%&d|0sR#Pb`x*B3*r8tmJk%la zR88M!R0F=OwJ%W}KsZ72b=`4Y_0_576=I`UffT^i$JtG+@VO=684&@9S4P#Xz}IV- zkYzI1ivX%sdkoeZ14^VVw@NR9w8s!(Y&K`2SBfA$_kO?X!>w;G*Af)qOl)}!C9O5Y zO>bk!$t>1*+2frrS&D7x!#%oX?ext^xa(}~ulF*pj~|sdz?R|u^K9B+h&1bs&v-zV z!k!qo$hgZZ;J zd|an?qqBadh0V=3sk)Jlj@=D2n3MV3H**Y79X~Ug_^}4HVVuevmOdvXJTN zW#u%9?YMRzs{Q0!D}*G4$J(LAiCbJ%;s!ua_!kO{{n`WQ-xx>(n3W-^vANzRN1$`? znF9tBNp?^4Js#0Vs&dukNys~H{&JTs#~?j7P{X*9G@oNSeC<`UHL{uKrnk5W=}AG= zR!lLj+zcdI=~uD02l6AuUgi65@ZP*MkIE*FIg2G&5wsL&Zk>j z{bx*=`d?MEPrT>^tg3gKe}&wQ@}~idul0{wxz*BzL!S+5YdLiBXha`O=?YG&e+BmS zj1w)bS|=$(WTBig-jmWT@~*b`V@9v$>EdG?Z#%`KHE%7FQ$`E~9J8%~eykn97BWy# zi0kq{YBwdhW@a4xIum~voUew#x5ksrBM%0%M}DKvY|pR zwtOjf=daRd+NDBGO*gh2DYY}?K3-;)Jtyw9tO3pBxD&N5=dHV!p+^752(T3aCoAqx zL>BZZ7x2)ucgWM%9h{$&Ha3=WYAmt#7Jtr&bEk|8^;lyb3!#bC<)T5g9%hD5PwVr2 z;IaqaE6`=G{_@yceUT?xH|A+dJ^&(qer$5Nhbr71IlU z9=l2Zr?NXv{dOE*F`|o+Vy-$6coHDn&3?~S=W+&;;}@6j^LSf2Q=4n4*b1gpbE>>t zQf*zfrD;tmV=TSW)!xkNdt7D>e;L(md{pk7@)FUjug7k)Cf%1%t*lOe;A*+r-&69D z$MTW<0L!jIOV8$VM$c!FxEBY}{MMTI=82`{)8n1tIMknPI}N0ZfetWPF0Va}npp%q z&WWnLxXN?guGP2kMIcIWs`6rR4%eExwbajUo_dPrCax@O7l>(yhTbD?0~3T-C;Mie z96wdPw~MSZIg(xuN%~!RY%uxZjFnDcXNE;x;Hht)zvo$$UAM9rb2F@Yp&1<1(Vz4& zmFs-{0$ocp>l-hQu1b{Jb+X)|l0B)65suz@0E3j1Lkd$#?saDm*wfT&E6SSosXL5) zSb8e?m3CPx7$V1|^gYMD+Q|8#p>{<_ZPIe;1Fk)OZ&qREh2yeDWplIwlV27vp~m{! zT)rlycf(HPM}$7FzKe=Ci3cGhxB7o2cOJdE_QUnGJXw9$Qupaqi@9m>M7LLJA&iS&_i$hDPf=9BustUL#|3{i(g*L`ip%DlvMgT zPDKfCyp$-bTP`k;dj-#yVC5Cvj1hTW_Grp}3wWbOL(^{`X)@_~dmyQ<({tk2l`eKa zjhNwrOV@VD2RVL?e-|CES+Rgd9Q)H`&8=p0A0Q1f2*oj#A@%T9-Pd$CEL#)g1-5ayuIq<$hxBF zzgJ3FoV>|7Sm)e)YhS$G=l2U(Y4mOHYCzkeY`#&$N*|tZQfmzns};2i*er3r)Y@(I zONC*d)qqEMr0T@3?q1NNrXUrpWy3B;6^|gY&o0TqHuuEkUSFQczPt00{wm2Xz=kWW15>X4{x z_nom=D=+~8K*{BQ#b|C$+&gE1&TA3nlfC6mvjYl(H-z4DHh<#KRZ|z@HNxq}yro5L zd+>o(tZY|_jnSev;ugcBdPy21-@(z`|3~L;Y@c+bN2P2{MubnN#|mT+bG&=pa6EoD zSl&10AQsVpFn{QDwT=IDA!->nKUV{wb~Q>qmuO?h=*dU;0pJ#3fSIh*+B4AIOL$GV zSCl`4Qq3n|^`B>GR^in{(a9S9)Mz^-vbIIv(HIZHM6C+`XTbSdjiiZ<4wfUQva~8+ z6!Fy$vw>rx=ody^o@qLDX3+;9+AK@!@<-i+!pC13?-(W^6im)e8& z>6LKm)Ei90tgL6Gn3RIy8wc{C($*>Z*}L$tjga$-QI`QgGilLB>C3;R*hMZ^i*GpW zf7f*Tj?3z=6dpzvY?L#*){Pv|lxhV3yoS5F_t$#fAjgRvDF$jHkLA<$r&^DBm3L(L zE}1V>Gklag;JKsyv6b9&mW^~CX;)Ray3@69om}fBg6BmDo*G zk2!)3cWb=7wBzlZRX2YnyMBruq^@*pkBUSHb++N()BiQw>PE2y!B@_|&WeD;k8Mn> zfUAR)jkt(y>6X)`5Vi*P`C;PM0<*s-9zR4i`qOJI4H4uF!}e+k1ZEf07R`NvNPK!v zy2T%cW%aUIg(%$^V0)_23N-5-@H1ss6#e5LNbVNQ6}*Icga(XW7f~>==nu-_NEE^7 z0XGZurUQ7GD>00+15I8ML`CaA2>sz?#NM-!SQwvbyyNw>_vb&|S<|_AGj1PXu&6s$ z>!!b81N*$Nf+xHE1q=S@5d?eLMlh7%$o0T~{LP@!+&|xmbY0WGx?1p`*-LBou<;1; ziiBV5nGJ;Qp2$_HoGN2`TKU{|n zr+Tv=+U{Mi1dunh_kIq;s=l*xGHFI<$D>f~JfVK7$~8sJ26VXRokgvqx^M?T2>;ci6EbPLtSb;M zl)DagqS~Qwr~9s4TH7kmY^a|m2o-Oz@h@WGxToQ~N~y*p2D56@BtNI3Bj#nY z8bUD#+`nybr(@qCW?fY9xI@#WvRM6sxQ(taJQD5iI+#@yf4seHS!?fkPl+#`7M?d) z-!jWAU#iuD$T$Pzl7k<0veC_L?5+XBx>0fIUy^2}pDqueH)kNQP8)nH1&_wHx^f_P zC-Gfb!2TJa;gCI1RwEVa@?)7fnv~cA7cAIVWF*989E7U91ILd;HJJ+L_Upq^(n#Cp z{mjHn_}W^{lW}jPhNjknci}URIcuwsYHNYB$`M8HoK7a3&pdx`eM7ik@Q=3>+~Ky+ z6OU(aM__vO<-9sbe`6s6DMwK0C?HEwT)?=(($0I8K*YHJJ%=Vy$x$J8dc zc4Urrjvi%ibE@}zXRuHe*BR zN(&VA(jN_>cYWm~_t3&gBCAX+wau%QT3WCDpq}d@0UAh&&P%7_h;ra79ZBzZxt2Ww z%(x0$aBWuv$V8v-@~|blp}NyA>qTQ_jx)(GK~Ce|X!{Rrm^mJv(bffzc0|T7rqmK| zhB4%1ZPgo}na?g2^uXUmCK6kqtv{|~wsE88ohOo32Fo#j;(oVwXV#>r*8U9B5}_cJ zEe)Ql)_VQ=FsXBTvkvVOmd<(7+%x0SzFw{}W32(?_8J<5?F$VyXtI`D-CiuGd{LLb z`j(`mef%OVKKl=!H4J$|*E@x{#|caAat{Ei@(ml;X83z0&SYQOXBAU73Iv32ZYW{V z;?^Iuc+ePqa;e6lg9}2xN5{k^EM9-smpQfb9O743SQj4kV4uBNqzw{7(lq(W-pbCw zl{{LuRHv~Ue3}K)+pj0Kz7FW6$XuA&2KVOdfZn&!RW4&VC(#4i0?K{Fl+Qr$@F@O9 zfAluwV(Uj_#VxD0e;qacGE|zWov^Vy$12(@PrHIT_`-2n`*CW!vB%1A&A&-;F^&=x zYnL(0Dn{C}tW>HGB9sKTmr&LK45`rTTX(wf$M zTUw8K33oBHB$ZD_0s^91&F^M??W&hTlZA#$E?u&PN9v6S4)DY@O(u_i)3dD-L*k{n zIuD94l;stUk>c^{Li3j+I;>qG9s}A=q3msTrdex-xtClYpJ1P?$ub_?@>(t=UZI9} zW#{O)Bvx$(RWEaAT?O$phO@W5cc0+OSPwaMoZ>Vd*0?2{-GqMVSb;X3`z6MY=)3>= z5LfngouABE!WWa+Qdl)6$|dz`wca-&S!XW ze3^KDtLmOf&ztuX)kpJNr=Uo0?xXK#6+SLhX-rgXMcJi`ZJ>6Xn9sPMiLa@_tOLs} znL|xkI$j8i)2p7PsumfNe8|&XWIo5o_>2La@Q|3T+LK~`iHXE&JY>y0Pah*$TOR#o zTG-#^5M$%R?TAri6Y&hY)&YuUQ}n%Uo%QtIP_CvLmmVQA4|oM$51;aA%s#6zYz*A@ zbMwL5D{NTX6u)RGi?qpEH?V0GPCb-My{~SPC(fTV1X3Pd?T)}KJJPe%QZ8TTVk|ZX zT0ekOx_agUnbUB33@u3FZ`_v=`rZGi+_hr@Mv(~-L$x4QMTdbPHhc)q@i=tf3oSs&2DY*xlRVr6F`nw z!ts1{xFS+}3XWTu{rbamG>l!p%BI8n$n>5hBoPibb2L1n+co=}G%g#TeR)AgqoAEB zjaqurB508(KCdD6!Zg;m*YAjfh_raGZ)O_?-#jMnzLg`XLf1n!nB$;H76i3Dh$~Aw|D9XZ|;|n@hjH zMqOKlTtc(f8#?z7VI-mju=;H)CmXMe7fVOBW#zhWZg$zhWm#6HxFU?oP#i(5^^0ot)~uk~KTs;-BgtLS_2s^2Ke2AHRXc zq~Kude11{bw%_V7-y5#y;}Cr9DP!(_sAv>C_jtd(b${Z5B#0;*b*zVsCaI224vaae zPPA>$e5;Kv$G>7#GrlFoXr<BI+hXGoULFW(8PgFFT06r+G4!WgwaJC-O-eK1 zW3>Q?WJB4DHun5el&^+vKOPx(yv|vf2;?kv;i;^P7@kxKz_$gc`)#5Z3(ykrC&x2V zC)-?|nCDJ_n5;3h%IMeplr)9$uP)xdkE0JeP+Tk%=(rzFSDh5jB}O!oF@e z@BSwLpjfhvhzV-Wjajn!alYD`HUXy^#DX_n7da~0B8(7Ari=N za$-DPcOcEIbxYPm(;$i1B)H)!gA70FUb(G5SteCoU-H2>t3RaH`W+7g11BU+BM;2m zc#{Mmv)tROlj6BE9l|k;VsB@QpKG#+1<9wF2|No)=sKV`YK>zjLZLUd ztf>8rz$%T3=vDNW;}^to_uIN|nf#brvSi8H@xj&6p5<-q9!nnUsZr(cRdq!U>g?K8 zUH`K8d)w|Q2UzI7~Xm00$7E8FCT`h)5n&%8hVB~~vm6tp4) zpM7ug*y!m~E)$niN3_?jRZZ&r~i=`CwK4L$EF{lvdk$1+pc8p>Bj7BqZi|lRjQ9k*%EJ={RP)RHV6cDz$FxulNtuH@VV@HG~Z^omqXF^q7%YsT(7v)blt%{gxxc&b$ zIoEi%&FG^f)t<+L9U5fu3Td#87}=zYa3w07{R|WhF?uNpvi+qej)&MGyp0S3r_`nI z&SfmWtLk6-bGixBySi4$I{;ut73A&bAe>A9w_JU5*$zFo`5OC)@FYOaW8`_=zJ@1s zQ%8(~z4I^UC*Q6cl!R6HMPePZs zT^eZ*aFG?qrFBXXP(ru!<8mweu4&%#?wTeYYc?WEO8XfR)PY?5S11&dl{WHi#C*cM z%ejvqLX=RxO)eHf_O!xOxOv1*mJ2=r17w|4cF-MqCCdb5Rr{z{4mhj{Xs}d`xlPq| znFZLrY54$?1*uxqjs>!V-(U?4eR)drtFjpVqpEIIMQ;|Z-mBVNBc{T3(1l}5 z+L-+4cS4u?E?8wLSz~>XyBXYr@lqKP`vJ_)Yq&6mSi#>Mj4?WpQ^^J9kfxFjW74>K z>H+lb-iJHWi}%rmKvQoh_Kq`DEsi+UX04R>9K%Cr*juc;r4q~p6$Q1==t}qQJ#Kh1 zShzy}4}0Go)b!f@8Bhd4u^>f}VntEFLKCH^*hN4I9fBymDosiN3s?{=C?G8qrGzFW z^sb1C5I}0^B2q$A2mu0Q&m-J--`~#e?9S|OXJ>Zy{&(+iA^Dc`obzetHVY8k*+y(R z`*iIa;{1U8{}S!9*KWfXu^m*$+ zn2>{-W53GPSa#{1K_1cp`nIYY2;{xFCJONV)A8pFpYcoRHyJU3P}CgP@q6D1 ztWIXBHk|(l4fj8KrVbuh!+_u+P~47r+yF0cC3y1rD5j?!l5-@2n$*=(~qVzKHP0a zhnd%5c#9M+)9|=SP=m2DcyVy~<@9wrck#ia{*+f>bzBC_OZV~6&+xY{4vM>sNRQ70 z$eO8CYBL!0yjXJif7uD4w>yI-)D~dYYy(y6OWWfhC>>9SO$Cd(B1q6vrk~+}zXeE+ zt_b<{@n=QdnfMfV>~M#L3G{(cA6nt^FH&&^I8#2a^L7B5@YOimK-DD>+?Oo(8&3Sw zr9$uwAh~NZru<7E%ea_K#@4z2>;uP{)Hhwxm8yoEsn>vZGal zOU=}kD%Y?{W!H}+<==42N53A(a6Xx>e;b8V>gU;y5nrOL6sXDT=Uji*Mzq4oWd;~? z4x~&{WLgb*+WcXWj~>k4OU)h*sXd>KHV8NMD7abL;YA9`^E2aCDcW8@=__(3Ey@lYzdGH#5vNOMqi|DrhY3*ns$aO484eAn`lE+@R_GCqGn3>zqvt#83 z>Ji^|V?>ja7OQ}S9+k9+e=e_N5pie|F8eJdZBREc7N*Gk`0j)#A6-*v2baX6kBPA2 zdVF(4;PIKNBE?$;aKhFVUQXJpv+}6U#{O&h?8eRi+6wFYTy$k49R~AY_5To>TmDoS zjAa&*AVtD|0UM{h!XzyKgzD1$+%3( zBe3Q#MI-henmkd7)P;`D77o5~l)K-CY&$S54=-RExJ7e;JCtulzAGy13o^P@ z#Tx<&^BRz91NjNYGutLPy^c7guSiz`)Q2u0><2-nrCs9Tg5&_JY9wc^(@A#j5w8!p z7rZXbiXl04eJsV%?T{iN6GpGk*Ab?PAjG6iA5y)?1XHzMcxm^A$F*crjdJG_12gT( zFb7b2E#^DM)>GTon_X&a4js(BX!gWlkUPx|KIh}5R9Ykm5cA0z1RR*VBtSEcU?s@H za4#kstw4N0QjRLsK(+56wHn(XIuRb%D%~aIfMXZFQw~GH^{q#4Iufp_KxfL)Xy(B;B?G}mz08FA~;2n(&WIW zc@Wq4vNl4M(T6cF6?k$84n79J+v_e+#Jj?nS5+VZ6Kx)xbQCD=UGo83_M84aaCOpV zKLH6H;$q#?xTJ8a60kn2g4=_+RXB2$~WH0-MZ znjSjyjok-$kHeAMpmx(RLt^)8-vj+K& zu_;PQfJ4KE`)&}-{}cw!wd`Wp95xw$wU7H^>H?E~;Yp&2ODU&JcjCJ?bG zO|~I{4fw#`r+8@h_JBWSA9)dGZ&u#XEWonp?h`#_J<$A&wrS8LGca9-O3tW=4p#ih@Hp z8e;08XRs9dI}N6}d^#Iz?>Ku9`gx~T16k(HwDDq)Is_0ii<~D0?4|T_MC<5xCh!~9 zR9Y~L&&{uYZ&)yu@2k(^`hG-)`Vlb0Z!QCH2kg_I+<`YNL?f@u)!^x5drt;NpXJuH z^)L96M<(B#U4{~G+sPL1IPITLvt&#&e)nZ3MLoW^2;D}J_IB%(707t%KBuLsG`9P! z4w5=ViDSie)(f@vkI0QK!p)ZjLyen3N#Kom?d>B+VNa-a>&{cnB^Digv=iFCL`RZG zn!;gWDXrPtZYA_q#apG-&|H4XSLY?u1Z8p&h5-C-s%C$JrZ5xwZQJh7_FStE9>*O` z%A)3(Exct*f7BgDK}oG^I+T4iU=uc2MBY)f{@EOmbQ@C1Wj@%W86eO0`wZu|+r5?y zmuh-@ioQXa9QKL>B?o0Jg(QuwxEjjPWS|Y5jR1ZBZax}bn?opS{!GBBo9ZmT!6n-@ z2C!))&4uJO&BvE%7tkXtK@f9T(iLV{7&K)ZlYNtY^HylvPbRaqBb(uKvd~M~J5}*` z?_nTj&PR4k^w^wQ2QWJJNOh~vu{(Hg*7NwK5#Fec09>kXu%*>2MZnf%IFtWY{oWHh ze9x+K**8BAtB$+ImIFa@McI-B*`&eSr*Kk-JAwr&B=(mY6n%;wn)0>z-cU3Vtet&D zu_i*Q?43HHf@Zj6{YkYIcP!TKAmotRg4fAyMNEFJsOl;$y*g07$UZaS);jm6+IAOc zP;a3!7H7r^Y7R3Jr7#$ zg>%tfe=dOcE06{othzzEw$JWZF*u*@RVP;&P7I!KpVBLd z5M&cBiuN~0?#ozza=C}KFPZ!@f z2pXxZbx=TxYHmDJG}ke5d2Mw2%bxVjXlE2)uUBkfzLBwN{8?-pJTHec8qv2gEK9&9 z>9%K?*Vt1_jqEz|>@r~%q+H4f%E=jSWD6-PakmjXKCi~;DwPo2@}D)|xd z&UIn-Ftt59alMd&bVUaA%=8Wzuh3#OR~W1eVCd<(`2Z}+d5DBCa_i!?dF=!Yh*1YO z-hYH@SbXP}4sRNtnDXLuFjq|V)2%!l`S8;Ys98khlCa0n4nZ(jp)*D8X3}$m>ohn{ z#-N|b1103bIl@X8U+2$h*{j?H@(%_8ziGfGrSEhD$zA$Aox4^L=QUa|80}OsG8gxF z$v;TIuFF%=76`E{DRZ^F`|L9yRSC5Hk(g~%zWs7ZoR}KHCat5}43btd*`|I`MqbyX z-6n05b@`;D4Pq_T5^n%O?cjiStrl|27iEAJBpYgL3v|~5<2n<+@-6RG0`^d328O<$ z;??|+9t5ySE)3XhYL)^Z4!0i9MgXgkC|WTbap?NJG$=4YrwlIpGoFS0cV1HNB6CM)PR8bF@|F?m+HqOfD= zgU`{54{1KX{B%W9u8|^B#;<&{h$HVxZ{bHTGN_A8^+O*N!Xaq~aY*AUb@=T$w*L{` zy!@m=PwSzw>Wjt_D;t?2xS~?#kP%-=gu!4CGySh0h4y!_3&02o#Dlk+h5jC1KEHO~ z__vPQ1PpzHMsRZemH)!QF!k?0Rd<|?3|De6c)7lw|7BTm_tD#>4~4J|`$j5s1tDnN zD`B1ojoC=1n};TxihFDlfB}cVTZ35YlDS_IGk^)4(;EaV8j*{LN)mX6hg)#BH4;>@ zud0CpMjhXKqz#7FW^>NsLvG<5oi)lg)>E}IZ31}Tr)0pU$@HSH`m&|%{-v|2QGkhl za^fECaX5|hsIyPj>H4}w%CdD!?L@S)*H8c?*Ea~%P!+vqmx}D8T8)g_(#0cdBul2R z7J&*h4XBR0L#c^H zWbjvhKOEu%=E=@^3d9EhKaz>sQAl$9;cWr+3z4SnQ0X^Mw=dt+-m=p`L582I1B$$3 zy zSP!lPmBLYs#__mLNLlpmwMe^E;RM)^1b%+nqRZT`DlNPzFHq1sXY|-@{dZrR590mT z&4AEyY_K*0gmnCAr>i|BTtW`s9bCw{RnmpGaOzluCwsclP#ZR@UWyVYr7gO?Uhn=} zFG6POs385fSjc{-U*%4qVzvra5_SNLEt@KC=+K@$m=j#Rp@=*L{ku~|)`sUOAPE%7 z*=X;(A5E^yIhZSmHU-U>E&SJ+`qLZs47tUT)yYgCu{sNpwi&n=UwDhid_Wk$l#;2g zH1`qM5T5=5{VQcPV9+)It9BaDqvLd>jfhb80ej4N#c{02DHZm|$g{TgC%hI!41%cS|G9ieik9Pm{Mu1F!jb@AzJir_wbIcFZ z&|-YYW**>Z{j%nteAwOvQQshqWzi7_7x8PLgjWbCfT}&p?)`*brzpk=V8FwyLly*l z3$TVh+?5;n{>L)D{YunytNn7`5w=%j7BvAHZjvhcIG7IFJP z>3$yN93X#mYZYjtOWjP{;7}#}8qblrP`fX2Y_fKLs#^wsCmy`ilA0c_pDyXHChEnl z9jE$upE7mvk$5NwgBxVU<*#tyWv%gXpFcE`E_*jL|<(|4(x>kw%XYDl@@Bx#R?~aj*q_wc|jwoR)cAQ zl@j|%drj1>_MFJh>$y`aJls4+__Dx>vp1_Tav(Vrw_7)t?EeZrJ*GugdADLw;U%x7 z*?u=S{O_taqXnVHB&7{A?{$m?eoBrA85R6lpZ*o;Ia~q4Wsi1)N>)fm@q-;mkh}Lr z@dz=c(<1gcPU6t!O7@QcZ2?FG1c}s0R zam0+~qiwSmtLQdHX-4Jn-etcOFmB}__8v+Wyw{xJyq`vH!<$g}ya2H5yCH&{8hAEm z(+-C_B_jAnebTTAu5PYV+dKQ4cAD-l4_Rs)=P+F<+I;Jgh41ltEO}aKnjN=)74Pnb z7DPwXvB-GQVANk6mLl>WwX2f5)MaDIs$!(IDV?53-@{o9`u#yl&}C2!(wbY$K0ews z-UvAm^b>g3*jM!%MV61rL3Z%faDha1vH*0J2)GV(E3UebyH*Z8BA}Z-X&`prh5rM5{iSZ^|{{+aCGB-UXoLkn=><-4^xG zqJdI(u;d$93=9C&5HN@TB`I(DW$FKdgMy*cOwP9OHm?blNF?hf0LR+Rl4KZ8~z=dm#8YMe=5SQkIutEj1d2jr;aG!sh9=v{VUN1k14bd;25d+(3u65oG+cgd}Vw*2`; zts3sFdKCX;4&f(hdz}ql7l3QZ4OTu?uK&oU!l*D*5sdi&=$t8?5UxiSEcmG<~#!^o~%ovz4Qlfpm! z*-a^@PDsC?Wjkkh#zAk=0AM~KDY7=mv(2rIkkt>04Al*2vFmbzDE8|mgWWIM5*mYmu{B+{3;z5h6&-iWE0mMrIEX8> zJ?1&oAZ9z4wFe=KGyQJVgJ#56+V53ZpnylDsI=^U8Z90yQ}6>+eRj|ar29Q)^`za` z=HjM#*McD#HZA1L6yz5U?2}%sIDv2Yb>`UKH!TcSBmNTHvv4C#%J;0J}k_A6E z+rZ{CD5YJBuD`x*^&BI$rb4HW3~@Ce*CWB|%9-&l!NEd^v$sSV;REUUmEe%J0QF2J zkk|#Sj|1oT+88vQbBA}MCFbNRh(CDUR4@y^^?>Rd~L-mZ?^fkF9JPiBc*$8-KQ?e7Y18$@}~ zB(5cVlby=8YRS1SWKeR%9JBDJH_>kTb88pX5HJmTBp^%_9VhbwyR8b&mT^k`u%siO z+#eSpnFSxr03cJNeM~sb{*Fb!7eAr7OBGMvJAN3}j_K0%x;?0AB|}Oa@>wL~R&!*K7X9i3Ox;^PS7hg%C2F1xl*? zCxdV+KedH~8hd>4OM?k_Z6_VVgeWyqWwy_hnm){ox7)?{5G>`ilE;M+`OX!Jw3TOr z1kB0ngg4)aDsmlnoVeaNU7%h3vH9-c`fc^(Zo#8pk!aSu*1+7VF=73QL9kVf`t8_d zXd-m^x=VF#t&wO?j@31I#&GwPZunKrAIvtZ;Tu>2L_;4EXbnmE_u!s>XBTFG-)R)k zY*#Fqa{;2Mga-Q;2gxGwN#nCho>&_y zJ>lyx2qYkduWB=Carv9fx^cjr;p96g(68k8F|L%%85@7C%u@5w=zdX=4)?K+X*Y0;m{J_-DT9KL?Oxu8Ynd-e@s{+P#fywmnbY)7N4}JFO8OehEggLwr zkDVBco@$R^t7QJgU$gU4>=|TrOfnf)636pnkiZ1)x0ZqLYiNd7hNyeYJvZ;l-&5sg)7r*^7k0E$@Sx)xty_{U+@}WeriYK_ zN7vwL!-nP$h~)%UK|_<~!(5V7)nNo6@CXg*DLiQzu-!XdYN{Z&YLZY!2}5hfy3Mef z+<&iy@6@gzU~Ab<2N<#gbelF15Ese|acgS3k!7WImuzg)**+i}@?hUe8`6FN-1m^H z9l5ljh~^;kh&4X{eX?B6DJ*3AUZ77q$;;pA^)33S%~oPl&FvO-T)n#-e>73u>AGXY zj8NK*R)*<`b7KcK_XB|8+Di0oVLpD4O4hjt&Z-U)_569>9AX_Q{hYA-N0P#Fd!=&c zQPKqtFqe4C#kRxpJS63J{jMfg(6_{yru8Oy^3aZ^GVr%%rF%>P)SU^s5DL~<+zaif zMbiO@IZVN@zhTbv)E$i~M}UswAEfu0j{JOP+L;(EbyO+phdmCLe7kpP6_e>8i(3k_ zQ@yfX*&ZtC#U>l-$ET~+e-44Kp%sSz00BC1QXOjRl8L_{14C@c#lCdz*ZYOh<&V14 z2z1BDdI-9eLh%8Jj&BF-#4s#@_ysV&cJLaHL7i5goJT??e%D2q7iXZ!rl5dRIDJVe zPqioX*}iW$F{yICho`azWWHA5gUEzv=?>su1>Z?LOtg0YzYEewJhG}d3~S)rz|StC zQ!<*2twh!Er`d>R%Eo!y(@Qdkk^0yafBE_h_|3tE=PyM_8kx*>MC9dW#`olilp5HO?DyFWh-*1F zcO59zvxZ|X8}uWEJjWhybwLMX#NiwMMi4y!+qKWgZAEL+!DP4YFLkXy0JOx7*4)#C z*t7gwq>MYUXKY?o{#Duz!TGhCbsT;gDVRnmzfI7C6!GOKu zMlyi@dz*}PByG-`?S%$hHu!hAGL(cU=JeH|+D}4^I{nNZQuV6wmm2{oeZNF5-blUS zLJVt$?dPc6tNe;36uBh?ps@^*1ftx?Dnw(jM1IZ#6OYb;)aD&*sPzQTYQ-0}=v5>!aERKZ!N^Y=|4N2$Zb1h<&_QX20oG#^SUw z7kgjh)Y4Fm993}rH4o?F`ckZTFATBXGK=rTAq@4}FW z44e;Y$~Mfr9>gW35V~x1SJ&^hjJvR=|A)v_&0aK}r$P6Uesd9}+^ud{xlJ3t?A+-q z3DF-#-FXhU$l!JV5(8^eqXFn8*|s;*?h8STm&<6|Z{;@Ae{S#O8C6UgA0Kkq=RqY9 z>Hqy7^uRpD-SBodL72$+-aXf5p)(s~Iu|Y1bWHClzW=EhnDv?3nTOGLr+M*A^f&FVO@AAhIHB|lf)D%<)5s=P-M`6W0P*<|;2|j;qY`kMx;?H%(iJ$T z*A08F$b5Vqw;X24Stq2>#8wL{sS3HCf^)|Tg%Z$Uv84xX#0(CX3-ullVPOrnWyd^- zCbI$>*gmobw&W20iSJQSeGE=h;Uo{`tHs)1M6dLLQP?0X-Jq z=>z0g0fXmuLl%cJ#Bw|G!)17i#sU(q0l!2q0YwW^*P#GW&=#C*-38&osYuu62{YOl zJlsK9SFnwVm)}6Pa1CF7CQ#e*li}7qn=gwUg-uE(P`k2W2&pRSI__rcmL`{Qd7*x> zbFe~q+`?z;%%3UEkQBXS=SXVf4gIq=IX;PO9#v0hhn-M?1Y7%NJbH4xl%>1VPcoWijhOf!3cOk!BKG zIX;**km07xxb!&X;*v#KZ^JHJp65itcWx0STdYk}E+_ASB45QeIR1x~*c(9!bmhRy6C378mz|x_@cWocuSGry_~b#!U_Z(mLqzF!bP7nWjbPEDYNe-2dD} z6qu;*YZNu%tG~kuGKyH~)Io%OSq~1esjPHj^e#p8Q-be7r$$T+lr3YT^e zdf79;J`tF?$>CWxMXq~$eum}JsAh=l<@YM(kA3rHS!H;q2h7#~5&%oCLqH5>~d|U-a6sD1V=OLf_eGz25KiKzdZsQz0svMnkXe>Jbd`pGNJ3ZP^P%A z5o{Yo*z1AMdl&KkJ(z4=8_x8}i0;5GWwTd@ zM;?|9f35+UVM_#ul}=dW!^JCAXpa3g|Kg8PGD10<)aXO3_hb zS3f0SAjqwl{+9y?IiMq{v(+%)u}=l-d^NX6V^WCVJp6;OQL|D)dYNNTxwAu&)ie|K zf3zm9VDy%)8x+9Nh!7C^EC|1TEOdH>r8?@_?L&I!Jti$)g?3W!)JMk2?& zL-BZDit z_34?im&Ga(pWO`^E1K-xd_ZEH45u*Ky2@xA!%dEvgRYuL?3Y0xuz8cTm>zjTpqm){9!u8vEo1VhQit{ELs2xR# zwNZ*k-(Sw?#vAcYSNB&_YGn>k@J0%{0Uh>PAZFBsc9uu_i|F!~b(DlFDGw>I6~=e^ z*oq)I#tu+0#A3I;ya8cM$bImUQE+BrZhLD#@qzXkXH&3|kxJ%&9oL>O{4+B)@>Y0H0wTAxGyz$&NIt^5H28&_N3xku?;O-NqDg!)r3K82)SXj*(c zw*Wd8x0aO{Akn$_|8&K%7}~PO(WwObxtI1_&kfm_J)BF*%TKj$7)MO-3#3my!@oFn zKoJ`Is}9`H56&?S=F1Y8E12LTK^EsDan|1uirR$|9noO}85688c^3(dOo;~OzqB7e zLPrih>t_uHJ;reW15z3x@wf;Exms|ss%yC3WiTeb^Km22$sw2U(ETXQNHA+DZSWial%@{ayPQsAEWT*s!BqDK09hajruEP zdgHD5xVx!36+2&eDB4t>^iW!nr*OlUPAor&75A>?FWe!JvyG>PCUv&9+G| z_=NNs&r4SN0Zjfzut6%ijA$YI6EZmf)^#KTL@ub8MJbAq?3e}P39C7PZX+5umU+qbI8b2Q%li+YI@5=n4sqQMvsZBBTM2%%|RC zXdoFzl2v*A`WB`wD13xak^x%5_?KdD)OLE5|XEY!A12y_$ zc6?HKG`@1G=(O&2oO}0vCs*oLT?1B#qeeF_`iQQ}Zk3SG3^b42v`T znNAweA3q|uAmP|}Apo!zMg(Fw7(F%*xDRIi1Ff7-mwfV|Pw2lHy`K!MSaErB>VV^{ zE>I%%%);)%t}%#okL+(*nPAKP4!~Dxgkj%pvD1m%4t5e&dW!fwBpA8m!Sxb;N0@@c zKNYf-k$X>oS_BJlP-})Ucfj%oYO66Y~)_g||mj9q<{a>l*r@{zwA(>Fi(`^WP!Z`BFifOrpx$3U{3DUdXIXt{Z^92aKej+~yYa`utP z27Q;nBDw>9pXek1BS#~*Vm1bCQPP1#M-;^b`5L(ZiOVF*t;K9S=^WVv309t4;pVMIzT7Yb> z*tq_8@UZB^tI|nEBxY-&y4ZQ%vj<+alztI0VDSZM{POQnaAKmW3DBrH5FYI8({Noc zo_^y}FWm9m+~>Ad-xOBI$PB@`z<{1}Ux)G4_$_wap>TMw+aiiiq{423HU!5Y(W#Ng z@H==o?WnK?<+vs%Mwms~_mL~S=BS2uxpvrvz`8|*`~ZRV4KXVy6BPw$NL1^2VN{a1 zhw#0rBZs3lHoF+&W#HahwHkY-p(+X$ZZMq%!_my4sm@5~#dv@|Y<#}mfWQ(PKCcQeoIPv3EqlK^j$J&ZSb@b2ltz%hox z7@jUcn$h3KaVHuQ2N&@e%u9>j?S`4#JLfOpqe$dAcrYc9qZf2fDY&vyWb;)Vr`th> zG^s->4Yom{WZ%3AR!lF5kA~&HptGY$ou}`~**;4Vc-fWPV@}?;1?am4bX!F*pNzq~ z;P#pf-k2=tijn?)?8Q0ZKK@nZS0iJsw)@^h=+`*cHL&ZfVjqt#m9{#P2pNrSphzF$ z{zHmH>j#kiiUII>S7-w8aJpbmdpQ{eAAJ|CMr-HfP5vg{H z`uqhAUG$(k?y3MY$e58}5a9f}Ko#9s-GRX@2yWAxYSzIO+>3h9Iv2Pz;~^WJ^UpcQ4Y7=xhZ_U)B@Cg$+FeL1WP_F<2%5$a}d_56BPh zw$}HTHtO+M!t3|RkJY-01*2k{gS+tAkcFZQh^gdz<_i>MhS49B)?({|dde>e0Vthp z^?NS9a;!m_2X_Z8>+YfEsHWi(7TIcbzcGsvg~iRFN@+<(>*F<0X34!gx9d`Wcq>NbCpcjpPflLXE>%h1 z&<-ZNR6umnEA?=%77l(>c;9-H9|D2Gk$hP4M9b>?BQMbQ%?>#CSaDCC>nBcAO_pxa zI-A~jwnKK}t7;5}quXJ2@fCOO;*xmSjEsT?8TP0i-?t%sCsKnUOayz(#%Bsxz@i!T z(pvwszfdq1>FJ-Q`$u6>n*GjG%GV|1oiUlFk#7^5^A-?qUZqm*yG;J!lehPd)-12lhZE;4(R*ekA7-xG!oQDJ*`()=y^q|9>o;8c+!=eCoj!1peVj8#y_q;HKAdq1Yr9bxZv4)k7G}qZ0~AO3;_Sx~?Y zbaj>bwj`cdZ)QJTin!%_Au!`(SKbnL&inBjJKD)_k+xyxfMJAgIDwMuASa`MCvBVX zdW`SP{)%d8YsTB)=#h%lhA4&McBuM!uCPF;48XBGyk)_LpPI8^$m&1Kmg?i)E9HHa0{Oh z$j4C|ywgGV?+Ux}3NNgcajO8Rwu@?sODxwfi6)V!v|hV{oQneQ&epI*UrhzfMm^rT ziX8Q;0*2uE&6B!Xm3<}1dJZ}e1LzYv&u_0)tcS9hr_5zv7K?x6A;-W!&WH}G>ltLI zKnt(p~m0MsN8X?%8JIX)Sz0tZCd*ox&ZRLus@%-=}z$QrN?D8|aj$ z!uz1MDhIxD2R&_9sVaMu!t>UOBJQ7O7>`ks+<5WSmDXo^V@DTmsx7Ieu-^2mQV=Nn z`SMaHUa7q0E)!1hE^nkdwBKwvMp!81)jjaGrQV9UeJnA@p>Zvit)4T51@lFH@~3yh zfyn#&F)fjD;fEX{lF!~b>54tzFzqP7$gMcOWt(^#T9RmJmIdni`Esu{g+d}ePa(fl z#!V42Tk`^!yc-I&isMxb^01Tr^oN+8JQp~9r@FOI)2WxfA}|+ZIrg8k%ryXPjX)&D z44hF#qU}D9FN8`)q4=Wl+~~v^!NM5}ri`;*whXoArT81H86RHIxjch-)Q>~q5r0X7 zf6Nyi^%I)}WYd()ttVm+ovcYIc}VEmig|uw^~AX!QwRpV{%IsI**X)>tHRi?+wk!h z!eBMKfUfpyb4uv2i4UY;jw^|5nHAL&)92mB=p$2lYQqkHkeC9MRN zeb+Iv^JbZR4}G3fRy}mc645YPgZc7`(5XHajn&lJA6L+AO(Y@rS-k%R_ig~KWMkRa;9Vnu7| zqjL=MgUc;kqH{Za*Gv)``MH<_J9Bp!%>RgExbPWT4D@(SskMH%ihD3!bvk?cg{oUri$lSZnM*G?{X{nfg-NYEkazow-VpQ`1 z{pM0QnoSGK|*Acn7cYR0c`Hnr(ot!?{|OajZ)!oGuDoRQmdd+>?tp)#Db8 zAxXlv+?VC<1t%6v-)Wy#nFiofZgi2Edml9aMvF2H2eN(@XeOm@)Q>;rQbvqky!C=| zbgo`UF-&H&^qmLEGZrW~GY{+7pRDc-O<32hSNd+V_;<=I;vVbAYP3fC>2@G~yWL7r ze$qpjt9H4Z_}q|3aK69=qq-ODWz79T(u}WWSi519cDm+FU$(gp?3c@AWJ$wtjj)0e z7;Dmf+CV4LTe=TXjcVb2J1IjB*lp|f2bJ?QBm?p*&^t!#r<#QuiTZxdpRcRhdx>!P9H zxqCKIZ<|T}aw!V>)P~Z}dA}!fgsfX*8&o~EV>TK)A9Mz}R#Cm~G`)v>AG#1OweCcB z7UyqoHKoF1w9g!|PKKQ5^{p3AS4n&OByMaMIXV{{Ztb_}GPY6wikQ~@K|YP^L6$v9 zJZaw3K{FBhWt20}$1Nrf(sAL^tx{4;0+$)}y)D}@J6n}~gfP9^XHE2Qx1IMI1+$Fw z>ha7CkLFQsTn?AsKk28+KE3S}3ipvqj=?=FZ5v80y=SE+pLCIBJ@qO!^pq9$yiHQJ zzdzaup_zN~WLAg}1YDZLRrGRNMcvX-i2}_nJ-_sum2S8QYnrt!ZTtjuw<*#AL64&_ zy_FQP;msW7rsJ?>3vw9S}QxR_|eSiixh6G%4e| z_ZdbxD8UP=VPoe2_a_-G{p-^x;$|0S5?k+mKBv-5Eh{BBHkues^YZK27IiJ(@p_&} zz!mBbj0mRkS&+KcJ|IWA-5U0V^=^_^XiqNdL5Jp_6GHKpTce&fmHq^29CwcEAE>K` zUr_9lXu*58Y6XZvh4n^zLERdU%ojG7YA_VB5>n8UV6_8gl*2FUl*GxDU4d!Xh{4^J z*e^x$yy}RDv4*YX9saF&4#|hrcNXODk3uI}xA)vl+hF06Ylwg4;+Ps;tKirYDoQhM z(U33-@UWPdIv6Kzv0i2d_ffhp=9De=2-Vk+C5QEd ze%R(754jO^W1Zgv>LDY(`V0>2ljKvjuy-l_o&S}b`6pJS=ZzZX>)mELVh`zF&z?VN z;{#;7fZX-;o}QszqkHre;>{Mz?SZA5k7CSZob^i*7fZkE{ z$+9ND?bRKVmfndybHY0+`d61GCJi6{|HYH>B-npx6L#mdro19jtfgK91%yrC07`)DYfRxw6ETdoKx(i;YpSpWeo_yPUYH_ zBg_oP_;N)s1vBp7!FpNk4yZI6JM9&~Mbf$IU{0#3znjaQihpKSdoqS6QlTIjQwYQT zxHqk!X|)eg{`;ReQN6K9t%d~r6ML$ta{i)UkA?MCq0cF&)Oh&jXM{YAZJix6Fh6XA zyN~Q-$5aL=yFO7m7pQ|40jp3|6M`zX{Ft}49J2_K!Ub>4O{V#n+!pbG zE4J3FgF>*L8+iXDRqBZ;$!tEyk`r{-`Hz-Ff>wPzMq%D{I84neWqsL1*741D~5i1%>LVHNmHhg?6*E z7kL<3{HBG?2eYX^`1z2HVMe3O|A`~W96ac;C$zM9nClhyAG`s#$Yq<^@KWDoeG>qM z?_wy*=~9OCI}h+MlQM$>G$TP$7=EBT0*_@mmi7-DMA->&iyQK6dP80!PWAQhZXfKoQRI$4}@ zZ$D-de?+mX#d|8+bNm+Le2?%5jBXbAp3+bY!|Q7>qQh+bl^=E9$Gh^!34aamT>#JI zc-RHHjmTT^n`*Wx7tNy%Vk7o-mBE;nXG|)ue0wzRv^Dw;xH4mE4T zx<}XGKf@}=L8n^ZhRJ9CbGu+%3(m3bs#QV(lVh|5UR-fG?4 zS2j6Yu6NR9dX)`!6yA8I?7M}X*rzlfp1z9u_;CG^eaYcFLY;11NpkpPE%D%;k3^CtFaZ5Mi`ivXObgi-i zAhfhottqijC)`MnXV-rDB?N^!(awC@fHQ`r?nT-znZ%kO)((BG85%0oMjM+jJx5Y# zOLA-TXdUM({2u+u(ESY>R%~4rIMy?~mW2ZTt^VzO&W!tWE;fX>i%H@Q(t2AYV&K$U7acF+mXsEIwqxk# zJE9*+N~4H#EYWLIV06ZNo#3p1Utct7nX8}uZ0hICkU7;f;h@zPfAJ>UM+(!5Fa`Lo z9*A_h4y!xP7DIAG`O>8=S5gwQIlY_wKa?3CvP+!TJbG$N`5eFys{53;TuzgpHv8zq z(#Zb`^sz-3&9GZjc3wUzgp89yZPaz8#JGp+1^EP6e{D#>cJVF-s2yUmU9y+}Xr{k3 zY&gdF!#ff;Hg~ue_u*}GzusM6@vrxWFGYy2%aPdraaFlrm-a5D@OYJYW9RxbZS@hq zB zkL9G^ zjGLEXI2OPVhq8YS;BYiLl!_0Yxh4Knb5Hi{VwA`&V!u$-p=PS3Pal1&^`4|n2^6zttSTOt{y=%kkWWVW@ zgt`2=#@?f#nJj(->zOQ);J;rTR1QLsHaquf(%ukDwp^sj+@lK?ri~mGYD%$XJnc{SvHNC=4bPwp`H1CeAHIjI<{&zGfZIE)lyD&Z;sy z#aI&<)}VBrXAV|0wW{47XNK4?-}1brB<}Oo4mhQbtya)(EM6_7;LMRa0eYhiTH*(! zXU5jtx7rAsO)5N6yUz*s6~R;Iy+phtsrx>LE3`R7uUcHmt3|;@QzbPxI(&;!HMTVDtYVDI zPAvw1*~FjfOBK~>;j`KdY2GDEeE5NL&CQ%8v=c@M(8n6t7z?vez`x}(c5I6V`!26F z8?S)Wh2xjl?`x#c*;wPa3y~j1v=}yQwVaQeb*_~hqxoHI{!R>Q34M!lA+XQqR?$HZ zS#;lJy{)iU37`qISxnk7%O z*IQqYeIZSuuV$vTuD#+{5EPX_~Ee@961Oft?Kxe28nVrEo_bT+f}`%yn%?wy_ha ztL={%lG!mM!>JlOO7SmyCaPq2@M1)ZS3gT=8mfgT$NYN&LUhY5iINn;9R+Po&y%%s zQ%E#duEP);vyr~7C3aj!q#RDg?eZ+!7WCu1zfl~9Kc<|G^w?$c*kO}m3DbKvBkP@ac({fJbq&T)_nue^EeFRjZj(wh zje0uGK0xff+8hF!!rP+&!R8DSe3TsHuJz^kAEMq`zh@@6ZhprH&UYf7qla6$u4Tl# zuayozTj9%!@vE(H;@&6S;(eL7&iYv_{#lQ4kB>fY8)uBn^=rU*u#UZ3xS(0m-G1l@ zw8#&aN~`7ae8*c=hbhV>K#kZF;##&UTp&KzEqMvu*7(_hA^CGTga!^lzwZidFXy@Pe z8QF@(H;Xp=!&>^q+cd*cHySJ~U8q(b=}Qgq2S1;y*kw{iBR>482IDMAs%c(p4}P-u zq@4QQndH(lJ2AWEo(its8Oe-$Cu~|;_Re3l23#y%^_JKpDRGzk?p30PhIvJel~)kI zqv?@^byLv|KnE%b4*1iMpBmU+-RIiMVlKdm6NCRh%taMK4dUbZZKlz8vSjv1~3;KYa`=>E|i!8ZhH`i$d`e?GX#+Gw=K5;@$>!#TkjAue&e0khw50^irL0 z*>viOeMgbESM*FIuk}`IJYAtB-{W9INP+E3)2!#eRy4f@XKpp`sLoi9$;y4Q!XE7X zgk{vNWtL&FgB2Y)R4veeR7(xE6B$O8qNxlKH=WyBu@u^b^KDDMcfRhgX^%2zJm1~- z^ZOPz4X2>$&EdNzRkDk0{Nfo5IV?@bspYQf<>1V`To0%-oZGNRRJ*spFHYsI>1{9b zTN3`nxz)3k*&~VTAWrK|s7CnY1s5>(M5c>4Gxwx6tdGsm$s18_LA{Z0yXZfbv-gcy z@b)pckpo10+Ck5Nu?@}IfNZXrxc2E7_MIi`kJSr1#8)Y;^ESLWldvoLnMC66fj{53 z=Yv<`C5V*&k{%XE4}UBq&>n44cqOhV(sF~wJjF~|P z8!oEqDD)g*9i@p4!nR?k9!IDW6+d_JUV={P7$hQG46kEYoL3velJo#{MY(sg_I8jV zUoO!1=>UxlQ{xRa>8IQy_*$A-3zTHbkf#82J-wWprsqnOGsx?;T+x_$K6g&HUTot<35>!xkc^T-|T`T>f6bu!Lrk9&OYV?{i+3cI7+K}NVD zY*lhqct^+7>##B~rd*Qn6WsvE3uyGm!R zRS>iKq*TTjbXV_mzFC1iTMa&afYP$fMkeYGDG~j#6TOOmSoDywkudtJa#9kKy3mbf z;1#Z#^a?Ri`);IOjB+i2sMhpa3B=x-6?KzK`0RB?F`_vFZqI@yDgr}fb~q^}WnMgQ zTgYjSQyC{m&H>oT3nfH;BkC5tV25cDb=ceB2x2#O9Y{oj_M#ez=y0gN)m({EyTI2i zflo@Yod0rkm8MaWx{e0jQuNA_=+Ij`x# zyGRqn{=Ly6P8Pzca*Dmc$!2~jsgJez151r6B7GfdEeM6lJC$<-rR6zO5m-O;nT6n` zi^k~l!JhgiCiw~{_av%jp->wG_W+HrdFE6m5+VeeO8YiF>RXw3~t z6d(Vh#CVbOBRQHzrf^r8!Y^7Rv#m<)iSHRQIY)&KJI>r7@^JcZpo9=1uU(uE`6@!y zmRdKrFL_F;6izGvtnBm*91Hy(TWWO53oHVq8#meFXF2liOX`$o3&ObY95GMHO`ZK( zv)U$&UVd?$Lep6&LW^waz=#J{BU7TmHL`0 zbBIJRY!Jxx0q3lWK85dU*vpp-(rOHQSN_7hG^9P23GTi3pQ9A`gB4%z-wGu>`ubgM zw_6S^q#%b%+Ro?(YfNfXB$y9;z>vA)f#sxZ&mBEU`UlR&borblVTeF9Dchi90wGTC z*veX;vdbh~aw4bLv?3NZiV+8)Z}Og13ULK>J&JQ>vPY_pLWOD(Yzp^GA-v{dC% zFsKddLyuGIjxr;X%KJ zYY%t-w(fo}{BTy%2@q^-dAt`tz;TyuG@hM2=Bvd^mbb%yYOU}=mYHv~;QYc$h6$IG z^;i>I@x(o0)abUr@)E23Cj)DNrO?4$p}Ez@FF}-5Yx%+yEa4ms@`A=QxXE`T8CrWFZ@^MVTkPB7=O!Xa` ztxg!0;KH4@p6zQ?I#XwTVhX=7Z`x7z8HAgJnqSJC-Rhh|XmHK`*|lo>+dXjDzcR(f z#Q{Pvj|2CS*?3jzqj|DGDRM5kr6<)}_TyN;8c9Y9TBsbbYB9_Op$Ezh0Q+OmW9S#4 z*^8pVQ+?#V3uiU7i1*fe2mpuYKJ@(cZ#;2a+Tv00g=ZHeBpz4}i{4Aq%ZL0PSqJ-) zOX9;^b()6EuSe>Z;usQvTEgpW2K`gd5i%9Z>Pg=dP0H z&W#@yFkhHr*u!DC<&5u58`d9v^kz!6>sxYI{J{socT*;C=AekHXEmd+2W}ep9N^Rq zn2)I0l^Ei|0=n);e@?KvQ_V+}BH}EE;5bJj{`u)4ShA~=6?_5Nj7aG|f=+N;hkTjr zjs5dA;aUgY4W-rHlWW!~lKrxJUSgZ~=*S36MN*;LL6nDh98L(O;;)E!WWE)i=Scwi z9q>^JBBx2}JlCM8xubY8&lN?oCc#m=9 z;xnqCCKbkq-L9A!ViVz(U{gXTXb(@!1ia4b&}L4%dIs=%ykb8SJt@7MMxlXQ`i^0V ziK7>o2YX866Mqa z(JuK6lS`T+taF&v%J#kG-qcxuPf9@%q&Pm4SFV!5>{=S_!uvf)66dj*Mm})9)_|{) z%bXm2?yu0a`xSKhzxgV9`qaVl1a#+|Ck|PUv?U=Phf_cU)*)d-U9RFU@XJG8a%6fU zUK^SW(FdSF_0wr6^r4{Hii1iP4~t!Qif3MQ{tZGu*5btQEQWJC-sY%N{$2GoecYO; z$`RaO@FWidX660QP9RiMDdB=Lt+ex=zF*r%c0=BSiNaxuE&c-f+4 z-u?$YJ?~H$64%zFs6<_RUe_E%pP^f=6W(Uk7=N>M{s~;;XLw4>3d~OST{xVXU6$7j z*x-}Ao$f@Rd|Rb8Ak~VV9f#}fg9Og^0-Ye$r_1qBA+>SS!>}0RRPbvdV$ZGSFnF&* zw>AKrsTy}8;S~a@oPV!kCu->j1(_J+*B{;xa)~Zn6+Y(%#3}Ya8wK;CqVhnz%14A% zDd04_sSJUFwwfjMW`Wdcd>(Wq)k;cma${;+e2-b_wbgDeol}Kt+afBem4?gG%OE1@ zf|x6xj)ysp!;q5~&NH$2`Z2veyvi_Y9<)gzz|LGu8zE&eqlaKP?xHzDCy*#NjTJ0bXLCGDzXho<~N>qVXHpz1$Q z_zCh2cik$-OVAH(QuZ;T>IWh==8BYm24j*q7{!#JB~Bb- zct~WYx{Tw?_olaKB9U@^Lfm4`>9bdW#u+NTs%17%O8!s+n3rQVy?nW0lmY+=9Cz+^ zEhd=KfMSXsLf&xykQOu$*!@gMeruL6VY` zoC`rf36eo_hMucD|G96xd%NH3?l;D}J^G$8&M2wcwQKLS*P3&FVXkL4m1L<7Fde{P zFjR8auie66_QF5+uv6@V7x&1PO?VNvm%eSUYGY#WbjQ{hqj<;uzNL-5rI``Cqp_`> znT@poFP5KI^c=gXz5RVVaXvn)e~#d_vAxIlNyniX&a(ghbuBv#h9L(1Pa@~=_!S28 zN>1+DWi{v5)BSs2v8ZocoMk?7-8*6b5sD9ANCQ4L%$^J&U#;nPcyRl+8k*@vEL*UAxV9O-}wb5fwcqc28;5UeiglZzfB*Y<0jfOg{A`{4U1( zmbt+k9r_ohmW_lRy}pXX97eD53K$CXdWsW61~1-6XfdSldg9kgJ#Mb^L;(LWj}?EA0n0bRC)LanT#gQxWy%A7?%^ zw786hY38nG;m_#P!)^P(?P@=}Q{|Id?y;G6I^O;KarfV!%pUGNcHO3IS))voGQs-t zvgP}xaJH#PT<|UMUzg>%Drq}LHS4CdMj=1FJ^m{Yk^N?BDa(BZjv)=2N z#ad>Yl$;N}Tn=*sl8O@Z)n>neqriD z+cnzMT)mIK>OQUw1;ls+F$G7gY%R8=!|BgAiBG-BR83b}cL~f*yz2XFwIouIGj6bl z;r!j}CBHvYb}(7Id13PULG4zw?D7N*T3lVcJKT75=?u*L7}u5QsKMl-e!&if&~QD+ zZ}z=r=|US?+dm#3%8D;&;QjdK+p_;^ma~CKzkP>H#Ix1kY`bf++Lrk>3-^S&Pv|$S zZI!~W43BvDE&FYX5BYH{*adub8IlaC(=mE>s=ZB?E!K1Gr>`IvHtk+%9Ly1?9qU?) z(r#^w)XlkyiZ%!H`kBJPIDvr^n=)9|l~fCT)YZ z9j3Z-7GP|JT{n%QZtxbTw7>N|_T#O0p7hl& znp%GI*>{lP=K7SuZggJ!-IsD)h2L#&7+Y2^zQzTMMSDmUOsS#SVjRHp!F0tD7Msq_ zv@pK&){W6AO)cXv6T(b>9c%UgLC>jQsC`-F`~9n_c4N&=xm6G9j<0?k^zF0$X!UAt z{h&bO^NXsAE^}q8Tmnsks-cA!S9)+U9u-x}7q?$kcs1qJvF-f!t%s}BWn|vn9NguC zuw=QiGX|?mrY|pT2cBYPT6EU8(c~G?`D)r2_loGvfH}{q(HK`AW(yG}@#Ric+4V?^ zyt0+sW!)J5a)`YNbo;zvB6&F!!)eOfjD=yIx_eAl&cQr!DW0p?Enk2o{wv|`fXi5p zk2fsFuIq(-vll-nP+X-J8+l$IV^gj*(N}0QT0qdxwtV|(Uzag&$#{}V?wC;kk4Ev% z#uPVPpG$d&(|}95?L=yX23pbebeg)@&oln9%P@n)e7(4Hi;PAelCx&GZ_X5k@s+P7 z*+MALxcbRBTr2I{@%599z3`mv*cYn~T^w27?Hk>y-Wy0 z>+SsdNa&bxxQ=zP4BP2PkB&R`oFEUQNvqvJ(I_m!z_Zx4Q$=?4`FZP#ZzeH9uA4)} zwVQ|M)FtBG=E_6{WlJ@jzCWltb2V_Td^7i5^^qHDRrM70wK947Wn3K!0*XS{0)MZh zM(AgGTC>8+tcQz^WXn!;_6Q)@H3<>kNWkx}JHo(ujzWCTw4dxeYar5a?tS1pLgD>(bCs zC-4mr1{bBStrg9dO#N!vR#lb^+w;LW7v+<3%h}?w=2am`7n=L1m;~E#+Y1eRVa30q zx%p`GZn*nMt^IiH&Ng3?lV8iUUVJpZzxGPD^Ui!!;1tJF;N}^r!Tj=hmH_VuA4#R?gC_1y6dTDnamACisX zm#ULKQwqU&?67FJfPM%}aJ+4cfF2H%R|cRXX^WWwR+47bR2VXoAU zT0WXG%o?(Xc|!aBy5bM6XR;m}6-%yuw@aC_$kxs+-kt`lTM50fa;yp-Xm z`+4;$&G^w(+7p#MYMs*e4?k4Gh21V_5h4xkZB5EFwzwKdal*gyEEfaEcNQI*?{tZ3 zWGaegYo67(V=X7H#Qd!C4Y7}bbx~Z>NJ7rS|Jh-_L|L%y+IorC_LM=8lnI#>jdksL zm~N{S1nDowKI!Sw|5U|=u$^oTG%j(R5x>PR!Z}x!QN1ekc_i&~ZLj>hv~nGboC1wd zb-SN|0?ipdUW)2WGKWQBt&WaDI>%#OscD>#h3sxj<%P6a$%HsKgba5_G6e9|%2N{b-DW6oKfC<%bBxa6eR>OR;_Jo_od>NZY7*}|5WH_m&$djt3$eP)@=#CWI6f5qe5~M^99(sXub&-aN zne$(w-~V_-`D=dP@UjP_kZ3*O+OGl2OnTT4MuC^?(Mm`VL!n(U^`~=J(f#oRz zUp$|b(^Dv?cu_!bk@&FKd?Bx`?hsbkAP~0-calcV>TLMB>=0GWz4A3nNmxG|dTfKf zZ%1C<5MJ;1*D8;p9SnF2(cu?XGG=3Xu6QhNeM^X-DV?Y~@BC93GH;DD`^rNw9Q(#?&N#lr-4lEBq1W83(wjtarzVA^ z<}O*h`+{6@h&q`-5JX3p1v3TPw)qzEH#iuTvGKJV)hQjN26MP_BCofH;$NyT)&Mh&2KR8d3 zPz6g@N}G1&sL_>}lh^w_H)j@c!F+sWOSe!ySj5#neCB*8fRJyRJhW_W-5x)mL!srh zvpyz4>87Ahipi1d@a-ex(u!m|2mu8^BKjPep(WBcZ)AS z-WO!ASIS01S~DfWCB0Wi_0HJ1>uSG~ogt&ZVc9QkQ#pmdxY`DjDKz~VqwB6=X{Vt4 zt22q<{`~qdi3XX}Q-^M?)0!Gk-YOR#c)Map8^4((o=IorKIA+j7tpQmo+J@ASG6ELQcz4Fd^S5~&ovys2*U~+*wey`hfzN|Pz#15!7 z!ghzYgPuFp&}lY)a-|8}WmS3V-Y$1qNb8|Q$zc4vRhYWQ*Z4M5o8(7AS)%k>cVFn9 z{n#$gJ<3=yY)FE^t8qbj5awt7uuXlz#%oi@6{>Q(>$G%IDzO5VJ#PogLRs$I!bNAr zS!6hpOJ&_XP;okvDW;CKZ<8T*lu1fSqGg$2aM4$e#^>7ec_<&#q&zl8*ts9Iu0e7h zU09&{C?`Uf7A#K|SF=f%SoK$^!b{J*AxY+#RqnXq@$Vm2Him?sK06&JYE?R4yS}vx z(~PEs!oX{LS*$bI|1IDK^OViwF)gsTS7C^fbj+W4wT+_sskz_XZFw>b70#1xo)Arh zH{$A?K77c&Zx>zSsq0l2W4rS0-q;Cp{~y9!^n6gRe76!KT^wg+i(k&Co120kjjYh5 zIVU4BS~%!=!7F}fks%LfZ^tLNx*c`cx@00vcY-N(~zf}0Jj=zh+r>+J8~A3qyPL1r{vOmQ#SNZ?95 zLpt{Ji}8AfIq9vSDFS;*rHrwL-(-cN@V6B0<#o5g_xW0clReo~g-oh$9=JPqRQl=r zxdV7d3i`c<1(=3=h1A@C;l7u(ZI>DVKsB?x_S8yk(1&4;zji2KGLIobpd^)k&P7!{ zvtY30;U03~&D)A)aRsaM0KWoP?1CLwXlaUMoCxFL+v{WTW0kzMN2lA8WvPdUb{QU@ z|2&&dGc4p&6avNnIjj7dW5xCS)b`b59C2fF+qwj)h_*J}FARE{hoIEhV)VlbBvY^ezmv7PT^h0tKRwoZLGdwVlnna%*XN% zF$+&Nf@Oa*UAB7?N8>GBVBu%@HnQ`sr9nf{U{VuPKjDD_o%*{Vhhe^JJ!{*mgCX79 z(t8~hWN2xBxX;(nuW#N~@H^W7gd`!A268EP*GH#08SY0(3vVK;*-d0~A{aQ{WL#nC zJCGP|{pI8EAS3L5P zL)WX4gnXo9oJm)0Qt(vCzTm7QW|jz5c3B6uDdrYiKgcO)&{cUxcUcjq?unJsCOb9U zflz3jS-|geoaTPkZ*jCn*Q7goE%IVp;vIX^=^`KtC6GH2elvcj`Xdwr?6vCQSH znARf{5FDTS=bFq!hB5Bi#sCLD|4NGE9Myr7IVyV@c+^Xqu`bui zjOLkS4e35zm#CxQGm@yJU@>aM&z~ht2e>NeYwk4Y;nZvRn5vfRhd=gP(iI+S=JyS( zjy-pZZ{QkPnZrC(x{gKAg+?Derfr%iL2`xWdW~f!Dx%8u$W9~V-)rBYn5`#=`$b4= zgfhD8(xx9raE7C-b=Ia0meZtkU|t_p=`J2g+2(3{YNy!Vuhc%FqIp%jdGyX$PjEvK ze5m<2o1z$Z#E-(uXvm*|ncw6|klf|y7#gwKHg1#e!TY7*p%k&S)=Vuo3bd%*iwXM5 z8GYokg@gv#JsR&?0|{&QouN&-nX8Sj-CHk{k8X-BHVV{@yU5U)+G_@nF~nyURTK?U z>`SKd_iuBaUTc)5AMnXj@yq=@A6@5mc<{wqHkC|3%QXe_qr9w;Mzb+Zj z;#Jz*=2N@qzsh5>mp4oGg468l!HXEv2V9s-0S|y2lVxl)-L)$E=@SOIU&z&-unhHIjJ3p#nMKg zrP@aBC5r*m=4$-mn~BBBwm%^`@x)ahwGTZ;0VX!|tD8W*1=}8P28c{!EmCz*Kn=>L zLY0Bp+>f(a3P3R=Z-JpTGmW#W(6W`vCnR?W%_Ssa_N1o~SQp+1gTwJ>q`5{(aRJSsf zDb7q?xc{TF{>Y7UZ6n+8rL8c;GHiI^MS{wL=OryBh2AWG;Rq2dwgftcD@1bTcIe>p z&|=$n19wgY^suh6B!*X15N&zQeluY5rOU6V(}$#Xq4rNf086~c12=q2nywSd#5nSd zkOx&pX@rcI{#+5#$F%-(Vhfs#m=_AZtwt{G)SfZ@%sejA^ki0n(e&GlC2fGu3k(K| zK{MS<*Pg1YS>%*VX00#B0+Y0O?U)zvD|SNKYWtPWs3;x_)0t z?~eikM7EeZ#_EHXgJq%7yC7$!4E*bJf&k~ov-i0)$kWIOYAB zH@yW(H|ztm*G58>f3@hirw@IkYY{6+lqn=+91Z+%vfLMEH0ZI~w0b-&cD|f`?jHU! z_NOBLtV(7f%$gAUK4)H)u0p8pM^~~kg}DYIGI|5#N7i(!XN(SZn8qXx@#+s7olpFw zUzZ^q#8o}p|1souvDq0ZT38CY);2?)!vf6e58J3~pZukw^6gcqvXoe%dYgya&!_aw z>V$c5Sh-WoGnWFl>Giru5>A=ZpEWGg1csVH#ojphInN_J1uqGvHvGUP;JWW5y_^Rm z+Bin9W0~)oM~!K%N1A~lX~6!A(MM=+U+Xfk?|e4Y+eq!Ew7Bxets^6kf90tH}Zy6;%jAJegX5iCA= zCQlUe`{SSA=yM(p>S#^8x_Gb*RHXr-@EgDmX(kgKfL45zcL;qHsBFcH0j}w|oe(?v z8^70t1ZhG4ri~O7^HzpPZJ4?Owo={Q+3o-mGbUWu_A4Iozfh$X4P>KlV1{*LC^303 zfnV^(Jg8;tFpkg*@4k%-DtSYTJ`UuY;R=&KRX}p@^#hr>QKy0~jT`Tq#VwcwO>epa zan^yYV+oCewFN8%Z7Sej_MFm%`sC6)c(5Wa{Xm*Ui91e5B5oo$9?3Nr<3&KqzjV!i zR6ZP@!S*U(dR{qiFk8O8T(Z5=*KviC=M}I81%q^$m5Gki1vZ0HK=E3!Um<*Ufb3Swn!|l`4PXsWOWh~_)K!PXgGKY;#tuB_1{0M zI)LSiMzq(~?tt4|>^(hnsQ0_^$X;(qqJk|;7l2PqN1{Ya>t=`Y#ni$9S8nAE8sS0r zMSB381UI0-F&M+w=CmX~(2%V3I}I+i-{4CHaK|SJpfB@Q<_P=Sdo6Kh5SWz_!&9b7 zd|smT?92bSOR#>$Ki)ozLX#7i`mfplnILD}5>@-ZA+0gh&%l33A@ zpVXE)ei9De3ZAW~*j~{o*f^9BSF%;%c&0biH2&{1=waS{k z2?6r$ssD6-+d1kttkbm9v!>OlHpbDky$m3xB^~^}S!b`*gS0V=Ce6Eij|! zJUz?@L$mYB#Wm{`rPfEx;4+jUm%9l4&Qar<_-@yth}6c(@5Oe}fV3Bi0teyluCFrE zB|R_z;=q`?m$-N*QbN;EHGpKe33yhPwjM(d;AX>@3M;mGoVY{AnA72HcX437Iv^f1 zmBH%sh=3;6`8-l4)bp?&P3CV*Uo=;f^})LAz~;GjXXi~$LASQp zVJQf?9Y}>eU1FAFkJ2S9gGcFW0aoi65wF$3JgwUjh;*h{K1zs ze}JZ1+F?}gbnGi&sNz`;cl&$uW$h-B^3_J4>ppc&dVnKD;^m$iAKs7C@3yp%cH9BX ztuEDG-1d-1#Y9?+tY&nZ?>t7vk1{6Pefg+&cjQ6X5n^6>yTVHU(e2Y!*@l`UPf}aMr zfty;~xVGF6Df5+QUWV%%s_#h+)+#d$pSs~*^{95Q*pmzZ+mogD%0har0On=z;hm@B zo*po~D4Co^E4Wxb@37vpWTQN14+}!7vz4ZfRRIgH@KKUEKp&cLj<&s`5=*UDaa*p} zQjKVKo@O|o<1BLl?ou{`E0F&;@XO6%K{Qv6{WKWw$;ahl^b8;FwWTwA;{VTm`_GS^ z3(t~Udg{rPlMuC{x%>Tjb3wsWFJO9$rQwum?YOBDOF#UzX3axU z&)|ZvzJw#(a_+EW(AnCzNJnX(qw80X(axjDPhz@n-~F*0VU?;+6jJ2WERk26lDK%A zWkr5u1Rtacu9;$KDaah_dquI2Ii^j7bf7g|IUZY5;anc%U=fdi#$F@)XTjKQ|C_}b zcLQ@99`q4)m_2(*o`7L_3Z5v9q5m#z|T9_(DX7{I=+EJvDz((fq!~&Ooc* zSptH`;zE;R@8RJMp|m?fcM_cg({lN^YW<&Gxnj>j-PZ4_Mw?zu#!#V&o6>mWRB95m zS0^aP$?q*+7As7uZWEvZ9M09&F~@v!iE-7#Exf4hwow~emnkq(U0ojzxhTA$FD-4N za!OokFxwRszEMwzBhtTup@E0f(eJC2zkGh=I7e+o zK$H$n_aj3H@1wUFL#{s_k}r@kV1Mc0jG}%8PHxod=VhD^sIyeEuiGw*)+xkl%^k)u z)=n|s8K~G<&XYJH^9I*;${9j#)9@+bW%1xltX{=>#C9ujA<9_l~K%W81D<2sG@v<$;5yOoH-0V-y*3JY5%P zk+3Vca2IQ$pW`D?-hJV%804bGbpTm8t}cV+9*br(cj=#IXnb7hwK&}aXR@0sT~u;~ zWU6Y>{(Xvp)}*?~p9-N=v1^_XPCB^rN?&S+BIPN**7D16R#rYDBjKmSrk}~3fRvT~ zB5TaZv2Wb^LSoG|CY7hh)IF%A3k<5BXGIF0G)d&)+@h%Rp=y}A$jjNspoiVpk4l_9 zsTtSZ0g=jRb9PU?kY$xakj`ArpMW)K&!G!VpN4D^G{DIzd)9;w2+bhbXdV%LU+3KI zjpZ9WO|DZp5qtWefIkt!_N_ zl^NsyUuh6Ww0}chx^TEjxbGa%)r68{m={fDDTFDpK!5-Ds!C$1C0k(E3B8vP#D$0T zc_9biE}scriV2{tTLk)9ZNHoda-?E4NOD^Mo*_;NQWgI50q&9cVHB#j+{?j@pj*c9 zaB45T?(@^aW@k?p1-4u+9cw!-+Pw*Ne-nt9&&g_xMyEuL7PVtNKL{H$MIi=_=GL4 z9`|E(@4JOzcu!h)5TDF#l1zs=Uqa>$M^uv02rRT0Mbh+0EE;jSeh?9!fbMrXiC=7bXilxhCSwHP3d zrplOFvnEIn!m(D@>)3(;GsL`v{^Rf&%Lk=_o1aP zRlCmJ&9~z}zn)}qe~YcqZ2T#`{&jLT_xswmUr%))%Mu}5r_kCAFoE15$)#3N_t69g zRp9I3IvaO3=i-|s78_Z{a{wAo0Ewdj$k)adYCOAy!cY5{1K(~{>~7E5Luvg>z7!TE zKe-RWJV>tXK=|Qt)$KIwWAa1ICk&tXA&(Fc*-sI&e~uP39*ER5#F0|5i1J(E+hmAV z$N^r`G8HjcB8%dUP)2s~FG5OEf<>me6EIc;5G3%i=0R-{5_Y&oJQhOLo!sX-4qfWm z5wWh5nZ?o85OG^R83Jg|CRjnb6SIr~T1}=X9u9O6WVFq@H_gxy?-4SUL9rk;F;UaQ%B4j{*-2Ud|xA*sQ zwV!-Ypf`joF~uRAK=Zlc@SSg{zC{L8VlkDg14vWm!0u-l$k1Skg)i<>T8qH)WG(or6n*yZD;Ex z?tXc04@TE9IhiE+2f$mn=pmPx34yZr2yt1p)-HkFl?Mu94!s@ZA?QG)eRblzL7#)f4z~BVui&Gnh-K4jRA=@rAxXlF_^7W z{Ky5UJXpXREACLA4je@vLw|1ZGBVh%QCTh!Zk=GQf*Al+ zuB4_I>8|YG6T@h_6!4u;3zTh*0%MH!uu_{k!K-Qy6DF0>Br>sN8f8UQ!_}`gP4;izFt4l-2Dh%P<6mT<(8+ZcdO9eEY{8}d`~2}7rEe% z8^yhR&4Z8yTs%%*LuOm1*aPuNzS?I6KK{w5NE7l=fl_$O8i$d?ny zuQoOYp#c$@if`+zt-vuFp)5_(YTKK#|HUcBJ^!&8U4W7wCMKSycIWX2IpU)EU*o?BjU+cB0*j|_!awl`3!2rz^ zIsr8kTTb0YQW?kxP`Nqc-qO4JI1 zhS-+?^I{b-ND_vW zj!gquXwrk4RYFv9;R{ET)GX+I(T^)Ea*H{6;F(W=lqcU`;=r6(G7laiWu!5nz|;}H zvy?vRfEMHicxA&K`yB|n9t>UnHN)26;4lN)-VEtK>R#YI{X!@M7QzgP0)wEGGnCf1 z+Ge1F$y=B+YuCK0;yz-8_;U;@JSSib7)L#>8KUZJbP*9Kfc|L}1y#f$K-irjWNYT> zs%RDg#Owf#`W5%o$=BHk*PjTU{i3)Ilicn;!_Ps05HUz@gkVR(5F+*~(tQWsVm`4& ziE_L++^L$Wr)n8|gS4PHTATpPrdsyKxjDx2og6j4?5fb??n{41WmM>y@XhsqTs z-m?4r>ljxl?APmolnit)TYHK5#G6KqE9}_1za>1C32p zM5x#)__i4k<(bGsgxHXb^~`WS13pr~X}b49&xp!^Y-6d;cTH7)^HQU4hXodVA{p)~ zNQ-=@*gi=S$mj}`buSc6%t3Dz0)Nt{#X9!d1EvjJ@t~i&UAm77PNST)Yg!QacQuEY zN)tZ5+AkJL3JgbZYs)izA_&WK=v_oPcE9Xx<1qECh&*I8UV?)S;vN9G$KZioF+flW zrFw2s3uJpJe#d9Rh_qEY?EpGpHOvWY-epRCT@p zhd{1MC~YLbQhui^xB4d(weqK5-~W8ZfYvngZ!Xd#LeEhS2>{wqLD`Pekq{vFM~T~y zKSzX}gy+DxC_?>~l~BXpkG3gaw@o!5^Mj()lDa*Kc{WI7AX406Qmsr;ZB}}(L0}90 zWdr;D`T+MA-(j`BAm%77+B}Q%AAQ-`-b@EG{$B?40%Uc=@er)m z#hRh?HOnpjRsZnLlb1-Gm~CgiJPzVhtPL%prsok7o5bM}6CrH&evX8|w}K|(h*yxM z5Kk=lJJBnsZAy22hns%=IG3Uz$zqmPSps{dU?4*Qnsp|iHf&feKzaf)i2AyzJS9rT z{5B9`m?s>eg!=O7Cony)tBNTO+5k0Q%GyRJ!pnC222X(+^%zPS)DSZ5t7&A_@-Cph zAyCm`KR{>~gcV#G2W+2Hu0$YjbA+y~-0T%yZq`i%^l1aP)`mmYi>s$A>yA6=mcWCT z-E?>uo~DXQ1c~W?vIDI(3nZ0X*YaDLozP6hB{Xx>YEw;_cn0kP>f_=mD>J(`tB$P+ zmzVXACj>Ds?e->`MkINd?)3=Eob>4&7HzIf^fYxXZ6G0d$8Xj{8vTGPQmOor@+B^YeflAW|?E@t2 zaPk~|wO;$FZXTk~c}z#s5X3cyDUBQj>pwsK>NDzEg6Z}Ofeq+!N#N$^zl_9MUr&{F z!pqMU>?jK)2pXkd*h#uD08Db@frAQ%vEMH#thL?pJx+`}H^uX52lA8k^t}y_8N#pr zj|a};J;@KCKVIK^>R&;tB$_yojhG?Kuyo8pOgt2!_P|_?{U%D}^>3Q?dZXH;7ZQIv zXrYm)LO><}_%Bcs^mB>nw^i-HibR4cvYW!ISJLU`fIc)VHCHA10rW{Y^1tALR!V`A zDEnW(r5CdwLw=Cq`3{&$)2;hUQX$g6|NBGQYjvG6EMS4j)XdXI1&AVq1BWtAa&%r! zKTu|+epl|9NDAPUN2x^{45?nL+t9*R_H_zc5VCcRd%#gMxGf6Grj*`ko3T?*#PgNO z_+8Os?JZ=hh_$~lxVH$>DY3&3=DY{%F7bAIU$$#vfCMPdbebEO08FObp(xTXTHI1p zp`uQ%1eTrQk$iATxD8TgIIkxtP1BMD#?9+(gD_mlu$YId3!XrhNv{G#p|~Tomj(>V znILl`kREl=}BCQ1e^t2w- z^BlS#fH6%$g={7y(_{y>{*O*J3njpNae+UH2P)cCr#!H2RVWj__POj&{1af$+_D(- z+;VKUvLpEcB~Q$*5B2eADWLuk&z24u>;%#!iHQQimom+K>5bVh98+ZnT1^xI?{>Oxrp5jzF+TX>{@@^2vK_wEGPE~J2y zvZ?FOrbNT+z==^f>jBq;yfZC)U!WA$n_ETm2U8ohVrBem5Y0I%Rte{nDb#C-2Pl~i zF4Po|%*AxRSgLJ#86OZkRHrzAa z1g1KUEicI6sQ_2rv>78`%o5yiI@%ooxEN#7oD=BqS1n|e8018mTnAW~O0s9)NJSdz z8_fad#{{VRuP<33LLD)K$P^*X0ApH1VweMVm{Qh5#K&r-6cD?Yf%r)(op&PaJAf(&WuWkHPJ&qy z5uC{lHBU|e?K@9qa1kAN<}oNMKyVvt14`&XhN;tg&6LQz(Px*q4`2Y%{6^hdX^4zD z4%oyDJx@^Wj3qVDP8S&Y9A1(i&`S{yRcJVJNFy@J^$mI?82o;RD6$doq${9S*}scF z223;1+GfD^E<2l|qfAvNk6@6^L=tLdL}DyEI}_g*fcFtnD?V^PR8d$C979%;8Ylo@ zdY_aikiG*gKMmHEKtJe{2o=dgTyu^FYD;x zpS?-;v_loyJ@St95;~u$G0Bh72m}2PUvy92!s&2NU>Vc-x4%QZ(Qz1Bv=apUC;x_5 z&l~kFg{-_I`nJG$CrWwW&Y$rScbNa5W9rXS{zisFG^hxh5>0}@R*3L`TZ`6x8=WWN zoc74MN}?MLmRSdKpa5GPw*P}Z`t+tSkK>W&2%Vvx&Iz!3WUXeQ4=!bQ8SSYIK7Jt$ z85I#h^>=7BEzq{aFhI}nD^DoxT=R%w7_ZGEjB$qswhq)}48bSdHIKNW@Y?A401;un zPy%ps>^G`PIdr!e24Qon_)?xg^VMmoSY#CQBbH4W|kCUiLo zy6~i|Awwzd^9D{`9*N@Gu%J8MHSk{eM4&>))Th9hLJ6tz&zROIk^Y$_?FTA-5YX=g zJE*%<$u1(>m-SDfZ=HEsqJe?!FH+5bnrccMymf;oY`4S-HlF|lj;Qi3yDoHG!qLfN zm{~B=nt@8D1Xixgam!A?)-`)Q*n!E}LZ~B}MO*7hE0I{Bxr@ZL2{`@+f@m9{X)O+- zHLnCKmSvv|-4CeH)u2}sPVIdw;|2!tR;g&bK_%m#0jCd2HCNfC%D( z&{n~$Aq6H(L;EaBy^6f=znlyhF(y9O5^Hp|H#r>usDM!fd2j1SjiPQJLvNc(ll(y9 zTQ)}aRQN=9P9K@pY4rY?WSF_xZ{EqrO{^O1ZfJ%!yat}#B?{WetsdnvqeEWF)wxPQ zOB`!-E`*n*5U(u`;;H~BBL`hPZsoJ!)|LY$#F%x{=A|5`oG***bx93k9+yvqCxdh- zYp}85p(iN~>{VFA*mLL_WU4J#Rw6^R{=Y8K@HX(q9WV~hiQo)Yx5P5ofeoY(^%|zB zF9xEdQnHW=F}Sso?x{RXGDsuK@&`J2zf}11&|isS(53(#Ks806*v}d9Ozx6V6jBoZ zxg|Z)O0WxG^mT)1s~6f(?En$i617;J{%>ZcJJMqjF^U*w!Q#d@@0+7hXUq~X!^*7= zDCm2i&xDi!lMC4yAtl^Si#+}(N}eM50fcfmz_;bAb>~5njANf7WJ84zjl; z>+*(2fqQ`Si#TyPFm|27=z_A?fy@Y}_n6fq+X3PSOV6Pv(dtiH35kmjxh|25aeWI~ zltX=hblM$q*F8q4IghyUe}*0>$+~pNYwcE0-qOszm6?L38$>kGCOEM>{ygmGI5U4?eIpT8iZdWlU0^|Ru8~eP$NwRGL zt@GdI9y}PE^Bc>9sY?HkD6|Zmpc1#z&q*VU-SZT4{}VKgI_!P|8js9=UTk z5k1cYMVK5#0Px3VE1DE{i0K92j^XDdLHl2bgIq8<{~KLws|h;Y={V<;i2a{{{Z6h! zNHx$#s{u_qXq}e``3_sFaIT3H3lhfvCT`l&|M=IUYU1U?5aWu*QH)FiJqbHH`OVLm z-~QaW8Ff{}ug0iG$X!D3d+)#c2a})Dh~TZ%)N|-!;ITPEOhKkk-(y1L8;`Iw{Ox-n zdyhH9CMAJ{y9ro_a(l?B6@XjxL`dPE`6k^=x&@Mb3ecS59#DKC(K5<#iWs?!J5HV^ zfw-en0y?~{&=24dU_PJ`@z%(e-sgP>nR<~5gZzzCWnAb2Tg?l4F#PuLbRY<%4`dU^ zFGnMQ3WGevad0t$U23Xb|0|u?fN1pohKUObq~{YQch-g{TD*4bq}UP% zkwz#EopGo+%pS7nuL@}77t?e8`4rh&mi1v*DJb~w^)EL?&1{YXGS=6v@Z)Qwuz ziw>+@>j3IY1&mPA3lp$xROX1^_kJLciM+J|aeXlW3w3#LpC*ymntTHdCg9a`C_xsN z0GeSk;u?y=KhyG#SpelP6>Ua^Hiw7u+zczecn|`GaP27p@&goVj6+2$2UHNAWSOpaAvJDjT-|)Nr@!c( z86a82!0y1F2kF3>j{LR(J5k;$7jaCTJJLuiuO5;VM1B~ttpMu^w330CM%P=LEli7H zg@O*P#^!#{r(Wn?Je`%%k>{w-eH&=lMxq3757R@anoth!vBkHKzI&aTSCqe4Dpfu=iN3A z-N^rE_q_YBi~ob!{y(=_;s3%1_oj)=8)YML=w@jL4{O9R-TR+9^Ko^Zuvdfq=l!QU z-H4T+Vd+F87k~yL#!0w#zas(_BT#rm_tW^ z;aBwYE^KZXHH{#*x3IziA^`n&Rb)XF7E_lBUM}cEgx%Z15O)9_X)*P8EAbs&KzB89 z>A1Du1ilXa@i+G+;N1Sr!#P1|UxXzaq7Q~Wat{z)Vz9p?7pCsY|BJ9XyiG(x-X(vm z*8kp-V`XyWIiROgGe8w~?E+>Tc6UXP5E5Uu>pB!u(P%fpk~bW1+n7KLD^3imemelZ z42}nu7rft~2D1xk9wYJNi}0~7pnTQTTbcjr0yI8&34QiYM*pv1^KbGK=bw7{KmX>U z@BsfSOZ=zb6}DFw10FV!G{jY)zBklsK$KR}U)#!~F3vJdV$_3nT{@y}YlhfIC|vA` zdJEb*l&BM+Zb@X}gbn(@+fAf9hSfmuuz{Um?7Ow@rGiO9f@mfNdP!Mi(A^Hzt{smk zc_R_EY(Mfsw%bkU@5)$!c{CHA;246f>$?XtC;#=aOr63c8We1%3@@W2|}=G-o5~6`1!dtFEl1F5DPI zN{QA?GY>(k9fl$l_%eQdwq!*zZ(77FF2o!=VHkyvH_w9Oa1+pyRi6&K-6j*BA2XXg z%bByv#UGC6!|&^j?7qQ#H9%y2X<2_y?aP&?;heT9Ah4L0O35H~0D76-#bI*_&Im^E zLQA3cU|AP0uuZZ0T96NGs^m}FQJNZfcUKETh5hhBx~{RgVkEQ*6LCh{$oU$Fns+D; z-`oeizOuz` z_to}Ff%VP|_#qfYRAz&GmU}Bxt#(H556F~`$@&vuKgj&eHT>TRxcrwvCK5$xK0Z&4 zOWy;-&_d=7KSD;6I0%M?SfT?B=pDB%5<^mAdZn!fgG)5pdVm{)HuNGIC<-jby2xd8 z@XSySIEX`j&EDOUjIKlGaR0ph*FCkq6uW4T1fl^N(W~IHs->g~e92=w^YF9s7<^VrYPv#7R)s1|;JzH!f?D#9-n{wX(!{Uo>=?_Q3+{?i6;L>@=66Op1S}eEye$N3ni$iLR zEBJI}il`IBYId4YOi`YLhGF*&Bqaei#{(pvQ6{*Mhz?FbTB=g57~%z-2Lj7n8{FK) z9Y~y7m0`DNJZzDp8RAJ2mUwpnn6U*F0p(=oeLl`jEu6#kw^EQ z;3KUNWoIO(9l~c0>_;psu@eJ!0BA=BH)WFOso0x-A5uX*0B(7~Y?`)aN5!(EdgTU7 zYbk*3Tgc=J4{Wq%FO1B&8lMH|rijmlW;Q%*+M0qK;gYZ#))$bPcpNP9Vwh6k;%r9; zfxeGJDn+p5O3$QMAdv7|;<5pnj z=U=>KsQI=RX4c|9d#r%IyCM!J?%0fr8sE`bL##ye2F#^cqB99tps`VhRFVVOJ`?>x1h6LY ztOMbl-2gl9cx5Y{Q&Tu_gq+>K&d(|#L)a6xPg!)N!8;tEDkE*xxV9R8=FS`tqs#9o z=8dk3W^Eu@lGwRoyE0BK0M`2v*wWY}s`P3(XcQ?(OT+S9X|z>;U8|CVoco$oG9ocB znE6UAMP1+F|;Mtn>r5pVicEEQUqEhc7;S3ijz_Z8Oz_*qbc6 zXIPUnmI5y0A>{o+eAC!|Kj>zsu{rojP9cP8VLFtu zc1oeFS(`bem^76M*^0!>Fm0r?h>{Sd82eIYMj@rLWXUo}*+RC6@Ad5c`6IqRx!um` z+&X4nujli+9@pb>zZCj0LnzI(-rFia7RxJ{4fXR6`+ZzNClSxFA0KX&=k>7H9}&C& zZ@uqV@d(for?zp{_rmC^Eo`GBUMIJ}Ol*aqUePYj$nXL#n(1kDLHCiV8N*BoMQgU2 zU_kJ0J26gTNLV#d_nycQ!X~))$;mfxY=UtSUJ9ak#KMG$)}5TQ>GgOqizQGQkt5i= z=uf=joZ60DM|Wjm>Hgu6VDLCtmxyUG<~`*|Hs?D@0d8s?@$IsA;7FN197K|7LQ-07 zb^CHWs~@F)D>qV4SJ^u*!K3l5PiPJEsIssXQSBPxg=(k%hu=Hztv2aM_9Lo&rk$?j z%v#h=geep(!Ap0ilRo?rfslskt3+0)i(AB`*<*Wy0t!<21% zNayN~ZG0+pA}5hduah(GBHs#wSNaf=+yz~T=9f;qwNccXAH9c5QL4z_OPwaic%GmE~y&E@Gxdtdj7%xg~2o+KXEly{%iAL zzQGi17VBMn^fg?B)2Lk?>p<;@Iqsyjf`wK`Vmc}AKx)YZobbBJF#xzjw(1K4iD~*B z$!NyW!xEQ?K}h>O`z5*;qt0si^6Rq;_{bkrxV^LotKcVf6Lqp_Es&S$%{N-6GQp<- z@AOKhAMMobt^8H#*ZKkG^yaChytA(S?kDME)LG3kNpm+1Ow43KZ3IAO23@lV0ja}h zFCdlr{y5kG^wzt}TNfb`&yVJD@o2BB$kF}K@ZMAe6y_>Z&r8J zcRq3)yVZ6&*RVCuVkXexBYV?c95IkCwbA%XbCpH81n-KK-31i>jTW<)QIIjRbdwyj zcfdx%WTjNzCIwM~MT))~%b zCWi};N?^GzksQk5xaHYd*dh96_Wk9$5 zxy@v{;>N{;MINCUi-CoFOs}Jbt1M@{%{e|Cd=yTVFg959a#bDcFrClEn6jnNl7BB& zfZ6;Tyf}ZEE2aJ(3YMqlqoqy#m;y@jAK?onKj>9Yq+Z27)Zz9~m8<42(d-KBmVzy> z*b)h?IX@9p_){$L7n|WW`pf#?%8cO;zbW-J!y&K=ZG2RpC-O7h--9lK8byV*fnaSysUntdW?u*cqqq$W2ZNLt`ehTc(PMJLThwZ31$vasjBjRI z{OtC;Qpf_FO_e4ta|F$rS<-WgV4RO(-apc%(UIck-0y867$qUgzxC+Yz{YhtOQ$$h z&0u=Jp*b4&;NRx$s$yc`THlIHwhf&dn~RKQ-B5Vb{f={7_>*9I`|K9(&eI2&;V+>S z8u|o-@UX{Z8(tOFEFP~cY)qRK?NEdFJahP?{N@1=ir{im6Qtv|P!HVg*K%T(;xejW4|z5FDKuV@Q%H~W2ygYp@?`?a`*FM>={$- z=nGI4b&cuyGHI&(>AUfLF;Zu*ITxpQAC*x zqHq-+Dp)$?Z(!oS(MB0G;*3QCFi8D)nnI|zzp#d@&)KD^{X78^w4J+ zA;x<}I2l1&qF2G7m2ek2v+TK|zF7GoI|DJp&$6bUXZ@%L`t3>1mh#e;a%6Ayj(hEy z+IsoBw~BdNEw&ZhmMPpKleP_pG)vl|@`kz4`7j(3L9eS{t$EK56@62KKx`%Ypmylj z=$Dsg!9(JR=27+VExVnBe6SBREBRq?Z?rg1LZi`i8`b4#d4XC_8+^WQ-+?bu6$T!P zXSz_LFZ_nh7Mn~dlk;%2+j zcSO2xJwivV^nlS$8@O4@>+_`z47mdrmZ6S%->HN&fdi{#qT^V68=?ASa@qrf=09-=|^-JX7c(Ad)rzm7I| z*2qC$pcc;U&;ZMa(!vYyBht@AGJ7Q3^nv7{?v5f--x|=jb!t&^(WGeXD`++j3-F<5 zuP6gSd%=*o^{B2gwNn%bdASd^-%7w-Gn{}NWs|8Gb6*9#>W+~g`U9crd3Fyz>Lf@P zNQ$k6X+%wF=+&*7)L`e9n^!&<6a8!X0#DwL1bQ%TE74%Pl(54<1cyVt0er*Mm{Em7Ma+aV}<-x!s1XG($DYEoYbGSHc#+Nn-r*(q;G_@?Jm-H8lGaZkEo&qR@L6Z)d(*MpN9! zK?wp12t(|H=6*|%VXrq{Kw8k(GpB>SYUbs7N;4_Pkl%6Xj|mtXrY_abFB?hUM(M3x zjgdfQ*;x4ypSFSTc-V?RhqYfJF0Qx0uNrYoyO!B3`$sO-aE)_X`7j&(z66 zSLW^@=n0&WA4@^WB2~l&Klw+$>fx342tRD@&OyZASlmWdW+t3LH+T451nh-uX3jtZ zFU_Y}3AR%h6dxil06pQ;aJW^J+;Oh}VM}k9_P;^Tbnkxf9xC-2l2?$4QLiR0on}^O zL{5dFb9k<&vY(D%RH5r+t@JC;B^S1RMmr`^{geHIoT-+z0@-3`4eiw3_IXl?s;5IE z`y|;xM>{kvv8igK!7=^SqR0E~4%*=;}(8fLIpGZJj_( z3HZeROoaUbr}f9-P^+NBxu2jXy=OE`Tg=*B|25yMdh+KQz6^f{;{kHPB$$9|MTECP z8g~J(Vih;?^*gSka>}b1ZkNRK%7~YhCvq3iT+?XD#D&wwILmJ#l52QG)r?$_1_N7l56=K1 zd)dCZ_SvIhOL50Ai2+L$mpK0T;e9hGjM%&=hUBQK9+0?Lp{AWy2fzRcr(^3hgkrY(eov^LH{}+@o$OzW z`w(RFTY}fhR$0W2TXxyTd?WAS3ew5v2?{k>EmJNlj{KXTK*dh0ajTKnOx`2kB> z=_~UlV&LJ=`^V5b_d@Q`=U8=J?ivA&TtY=vEb&~cwuT^L`c30Q(Pkv#ET{1{&dG^h zDaV`QoQ#_+HfwB@CoJ7dNhtQqAiQRz*a49cyLb-Hp!Whpb(qCIltX)36n&;{-ks@Q zco6z6-v?xqjs!x*J(oIY6n6J$jBjt%rvqe*lM+?lAR(>GYe3glBZ^FJgTKpC2oiO6XOSmy)SLa>CeS?8O4V zIawF)OFyPb`1oVz#&muzvhlB)eG!B-M2(KzVkkwuP_B4$mc7T;rbCMO2l-J$jMLf! zskjwnrm3hrgEqOG?_s`^m{a79F(TJP00`DM^sprHJN2ML7a-mD(GRAu; ze&4VWyu20aoo8k{V;^}*EHe&_hSfDvaQ+u;`X#oH9tK~nde@1#gzp4_;J-}w_T&0` zjM>dXbqv-fN*66*D5Y|{#-I2oeKWRS^hTjD34SX#shbz1`MjAgD?z9q+N(PLG}tDs ziaK#(G^Q@blvyd>=NYy_+c$awt`Zb+cnq3=Zs2=%BANMIJo#(Ud&Rct9&5yNa&aeFMTHfqX0Zs3`Bc@xLMfblV4*&L+5#J=xH<7qilJ5&bG2ycyG z6KaioAJdeR`s-e{Z%J$bp~p2*xl9eCOZh--?Jn7qSYa*V(l{V*nvKXadydI&|IU%f zZZAkwX18Bo0W0KD=ys<9UXvf2M2OOq;mOBhR|rHS-=V|_Ux(MyGFz{7e9*sK=x0iJ zYGRnC{vS=TyTf}c6w}|y-N=(^USg);pmmwu&m;;MF0a51WtANp&(bwaCt6qHwd@Ox zVZuQCD#B2o1?}wYPK<6IBVoKfg>fnkZy<4s!Ln9yZHyJRDY-voSGZ+ejyJ)#9#Sbf z$mAa)uW`8=?x3A$3)q6O66pg>f|82K(41%ot1Z#8Q|4*W{P5(T-18G_JYb2zcJ7+I z&mcV#S)Eb0ukaB0jH>qKQyj6dfR)cIOY)u$SXN@vQ9L{;F0@|Br2d;nW8oJEEFWFI zNSH6x*v0kNj!;n0J^}jYju7~Ui=F$Y{LItW1cJrrZBiJypzoTw*xTEvL0yQbMrp-n z`75&I0(c(&f-{wM;{<5-mM)F6@+6ype0R72TUZ&#-NZ@u|6_$3>Wkfz?X;LoVeUTa zR>QZnXJsW>xKNGxQYq=&q=2K2;RAICn%y|6hpqDo8Agf6>aI`Ye}_Y~+~GFk{>c2R zHIucU>;(BIYh_HYO=d`+GHUqdDBXBVvBJKOR? zRUmDds`w>cF-~ALbE|!4hDz0lWzkPK$nND_w5W1SX!IT<{`9olbn!G={|cCos!C6nNEdtUPv9+Lle?saOfd zJz|aNWXEP4DezM-+F+oRK%npSyH2oM!eR}DI`r8!ZNQRf*nUD*V5p7yWvt&2kyr=e z0>}}&p}5_LJBaJ_ArcHaQ&t6{zT;?<9CcqlQKJd$8Q=I=C}98NPb>8mg;+pJ^NHun z<}I$E8OCG2d?L3#MOWST$L`^@Xl>+ldk}L1`SgqLKsTs^QYAAKTvUX61YAWs|M_lP zyBG<-=Cn;eZZD0yQX+X3d6s_Jo1Jiv;DSFKwwnzW{tYIe+^J9*u-ub ze%xPST$=XRtS$j(>eQ@7STXU%L-C%86a!s$KuWWxMWRsvg zH|+5HAX3}|k%#P?LY#rioVm*RraD#~0&d=pUle7oNG|0xI6`$o|Gp_tM6kWK!3p9OFpjr)y_YD(+(EG|F17He7V*vDkbV$MTHJ zID(tCaOjgsiYEW34Fc*P%bm6kYYiRf({jZ$>fMeR5h(di?Aa#UO-`3TL$1COyrku& zpI9TX)w)?xql{l}%+g_q?XnZWr}942w9AK`=H{wbb~}HzY`cS}DviBD6wWcBqhS*n zW>05_Ddv_kWQBdG`8ylrMVG^REK`|g8U6UO-`7KsK@yUy3bQo)R%i;0C9HJ8T%Q@O1X4N&5k&%#wK?y-L$7NZG zCP_1n8cW-Ys!LX8q>sgawa10oJN)Eavm(0&T-?zr#5NFWp4KiCKm%I!Uzt===5SEU znOvgx>*=u!%lLH-T3Z;(v07e%T6&CxZz8`5zVEc`^&>u0m0zWVY?tQw%b5D%3`plM z#xu{UWzkk+%mV(i4ha=aQ8xa=`6H7)awgz>9g&;1c%?r=y{i-6m~(CYT%k%QuGm)R z(SSPc$pE{$`FotXon;(0_grJ)29GEjMYV(BCzXDmYPh+q%Az>g?tDBgVTwR`cZ~VU zU0#&SN5@WYFB?Qa=TiB@G2zTf1&^CVVii7NCcBapu)l8fmC)NC(HsP*@|~(Zo~LcZ8NPf{1B5!vv3A^%Ex8(0T4qSw;D_r{gXFMDUEB zAcz}$n@J=U_yrPn6La^HL~XE>;fbtArP}=|&y2)oIRkqnmdD^=-<3j}#Ao#DYpBuB z=u^cB#>ZjCnD_O<5{l+5r0ug2aT=Yoke4B1xR+x5l&P$z?M~0Sed9gtp`zahTmr3% zxF4_aSrWyxPa^6WKYl{;to{;VbS*+gwWt)>{p#ZXVT*YqC}Z3}j@d=YhN`_R9l7M+ zv;=cvPl)CzciW}Y4lbohNBgu1T_rnY!z8;!yGndcPQI1dtkAOK(fskPhV?vaUADs% z$Gekh?;c_wml9z)`a-iaALE5G+iikW7aBmj}tU2@a^LAow%sT7y(>wq^{JQ+=!&G zY`V_jF&@9v>8L+k?Ikr6nkQeghY=+zV=c+1N@C1vY;FhZf?5D=O;5+E6mek;bykD@ z*xOP+Rs_SVAqFtme3$Hu8!m_{iYGPLha7bn`kd#V>)a@<LEE?R7$*3QmvPk-JLarrsLCw=Fox9TAqKuRl23N zxgjm9#NS22-C>Qji(G$1e;Fb533f{BaydNXvzu8Ugd4xM zZnn=O?9t%w6=Ch@NtHw;&U(iflEe^4x6(dS$+|$TfcE7vLEXsk7H@y(sEBp1Yy%fB zN&cmCpgrT_0UXgI-~%_7a;vKDaOh;TjQqQ1NBy$zT<-ZLW%;6_Zb#~xDy>I>B{&87 z@yXha-56r`DNE<_d4YOOZ7=GqY6smfx*ua6C$Q!bg=lD)n=wLS_9@BNSzhr}xWh0U z06nA8!v{{fS^dke{Za{ak&uFJl-c`;HF8LSuEqVsuR)(5mo?@T^~|(4#f>Fp-G~?0 z7c=0lG|%Xo?)hLRugi5gCNae!a^X%r>aNAQ3R4D=Rd@YusfIca^6v*>S5E0H zWPGbZCeHOjTtn+$Dsk0*>(?;vi<{HfF4efEV8PQc7IKrQlqHyIa;NEfB^&aWp%8|;a z9?;8Ww1HTEEk6}>2yIX=DODyB3jR+s^LK(D&E$DFC>&kDM(`d4=tBM|sMHIS>X-^B z7_EX8q7ZMOSRP?ZBXHoVcwP_h%sqYD$i`4`BIq1FwA1l4s;uv8c-6r8B4eWpztq>^ zrPjZMwDM($8aW`qQpL*(v|1Q>BI;3!K0g(HFCE0W`8u>3tWD1!$Qpl_Xp~Vi(52+H zL2=!c*ad;)(%pW#o3cFKLvN-t5d)C3rX)zjIO*%NUXFtT6NX7QJQJYR+CUNeypm>i z99Ej{`j19A4!vLQ5|og*A4$wM zE`hkitnf+lF>Pg4sxo8#RosEwS!uKkw<7s$q=tHGMELdB<>SqqmN1;AbM8F=>$TD$ zQngd@|H#a3ND7^dBsqGI@i>e=i|>V<);2XmNsA1q&iD~i-*@W&E&o+L!zOs; zRnVU}p>c=8XPKTJl6@@{@ zQ5&2kGLgI$mXG>PGko-{XT~ou{i(gQo(0*0h$^{LMV`&GpV2NJm?r=rLnJBGUQlAu zb=^fnne=zmPPG0+-QClKixxHJGqMNl9M&lVBXnU3gbwu;Wns;YZu)8Nf>E$5%G(%S zJ>2OQSpUG)lh6s+CDbWLCi? z*-^N^p8@Muy_AqQ8}*;V=oP{VBQY`Ao=MQ+i(pIq#m%wnwuH*=ki}SNVbLI<#c4@4 z>5_zQ>*<~tzY|I@kgC^64Y;x1iOX^%84ZXi!g8^8D}g#f_r{SBKDrw$AJd+vO`HgO zpD~=~XTA(zK_}!Jx_-HK_`7tQ7w8`9-S0tAa0ai;xoq=QVK5A789}s(um$AheH|*@ zvqIP(1zoA{;~6nnW_+}2&*j%04ZNuROEM!Oh3hoj9RUGGO+@EBHueEU z`GtGDfctT_lL9izQ)CHSzwBk~azQj#@ns+1R@(qucH=EvO}MCwGPKI2MjB$YiJ-^H})Z$qE*pK=S?;Kp}AJC#WhU8q2D!nZU>x%mVyVHxCviT{Ugs$Njw@x3RAe06hB%nm;)(ye*Tw?xE`}i#?dE|Qs*t?%I5X-VMv?=G_c*a^-{cb*2NtqCfT9>!k_cU ztk4`k;Y^7AW*3`Adr{Mm1vy=C#NWxpJj{_?SaGyLk>@CS%Rz4Z~*dQM{SboP;&Ie^0-NNlnQMm{ojZ zX=;aWWZBomNOj8EUqJ*RQYMVI$a_L1*_78{gl_R`Iq=J&1iU{VVQ7-1GxZm+my~dIx@Fp#a1mKKucV)P9$KoWaQl!R*Cp2Xig)6Q z91ZP*;9?if1cveUo=69NOMU9Rd{X}|0ja~V5$hvBmG0?8qn)XRNDx^m2KRB7p5+xU zhb{zRM_YgnK^gH~r1Eb1U89aPG!>f5L`=x38D@aPGJ?L2!Iy9nCNPI-TUjor*QWm{ zb{ch-A?^S5n(9NW)}H-zs@OjLIepDQ);O)x;tFo>OREhMM?%s5@ICx8GLbk>&!eGp z;~swlvhC8i%)_-CF4n>|cw4hGj{M?OT}j*^?~Q|v!pBi$5NgY&YG7i_d$dGqzNmS$X ztfXpbR3Ct1lB}va3F~RBPNQG35%fMm^+S zUx=k2;=>Hu|5ETnd-^^Crst^QOYJ6v9y|%7>L%Wrx><9Fo&b2?OJ8S-x8UGcBnBM@ zcEQcQHl3crt(gDdOpbQ zyr=rG{B9wPuIL`ONT+P#s-Vf-G(M^*y*pl;H!KqbzLde)T=AATdmn|xqqFf*J?5Bp zE=Pzgy2I^OW*cP$=BU@VMEI#Kp#=idLeuDIF`FwXIkaTQ6`Di->j@+N>V9jS(A|X7 z9MnXhjRo1aul6s)E~WXfFECj#edQ&L@g7C%qyB;{3N@;=j7^~l$imnwICM>ES_Akbrimnn7Bx+UwRk3At9ia{_ zQhr~oqp^C6^oYA?q+~bUUtb>-(Jdj0wAK6@j|RB$ZFOeqPl0K&^mn*kM=t)2osFz? z_iuC3HRW#^{(;R+()+w@2*KcN8SJe4W9{4rO^=Zx7&uA~p0N=_b>aYJv&L@v2PWR7 zfR|`Iq{nF%I%)cUpOh7a{#433he|$sw?m7eoC}6s^#w?o*YlQS`s=h3OX{pb6kZ)W z^J>-fXl2oyoM|j)G_1%pnKLt4D}pK9GwHUB%y92L<&S%tLl0QJ$aD|5V38%qTVJ=Y zmY`XTPl%4^lpM2{kb2b*U6*5bUL1+uzE4a7y23XEdUbp9Jbc864zlOtKkURHhaWP$ z30Tmgl`TMIHG)95u5CAavdSP)wV;?Fo5M5*!hECZYYRd_JMDA68y;pc`)Z{*R zUMz>@`n;Bg*+SDUv;!p@Ew% z9K3H=@+%iZif{}bW|^M!LR_Lgamqhh&FAVUX+YVoEGIST^^q4pjq|SLwjbMn+?uJf zrTafsV~*Lss~f)mc{cIs4sqOO`#I4Ei=i#w4d)*0n%nlN^!@a5M8vO${=&37Uv5X& zIyC;5>U%8oX=dh+k(1E}u_@OI99ss5c5M^(56kq-J9O6kL8;62k?R^SOV6HrVE#6s zUt#?q;KH2luPJ>H^0RlJvZ&STT9a-5{yQ)VNfX`LVwd{rSrg6F-e$Gjm8JYg}ey3Zo zuWx3V^pUX(5bV9n3|g9>yz75&L)5e0w?T(JjURc-hIN&YGrZKc>9tvI2;moXlQ;js zU(p<;e+<(uW@4oiIsM?qlYN~mQqJ5nr!S&EGyWty)|mdj+FMs|IRQkn>{XgiM|Vxu zZwWtkd3$+3$Rj~F9(*xoD4UMU>U(xQta@6wTl3hi@BEmd{H_6CA+NmEACt?&7N&lO z7#Wocn%P zezYy$tBr3c+cBq1PT%wM%)bL6tNA{8%7vU3)1IF2J$EJ|nZ6x1IUIJJc)Yc=(s%LP zKxxNM@8tuQXSHWqb6t;)m1RswTwi?bFgaMDn!&eCS-SVcQ~qmfrnb2oNBezhy-{Da zBzbUcBP{p)vB^a92mCF$UNJMKO_x{A&jDueP>n52ZNT&hf?2OfK{K%9qJv2f=@R}$@+%FTA1*Lx~R`9-J4Rb{h|T5pN{C4wBa+MzKcE)S51~c_oNq z=D&oMSJ6X8AUIeQq+NSz`pxS3p?9hNhL#->b)|SAjFnHOV{C_JVdu*ewn1wibBsFf z^-=zO;IC|y^em^4ihd`ao`9jHzoN;|-AxUa9U~SwLN+D03nqc)v-4a|CB>|wQKzZq z&m^gG1M!BBN=QWNr&R`e>H^!^Ob!}DjpdTmknz0gi<#I@9SO|GhG$NPzfEKH&k1AZ zjbZ!ulCPfE+VLa*O7Y}(emZpRKt)XGS~d~Ecsg4Nm-mA&JxUw7`URm_gWC9H~#zi41-&82@aeP0Czg*q`4;*=zVP zL&kh6z|5PTv?6Z+3f1N)KI=Ogeew!CXT{jL z+~y@(;ey7}L6sCzH7kH06Bda!Sfyw^ufSkTeU-Zg^o$AEz1&A%6ZV6X;!D(I)^{{{Gk{tr4K@PC;~zW+gN|Nr{WKdft;sMR^J U*Qm4bFT?C#%fl9VX5N?oAKPLa+yDRo literal 0 HcmV?d00001 diff --git a/benchmarks/results/prepush_smoke_20260423_123335/charts/locomo_comparison.png b/benchmarks/results/prepush_smoke_20260423_123335/charts/locomo_comparison.png new file mode 100644 index 0000000000000000000000000000000000000000..7f5f7c91e81ca4dce1506f440423e88b57b953ca GIT binary patch literal 43742 zcmc$`by$^ayDvIWRCFqeiXb2eC@M%zK|%y2rKP(?3F&TB1ZfLIQex1lbW05-C5MvI zC7pxZ_rv_wxA$55T6?c^{yFoy<~2az9dA6(egEnn{tx9OPMx4Sfx%!-Nj?x )rV zmxnKt9EBJ6h_-Eb;kOr4w^y=yZ0}@XYmAXKu(y6{W&hOF@Uo+^t(~cr{Vp^~^sOJ!A6)o^%IjJL3vHvSm;3L|sp&^`2e zorCBydOaV3IfGtL$Y4m&>k%dlF}!#mI)^y~ulJrE#}L8mr)pwvc)>i>KXeaX{{Q^* zPlJ5p@0Ao2#e+L`R@(INeur+opC2fjT!_(e8YsvX`W{p5to%Q(&ikUGhK5Gz?s{LRFSTXFM1)aDSEj}nuI)cxQeyAAC7WlLFa=+{ z^*EuAQqbL*f1!NEcGM^2(w*dwPh$C6?r!v(hN-AZro_6MQE}*3D-Stw-S%fX)joBl z-)*77G^?PeF~@44)M2J4OKVX4PvWD+-(RPv5<}$zSqs(8etkC9b(+m8%3j)J-de0Z z-6^>J=iLR3M{jm|Z(H;xUHzc__a&P;GSeTQ?wguzPN&2z6EgEVq|!wEND2I{cNWSC zrLkUngy|@&%CvQdM2S#2H5~_)nf62p!Ia_wEe|R?~U- z<;wP%?NIZs47Hrq&CyW##Vys{$db{JJ01M9neTE`-Z9BCbMHi`pL?BCxy5|R@uz_) z9J>`qhj-5|nUHE?D|^}tw@|6dbC=1P_FAm7;T~1+N~ql{qdu3(m`?MYirZh)>pC3J z`*yw}W9%?Zi3tzaG)wDBSIOA%thZ6~sKR)srJlrmisd7Wy<;TYfZIe!*AcXz`th0e zuIreCS*gQZZ^0k9`V1SoHh~qHU$8&~zglFLOhhVq4B5!#$wiNPZqKQW-d^uD&ne&Q zv#Qd$JsEAMS|-<5XwkH{J+$BB973VzZleL0W7aPbc!9NXzG7qW^9>ug?coKM1Fp~U zMjvz>FD^L1owfM&chk*VU-4OATKSD@eNSaLR$(2oE7p5I2lEUTH&{eF^k}&3;^@4} zpN4BZPIHWLnfQ2(uFYrNu7w-_9u|}4xgAZy?#58YK*Q%IwmV-}g8C}A=e0(;**sdJ z?K>om@66?Q66aXaNXX)Mem#j}HSbRECJto`SDP%EjAbz&s@x^kb8F_rb!}FtYEF)S z&|}kbU#d6nZ-B+6JL-VJFvUSdh!{Rpy@uDT65x1sZw$U*--h3`rF8s8@AsbDtm``a zEG*|=dS1KSgR@4vuoFrIMDL6S-D(_!CsrN|w>kOG8wuL)!`j5sN^V0~jwfG_P~1H2 z$uJgZQNGf8$C%n8UxBdB7#+Aqp=lET8Sd*{zknCVPP5H+rpUWY70#q*^iVd36@HmY z2n-*vYr9J*cyaVpV-EYv0NJS9tZw6BYD}{prTcr^O>I1dPT#M7c<-^Zgfp(%+cN)E z{p|3{5{|p~Nt_3Bg=JT&g0W7i{r4R6{$iV7@qVDB>B{CbYk^)2q(D z&KJLXGq?H>VbA+8>9>V7_;>B~nT(w9wVCb=_n64Syy@hq@Jk-6?Z&6?u0H$)3r%Ys z-X`2*cMZ>{+-DSecbWSYUeBRhrMt3nmAkXVZgSb0ef<*d4F1mZ3jZVCMz|2xD5pV} zmCsOe!gV~>x{VJXJ^5`x#a*XjvF0zr_T_8^l{k-e4FXzJU845%2!unaYR!D4;WHvIis+)XXBG;HHg&S{~Y+}>Ge4T40MF&rXC zyf+4%^jspWe;W9yY;XO=R{Y65Mw-1MzvHvbXhRrl$=F+$UzuN8HP_oE=(?S0pyDYNN4o@iD^Y-l$1SmD`3T)u6?GD7wRu;agg1uv`rPxG@>6_;uCp$v7|F zo@|}6@>bg>R<+w{;z5*eUPN~YGAUbVtnrH*8&3aZlu2*Bf9#TSrJRqWib^I`*;Pn##k` zW^iTNDauS&viS0m{egy&+-pGtr8~^GSoNAO!-w<*%hKs&sit|@uFLq1xxS)?6{xOu zjd(Yx|BXyy^_ahX@p1u*1#10j6wYD7SP~PX3fbGg9+i({J#xyI8@cD7G_e-l)MHP_ zWGHb`y6q7r&HLN5?OIeU#GbsWW~s?j6q_9LN#lN5LSFUQki=3}&*|J>n9*I(Cyt$4M|8wXXyw>M8V^LwN?B zNBey~@wY#{9LeIdG3dsA>f+x??!Vs23w~U95^0*8UrTyN2JfMk54CafC~3Dp6ZzHI z>@v%O*=DG6B{t%ld)o^vtZu_EDV9CBMt?qfCu=5oK}9oK4J#t4fMk-~|G|-`?&vzPJKapax^?EG6Cm%3r2qj7g({;u^o|dE8mOTiyN* z*RXRRD#g(X@Y;8agqp>)RlpU$!4i_!wR#j2QA+HMsXl4+GuMFc z-U;cOd3D72V7Vgf@A=uBs_d@gx8IkJ$hCQ>osUs7KRb+1%H**f`?kbi5nhuuwXFGW z>bK^JxP&{iN!N97Y&xYf9kU~@?@PCP*7^6gW^)KzYALtz*PeGo)|3U{OJ^K1O_u5@ z;(q6aw_;0YouRr-WS3Fs*ekfMwD8zn*H*&+ju-LkDGohCq0J$iy{Rb##ddjAP1ohB z({AwGTw$MOntYrfx1$-G2dix3Hg|ju_IQOJb^#}}w`s)izZm6A>BGD{;d1Z zd06h!8T1hyqj@bJN*e=B>flRhBHmhUc^M$%BIxZ#$b{I1BgVs zKVj3+DYE&t%E|ra=3c*B@qV2zZ#1*GHHr3#NO1`Fr6!+nwNoc@ibn!yOjw}~PeYf= zzFo(n5-xXI%d*6{aukul-$jI=jpkfLNm~gmtK7avcwA)AAOkullZsw=U&W?WNmP zP^R-4*1eL-Nz3YR$eh!*y=rq;YGto`oxc?z&iFGUoixphYTa;K)aM+6%&Kys8}z7u zrLYr|v$}N0u^DGS=rTEOR5c#1wY+^+l?j@4Dw^H_7I!%^Zi-&J~S zhipXlGV9>1s&)&@=^t2#zAEiflpe=z^xMe?&(^!UV~yB#HcD52zc@XS^KoVeAoVl= zg{Dxzk=;^FQhL|8TL=`Em4pdpd;-0kX>4t%&|dKlK}A&q>;BVndbX4wZiOqs{n@*% z0%DAe+o5|Hec#WMv2GLFah_W@VytV9cW2w7{;b9_2f!#SeC^M#F9P2D0JLqxv}My< zaksH>*0%wvcBLt$EZb#fpR;M+cU=GVWD*AjPkW}fVD`$K5!4Cx1>th&_>+8dOl$mm z8&&(VAM`w33Uc`7i$^=yt9BGq@OcRfmD|}nt}5gM9@1gq&~V+b#@P6L0I})?@k>hb zcf+y)BXDJjJ-icd(5!GuYv#EC0M;-ex&@oGcdOiZ?RSGvla_S1p+_YH8sHTQk*h_h zii`I!Mqi2qL*#u?%7?5y^KvvG?~*`Md(~jMJWk@J+*;D6XEIYgS$)pK3d+l3qc0+w zZXQZDd3EM1v@lKzE#j+i@0|IkKE3F;p!JkfW#evz=;TlbM=J?$V!$l)g(i9(Vht#} zM%n3sD;B-^ror_6K6TbTP*mCU+I~v-yh0t^ljfO`{kF&NXUCSg`4_7Xvs5?&JZD{? zrTMDBmGWwq4$qlA3EUu>t#Wgu+RE*dX1U({BjcestT_!CBrOcpFBGFD}ltTn?4(OBGT0@_e%_Tls;{`^ddP+FImNK2hEN7n86vIVn# zT&KK{&TT|0tf_&FO#sD;R?0j#ss0WMaizP~xxe!)bl{AQWh1%LmUy-DO%xm#DmKDp z{Q*^0WEwYxpDU*=1CHXHL(OIK@IMeZfbM@I@DWYfdAx!}p^B#CTDRH_^y%wtXdv$by&K`E|q!an>N16@B~AS!O2vTpbfr>oxsL#smVt$Gb!S4 z4)v_v2LZxE`NUwHRgsQThPA1?suS+lk14fL6# zXg`KxtdBlS-$9E_hYpEK7$t?t)|k%x$E5ercmJ=om&l$qB*I`iSg7sK=EI9-k z0hUQNa%_9+!GUyhgu-A59!H0f1fkR<#1RIe6uVELyx%=V;tY_G2|pFEFrk9*I8 zsd?BuprG>Y98R|QAqYKexeqM}DxPe*FAE2f&!=in;3&(BITn3|X+XJ5Cq6#OR_m60 z=(A`AjLLL#aoBX~M?Aret#SkQ^y`_!M3>U_@^lOA3MhO*!i6~QOC6a&L%)RHy{y4^=rwc3n z!G7$GL^5bG>F#uS0(Ev+7xHUQJnIwwx$9icsLyL!RjhTu7m*pgs3LQLl0x&eVzon> z&)N+dlH!-gEoKf`TkCUCfJVw_XxIw66!{)(LwWCrb({Yv2o(B8>EH)#Tgg%&_O_pj z+r_D|z?ua>N3$bbXkw_#oQ}wbJDC5|Fc1rJ2TPnevF$K`c3BV>oJfxXCT`E~w{60L zn8JF@=1Gi$Dipo44wjtqx4`ELCHZ@REnmB2`;8tgc~jsG&D+o%C^}K|$q~`Urz6?S z9u$xyptIPj-8n1MPFWsVMu`v^#H0B^1M7S%&GxajisT-kzzzGhyBlM775h%f|9|*OW=;lIwtGMO&7Pha)2OUdRaqqlxMzPyyxlH*rF?Q|jHIv%O)1 z^^y;rD75Hiq=^9NScC$Fi|S`rVEYbvx@1?qljFZp<*B$jKX6CwOhbWrN0OxHJm7SB zmaKfmQb0qtVPlOfIog$;KA94i9F()S%9a|aJE21Bn7YwZ=U+7B)K5>7h%v9?Ot&WORZ?U{+i81Jn)or{wU(XX7S&;5f;^)h1@S z(h}Y-u6D|^pWYYtdB=H$S9D6f9RZaEMw2Y6y_Z+R)frd;%Di)@ITdr_pWDyUqE`f@ zC;YU3*7fHFlr@@EXQ@{X^*nY{5h;Be(Z|L-%A{6anLL?Dl|AF`ezo8t19feTQEW9V zU4FHFXba<$MpG-$J>Tct*ADCrFvHDEjIz9RMiL#i@jK<;H1v;@AO5`tXb#=dO6g(n=gZ|lgwJw|M%L$kk|8V+S^Zm zUOD2;?QlF~;%dC1n6XcYK`Y;6^C|TR*ONrZSh)ULv0GP#RWm$y4Xw~E*E8n>ut6)rh$QO2a=cV3C7{4e;MdoOOoM)|`&35kIlHo!&Yt;?`0X;~zF z2}U+2n=J6z1I|r}ql-zZd$Le$A{ZT}JFcScvNqkBqMGAha$JNb(lp}GZg=Zhss_Xv@s?7aY+v1Rbrc86b2^-$yZZty$Iyq{d7{`=84=iw1l0YU&laaFu8#Xy^0BDkj?Nb0~b zmG>54ao+XwR*2sO&?r|l8r^Pk2zB(N;Lr*?-?9^@V=G#ef63YJ1!FAWf~A338`B;w z>yTo>4%hmnq{Mmhs~yP({4+k>Ro4tG;h+-VnJ+ftAA2p*QMJFjkdKJfQHFMdjUOVX zWl^DeL}&HX4nVOq5?eGMy%FmGP7$jMB3}6)*g^75H@`GrvB;5p$26VvPEN}2lB0Sr zlD2`x#$M32{@VYZvGfqmMF2|DbVg34RA(;~(%4pkm1g^zyd|$eS(J1_R$sy z*R~jEXxSf}$U+Pnqn>`2^xO~Ear~9vYnFixXVsc8wizG@G5@omtzMF=J^(92j{mp! zVcW|WR2<2AKnmyfdyaHUf^}!_PsctBy8=C8KeFfnD1&^U995CbCyNq$I}JFfWv`=h z-?6<=6^H2N0ew7UWh;8!1?nLd$l=vflpL3>%mFFzoh(Q7X4Ssm1~>kjQ@FMGF|Y?K z%ZFzTh7=?&g}JUyDVXlU~no$)SQ<*HJ%K z%CCT$eFr%@CES}xl7OaI(*JyWrA)dm-MPL&kR*yN9$?A4qF-Z+})Dh7(I+G`6$mZ_$8>oNq{UE zg>*qz%h0(N8%n?!rJ~M&QNp2hi1hKULJO@A`QgOg?+kudKO4=6ETI$(K4rktAXEimVY2hm}ckrj^j_>+h006p!s#PJXS`D!`!Tj0}W ztW%;3wge2pB@`YG!3h< z$wjt&!KU1%wSa7r%7bxa+v?kEdcwAqFc4Ut`U++y@dI}8T5$`3IB{x=_b;kUmP;C$ zGsb=}AIbrtHf3YTt4aWK(*@vT2RN2*kq1Y^h6;fE@xnpZSuC&-fdhdMBob+_{q3Pn zfUz-y9vcHRY+~M~#YHmshtIj055Sxc?|GDv7BVh7D+N zYI+`FZhUofs5&D9vLGjTV*&v*knh{>>z_9iyL>TNf3z3p5TS4!b4|qb=K{1|Gwn&Z zC2bq<*rl@yFAkZU%OFL@-nFF%fF?}7F+B7?k@_Rww9}pCZJLW9871mCd{Y_#W;*~t zysgc6OH##<$~^Z)S@vSsW1azfCDJ*m105Bl^ki84cl^}Jh{b0 zPz@f&-F05Soks7r`1M#P5TiE}(RQsg_F>8E%>MLg)K7E2I_LDkwq}0mMK7F0vY602 z_l+_1CO}GJY1la0gAzWjE^;g3in45Eq!iDL>YCbv&VQPIw0VR7Qu3h5VQIww4 zI1SF7zsSVJi(P*if53too}-9S|BRRl$X68|4+*J%f-FE&RdFeHvN%+|Wzs7VbmaWi z&k84KMrz~vv6JRhZKvTvyXc^6cr8O5!AeMXlZU?hr7Guhc3)C_zWW=o3*R?5_jSeE z1h=17=2*&Atl96C5Yq{^UtihMuTP{hZ0nd^02JnjLVNwhe_A zzQnT_Ip4=Ek>i}p!@%jNtz6z$H&8Aqnw!DNe)0?)8W(A{;khvp<1}a%$NutFH!FMa zm7vDbndDvTXgciDqD3~Q9t;-(QfXd!H1>hXkRkyup2q@Te*q^>lpNn1A0Qp5xrnK&q~iAfB8dpL68nGqMMf5T<^(wn%w zL>8NqUxFPNKLu@1h0_oj5C?Q>j=HYv%F04}cL#W|OkwjK;lc%=-ZM~}PKOB%<$(1* zC{!n!#;Ge*c_t&^5kJ#x&^(7iEA=S;j^m$*rUyuzjDCvSv__9%k441`+20P1Cjl=w z4eo<3t}(Y985>7KRukqs)9|{_q@T=g6B(i|y0u=Q?VCN6Ux)8BDZ6xH)F zX(TH;6K~Y~fpUXWfH%_OJ&1wIqzzfh7EWNe8;3c8gbEAc#8yH*Ta3 ze;;Yw4~J4Jk0YUxuGheNbX^fHO}(uB_@lAyRl_m1b4n*pGxBE2#R+;Wdz`$*rYZ5q zA0UsuJDDi;g#Q4u3Ri$ zVV;Sp?hL%MU0wac4%U~lQ%dH>g_mOBGuY0jYbiWBr(~35puAacf=fSonpS)82UqEK zc(nWS_Yw->=1b0?yNB?2={W<}LU={X)P5Xbp3q<^kYP5fLPZ-o1)WV0hC+)5Qhj%T z|KhZ&nEYsLACRsmL@tGES*GRx0VLO!c&`=#$*vz?;kE5# zX;rC%|2FJv|2+cqrm-(HzVfc?>Nf+E`WrtOWCB}DZ2}u;!1okI8kC*W5TJJU(Ci?# zuSkFLlx-fzL-`c=`)}dyh`%k6HWj9(`#t!+Vd^}u1Tdll|$Aa1===tXxaz{xF zr7t}*GO?w>)!hWV=CW5aWnJDWcW2z3yl-d5t3@;M8%sb1py`;d9|x=(v!`Z!LB(3( zDn+nR;@JgY?u`{hhR*Y-Ft+4$BDV|l_-K*5pI*(TYBRTT;r!%nFwO|Lf#b&OXJbRn zZ*M|6wFgd73WCZOMFWnbr$U1RzEM_EtN3%0YjRD#nn;P^caTDa0zsygGb3+FPB7L5 zVio|NH$T&dl#g9175(FA3F3^aWW5MAx$qaL^Z4rHxI>3=dGpCpmYTCGfoo;<({D3Q z80~Y~7lUG2;w9pfj-_uoC7by9l~5Xfgu2S`4%(`z!IZWHKqTWhvaV*(k=bavz5suD zv&|L3_fOJ}{w&gHL8Se781V`Hqvv&O+aM{Rnd1eb)mSnmm%yQW@B{CQKYo9G(FwUB z_1U#VK0i#Z!9Rc5>!0Ex=DWx$b#=wwJZv_fGQ_};)zvJ)8R%MDYtGSr%^)^0+TMDWw2_v zTcC}YK09*Z0SIGy2&^(^K{{)XXW_|5hBEyne&m#=pEEbuHRj819yG@Z<)ZR5CE~t9ly~Em4yj z@zNjIrU9kM0E1Q`2pYP)nv}wqj=1Ulqe^@4qtu#Gj&8;B=&9R3FZ*e+SF9?6O&Vzo}00MB1fTJsHNyxtrGDSUo`ywMFyObGKt_pw;ZMgiu=A*(wd{KF(9vcC;o10W!YvU{YoP-9y1fp3gifcNv! z6xdnTIyV$@T}^01D~lt5DzLB;xagXu+7mkgZ^fbCE7uh$0z-TPN2V&SufC*eV%+71 znp!1b=jtRR-!vlX=N^dby2>^Y#R>Um*Kx^c35S9)$5a3e(+M>Y9O& z5x9e?xS7k~q?lRDAiuc0au7iSkCB;G3tYCQ92_ii-MPXlU|FB9#kqfe`sZg}SS%f< zq3ADU#OjqH{dS|Yqc9FGP^xVKqI=`p0y|{ApN6r z3n-4Jw|^BZ#O7ICLE3j{7;$)iqe$8zr%uKV&SRHfyAG7U$#^KgIxrnyDhSzKlOJ;6 zR~~r>8Mg*hMjO$Z~azVfe;f2NRB}EqbZ76LQT}7)zoT-TwG+V^N7Dd7s_?D zbUV?fPmeg{J%!!W)>Z+BFdh5$#uQZW##!8%UzD2tz+bKp9JYb7mot+6&i~F@V_qN>(}uQpup${jC@Ja}REwpd_yJ z-&px|KQ>&G`SkISfW}lGZI|&d7L}*Isb~6KXR~m9y2^JFAf^NbvV^ixQ{|+aj`i0g zD-Q=zz(D{L^ogye1n!Yn`C?8I7hczt(x%>^9Q5`sXAyU!h&f0shUt3kuC0PWGeI~u z1g63&6#B8@TId3iV@?^#!&yu2#NH~Cqb&X$wuU3+hQ*f63X}sL`#BMRkH;4a?XD_q z!QxWfUY|4f$g~Mgj|#Oqz{)ne(f|XZ{Z8WgeGr_^+Gsz6VkH>NW?W?CQ_+>6ML>Ds zLKJqRFOi`?#B`-5H(>i!$q_haCl{FQ`OumtFdXY--F}>EL(n8*;sP<&QWN`sVH+s8 z?+St2WcC3T>khZ&tqJxZ!m1%@_3?}U=*;hIcNy(Z0!Mpd1}zJ{_dmhyy=&>;&5%D# z5R6n6#r-;-#-GVsQ%-_0qO=CMdiYJMXCQqQ*;au`I|F+leYO%6MouQpEGM1UH}Ve< z2ELuZ;bP=v2=0??J%-R%xqz>j#7anH;a%=*p#qdo0j26BHoltdCzLkhcdNZ5D~8$S zBpVDj13B`}FV=$xtWG^(p7?Ga;7{C(f~kNzd`C%X)Gh9TaIH77Ju|T!^_q-Pe9WMQ ztf$_i%yF(ki?-G-r9iPjpEwxg5|tyoM20HxHs0UQ*zLbOM(g~k<40HTtE1FppTxdx z{4@#;Yui^lYoO0XgaUQ8p9+y`3!6%|XWJg=o7(mcR2gMK`QfBjxfZZ9UDPeR7ujLC zvA)wL@PLA36hiY<+_^VJK2EbD8KQ&)!E6f=hJRfeNLno`zr~*XW-a`gP@a&TK(b&7 ze7(OeF@dBtVbr4e4aI$JP%2=ElXnK0xsHKs=RQ=RwhHJ$29U+Y+Q1 zxg`4n>6ga@*lK8ffX^Z zMw*Kr(I$QlCXeO(3^l!SH)$o=j9Af7Ds86vuG$HTC!mT65Etjqj~%GgAnAT!D_^YX zcOQdi(RRtV78PBT;gamth5T}d(p(!}wKbE$ zm1O~8plOJD%An3RuSr91R(|Y1+%r-P;Iyq}wi?d&uJ2@Tez*_2e(x*CyFO>I9qrqhBkdx9@k@CL)tpT)Lz=DR%CS(SN_*-VhBWZvARMl& zxeqm>b=b_Okc=4g!O}${{lw4HoY$jnC`JZ-NiOXFab{maPkC+l7?{BUjL(BppZ}vv z`}HL3=>DF3@_6+?n2tDrasP9`0zv>Ybq%<|=|((b%ZJoiMtHKAWOAa*0Qs6W^8cI_&_CJ^+Jw9LH_PH-U-wgv0}l97!S@$^|6_>b9D=33yN7N zLNS=#dia>~(7k_tK(=#ypDCEW8hy9F0>WHUAbTY*GNBLbQUK3#Nb%?e`qBw_S=BQJ zfyv2uH;7|(B8%lVC~#b0{bJF40{9@ePz-r|c5hN>UlK{!edZvdYJ%z1sby86GRo}* zySx(=zBtfcKi~l)j4J^xJTGAq_YXJA%)8AQIG?$N511=5&{@~ zSGmgyO?6gY1JyJlMUK1RQYItXk7VC8l?+uEA?U$sNX70%>lb2)I;OiwM@8`_lvhfF zEq!3ILzY>;7q)#TXw5>Bc9=@gl1+hZwr&SnK*oc!%1d5gNwi=V=V<7bH}LH6vE&%X z-Wx3t?jb~sx{nHgdmi@q(Se16QIr~TUy(iV!+~u?Xu=AwO($8a1ms0C1$@uV6gp>) zeHJa?4;o7GAAr%ki4J$58c2r<{xmcZxnp$h*0i5X0YW7q*ZMA8INy~ZW5hZQh0@P_ESBtqBlvn*pxF_2M%72Q8b+a_=BrLRf%KdVf*}vG{1EUwR)Daq z>r}ko^4xnRIZ1#Wwg;l=YZ21#_btkWpgNel{@afCfCN!TkkkQH;Bp2b6hcPOaXzsz z_c4?+`!BLQ&rWA6{{7q@9D3{}r~bU1666=^z;H=PBb#t05|;h+>1#4m?aKh)cU0}I z<^T-t-PZ__8^p~Y2Jpk?kcQHHCHy_HWEUD zFk1*;wt^(Fxq=3T$_ui5z_$l#^qyEMMYr3n7A__1ERBH~apqOBT(M0GygM8^HGiLk zN(l5$%@{43S-{zGjg5`VZf9{i%P`@S3@GU4G4G7b&r+l&88eW9t7JMz1&E16OZXIl zeV~`!9As@Pe&wH%2({4chuSbm)-qP*OX88x%hX%OjUf3I=t42tO_hGIV83fgLYh>vy{CnNs~Cd);kFfcq#bji6BOMt6)fm|M%a zg8>ZDfx|ffL_D$ZJGsNW6KIx42keV+B7IXjvd=0f_k|&EV&|Y^5qiUy3b&RTTzuQ8 zA?W+R1Wq*EG7b1u8UgtNqTjHcKFGm>rg01xC4c;M7Uqf?!=i4PVN4FfPM^L8x@-xb zTmmCs_JKouHw+F$`u$tX1hY^FJ82~N^_?DD`8cw1qoy<$ye=#}!JBHgd#ZrXOXL_(+{DrUCCK?-;SP%h>`N`jfFUo^6v_zp4=^`iqX^G zPQ4ez+?1YB^Vx7y?K1$_$?epMXprd-9DR1*FTN3@mWs^~2I&^mJ3*GAMH?(E+}&`N z^R(Wp37YP-X@VpfxxaZvPA`YrRlO6VT>`7AU|+li?Z|P&wqa zf7kD2p69FKY7oZlw?MbCGx-IDN?=I;q<;wg*E~xxh)X)6Auv*Ea23|vWjOuxxrS2b z7KZ|ebY$_SKouJ|$dtoF3T`hL)5uRd++r5-hEv5bDG}={eRg*ZEZIn)w)F8(Sff0} zH*%pi&giFYbG#Fr%cKL#QHO#w!JzDts{nxHEE~u~G7{6-BTehHpV_l_!3715ilHe-ac=c3M6Npte9t zNSESCzf<5;xlsY1iqS4ov;FRbcY(fot={YrID+!z|>KZ5-gSI8t z63N|UcML`+f=641C{T{v9)(f&%k-)|r{O~BtOkG!ec;hbD-KO%+kpy%>$6lA@!XnD z8O_prCql}1<(Qj7i!&sjl{MF)8mAy@TnP2ok5rMqoViHHaOZz8Lbb0Mc$K6Zl_Kx$O%9)uEd&2_2 z>}UKP$XuNI_8|Ombr6xbDCN4verTbdk$Nf6ByT&+_w)G~=Iuvxvhh!euML4takrl6 zHwt|nbl`@13rH*SecA>P8VjwlC?6VRFpt1(W(K28OXaKLhu@gAa>9Hg%An^1q)9n^ zT7cs;J)8e04&ic=MI<@UkQ73`&o7r8OAnlZ-bF9;e_yGcoy8ToC1xceiuk%#>XHa-o_Vo^w5#C%sTewVJb37wcx`Fh(sfMmKL=uwW*A|Z z?3gMP$*zExfaEKC`_&Dw%F@=ha5Q!@mI3!`8UGnOH7o zPCcFe8sv(qYTMLzK{V)KFc-34Y-ong1p=`eX#TVVT2ho|fms;Jq9Rzl3Q?*a=Tkxe zsq(=uOZtzI)`N&9fJqmjZhx1R5p_e_3S{$J0G>82{PkEGQ4!G3AG?Cp+KAt6^V)mo zy8h=Q8kG%2a7Pa1p`82B(CNx;=%agFFzR9t^dcDz>mnK*SBuimf}rl0g-(JDFO5=n zl}vWcoDya|nHn>&!VcgyxDI^Z_3kW1z98klu2=hW0~$&+lC5Tma9nWFp3{)W2ZYt~p~I!1n2&ieh;d^Z9N8iw^aoFR zGY-ZKZ=(?*zU!Kxeb|{GVwwbaf)%-pNSA9d=!7-$u?Gy&t|ZBK_TXJi{GJ0UN43BY zd3K&-%E&JuZAI%iA1;0JwPi|-qZCRO1mpK0%OwXz%5@76(8@YworYIIBA&z{Nvjlo z+m3e=Zd5XIz|p)kpLXEuMOXwGHf3b_Xg=gXhXSUmL3(ABmyyy9S{hDkzj!o6nnK(9 z{$Lw|7<%WP4WRLtP%H#--mR{E0nR%y+YVw~K8%Vc2jA|!PFD-3AEv5d9EKvB%G-!c zvirS8_IYJEq#PH9^ap%wKL~N!f??!Y7JEu*RY_ zC_&?|*_js@x_S)*SsOoy6=*8|(18a~GGYbbiFQGq!9Fn89G=BZulZMuArDYWjFhwr z9txV5o;*>_0riMxzP=Jrt=G6+8uR)cNNT%>fEF%8m_Kp^9QV+F_^xePa_dOzB+ z*K~YX@6{P*-`C-XDu75Q3GDpE>Ei%b(G(izI+G@aUqV^|jO$B1iMA0(sPi_;(e3aS zYQ&IqGy8fC!^O63kag61O2d)N>QZ|$=GB=4W~vl7)m(Ly@nYT-LHATXtp>lf4}xYuM_?W^z5t497AsryS2I&0_xy8 z3xokqriAD_Kxn5?X*Wj_MdY#K=|p^3~n z{G8``C+6IM)2*E8PPL5snO7I>M5ZrR%pGr(<$P16p2k&0tX_}ii84h){|gvA@xYGf zRZ+g(fju!9{{tKU1x1HkF7w?O*)()hkmZBq^fF-+1)`Vd!dltW2@t-Bp&6Nj>~KM^ z=TkRfyeAZuRy;HFu~8Z%YGWDxDX=?jXn=aK;MHq3QF3h`ZZClD4J8ehjL$_vSYB>U_F z8;EC2NL3zhsZz6X!ydh9vul)H-3v8^ec7Wqo|h94{<{N+^eH}-SJ_@MT!oz6`&@c) zM*2v!Tfx9k5=~oDD({usrSlCtE}SwzD}r49Vss9gxXwAL%eC@qyx@{Jh^-sRD0}1= z!MRfib3P>(B~KQF%+;Tni1=V76+Gt%tMB(FGZQ=a#>ACu zKl^mT4e|c;j6(+n*u#F4)xUv>tU(OlZ&IV6VoG9dA(k?9hTI#a#j^_4((QvRzj108 z-&XNfaJbmV4_s~K;jFLRfllI`c9VIcN7rpmDqhWl?2<VS|bs7~@RjLcxXJxq!)&H#81J0V<4X?8i!)iD0QiR!|IYq)}#U}&2dyq6$JF&kG zY=R3WB_(^EOx-x`d4+LrQ~w0Uci8B~C{p~MdakWAMcE=C2$!<#1tugj&cw_Xrt6h^ z)g)=QQKzIj^slq{k)>6%?2U4dEh<)MGPf5pw03@dBz@r!h@95MUZ^!y19#2+X?wdf ziSDUIB0>>6XH z8$1)+BZJTv!)akSzn_b-rq(&zYk%(@FnTgZ8G&J+a~+__Bc76R)% zo4qDy&dQ36Xg1hLj6br!8)D5qaV^N0{6L;U>LUFRighB3q5o0|oERTTFh{OofzcZX zvvw3R%h+MR^JEkb$G)*}|Fhr1a&PyCyZ_Rp*C$KxZ|Fb7UVPiOwpIJYb$7(>QmQ#X zaXB=T;7Mc92@?j&qos$c4FiwTkyVOeb<%zvbJ+Lbsj#K{hbR8;fJ^ostVPTTOXyDW zz4y4Z{}HbK_uo5MEts{&{{@)o;{6m!4&8%42V|ymz(Z5j5CU6;f{4<@;M$%SevyIs z10pONoyAb4hgo;J%1j4VnjaWuCm4FyT(lU)gB~E-izc%>A?y4W83A45+MQR^{owmMbzIGuThH51;p6y{CFa-uhRC*^~(~BsANN!7}*lmn5 z&;&s`h&)O)Ayx2jE_inGimD3j$P%^lM`I-}ck(Kr8efBH!do!_5xOfU05C~|9^aMpkoWQkY3n-od;-8 z2R?(N(|x7o1CnIjTdoB_XFd$eb!WJ2+xaA@g;1o2NN^v#}O}d29TIGH{I=&xbA-1>?>b1iN z@Y?Wgn*q$!xa1oWyOJr36Dj+6FD)SP6b&^^$L;St!QxS19z_tPq3df!B0&P>O%GS% zb=_F?-XZz)F1P|Nt46Qx-O6r5Bu+vs8^TuvcSh~NPwdaX0DDn z48X2TeJ7DPc8co##xDGL;$F_}ghdD(7D~bKpA4#=e}rbu`4(KDzIUR+feA4+H;eKz z?)P7Ukpsgto5{ge9q407yhskBF{ulhv>P5ebix(ZAdHTmNi7{3hax{f^OhCnt)R2( zvwlV$kms0ph3gu#USW@>`8b9{xH6sSH`uXZB8344`lcGw1@A^7EA$qRL0}MnZHimC5!F1Yau!B6S7g zSOpC=TPWNUiwi-wS|320bjLa{{s}TyrbeHb(hG>#)|#R(;LS(uN}xwJeEJOx{P4wM?nkya)R7yg>M8D#WogFT=mAo>1DqLA|?t1r(_+o>p|Q|rB7l2t4C zQ^@ot8ZS;91dFXYh@N@WjCssIK1-#x>dbxJRf_9tw5h4*N5tMdlvE}sg;_YYU-hYV zaEDgqWaNil zPfZ8d31T8%^=@)NSjbg{@nQ%ybj|dm&+S1IaM~Id?;F;`b%oc|r{2qyL$jVJ5C8}3 zIP?1#cvL`t-prS;_5Ay&lMi5m`N+dfoSJVUQ^xN>I)VV0)oS49N zsDSc>PLxrAH>dBE_>rNpS0zNXY-N8G=a$hp@(|8OocxgrI6IkzU#y$m)R`Gr5{_~b z^D3@!#J6%{KX-MMo;xSvWklod-rnHzZt2vep=i{z_QZp|aLy~eC$AlZ>ii=1vzd_H zN%}n@-bB_g!dy(j^aA7Q#6yb^{r*<_it$fiV*$xN`XCs5;H066xeVWhmD&0(Lvo0$ z){RaA=tb+-wV-Bd13x>el{7h~_>VumqE3z^^-kkfx;1j+_eC}YB5h*xB40QhS z`J7dHCtrjdrM{Y{dy+b2C8)73_4*&^RoOdzXw1dI*y8=6(#R9#3{n1vakg?>@7~+0~u?BUEPzin$clNoK`{~=_S&~m#&hAs1djj|xyPSJPN_#Yl z_blWd&vr$M1`L9Zw8VVs$)ihJYHatDExFA*+O;0apP$~2_N||E-5tJF;kks=9f;^m z%x95d>OH!LKLxQrPn+Q%6kpSA-lq`JNZh(P@m&kt4m<1f_ze&YB!OW)qGixpTO%tL zyG-%xzCg%pcHgrSo@a_kzA#V*;_J2D=nqxTgd`Rshp^v&1NyZrBI1=g5TqbnAkt}e zpuk;{x)i{8Gc%h%JK?%L^CdOqmHPf?Z$j(J{qsU$ripP#PLMDrz;BBN)@H>iDRF0& z^6UA;#UAw=4pv6cairgXLv~2%mBp9NYG?}P&7NzLo(;f6$Raw4^^Ae3ztl-9%QoQ`7fRm1#-*jbW{*Qt+bTb$UCPcO7#ZRolyg6 zAgdaZ*rbqUN6KYrk$^aHPm;0?h3qZEnLHSAO^fr$I4~$=b9A_U;GtXwLg>; z-*vhLx2KNxtKR?LSHjfh7TAN6I7k<@@Gxa2o1g!x_{t!=JIfAeC%!oN+lhn@_DBzu zfF+brp3v50=f{gqG2b(_^&h#EigNh<2MmFEAGTiek0nCs?o z&L;-tYFC5z)eI;$x-TioZb#9O6^0j{6M37MZZDkdu%Z2g$3X%@y5P_hT0QzFilf2@08u&TA^fLDmqDIQoF06HVG*Rd&tGpq0 zM0bqg4YsaQJcQ=sixTt+d6NvFxoKiwucFTy`zZ7DjxNP1F+&rFa<)Sq5Lr%A6$wc! zk>t9__dnWu^JuE~|6hDl8l;d6A%rL@oXm4YY$0Pwrf4#hOv%g`P_Yn&%NKzUF+WUTX%ie_qV?1pL0&_={>z(&*x)$0`fScH+Wse z7l=+2^$nb@kw|;hx7ea8Lg?R8B3ZujN67D*5=vh={NXv!!rHE0tfbqh@922KV)H|| zJr-B(ghDlcD6NLl)!EG@FK%x6z+YN_%q6Oo`?^#%$A~feWNcEnR^`4#irCh<<*S`y_8zqGby66Av+`fv0sx=eBS<$U;dM>!@<9-mi)SPqP}NB0kchF z@h6;wht!re<$M}B-?X^?twqr}3)C!t48nper9a4d{*8Xp8RN7%X6|D<9d7s+Hg4M` z|I<&ynumV@5ygyea#US#;8w!XiLNr2kL)a0%F%g$#~f@A&bD^9&O{Mq?L}*IHRML& zg5AZ=)@-Y+U3-?|6J`0uG4NKQ*dZxn2wi>zmNsazm{QCx2bMtTDAjGZ-KGb1#@_h1 z315AoYXlPPh4sOAfD|`!9j@9&)hA!r0sPE- zB~JM1kFNYv8enf$Z3Kl>r%sTf~puEg@qw4Ilpdzh2lX{OJD;GxJ{{DfYjAJRh8b z|NFac>p?24GXiOncK;06@p=Hykl$D6R9Z1o2oql9!IMN~ULY?@r{j-vn%MEbpg7vN z(oz*uu#YE30)p8N2$v)Xafn~3r0&a?(0rwXX6P?WnMAR%*6{JEZl7vSSjvQUJo`QBPQ9=&)@HG(W)$lfaGuz00wL^x^Yegu)2S&)ql zk+2g(J$3`d+!^rOK|S$7_Vro$Doyv>YH~FGpbKzjDtFj3(e(U-YuhMa#^Li*i2eYB z**?EFptDIsv%d`EX@|lZ=lO2bQ!?)9g42ZZzgSE%4!!amG!*FNvHVsZFL^`kwg7`$;FSa@%M;|LV#Hf;Lx+rnDzf`}kH6 zm}KYY<>jqRVO9|jh^!TF$R4SNbj36f1@p;l49%$o(Nv~x6f2wq|KR+?MuZo(Z9?5q z91@Rf-L#HgM=OY_qu3ZIr98SS!tP7jMuV;~nVR7Px81XoBv}wyho0P6%N?vf)jvT| zQA__N#k*`{&Pz*ayYjO-bGOx!?deC&KWrQSrH9+s+Uy8HeG>V(lxbyyb>aX$4J}`q z){kj@2A%OYi_~KjnHz4x_gqTH*IOi*c@}eR1xuDCP%ot zOAIGNg_BH1!8Qc{B(Py>145EVUxT$n4tmYwC7xoz8#Ro3VB=3TExIH@{B{OKrr2zO z>^vR=kX(^a0Mc4M#24fK5U|YnQi30-dRbD(4Q|}CC?Ty2_>RRx$TuF42zz{i>mH3x zCaP0K>vi^7sF9jpe*Fwx*39K2ROXEupU2IGpR_}S;0|#^M@Wa2*UKp^|gh;v=w8s;GuvmXIQq-;gL{+bWm{NIMn% z^#uOYxhe{?o&E)UH=LpVumt^el_)pH+b|JuZoJKHAwVT7_b7h3|GS*$yDPE(<&?c@ zWj@Y>pU-&%4tNL?;X!p>`c(=4OmdSO8v&uelKu&xIYvjQlIus4Z~_&ao`lG zHR!1j>NN{Gc~M6dNrxg#L%?7$DmMYCiCWO```NWTuDL#1tJe!%QlV}n3x3Omb2_8X zXrD@M>;VB@GvowvjcTBA=AXDb z)x!_`@zn&nCxkpZOriqfcage!4nAzVO1&= zoDPebTz4?T&S=!~a&I`s+5;ED{8R|_2OKJbbBxyKc98PT!s?X<3>qe>z3~2moDu3A z-sC|>j)(c5ub?Nu&N^NMkq{ZM=Xbo)?R;WBBagD2f4ogK07!(YM7CwlI?;r>=q3cv zR$^SpDH>hvyf%R6$j;jMiF%T7LD%|uDen_~u>s=jlmnf^j67hT51){*4<@L$6g|1- zjhSIUP#xL@^hY+x`D{VOp^JO9#c$xK(jb*UqItCw3F}HL6cP`TQSEAows^R; zjB}6e#$TnX#6UQ@-WzQgXpV!+Y)j2FBJ7YQtU#bEH4q=eOgTG4h4=~~S>I35JPLDk z2w{)mS7C({;bV6JWjZ>3?sJm?M_k>Uec zM`p%oA%3T-e*0+xZkk0se{a!sEi=h%-^GF7kZ_o`5*?bg5yIumCPTPL93-Jxh@fr= zPdURX%D03|E%FZGfyUe5$?|`k{haU=gaKrL=9(H+?T~ZDL^_cI48NkTEzk+aVa2%( zq!TNcp@Z~q2MfUVy!`%AV1rfFzY#C-^c3|z};IQlPNk!Vba za~=J3U=EV}Py@dTvarPxsABU^?)n?gkH04mAx^DF2JWaK63CLwJmIEkMb<8NT~GiX zl(@%8-Un30zVGmZJ-uLd1zGIZYsdPdFrfhM4$l&aP1IMeqw`0yj9L$4y8GI9xaaO5 z@`WY|#f?2!7bU^fL4jBMKi+^)u&YpGa$t2l=y&k%a!ike+EVTZJkGclH=MNs66POH z1&x*D$}sf9=|ebRg6h9TJv+qIxBewuYWo~+B~r7LuiCErE!H`>2{}2lbz8U#PI|@86hu zgYMBZAkf`&{_KG86aInLC4v`9)UO;&jAY=$-$#i-juSG5#I9@Te8H6;wVP(Asn7SE zI#lY|1)}(Khy*+5>h%WH=SV`BRvMI<10Im=hbw9B8YbFfII~W%4en6>*(rzD`(tTW zj;ekX21EuT*q*oxc9=OcQ*`If&FBV?<_UEYY;n8OoL|wtJd@)U8dT< zrFlWGCy&AfDOPpTr_w#0!j1P`sbxxVQC7Luz0#?R0m|-~QpuxpAkyS{!4by zn`b-q+n!17`>tv_?AXmaj zuC4iMrxk4_i1t;w{h%(EdSYmL($xJ3HcS(#&#yxKRPcrnaGE{$KK+4J7WLI{cjDQq z{AmmA8kQxiSiL>)HtO8G06`7!=iz?Hox66tAiD|FB(1RJF&q@ev`=)i&haK!-+=KX*%3E0oQKS4;9>j=jLzFnXr4d&+Voz|3-+j9LG#B@7xF~>MT_A^9{Ac%81=y{3otrYXlWW{ddQt z5P`be(*BQ4lzZeOP$E?@A>h$R$6}y;zHDMk_d_+s_0RHPt(1L^bZ&3HY>PG!^ z13LjF`EC%qQVjeeKBXACiY=|}>~Kp=Q6e-93W<=}gIUDT=ICNdVf{CZ%tBd;a@{w? z9Q|!mZ0%IHfe?y{qC)S5U@R3)#fLUEN?BeB)&nwzNkTAM!tgZ&6rj+G<0VO-G&vi@ zO}h)Pywd__#%n5 zs9cc(3>2_0D+Ao}AUHP6squxo`v5>m0uMou*-lw-vt^)YYZsVk+h!l)hgw+!Hgp|O z(t%~j^2&JQDF*xLZVa|soC2dnTrjX^Etl2lda*P{5CY**tp(nM(!Bow0WoKyE z3JNBBQ_MFrzX&?Ef}&q{Z=S750j$O&8GohTsecB*Izwn@EF(=9#FEr<@guNDXfq5P zrd9k88>?Ya0m#y%+-W1n69LjJ_BH>vZFT4%hGg1{ghl3aLAbr%;palT%%eQ5HN!`8 z?`CuPu?@Spo6eM2_$_D`SpGp$1NNV2s6!rP z@}b$c1rdDq=ute5S^d^82t54|CtAlX4n{jXHBmYPVZ;HyuB$<jiC+DvTVa?l$cM@c@leniu2Z#US`4;e>MOZ^NZ zsL!fUYdVp7XaewU<27$1x?jLnFHT|4_VvZ(Gv#>*4B)H;fDgDGy%6#TbTr?2!M(%y z)Mg6)`Yb@_S#Cc?t^*{)vS_{qB=ux{iUmZYwxS#nv67Pz7Kmc;Q=W}LKu|j&I5nrq ziy(WEK%+DnBe6PysNx>uA)q==Kr5e+F+|-?vr{(ur$8YqH~bnMI94){F-nEvFaQr{ z?V%~Y_-hk<*c}Xo?<$*z85em@KR}Sf>ZaUs-$^cLpzX_zSrh+msISj> z+d{1y+B>WU|7^^`e+CfzHzNq^%71c^@}FBG{pWo9r=lm=!1*uP{HvRCV{=#S93dlt zUKJm=z2a0fRPV&Um@3zXj-C2@=fC|W4IjSf|5erazn&6KBf{sGj@|9hnzX|zilZv2 zqZa_^%K%^SiBG?~035=ZnP3ylL|CV&l6FfPS_DyFY`FTFTIV|h4>S1@V#Z+=a--%c z-~c;RJIw;TNZq~XhFb(8AR0XSg*&tiNL23YZiZ%sDv1ZIN*8AdQNcaX_)7mPwVHc) zJ$28xnsScW6tmRjb;=aWs-)2 zAaHopKr>?fsANRyii)Iy&@ zBcxn+yeiB3>pP%()BOl#(??N}{tWHjb(L5+bt4U2Ley}J180ENsP21UKO2VTMa!;e zy`a|rZ3gg4$Oai)hJ$=r>-L!TX-FP9l@0%$3d&$iD5DATtUc|y!Yd$ikjw~@{J<^X+!L2;S$%gvBE$3?%|D6u~ z)0obFow;XGv0zr9;+@ecroLoGfz#`38ETt7Nn^z<3CTEqyVW+C>=^%6lm**x^iIku zj^UeG1%u~l7?!&g9VXQm<;Bm#`(+OO9x6?4JKM)Ua}TP-)*_5*Z{H9Wd24wV#7wBvj;+ab^{0Hs=~EBLdIW zN*Xd$X$O*psSWCgLFzO3M_vibJD8?p`)6GweEgTFu>YMg=szrt)CKpEJg^`d7%_n3 zY1%+s_5j>HLhLy2NGOnA^^t`qMl~Z`k2NM5+E&cKtZmMV=+lR*>-|*|F;vJ96eewO ze;BV}AXhvT_KDnUz~!7-EP=YDqZe`GJs3zLVYLAwxklPCK?eiV5jc~C?1kweb`z@r zs{Ix~*#o#VMF8Kg8^z*Z$I|f)s<`Y~N5X~g9}8yh@fgk#4sGi$R87Q0>$FF4kxl%Hf4Ev8% zv|jXFx#f#94biu<3RRe!&!tMWO@sh>mI{s+U7D|gp6@g|c22)eK) zu!;fpIQ6XvQ0u|uAq4&$8QX1WRUb9sb76}r670q>2eUpI@BbOGTM{37aI6qMcZod0 zsz*<8{=&J4cU9PFEZlZeL$7hT3!~OvL8|ip#DvwRovJ@Vn*Lc zg8^zd|Ar@2yxNDRI;&b&PDmMFzx(%F*mYI9;f}1rp;!YAU@Dp7-d4-V2jrEVZxdYT zQsy?v>Wf!DG{}SAX~pbA_Yy=49Ky)F#qOzwejwYEaDd7RB>?qLyAZ_2yzk^iYF<8K zx1#Hi(C>*m5rEIY&bAp{qTo`pL;fOlU~I_)B23iO~3m>LK03025&Nw1no^G z$OV*;W|b3Ukoe#Z5how2=#u~p!76K-mj`$Cl|bT(9`j5%2)PwKmPKf2G72trF;~^} zvAGSMPSgMWA9^B@mZR!OP*?(D+6l4W36*R-6b_bdRm%R z`)E3|T*vCz8)O&(bSPZuh8xHVu=&L4A$O3C)+dXVNsG{+5Ja(EX6-YJOGy)$hp7`Q zFmO3}RKIs?y9%wURoLL1DBX3|8Zlp-Yl$V;+8!tDWT|6xQMVy)kcGxnH zKX4wh;XBTst8LAd4g={UZH%h7hiO8Q2bU^wrh6g+nFR0(tPynY#bEDv)J zMqn^9DxggKqPKCn2Tgx`)k4i*gMndwBwp9P_{Lv9UJup3&6?g|^>r=J;)jvqlm~A^ zALHcIJs!VHQOVD3mJ+l2L@x_mnOeHk#7^Zp69GY#I|FY`v_*l1IYm{7cVnQ?-h|yB5Y#FZ16&mgwQqEOGz^i`Iy&g#IEbhIYU5t1}vq51qhh^lzVLb0G0?`QZb4| zl(xSZYK>Ei(r|O#_Zz;sih%%&a$p0_Fv~Wb01;7>-^0-QsI2yIm3pj`fv4DFNj< z3tyJu!yDSh)PlM&T#v%Rl04455SwAQ*QZ=VoFl+urFRg(n!MSKkC|NDNP8bL?FvXu zG3=Oh1bWD$G=u|YenII1&UUZkDLl*s+G2wx8oqZgIg+?4)+2NxR~=4ii9b@-YPrd# zwP&FQ$B)P{0FkXU^U#lK$$iRRwZ*#WHfRm=S{V<5b0V0r`gh;P_~?ZILEr0(u=37W z3LRG|BIqeFH`Rz;HbxR2rJv15ogwOWcx9p~QDI1=eBoB4C7Zx-?%uV%EN$}DVxjtE zeFJ*h{zJZi9yL)WZa%q#2HW&bjpvz{xSzpD&NRS1a%**`2-+({GhxQprcU#^T)^cP z6N>7R2U@a~);?};8>@c*@Ksdva+r#}MkTx7D+(Z(BCBSW+rOcGvK1Yderzh>s%ZtV z>Y<-dRMXq3kfem|eONqhef&7=X&?9Bs!Dnr#F^1*Igj?aFHi>Pe*!JhGq6f$je$*b zpBJ3SdK}&L37b2AgJl8PrDu z4Y+x{$pbw3Jk&mJi!)gJec2ttZGdsfD{{Qa^TzXXAi;YEx+u{KUnp@!a8IqrE$tc9 zrm+b4o^O00BX$PuhbUQicD5?mW0@etuWTvm+Z7C2k-Hd5d>&X?%SD%|L+&8HXh8F^ zR`Pr=l)n2$fs+4%UCgyV*;qJ3rx~N4=KCwI`3KtyE z(TP3nM%0<-<=ap%)M4yIXC^ zU~@2w$uIrZ*^TLP8P>}vv>Cw>W(OU>>uym{#O)uN`}Kg5$nq0NMqK&AZTwyT9=ox( zgy8-`=Q60wM5G9zDP@=`i8YbS+#I@}LT2J#d;tic2or2d($IITC}WR7Wyi)2-S<~J z{~q8zmvl{|95rKi?cIg2_8Teq?I5tQn}12&E;cx@1;x8-B?=-7)^>>ohoTmCEy=33m5e)1g<6%C^(iaa?#`UJy>ma{nj$NBj%bwgQQDCV( z!eh=O&=W4V#y=ZgtCimT3kC$FzFBtkzDCk}vM?a-OT4A>@vb6dV|rXSSur@xxq76uI4gzc4OjY&#=WgxW-N$$g2y2%l1!-8en&*dqSr`-tY$9}odl=f(2Xi!W zHO}_Alu344YWFiG*QHp`L?o08`wB%p8G``7=C&z{KL%h^4lDcCiqQp4ZM>II-V0A` zlg?p&!YzJ(EKnSU*DkRb&Z(`t!!g3eubKCG^`&$Wo$8NCE=ftNDV4#6WY-lvCYHH1 zxOL>R&H#YxVysaUd8onGT zuiL=z-p}J!2~kT$QEk2_fj39!or-^GxX1kCXaV`>8pY;DlQn{q5{q6n7r3W9cJ;X^ z`QCd2wQ3upErF%5ILct&Fc}^JZLv3#M{xREy-HhZord-%-eyaXRC;t(LP@E7DvTtS z24WzyB*8UGvAgiclW`sWBug%}<5!a|ncMCZ+1AUnEsH7P^y9tjZCmSU-nPEm$^DUu zNsnQtpyAVKRVS^8r%X~VCj4RhooFTR6{*rmsJFK_`T5LK=DEAQWZ4fsa#8|_VDyQX z+`RUC=j8b>&Az`cye$Q-P&IoU=|rv4{R!;G0z}hEC*HhY zRZ3EV7anZ{WPLmPR-?z0_qmyStb?j8b#!zjHx1IJ?wI&?kS;o>F_U%v8ceE7_%U->&)GT(_$(=?gZ!vZdg>(@Z7of_87y1Lzwf(JppR^Ho84v6`cjCDsfa>w{Z8~GL0oaNw^ZTbd&CF^N1-nPTew^McrqUD)6 z_N&7r%j*)cDrL>R&SlNbwgNY|dYVq@F;(hYNA8SK{HZA}!MpF;P{o6V7*1tTLOMNZ zqjOF|!>Mtnyj&n|TKdVjPy0>=@}2{+OxUg(*CKy<<%686DvWF{MnShf*wCE*YAHh8 zLostTgZra}5>+Fp;dlDnv?8dZZmA0>grNlB%lwt}j^jZ z252<0zua3|Z&!k}tDsN$X_~#p9MRq$l4io2HTGn zFG^ewRR2CpZnJsG$W3RHqQQ6@wm8`<{rtOQSA@9*gq$|9MenC4u9vp|v>zv=n)!bR z_k$Dn?gM%r-sI)DG&{BZf+@#3=rk+SMSc}$GicB6iOaNkS$*;D=!}--Nw#1HOa8@p z&o|G8QzzOU%DMDy66@$&d3fTdlb>|-=doUeI+gxXK_BVi0xgx3X%S1#9qtZap9eJC zJ}@fLSrJ|&)iULC%H|)9cAWlMS1j@(bOOcCpFoYVz}@q_&fbkY)4cwX(b-n32=#Eb znq*?RQ(AZpUx$wsDM~8}0#|BV0u+}NT4ro}-vvz#?K8}%c4xcIJ<(UlEnua8-%}=C zBpM1oWd)dxA<&y}b>&ue%>E0DtFYs+6g;qeG`R7Orh!x{ujuN%rTGb7zl@jLjduys z+-pD6t!AawRLFcZeUsmyOTdL+9=7X3f_SoMck3tg$;a@Cfn?X@YHN7_odBsulP#`l zh;gF%)j?u%b&dD5gU>g`+m-h4s0q%<607vP+oQxNE7flF1Rk&#e4gCMCC@NqBhI4q zGJ9C4+_1y^R0H4VZzpmXcxd887#e-aqol_e7;%%n9BTQ!3g88QoVigpgHCIeJWjeA zg$l(S-PVb{^O>_102&smrk&t5_7a>Prcpn!cj@NUS^peW&P_2!f`{L)FH}6JX2AG% z<(zm_0I^m0W^uI{n?sSUl|V3kB#5Wft#5zYYu~X`sWLyNvF&qykB1p-hZ>zt$ayj{# zXAHC^M}2OZlTk8331Z*Wj~$(c$rE6AsFu8@OS#JXWSO`oF&SEaC0LD^vyF>&?5UXX zj>v9<{FlxH^{+Pd9Ib$W(c)iKv{661A#Y*3B{^#+@u;WTR80`g>wt822^9K3xc^fJ zNDRz$n<9&Z5E$BL^sj+52?bU<1DhJ>iaK_^a1?F713=tGhgg+C0J$^T(VJu zJayHrH5R43ZG7ZqHLsgD0{AdsZg%cWyn2qHu|lw$xKI4aE!Ahf3c~K8%zS2sot9>gA|n)~wkMy-uizL^=H@!y5!f@_$-6cg!m;5<5PgS;cKpwT zdxExet@=+}?Mh%mxq7;k(*s#;+S_M!s;_SM2X?|@h*HGh{+RQjdN&$TxJUHuruu3M zHPgxoYwD8Gg+>`y)M!xeS=69IIv~5%{KW-SI^DZAApTaLO6}1*zf^#@C@A?(# z>#dU+J5py+@9J2&XJ)%Uyy zuhppc!E5`!l%z#2ukSdosD6ZHgO0pDwdNb|Z%Zk;@+Gh(UW_EV&v7MXZ1_kZlR9&tzQ ztdt{u$g+jcu3Gi6hilx`{naPS@e6Ragpmi1!{x5zRugZ7otjeh2$4JNu>4gv^?G*M zdx9Ar4g-gW5;n&qt%mlpmCavzP}z5TghAxh&bVrVuy%s@?sCHe&y)*eM}A=(?U|=m zhXr?yh7A#Y-@SG+kG5+domjf)k{9>03j`Nz9p8bVw~z+<=M~$KLfP|0rNUSm9>I-r zK}WfGp44>ih-v6udbqE==bUa>=Rn#})4bqaBOh&o78gfoNzr>P-VfU5+HLN~S={Fm zw|xSXX5pTig=g1_yU6;}ms-Ck`uYQtervE2(2%G#!)b4VTkzi|>;2-k*~ETJ z!Nuv-gz1~j`vq;^szzm@#-%CM-2M>5jcQ(>On6Ha+08ugkT@Ayy@=2e;113()IN}P zvYyq^IXWpjxSTlQEo}eg+4-Wuln7n23x#~bFRYO_lA#-5wb~2BWI3oFwhA8Dj?=q8 zm!$GX9p*%c@p*yY>ZsF>{W@RG<3&`XZyMcuntC13hw97er0c{5Cx(c+a@56NgKF{| zLd$^s7pON^++ArnwO7laI3ubk28W*t&(36O?$U}FDk?xEGL`EvHU zirMQM{Kc}$Nr1Q3edxVMAub&hZ#iL_(PA@UUZG5?!_E5$3yj3()RvK@8>TzlBV1)v+T|;O(5J= zw(;hhp=s2$@(qbp=WqOBTi;Ye7wbPu*iEB3z+-atuio(RP`B()Rx*F-aHS3z`iN+> zj!u^AtPpdKg|mxA)`dyB481zIYC!xNTdUSLgI7BbKsx;zc&;XfQ|r~b0Ynm8QrQDz zxWi8}YM;-PT9-^`A65vG<>S|YJ*^$R`jY{A)+v@+o7rwg zo{1Tc4*)I1uK+AwpNZJInBm^)^_Ce=8F`fzUxRhUuK2trDKvC$F8`A(bfEpqVA8d(fGZ`Sv}${JZ+6u zokTK#S1gokPL`|x2B8!Sa=r42uXO2pV0?ywy-g6*5JkUxp9gMT1s3@NI)SQvpEN+X zwl)@Dv2vb!73TQae|dX+K-&4{vXL;_FYcuMbfMS%9cv^;M;sjkKgB_BAVqS^n!$6i z1t&jXqG!!F3y76_hpT;7cJ{sx$tpk-d*F6X!eJ<*b7GQ!VZQ@%pwOauH?gA+xS#zD zT+;jd#HJK>M`D&ljS8+3PRUi<6!In^a4q$dOP*QDGbFJWR&Q)(>2t9rx26Zd9;0yO zfuya$RB`mm#X3|%jV_`P`7==B{e_sdn*!pipimZJw&eZ_n2j#t7jo~=#oD3w0DK#M z1`MLV5HsL=0lcHMubm+p9My1WB$>Ve@p};TxGz7=xdS!Jue;q55Y47kD?gasu$w09O+QJo*=EE<j~ggS`^YqoSZIK zfTyt51`Gjx#yqDu+q&UrzJsGT;{fYg@3-!?Ybj75PCW#jg9bQDNst#NtERZ<#y&Mie_vt@Kz$Gp*F z+Q1Qp4HB2(hw86t3~cD|Hww}FL2|N;Y{gi}mWpg>O~1&LPeqtElg1TEp!Z^p<<$N= ztyy$k^6(xh;5HZJbSyHR1&kf9`fZ35-}#qRP{ps((Z_y##=S;vegmSG`^rz2N8c78 zaWf9O1b$*PN}t?`hBgG~YJP8zHmbWh+aPn`L^Rd3Zv_T1FuHkw0nz4^_g;S4{kZ3q+|#Lt~u)yZDc7#NcXJw z6BpiJEBo%Xbxl|vp!O~AWqybq7?E8IhvmnzhBOYMk}pvBF!7FY@9OzGbq70_nH47g zIlVpNm|aZ%cJ~BZoYzEj+6n9Prh=`Gj(ydkQ!|hZxg%V0X|iyiOEt4b_4xh|c;h?3 z#&oMoXLFr`P}ZTMb=-`(@DQx6WvZ;&a&ujfc;C{C11X`p`_VUJwKPj^>5L+^x#WQD zx`s5hmLPvGO~B#J#f&X3)$<_cyP3utdgP<2H9S{NMxRs9M(1pnIAp#qMpmF?o}{uS zVST{B>&6Y6VrXw{f^YDgd}6keAg?n z2ddsD#fJlRe73ur4Gm4iROkxp%K{g;pv@;&=i>Z5zWMfD@w;hmXKD3JBig+Ly4z(O z;ePu9m1t__#@D{_X3oU#VoE~EPqBgZ9+J-Z+vyH8=2{W$T=e^#-qXft9Zp-{bDzzL zJg2pMGhrnr%~+7yb^#1QdcqQn~=`^9jv?H zSve!vf1P{vp~+;6tS>-=VVylvp{s%f59OTAr*F6JO^V>lSzFkD*2x`6KsJ>_S~lL~ zb}Q*<-`~%Z)mc2FLyrbC)al9%+%=sz1!t?Xnc)(@ga&Os2u=$_k8ZyLwvI1jU!Dsb z6J}_8BqJKzZM3lZ%e^A}mF|c1>4|p36A>hr$3n*5FnAX{!Z6j(yxRH65RBE_{$MG+ z!@K8DpvB+bu~&ZHwa7dq9@faMOW)#tJK~|MwPy6E5UA#F&Av@M@z}}CKYSgViP4U+C|Y2H7MQRjahwr?^1nt;dFoD&I? zb3h`KbZy8IJ?*nfQr;VztAf@U@5TUk?dZFL9Zwhz+hweQX-r*FSb{XTt%F;gXDybg*9tQgKF!2!+e#BLOp4;Qlm+iIJplYsBpk@M zc3i8Kk}~B5X}$D7OotDSAt!nk>c+jdb{-P4_6;kP^l=1z8G}QgZ#qZy;aiv6GIV;u zscD*S>wUy#0k*~`&I$!pUiXx|t}$$_6J_t-L!6s|PW9M#U$~<7KYVLgPpN&splXoq z`r|u5Rqr5l^Q1|kwfdJKp$muhibRnnKLdX7W#K^f;pen^A~vvdeF5%<%hKq|)8p@% z37iwHQT@=`85>QtRE4HiKvZU!^C}#!czIT9YFMC!fOnyNg4WuXss4FxjaLve|4i9u zNo~d|M<%Pie-6#4^j zpe#IT%XwmdX6fzDo;hC_*v#&AJUxdchcP{dK1#c8 zFtED`nt9W_u>@!$a6=Yb`vrM?BHp8Cx`TDl5!%MM3BXhZ2yeWDm?UR^Z$8Bn@c z5krT|otZG*p%YkDr1OfDPisULT3#QKE;V25Ar-&nY;{h0}MYy zR%5E^GVB?-a|%hxCXFtbX~mq%e!$e|I@Cj{B5we%)mJJ9wF(=AR-DW|B1rbTX_72b z`j*VuC)GKyXp;162Kp9OG?l+P0qDHI6_z&2bbQ4Ql(k&~Aj>S17cww{U{6aMvyZdr zKW=c;hVGG3EjAJnUu~~pCwzJn#Al`$=fOrxbj$x zPifuZz~9bFD#HrzA_dE6ZDmhPWZmd81R;r|QCM0=l~Jyq@sSvJ&)a%#NBm-74)Ou$ z+0rRV(K!Y=_XXArq;yWGEXhi=2lDvM{#ZA4Hz{}4)f4n%;XX3}X1MZxJQPN1IUc5* zm(93iK3)DCgi|udkO#CFGIp^L~{Wm=OJh8sn&30h} zo6Aa}?yOB{^2H8s*2=RigXby3b<`hO^KHufyWXjGe2p zzvcN&lv6ziPL4vM*jCJ@Yc3T&O*=d_!dZLVp!gB%5jgJx*PiM1kHT+^4F2VPXGhu| z+NrqRJPeNfGz&0Gt9E=qD6_3E;KCFLDcKU-@2qv@v<#7PkF219Xf0hd&qg zZ&>2eY|SLW;ckfKvf8b&r^nHE=E3lcS~BPb?};2sX7UX9SP`|JFH{cixI#V$*qz4s zo^2Z75A@aBo7mS-qcL(=%Bio~M70SxPPUBfX1?%-l^XgOp~dJlJH0C@ag<#?XTZzH zOMHb{csY;B-5OF|co@SW^%J2a?PC{N(7hf-Uk>~7F>fq;?UfwEIzZw9z6TWB(NX`Rqwxo>nDC4UELk=x2;4Jh*i7X2($B@xX8-9-pyl5kSkT;ZCi z!_Z8ckeW+~g$p{orz0Rpph^ibE6o+BmH7Tw`+2hIciWPJ2ps~F41<6B2km9&lJvY- zsR4aps;xMQ-e8YrN%#h~+hc!CuZ+nXn7_A<9Z_M+vh~+A-(p3~k<7MT9@Q~Tcb&Q^vct4q9v52ckDTpq_VYJdCkOl(fs_$3FFzt*ar#KQ6xW{WUdhYxpWpio*J(Kl3ynmNHF!b% z0Kc3jt4y4jC6D1JGR^y11}^z($DCq0wv+es`lYU0e# zNUVh{w#N5`+mz0dmj`7k?lmSUDR}h+Y~-g&up}64KGJ+UZI)LcAP#o;aXEtz1Go~#w{gRMU!E&hzZ0s^>&WwU9ksvZ0 zPQSn5kiKmra{;9D)LP$uwN_l(Dx`xgM9l=~X9BtA_J> z`VM(C9>{L>SZ|WEj;K%CDSyv#037#xmHy}z`?-$a5kM=6 z{5ux;Mw^_!hpSomifh2kI_viv$Jsu=CmP=*YH9uzlo-TcSFzHrNi?X6lkT@|t{Ng- z_PRL_C7kIJ5FDSgdSdg8*eLcN_g{ywKPNT+pglfEWtD%!;w=0c6qWt&4;qWWl?b7* z?y*(6Vn#`2w;|5pk|#^WeRXqR|%ILS&jg6k2Y%(#OsmL;Th9JeGzRg5))4 z7aLBoK6`Id6)e5Q$adyp8NjtQMOhm_2FqeNbqs2*EMvxxN4Nm7)H^pcpmW0L4tsG8ilnMDYD3i@MX7X`XSt`Ivu?B;9b zyqZ#3tsz3`DWoATIWXDFy;z2Ih;^gqU26^v*j}H5%ji5#cKrlqEzLe0IpqbOCMsm> zC+xX=XC+l0IPqisdd0Gzkakbm*BMODBax` zA{i&FV4+F9AJ}gOGMKXt?PH+Y{{nhhR;6|E2hBN2KElw@lqEwx()Ji%+~qXIKe2w> zLyS(-I#2gVkZ32)QA;B<-}*pIrFb4kHsCC(1D55A(SBfxR(DF+xcLqE5iX)UO#6JY zg8P;4$E?Mhee>Zre&hB{D;iYq5 zGPFDWC-3GX+x?ze=EA|`+}W&3jBbBq2I*c?KCoMV0i*SH6##vfzW}$=cT2hIy#OUgLf8A2F3h{I2R#ZR z@D3myn(8LvFWu-x{iQ*6WvCY5f)M*p;O}&R{r*+}kI_74*f^<~z$*<_fnX&@b3UZo z!gdZZH3qtS@@{rc1kFBiM);`gHNfiOn5nbB(NOLH4CHB`&2dh+I;$1=Bfg55JH*5U z2grw7c}&L`ulc6EhJRt-Opiy-ho?B$-vJuUFVTCBOV5701E)p#B{wZUe7Rr-!WI%s6Yc(cj@NN1sS)+M`gE(a3*9Qt0O)C{z`*l-OMr z=a}C^B<^x+yNwI-3|GFqmOI13V_}nifqOOOR#t}6HEF!Wri4XLinL$YN2Ta~w=O*% z^B!phZ9@jN9sfD?Ije?BuGi5q6>>qC*|rKR@vfb$b8o`gLNF{j-{yBPgQ%(*yMXyIn|o^uo$m-laF)?-Dy{o=;{{ z|At&CQT(JwMR0KNcx$x!;S-d!ckkZK5y0bknB<}u7-hmjLRggFL`O%%hq&kZFA|r9 z(fVu5r+drL7?*+ix91soc{L7`Qe8cC?A&Q}b@lDIi?=MVlAOBy?#W=0)xfgrsZ*zP zdjwn-zy7R!&d9*P&>tT_FY*2>bG-XUf5jEmoM2%u&zYg};)90|H>|s}Yvzk(sA5vu z!dX+D7e)y&J~oW#Z322@v4xXSzsh}dboAqz)o?{=jF1O^zw2 zpE-TnpzvK!<%2-x3;ZsNNjsI!YePd(7X;MxNnA z^&GuI=L%N!KZ$y&aIV&JRVhhQc*2-Q2yC&#qk* zQT%7Hl$%XGTZf{>*3RzOz^BK-LZ<8UzaBa|I#R4*^eSC@m!{g-yH{X_Jk)nzc!@Q_ z(wj7gJjAHyykB)k&vewAUx?*LcWlxKx%0AFbZ1hmNd+^>_5Mhd$kN%!&4s>wY4SDT zS7^^_u)NSwx7FX8FUG327G;(;o!93Ru#4DRh5r4;^6^4sMSe8GJE1w4%PvckC(E4| zKGx@Z5oVY%J$F(#^sA`UK8pE1!atn3h*?|8FzQH^r&)y;(QqGH{WE~~U{%dHx7xy4 z{en?GZqV>ThBZw8hs8D{q}SB5W9mJ++WLO?6(oAZ!u;^*8yL`LqetVq)bNC|9+s_b zWc@t+{{6dJ+Z-l4z3jHLo127o_{-zxnZCMFg0&?>R;QGt1CasDgl}FyVL{xo+}+vc z4^XI8YKs$Wy{itZ=}Vk|i{Xm@l+%bu50y@2Gq1~{ac^q8r#Z*2*Y0L~J^v@`q5cZ_ zmN53#Mw&a0vb%S)aRJ*0zEMX+pg*);PbiZQ3} z3fc@;xb)5rmA9g|=f9k4ok$a@Kcmc>g6qoGy+fTpRAPHiSY=v0MfO7FP<=h)@AZYT zA90*$PnEB!xF_RZ19QvJ_$Yj?anqT~YS+2y9Rh~*dbWP`xE`lbuKh(yG%`+ZjrEhNzm0H!pFx)=)p^&1v1odrCZMycQCw1?)TU`_|vkwsa`u9 z5+wCFr!Orn*P8R_#ot}8E-H(|cV~4rrZ1%ubCahTBUrM!_4yL``_rC;F+C|xCodyxog92^1Ght@v18tpB^c` z%bro!_b?e=?K6vi`DdV*^LKxt1;GOoFM+nKbX{4V6BDiQAwmV5MhtntiwahhPLXBg z#wl%KVd1xp{uj6#D%z5yf-`p(GB4kIbs|sW72%7nl{%e{G47OA=K|P=y!x>(BwRlZ zd9zCLp>Zl99x5LU`x|R(@9rwvNP(eS5t^&s^$f8B^Lx|hXvY1IjI0J%^{Xm|C0@GZ z@ay06A@$1_ym|8`o5XvnNmtXBFgX&guxFQ)`BLTM-{)6?fpd*M7B4C)%GQl;4q1?YRYxGZ~N_@tpjHv=|qH%Jv>)uP{Ky8gzrz){LYNyC0#Zx{rceX{B z4d=MVK09<=%Q7~_b5_p8;ZHx$Z#jMD&h7hxDh4id!-5!-N?cUNlLuKgU1^G$xbV}n z4^b!vm4n{!f4ZDwD7mawmD`FkcYV_D81HOtWE~bv5sp(;FkzWiP{B}l()4C%n>w?G zDBxA6KWe3Fer#5qXkK2u#Jf-ZUf#q-0Qn+C|DrSF`f{%*p5J$JxP^9Hh8Fp|{ zwovTtpAEv+Qd^>xZ(5mP)U#MV`$-J}8uQIaZnMm8<=C^znAD?ZnZ$Z;IsVqD)I(oW z%_5~&&MJ`AeV{6nSMsPkGZJgD%1`ux|8~mh9B{>X{A!iwJMkG$`Y$SDuA@m+-rljO zjY)o@rdev2>_S6+Sj*^GgAidl@#?_m>hItVGsAwA*cw&(nDJe z-9$$VvwWObQHkak; zK=1i ze3Ge}T7rX4`ba8-Lo=I7;1-f(!Z}?-^Y;&EdH)NI&t|hN<{SjaS;^igZ6<) zvGZqMj$Ga;nb#v1d6Bw7lRNHh+ zvMV?;HBQxQ;>u}hF#;}1q}2RL@x>NB9L=^FsyZ+|&}m6psV6C~i@=2`&(D+hrp42( zC}f=oHEWA&icWe)7aVc;L!ejH$mE=uPPVOc-p0^T{hr${v7bqdZ4z za>`9xFY2bQjd(3O|S;Ul3b=-h1w|=2=fS%LAd)X-`BCsb$1yHs*)%%lg9XqN)dBW%t`6xg|`$ zMI@9bVY0E%%^kY^sx7kE*i8s2I?A2quFf&u(B(v-hAFg=n4LN(qG9lk?P%S#w%05S z3cB+Z6&1tNeQr~+PKGy02cs`Da};$&JcSU$zoY&p9T#+;mSei@kumAQIxat&)CQAH zCoVoD9MwzJ^4B!qK#zYVt5gxz?-ph3bhccmy=kqtURI|9$%N6$9kLxtZN(vHRSO}D z9Ri;7ExK?8o!u3?=u5Wuu`B#tG=ye&y)4r2IXge~3(Z4YbF};}@+{jm1u@=;N~%$l zJu8}K9Rp#vS6064khkh_HlDk$${wWHF0!F_Rb_=`a*ic;ji0=6!HI*TtF15{t3Pg? z_#<5*;hA1WS!5ZXwqX?IJ@h4rE;DFpIRbM`1vtywi_F7fi2aHkUdqSys)}v!JY`y~7u~FU- zcbp@XmNkTlGGYbMgyk7RDULXcsV0|arQ9eTlZ_K9cbHMKkBEk(=%9V9;ZnJwIT}OU zGtceJ{Nmrd6RszR6AW=jn}q*mb{fz1gv;Es5U!N9#KgzP^DQflHPtuLv_aSxM!~K% zJ=K{a*Gcp)g8%+yq~&a1Yk zXf^p!@hM4%T&0tXzEvnCsM>NByC{clQ_U*wk}dh-b1GbTcS~7+K)cv_29Igbwt%#2 z_LWtZwtc>FQxY2AJ*j4s5~gAv+i&WNdtiR-ao52!IJAnl8{}&q72r8Mx2H`#J*mwr zL|3>IwY)s$Yf1RODj7AWuVsJ!DCo92J)2~Hcxax>q?zG4$*Jx(NTAa1eLs2W*7TT4 zOCzKMGu|t(D9VoGAT(w!O8KmTy)+v9V&aPyWV}zCWtVmSD2wZbP}s!k0XmarlS(k4M`kjTh|4Bl}Aa$N61ss zM5F{W6{lL&RHjq4&^#91vOfH*B{yG&hgIPEGs27##r@v2-yhIg)4%^xnw3`dVYaz$ zZmG>hl zb>Fay_TNoSTUh978$632;uiHYr0J;tn)#%J)?BlaFLF(8~RFj={Lii-Ec0lF1E$ZRsbinvX`rH%M!JU5>(gUl_=qulXN2RpSY zUxs#WSrx9*YkQ@~Zt~lOrS;IGcT*q4F3$x-mGa4iu@T%cVX>^|REG4k{FEb3X9s^2 z6({Gy;6ot1_sGT;D)q=7%FSpgEIXC%#k}8Ffw~cdHz!djS~A74y!7h|Cr7=Orxmr; zNpkskJX3Hu-E<{J>K#B^UPvHbGaxOk>6XfQ@%w&RBq$W>Cl&#!M4o*#J9bkwP2sM) zhliD}F8qeJb`S45`aUXMO)fj(a#)%a>(=)6%0Xx%3Z>_VSZao=8;cX2+~{oGa<%s~ z!W_44hC{$i%=DpE=X*XqR^EVQZ{fYZCz4cDRCV9KKX`TG!lKrM^^H$S5bor;wa`em6U)|z zH;G5$h1}fhh~-IE%hH-$;o#ulvY(W7K`_f}HOi?EW~^)Xf9uW85J}-}(*X~6zz8Qp zMyyy=z4ZN#Qnn7?wmT#_HkF=#uRKa*Ko!GhFMe6jH7P!vO|xTEC}kxBbAA77gKG!9 zZJ+%5e5lgF-ac3*T}gA0S-Z%RY!%^8;IO1?dPlU*bclompA2>j>!Fk)n;s4b-woDI zIB-RPvd&z1H8)s#7LVk9aM1p8g-)|*QzfG&7OC+*ocYHH&q~?f9+haNd+2A@PrWnM zCE>IzzdYeHo;H2|Z^}#1%!smWA@}umoY1Mu0_`nb>B^01-@LuOA8C`MKo%Cpb6*LK z-#DqQUfaH|E2dOi$>3?rc~|N4Hhc=-{rh)b?7H=Ksy_X&eyw-DSzC5E)*+pLk6eueWwPpQz}JD_ST=uqzt0Eh)X=Dnwo*Ur|} z0Z!dA2E>WBUE(occ2)noJe*G+^v<~sfFm-`J|P)gk>@tSH#RnQYq?WC-|>&u5&;~G zRO5T<3CKb_lBCXh2<$(^;F^@_nE=3)q#(_2-G!%A=sk$OCFi-j?X*OIxKtWqjFlyb z9-APD115n@u~}Kyt?}Ij`bxL8Fu=xSlBGi#lv8B$jT%Wy9sfklG6?{fLR5VcCL1f9ER9dSVe88k(cWkH5v4+iD+-6!MjMw{>z&{{7H*?A;(Z8vggrJ zw0T!h2fa01Z?ai@;(*@P$AQZzRM8%ef%MUTX%8X8ygyLA-H8u1&te&$%y_kS zt4gZulbEz*WB!gP5r=8w6Z@C_#{t%Vp;!9vNB;-$f)oczr{qb(`ue(L|EI_GSgb1W zi&r483j(~np)Awq6ixH|uP@2QO+i8YXmhDJfl@fVkL%95c1voojo}oI|@v*2k7LbD-QQCsVul0R^WX8-Q$~qA!m%0a`GD zWI>oVfLhhsiF;llr;k_Ji6lX_qI7;Eo&Zrc{O8F z?YW%eEeMEwhD0ldq=Ny&=J%r z8llYajAXs{RD5i+iA;+}s%;$1u^GwXKpThLahetDO%bgFlxX_sU4Te-qY9BM431F_ zDOqAWBp6s;T+A=?Hz%U>9GOnbt_D=D(`dKyZ-BAHKx&VxFk#pujnbYcfU!55Ma{VQP3`KRUL$KKSpX(n)`LNA4;B>i+znZ3HcZF?$qzd`d6wDkBygP0R^2T@emHYF~yC;-^xCYRjnF-*| zvK}nj{;VOYWYeo~{xqRm<=(e7sP*=4=Qxe{#o|-@Bjzg)vqkm17M4dHuu;#DKQKAE z*ip(!l5~X@J@1Pu=7ZgS50wKQ0qpUQ7An3oJ>C`jKfNBtK!##9FVZuqdrf9H-A=dU zie=4(XprCnxZ^kWwGbqJsdC>)f|*Cwytuf?Me+s^tB|+nImLU0oMv?v*4BF3vCK(ZITMJy7MEv~ni;x~$6urISPr~tGc2Gt^0y;T$1;$kSXo2fKEB?QaF1j=m&tzeTbWFr>)(|D@zI3? zfI1+D^ z(c_hWzD07|EsWkz*U0BvyMR_eBv{o>_IB@wk2_PyL5iX4qnIe(1oEdbofPu~jcQL1prGE8 zvU3BEfC#~Csx~J5u$TTeLWSr<)I7a>oWPISwcmZ*E+A6607SMs;MZKP z%hQzMed)@~Lmw3mdBY2ZgN7}lxYBt+6qvTP{*}GN7xGe*<09lW)B?^5#We+!pwV&N zss{YEBJc)DC}iuAUxe?!bc8(mM;j3Inaqs7zR=VrfK41Msz*FG*+th=8Dzp_!5!(B z|4W-spoD=m6Fe#j4V{4KQ?UpW=3-96!=0;DMT+fs0x~6n3q~#Lqa`OONQ^-%B6{zh z$qEj8gVi1)l1b}&aN}NChzb!oUg3!|8=67K6*(&!&29F81>{qDISo_FXU#!Jp$hO7 zZI*U1g}(ksk82iSMR*g?(=dPzg<_2bC@P&&yZdt((I$15ovl)9qBKO0KoW7Bf-L~L zMFzs|B;aNc^kHZSgjL8P8ASAf%WKEIDfC+SI~rj<`Hu-Pf^KTUU6y4K*_6JcyfIp+ zKlS2?Jq=|`B;=Bm}w(tGMV z(ue|QoDM3;(iwyVTpvoGLtb(Xkfg#Fh3svWXnu;wKOa#V?^u;G9^J@U-MdCK;|xW zQ|!OBRAei04X|4{3Ok#7AWB?b1R!5+?FRan97y!uwJ3w))PQvNbFt;`!O{YUx#6L~ zLOy;XBBCEfq$);~7wi2`1W7U6Ca5BCL?2a)=`6jpbN6eeyJSv02 zJH>Wc<;c|B9E}N0p>Au*CKGYZtCN>-*R8=kjsE$=4oMecFsUI5c9KWvf3OBtFN_e= z4ZRi?5D;*6BK^WU)mUKdIpaME>9SEbrH0++1o;5WqY`r0m`lV_byB?i`_Kg9n%&Ln zT{nwh?+8Gi?~-_oc!I0DxEOixby`o@NDVxm!I;|)+u37dfRFS|Am!rB$K z&#cS%zJgTHZ_y7AAOE}FerluoMFEIjp=W|g_JJk`w78cyHm2f76#ndAf_g4X{CCi6 zYS>Zq<(}JR2M->co-*(7%hW9BFO%NCPUNH65`{OAhav78i!<#xvXNYbUo$i5%v})> zU}s_eW{C{uw@#w&pm7ZG?u)(%vfQ?c$S1cI+OWUN&CCA@c_odBeuC{l&c{)ya?wnrq@?)|zZ^yod1QDi z*a6fWotjE52K^}hpDdf|u2It_0AdkI@ zFxrm7Aa_0g_f@YPLlqb|ebgorm-rJ5YP4b{=KXJK9$8YL)F7E?TfSphcLcL}f<{PZ zP8nnb#ag%jUUHSna^*9){AY{xjPi^Va2 z;bzbQF&ra!0SWQ?D&2~^a`g3ebaWi1Q!j&L zx#Pp#1AT=S;h(Gqy2Die7Gj*?i7rIlMCKzhDIoQx10{4f9??uEUU>Gmvl-@5?P7;K zL3rHNR%|uE1U@{_Y1l=gn3Unf-}{Uh;sv*6(mpa{0?|0D&JmBGL11#nj&EtLUj z5(9{BCef_Z1qB7c5WU@pnujY0-t+J$*ak~gx&4lyRV?VE^fv!|%Eo5y2p%924j@Lt z5L4-827nzaux1cBeNtp6LcxIf#_}@B(5*VI-pXp^ZoC=Pi}2m=nt5JYd0jpeJ;nK% zVR`2G(E{{+6iVbXDGE#Sb>3+6<*QfC2%-WG;O+GppD0Q(xnnSD^ZWua1ybKtTo|HT z?u;6kZMw8)8$V|7l>=I?E=?=E#T?Jdz%L)~(=;*QKk=kSbqYrrt2?=qzty*5V zVfdmeLsbS!Ino+;l?!&nFJEg1j!X>;V|w_}M(XG46WJF4qSOp3=Y3t7ScyAssoWPQ z5LVY9WY2LUqdI4(08M@-JomcG&Tn7dmjdUe&iN<13k7~?YUH>3@6puXXC3!6EEjI zj29@SRLwVjM?{IYH=Uy;egFN6os1H{aPXj}4w8yVegNw?*&HeZ?v>G~;awo2O8`1C z*=D4QY!w!I@HrMGF{qg^LgtdUn9whui%at8O;z^YGt%oMlrIUfc2Dfc!vjvcFh4J_ z)@Z4t1*wI=Eo!22k*CC5pn1m(fOhL64a_Rr1k+8`vz}=;b{&fKQ8qD^&l4G(BvZ5t zQzYYLMdP@t7-4i;y*mHEB%pj(In>fFe|sx&f#$UQ&gMkEd;{mjf9eBJ-X-))AWGVo zeIOI|e?lPYnjmplVatWF$`ewfK0lxt3G%gU|57M+U^r(Lg$)u4Dsy2F^DwK(9e z+sL^sRDf0m$%o%*M1Y9$RjCPW;UC0od4jIx5WJKDX;C)vr~nPWi^530!!Ua}t|%Lz zEN|6=-tdJ8710gFzZs`rY$1uIKt>H(M z)-a%q$F94nbLa;N2?~ei*N+4f^A$*A?HG**(-IaLZg$}|?|3f1cK_9*ni0azbjZF` zCjA*-vYy`?F1xQHM^BU(dY&xQRm6AUUwXkiMzkZUQdIu#SKIVX0(YtXT9vblap>Wy z&PuDpN${szA`lGTQ>YvX?A8q_{hN>iW%ne-cXV8!jg zF!%6}L8VWK?O4N%NFKxv2$hb|>Yiy<5RVmWrGgRK!x*&+&5;B_D)7q7VMkof*~OJ` z&mazu!%Yj;94rcHuMZAYGdKEgvnVtB2lG^OhL5Oa27}kC&&X!p<>$C}cqRPih#^MSYZZe2LD64Jy zAeyaoX*{%{Ra?rpV|a1;$34wr*W{9tk~cPgTx|ssfmoZ~Lpg#<+S{2sMuV&PxA246 zk?(d_k0X4?@akGJD*eaSSLagHY@|)VKImqVn z$sJlm&hk`^)a-Z|_g(RdyzW|Z;vzve6UXm1d2^MA~#rI9Fq5ZGwO@*#E_GKN!RX5cN;Sl#{qi#BkVSK z;-jRb6o_Z}6|Yoit~xVVx@3VZzWe+z*=OMiP^Sk#Le>K7)!~|m3}jb^H@1}XjaS+q zYOT-9+d4W1)_xIx`~3OyOx?mO-Dq}=wAahHff1@57MI0J=93q1ceXkG>7{tM>uV%| zOCC9Y;NuG~>1=@w2KCvByln6RlKqc}fCUJsvl>8)a7N!wV*w-gXLKOVQB?{brZqc! zJP6d4(l9giG~+H>$oo=tM1BHRfJrvu^b{duEGrAN19=-O^cFyJeKMip&y^?rJ;1Jh z-~hw;f)Q}$o%mp8r;t0vHci`ZYYcwT+VAi<{xJf! zvOWQ=TAzFuETA97d=RM`t?%&)Pf+U{FXfi#9?~$6K88AQ;6NffCoSzy6Bj-^jX`TG z583KP;TSpFB*VsEpka~WqTrAUr#gJ-(Nnnx!h~FDlU)LBts>LUoPtEfI6EdB|gl{Hlb`MBg}_W zuOi_7PFS0+aY}%aoF`a)gOfzGV93w(e6mI_EPiEG&lw33dM+VxB$z{oG^FZzK7!7LzyU!+!dlGHIdNL}O|$3YnPvta*BawGApgQ{>}0fYt~E2DACy1@0s;!>+`E zAb;B9TL(*QZL@P&3m|afg4&#>*gM5B!d`l-vqP9-zsrJJNGTI$@%NuU69sx6CcQt2 z7Vqa|bJ?uQHcf`=X6u~{_(HNOwcXG${uuSWk+Gua#$?XvjPtDT$A!u8XjGq)A0FH^>$OX z1DKJIe z+FiXV+suwKY8J8}yOB{q-VW48MLB z?6Z(1;gomQ5)Q!>M~?rLqIFO8AVM|x7AGMZ1gKLUWD&sd^_Q{29epnn(ZUBD2LD7$ z22DU)j)R(M^w+Q2qp-vf`4n6r830=3$9sw)N@87a#3I$cb9tNm$wo#hwo|)f>KZ3{ zz@5pB^s=90OcgSEp{%M5fS$V!A%UF`BHu+SR)9!ngmh{YDu$5+1&N`)2GZ|UP3M1E zT3EOd?gf<}j|WTgjK2olh3u6Mu;_3=BP2C6DAmcqnCb7e=lZ65`=gn~`qSWEf(TVY zL9K#sGl#Z()^8Fd0m3q0kxg&U0*e0(6%`xMgCWwPtiF|%mBU3)b1?%L4V9p)%CO56 zb)`N;&l8YVcI4;_Q(t`b>Qw`QHO;OigoPe(vmhK6(*mS4r`h+IKp_vp>-cz@{YE$t z0dno#=WkEZ#J~GkDcSw;QrQn>&oIw>4J|%2o>xJ;9t`-*eLf>Z#Aru5)bNRY6uJ}w~F1G)c@97;Dk21%A``3C}o(k^lL=~TfP}9=ZL*(gB#gVb z*q&m#z_AslDR9Xus=Jfu!$qeyB~&kjXX1ha)M+8c-r7be;T|kLDpy`eYPe~ zHAhbn^E7eK7aHk8zV1!7*YX-RpKzc0g23&O6~I@+ip|=rSJ=R~sX?I<15m=bW%3FN z3Yj_2!o$PsSInQ0)%TBIftE;}4EBZ(+J?;5Y>eq@p?D#w$}jz)GPDU~!*=*?c9+F* zUyf@W!CnyzG7S=%@W6-TDs3iAD*p)ZRvfGkzhKlTCkhpd#{V_Gk{r& zS|l*lX+0MTS>O@hCOPy>7A(K%Zk6j1*4OKBmP4iX5}c5!Ccf}0gV2s|6Ruj=-_pUf z)08tLt9(_0mwwD^>;5W@5C`JwAT(^iSxz(sOCYwlH_yoROu#(ARxH3*kNt+^w~l*H z=~a`dJiw$>*~hF@C2PCzHAwM6j1XtzgE%bRFyt1%X|Uz-Ya`+mh_0Jn96gJ$^-2k% zhlVaHv=ZEa*5ANka$u>@>A;fob{0D>0!@qXqaIXeIyJ{xTYJiQqd&`z z*tf1=m%5kc6G%+QqLvv!1&oVYjvhNgY(c5W2FPLmz8r-&+$g|I-9MsDbjq=HnVyM5 zuY&TA5LCY}B4E6Pr71;`n&WtOUEs-!HzkGP>iIKBNygss|Ne#+K{}|CC%;4&78U|w zX1)O}ERqQ$gaH)Ld;H0Vx_@?PJ3$fZ)Nd>4?>F-&njsyP3tx3qxY%z1ikIXt)1g#g zDhCxQK=e(7!`=ntE?euiF)dX3Jwv`KoAju+eJo&0qXwqo#ym3i>+fp z&!O=>PVgEiHiKgCI5opw71UFUux0d|f2{WlSpAd@bC>Ak{;qe(x_bnq`jZdfJ;6IFNC{IfVWfuAd?wnsNL+xk*QHk_p{lAdREo5`YJOOq{n}N55BGd z98OMrF^Hj?R{_D6fu+xI^QMab*!fE!66wg=K{U9atZKc4;f5xksZtzE{Tjtyp8Vv; zZj}L8K*>izVkp?H;9;d~I7cj$(|;Ki0v+tTTpL*^<-G5Mpw;If{JO264ZmlgfD}B4 zTm3Q^?K~uTM+G^eoc{+sQ-nsY!BaM^Pm=z-u=zsiZ3ShxZs2GM!GD?Ry69pkhB=#0 zTBfXGB3X2Mb#aXI?KpUdJPsQ`t(7op?6&syonzNAoH}N$Q7huOj8_JVmAwC&m}}vi z5{lCE4qzi*-f{dL0Q`zn!@CQNn)$|y0OsR8E(;u?*xj%nFL>}L$@Qg@5`+A$?KJJI zig(m}V$U*9oHepKA;L2prTs8ckXMXYDU;Kx-s#Sq09*hY9k{H zi4x<-tFx7oEpMhJ48)?av#nR6WiuFa3l=1f%P2j@m#RFLw3m6-j zB{pRjv^a787RZPZy!;>O5uqRyGQ;dIrOM3Xc+PDG55kEcdNU3dm6F#mmF3^lY%8ZcxXf&! z&K}Nqtw#zn*efAcQH^zUk&fQgIQ!9~M+0Tf?}kNoGRaiAyD)B&WF}i0I8}HK*`B!L#|}aH?WODeO~Be`i-|I>Z_+}YxNkOV`58~in+(P8R*6eq(0Z0 z1@BRrww!s?D_&IuLoOjkU(|IWcPQqxTbtDt49Q}L8o(N>06lGB!M$Nhmx6`_lgEGZk+HNZFn9&Qb+H}D_C8d z?k8dGZzQB5BKq?mF=05_EF;G#Yuo2I>4+Ci+%8ccv{3!k&R34scaUSom1HjA6;JSJ@+VGM7l8P_^zhZO3nQ0Y8b?F zV$N9*aYzDqk3KHMxMh1InYa#6)%zN$pKHK`O>>&1%0_+VTkO4QCki9%fEX_t$$CFe6VV_u(QQxS7@ zAJ*4<9b2*Pv{_y{5`5S%x+%xrE9}oekyL`=?ji3nasMuv>Z${E#Sgw5F=6eqpX`+B zB+L)c&si*P)qA1yi`wuC6B$eMze~Sn`8-^_e)C(vTJVc^ZA?c=;^p?_mIzZ09J4-X z{1MPJJY&F@C&enoBk_{GxFYv)2|(~~4>tp0)H=|m@f2*Gi=Pu1)ze-@fA~Lyw_fTB zImhzRQO;VWJR=D$!|pV9Xr_(#ho47gL%%X;^Q>&$V1Bg{8R(<|l%h`b0dS1}5J|CL z`!(J~J@NgIw%zWqpw!qT#aJ0ZWr>F*;Mc^rSP9PEjj1S?q1H>gUz^T0F4GYGD~`ep z!qoaVl~hz4APaE~IwiwRM8iA3_}m66Rg849XDDR=d`)dkQ0$6ehn62z{kamJRbcn; z5f`Jug4zzVZ1pVv^y%ip?t)QO{pH~*v~rn8$6eCdktL|k41QV1gmQ2y7Wi)89Kz|A z_TtrMOq!?!qfyGYAlAc`urC?PkoDK{b6<&!<5uZPoaJiY;)R8qSHY3vD7;yek^%s? zlh7UpiSKf8ZP;bu?d9NMiIZ$rTIYk+Uc#@Pxn;9fuh=^}7P)@&uBvocSI-n<*?2sB zrO#tMBxZfHET+^>TNtD*Tg~}dQep3W+hE1~0b8LkGg4ix_nz{GC~P%gte5kbL09K) zNQN?DQFZ-JHe0yA_(b5u#EC2~{URZzh5s1CkzSd0RxM3g2sV=TJ6mbha(HOPkznb? zqMnzZ)E4kVfUG%D;?%@s>pIPs??YcY2g!MPuU;M22cU?7T8ar}*_)fN3@EAmYQo{9 zY*kFyWfCTE#CBlg1;gaVHUIp>t%G$R$V4KUg)F7MAlK3-_55H^{qjBfqFT!~bD}h| zfcN+juAUyG_dWV{+vqCzu?jU0F9@sD!!8X-^>>*G@fg`QtE4laOE{m)sr@V1GufiA zfN*nfx7g#$zsk~0z_cDMb%YT8h71?chmVJ&SJ&3Ukv$U-9t1XA>K-|bQF|JF+s4zR z7=gGG&9#cl5NsRMVO2RD^hmAIvOUl9?8k?Hs?Yo@9>fkH_Pbm;x%M02 zwR$pgaLEZwn#XN^e7x^Ud>YwvzbCsQEtWH@thj&S>k7 z>&E2hy}!hx(w(J8X0lnzojN|)*E#8xDieP<=nj8#R`l)h;h}PytT=cBTd6a_-Y|VcnvWP9#LqdU@Zb%bwa&n3k@+gC_SKM{=Hw)Ai2Y{FYcT~*rcP|}m%M%3n ze0?eG3kEoTGo&JnK(Ce+H$auqN)c>EB?H@V7^%iVcI*!CapIDINYyk}b+R>%rLQ8t z1&blFTdxsHA1R5X>MhqMK9DWxj#0#9uY24OY4Ex&TjsQ*Fx(ody;!NPAbRg|3&`ci z2O~SV48Jinh2F_2J!F8z@tFL{An*8K%E+SFtR1tnt>o70WE=nZ0^pf4LFNdwi)xW3 zXTM6)vnkUzgr_eI;sfFJ3j*&?Opom9Q+{LeC6qZ-r+CmYt0H`i10_@t6LQiJv=_qdx<5_|Y>Q z!xOCHy}QxBjFRt1le!j}V$GlYcvgyN!wd}1uiiR*v)1CqNn`5M1|9~pbt%KUR~&O{ zqbEW}7JDU~xlQ%?BGMit%j{Yhs(=oD&#SFwcYMPXI;S)-Wk>2VL{_fM2?g67uZhTS z(-M1!v#g=kR;T*`m3pNDWp28})(4v^gtMEzv`@XNh-WZ%za_=HZiy7?)p5=;HhlNk zhuWYc|DUM4q5Y9tP|%4;r|YPQXyF5n{}uXF07V-uFJ|nM4WE++{i-}R8dMolp`8(S z^HI;e$+mc#;kL#`3IFZ3H@OJG#;5?*C|oZDzap0ix&<@7MER)PU#h7zp5Bs+fnv-GH8Yo zEedH>;DX1SYV1 zuFbTu?X9hE0}z!n0O91Gv2Gs1+$IGyoDu(ZpePF!CDks;5Z~ zdcz|E(ArKj1??XnvP;e2&Wv#zvYbOQz{{uF`&R)W!B1c@>hziFV@Fvb?k`%`f`W%B zX;>_EL`1aka|qp9u5;*T|IXvT1S%DZ9!%4nOe^)XFRw!33|Th`sGm%b>X@*{R)TQ( z{+0kJThKn__w<7}&3nC(nG3#n9WJ5fLGd8A{82%{uV23s!q0PRWj`-mfv=o-h((Yq za^y8VD7M0L0)6~14~)N4$xxBvoNA3Wvn`LZLd0@;5_ zr`h|w*XZaHki83?1(lJ`HX@lFuGD6F5JPG5ECAq3yJYzNXbl|)3A|a2Mwf=iTT{Q# ze)jU_{#Gx^_ewm;Ix~hrLRJ@HCMv(*L2C_mr1%=-C+uoe z)2DTn#NSh0m@~_`|CXYxFdlF`oiYchFiw3AWB^x2U`mxDp^dG`K0K7{fDWvR_ei4& z>@#CG^vo75Q}@kIMzTDt6+pM)P+QVK;DsM(i0}OX&i+e_5B~=dx;GujTXlc`{yjcg z|29bB(_=OLMPnIGAYDRUm^axhp{c8vH@|hO!l2k3A)-m)4mdPMh(Us&%t^NgmXO4q2O~R`<;hDv zSxK^i0zp03o?oVscbTWv*qUk!Y~wEs#*s@)1q$>nCt4RSaygElK3e7_+@JqNdF)U6 zO2*c;a89ZCxOnNl(K)H7F`CKdrDwQ+x&nH{d}W(j*j21*9?z=UOcDI7bTC<^HbyB& z{Qd<#;|~W9>)fb*)t{Y)#;tq?Y|qFNs6=sBl5AszQ16q)LxBN1{ z*jqN(CN)DnhXX`$2px+v$N}Xx|3TOkEaTiYHCa*`AcPs_5@|*`rxW2 z06N4b=z&6gxr#Lg5$&G4svt6lDO#ESG3Iv=w#J05RYCzpjP8MQM;TDLC&R98$Zn}p zza}qFlx*0WHbHokgx-9wSma{b0>YCrv5FS80OO{e%?G-EbGLK7Xs8?HC#+k1SzY0Jm^x%$)r#Y57#ag5^gPir!AzX9SkGC&g;Bg2@lKjO$-{`?v zi065yFbol|*ze!VOOxyaI9mH5TxifOUH(d#HAK}bh{zoZ>y7kmjxE5xl?HnM2Xk*7Q04lqi%vvZ zK|x9-6i`A!B&1PLN~A%Ugi3>SH)0}4he!)ZOqxZv3P>v*!XyRh?l|LPt$pvgd;j)6 z``ml}xbq)aESZxzzxR9JF`n_nHuxKT2Rd9D?3k*bq>1R*vvM2cVY>ojqruDcjhY~M zzxvYW9WjdrMso9F)~h5IDAyjHIgYrYE#~-vO8h`AE?&=QCI2U)O@(>oTPHmdD)y_@ zds~8UeB={6m*rkGwr{NKiIm(M@R=A0b^o*7;LAFm7{;L_4ppp8HX=w3e073}^I*^^ zPa}$e$&ykvKh5?gI1xFjo1-{mK@9LiZCn=s=8ZD z)?yUqIS%>SPJuXs{MuBI{md3;xzBP!s+~iCot{_L?p)HX!npGF*=exGx|OG<_d+S* z_GhW@CY(dxcK88$>XJG+>27fcXH#|F&r;8IrzIzUs=p0=)O_099)>>89(c;f`d?cW zfUJ{)#!mp53?RT5TuJ^rGCm>?>o*_6ez3`4sg;|Uj~lwbLO_El--d(i6mcKkXsffhucsDJv7+t@b+4$Hzl*7IktugtfBmb{V>_o8R8#|0 zC!3~pD%PS_+KHY@{SrG-y^_QsGeSZ~be+wj>RM?2^4AtP>8I)ux3uFDdkk#U@~{^g124;c4Z)!2X3!wJEVBg0@z6H7srV z3wFk+wXql|g6jCBv@9u&ilL>M3frR-wa?!7SpATu7k6d6)OW-M1SCp=H&b1i3NEW$ zS~tHOVb3sda1J^Cn0ZuuQTMOMx!!QAOaj&R5^ls;@dy0Rq=bY#Z86!~wQom^S4@Pa zOGW$6j|L8EkzUJE`_py&kp@xj7{B!J52(iQty^#=m>2q6no!G`*c?_W>7ed+iqB(A z_Y~WncAq+T9%Nv;^dr?=ID&pgqYV91Pi7sYMwQ5U#bcQ-dkXj){4E(boDu(*P8p{p z;ctV-f3Gxs`@%!zP*xT!XSiSth$se6sy#*(CZuaa9%tI>`zVobZXi5bgb@wSh_~;+ z5AvR$xc3vO0*4noa^NRYNhCb*&J5=xc-+$fQ*AN}c9MgDlfTC#k?bO;UJcDvSR<;~ewo5@)6nAB<8z7k z8JeN=%t>IQtl)n(fC#$Mpou|*7|`GK4@1Rp9Z`CNG`>Y-xhiFw0|bKLHm?I3xpb+o z;5qZi08hgjHvB1Q)m5wAo!r2%7CEyKnmm*$|NIdrmGM>HPg`ZB?(~HcZV`-~`~Pb~ z#1gX8a-v|k#bAu~CucE&I~|lfbWaX}s+9&EOriIo`^epVU^EwG&4bsR5zv{t${kGC zE81fP6+q}=1oyu%;l{>B;6D_Fzq=YIi(zKya5w%8-I1qi<^~h^Xy8CR5=A3tD$Nn1 zJlBT}Q!Z8@2N3~D7?jyQ0h+-`dC;^y&R}#e(Vy0l^#A99*bx z&eiS=U4TVx;y|5O=qm7=>qw8=%hU6zQwhu(Q=*|xQ~4sH2j9neK||nr|8HW5c@cF> zpWZ%GT1a z(FF=NgU@BIM$N3(A0|c{UhkYcX9E==u!Eu=#pfV*0gR&&ZxMujiukEopUp5i6my&n z@!af%djI=5gz7CR`U#H_zP2@zE^&mv*^V9?BPjG-)I=Dro*dEPoC+p$&7u=`i`Qa> z96q1)B=n-JfF4s(Q&Vfh)Y)@?vGkuece13u7hz=DOp_fdP4>=HKzS1A%pf8nvOAj+ zyul=Jiv-OfVy`vv9slgbR0eoiSO1gx5Wf5WiOFBo)Tugf>VFp;9GnUn5W0GLNNI%_ zWw`Bam}j>jS|da~g}`ozrP@?;_&hJ-uLhbouk2Q2G94TFe{;sJeNY?ztnJkt&RM-M zQmsaw#Zuee?{v_$eCuQE)zu`N zvohGW1v-nXVGmGc&jxwvjxy`4L4N!j1!_aI@EP^w5$KyZVRH*oabuB>_)-4^q+l$F zie?j*B?a8(gQYf!8%rZ-*c2&kmWkjM{97|t)uvV_WhR!!%@OZLUuNcJM;v;AXCL&p z=KDK=tNiaBVr0-!7I$31!NVL@>dO@+1XNs7&A#y{-#51oo{~qc-k?L*?c3nst0LI5 zM?Zoe2R=T23&3le>j)e!#$IEwMXyr7lGyBUL75+ z6*r27BVyCE26lLavhApsz6^x12;l5GUOr91^r74b6?03~|0|usq&zuD+ViDU7-W0js3^!s-JpGf?YG-n z(5`lLbnMB4D;DzT%%1f-&lNd?!z#_lwJ?!4(TH{dL7d;1g@PTP0wTPDQMmoe*d4ls znAfr~fI}QTNJX>bB#f1@w`=WpmIqbb%NBp|QwUi-;L30|kQ+vcjJAE9u*-sdE?+iz#CToiWsSE-DABpXLsOBJ`E%>k3d zgNq#1;`;*=Cu#<;0BkM1cjW%GBCFRDq1LVeP#7a*Rk$#+$y>##GKBWD?C1B!x5 zH`RH1xL3=@%O)mtH{;=HicUz3`uUoVd-c=xXZ6;vwMC4-^KsY|~ zE)OVZYhb!Gr`*{iBK69_;=%7zc*L__y(IvkSB0XKdkx``=&EqCTo(obrukrD8hsJK zU@c)^+QnMz)@Rzno=e@Ao8O{Kd4^CFypNpxU88dto5K#b<0~#=Z5|(s3kSyjO9h0u zK}(F<^)xqJgTXTE;>+S?2Ja$SjYoaD_xO1&W*Ox|zsTJW+Sj+}x1>p~axi`9KQQ=b zrX5|d{16QwGIb{_UIwLZQq@RFrQ{88I7StV`r?Xow-Fzw? z7`(pmm-x92IKnq>e05p+c7&dK$*$HC%^-bV;_~A0{v?UbsF-Ohr$0s3IouJEtCL_G z6llwBSLi}TP&xJVy7L5MGmG}Axq}tam=tE|l7ez56^lICk`FuYd3~m9>H*S}cVA27Lfa?Yrdw#_PfA$3w>3>& z)>7*h18et=u;&GS5Nf!&&&n7zt5tl@Kc6zaZ5&`cl10huDE$rfL%l?bH*|`7U93Ig z3KhGq^xoEd9u_0_{M52s%I9{Y(YQq8PKIIN%pISqlLPIBaiTYF>YcA|PspabSNl3g z*MPPVeyDZba{uM_1a^D+F?U#f??c`2VcP%5bsp#dV)9kbWEuwGlHN3t#(DE5&3$$Y4xb@A=-tj~h_I$PH344A$ z7ZCk;3(P#*_JKd&T!ol2xqm5!?iEprOaUmq1}5UN;P$*3{i&e0#0rg>*Fa@_kx{Vd z&1A3|gx;uo5;B|cWdVC4zxKr43jtmB|7XwzqrMI^l$Z>EsR*B3gA>fI6;}k6FkSQ9_$jlu3)JJyGxY`M&z(DGH808Z z{<`aAeJfehTZV_7OhK711zQ(a=g(5El$8IQ*T6}0Rn}6JpDrBXc1?*wokVh!T^pj?D-40M3`Dujmc|?AQC{J9rR9S z=M(l74=8*C!79D-&FrA56p;64VsCAIVY2DU@5N}Y z+Jl1+H`Pq+oE5E#bH`s{n}&`$4Gy%@KSX41sV&bBI)tLhzD_+c`*3CXIuUK6A~~se zEHd3(4e;<^{N^%W3&ckg0RFj|I`2{^n8%u92b8gwF{_s9;q6e4$Gnh)QGV@|i0!(jm&K{`{6R^?bLxb%3k%t$XD{DgDQ-LtAnJYh zlN=Pen5Hwc!wPSFEJSPQt7}1NPa;3ZXYEc-n&#<$;C}aDxWi(eOkvVyU`CXWzSG`4^rk_xTNUhEv=dEV&@lZ229A1Y2^>a3wChIEuyZgWN9 z#M?bIe(VqU@H{QZi{WIR=lU|Y9y|95R-Z4J2tDo zp<-DgzI^!tbLPZAI5ej;24DUVPz3k9sZdSRCyf$?=8<){H_#yqFKI7d8~B4X(Lmgj z7qcy%BTf;ob6)zj`@RnZ`HWWpgOAeBeN=E)Y2f@Nu<-kd^O1PB$B6MWSIsr{GwO*2 zJ*gHJG}Isj(|S0&#a?@`7SOvIOUb~H_$V^{i@@fz4UjoNyw&oKg~V`Vjhjoo-=KF* z`Bi$r!A!#}jT%YAz{6Cig~&@0Z!&14`V}^>tihD`_vRdW8k(q&rdbLUOH#j}G(lwE zzn^D)MOV8Q^H~k4`?!7-c48lgLR;2eotuO|Hh1r;M*FCxg$i{7@iMg5>DN~YJKNh8 z5%M9pmI%L9GZ@I~)Vub%&-ZiGtGw<*Ew9h${Kt1*Iw4|g7^cp1H9^N2tl@?n*B~zh z)uoqq$$-R{5~ep>tsl-~TZX%Gv<#{*3tD7kqpPpBY{7xjTwavke*7?6`~|a%V+2~z z=6iLe3sy%Agrrt1{#?U6yg9(b$dJAHuC?U1VU5DEw^ud zx2m5Sz`y&zSc?P;q1mKmP+9ap9|07P1+*537z1zy3K^Lun#lFppG-t&&+>O(6a*DF zW}oyT-&+L#pwWc76*(j)mY2B^llgGB%R=_jowHy!r)fS?QHIYuVe93>z?3eJ5(S5YCX8CsD>*Q>h2ST-)n}6$39z(xrUKkCH=V+)pQP&y zO@EI;BS0GGus5qlOU>mMdsed`%B(WT^=pga*V|eiVL{-u+*~%6fXX+Hb6t8>tr~&( z$AVv*-l2ya3%y$Jk60*sUDHHHnLJ#T`K31{iSz@^A6rlgT)bC(d~bp4gp(X1K15X;ZQgK9-#lLe*{1Krx&hU z_?gng`(#fyb0F!3k!zk`nGrY3a#R+}*`du4AaKWSgZSqsB#(fT5)k2$2)aCfJHh+9%kd;QbaV zrAr3ji*JE0Q5Bbyj!R^1#{nl7;AJK1DsqW^il_3hA@Td9&8tO+h)ehRAZ^HfLCltb zJe2yP8rEYgFcLS{p|Wa)*#~lKBepaA7Ck2!kFa>0WHU;Tq>8(4|45}<0CG+Pm0ra! z06f0K=;{WjT3lOA;m@d}IT>X$B1%RRhV8y_gH8G6*0^RV7Q;Qq-?u2p$=Qb=j=r=1 z44iZ(K!HPn*72d=3noN}xDNpy5Q~w%JcB$mxET)YkG&v64A6P)r(_WZ*lro1Y6{Tg zZi>&jLgdGWc(gRCV5qH6$ zoo>1e#9+wS4hhLXDHY={fWfBFff`6!(!l3H7^R-9?~3@*0C{~xhsSd#J%bmNOE)O4 zJ#dv;266@CpT~&u6!pC#r*cEv>}S4a#LBY9jaPmky6T?9(HTwX zZJ|tW8hCS%w2C^+$)uXG1Hhv#oncr5=Z<%z5flLgqFJ3^G99R`oCN^c-B_Er*XopC z0<9A&?{6?`b(0O#>36)wzqazzFg7j}CO&_rAwcVfwN}wtSn7;he_y^$Pp{NR^`7>d zdy2bb;o|oD9o2-UpkY%7B!D8%Gz$WA+!yNLaA^f+i}Gt!J+q{D_xQ4IS^2O);B3;rlcFO7#z1`&D zgJQFvcT2aRWR8RhPcsxBNXlvH(IpY*8(PHhR!RRE+(zVbb!``sZ8GuhXWwh<)R5g~ z|5<$D3<*h5MGn{yfq(kYBL?`~QLX<+DQNJ7|3?Bc>MoVIY(GRWE1-!aIsmTwEa(J0 zhv)w<3=w^Amr?hU8IMkAOz2e>r#5fa7_mLl)^YiBJsLOnj}>Io@`9s0o6cVu&_c>X ztU)Tc#6z!t7Gt`K;N$>TBaWqDNGJymys0pJ;{`|6fc^cQows0}h4{t&Z|-u+5xlT# zWV8b9zY??<4-jP&5=aXH3WxOW&bzW5i6Vvt;y(1^tSPTa!>&1D8`k7z z$k5FUA|;J`=`wWd?fnMwURZh63R`QokXsmaRt;r$=Rd46!O}S%uK&O^@?bZS`k2dw zO@k-B2cx2LpOlvMIgZw+h2zdmOWtQITB=2br58FD*&5c|JG6<*$@12cMXVXhFiL{@XSYSSC?Uzq`&Cv1>(BD;RW6=DOAG5Q!_YT;n@XpW# zHeh4uZ+{J73|2twLV;n25I@skl_7Hyx~^h~WrN&6CXA^MFeDHq9Ac}_B>@a&1PDL$ zMEqi*VV<|;VP=-L=*hT3PfuS6Fb^F$>@^)0=09me_)(2Q8)Wco$9Cv)@e>~=8FWuN$aTI zF^)TYOo!yG*lXbc)w*W!3}9=|F@BRh;3V3 zsq|KE%=LT3Q@zLD2e+jyN*1pAEQn;kD2C&y8OSnI@h+o3hXKE00TDzEqIG@a1w1@~ zXeVHC)2(Gjxj?HO1&3E`DM)nX0DQcTFxjnF;nS;Rb%L?UerKqg7r>B~)*s{gRFhN)K_s*FPj4f)<*hcqf!Ci)M>qvazp z0pUeOZo{89Egd<&U0{ndnkrjPbFO}UK1d9v%Wy~SVi5=}(W^_}&u^(UDH$slAVB9h zmI08POl=1U5{!nnClTfT%`8FaAgcZ^R!SwfaDwHJ+<|Z@KY=^&!1>;5IgG|-wiZTdpA3I1VA|{gDL#x{ShNy6!#K8m`WI(n)SQHJhFu~w7sYvjyfXCH zAB4YLK{@^_(R9{&1Wv$#f8L#rMyrg_rE+`M>VvV_v82kO`%(WKw^?o@umFtOKSn2i zu!uDEpCit0Toi$Ez)~Zok>HGy6Lk>SV>!Bu)&W5l{mVL&E40`KE8Zg9JCWyHeSfWc zgRRh|lR-64r5m!oFy%S1eXWDt=~G9C9%~EO+@ff%qnv2!qmt3-0djr;Y+w7OsbvRK zIALTHjz#FP+Pz^)kuFDZJL;L%*6l89-)_=P@Pjt~T(QaU=d06(6tR!zk@>B{B!;Zh`5 zgXi|&d)}*M_KUGrp_x>^c@}om+V5-)ml`%UpQI>L(yBfOj6sq(G_N0y*D>qrXpoLGDoRWk|^RJ-w`inb+p}-=p5c--Zd+yX!3Ah z{EOu)hU>PwVJ|5=;BVWxL;V;&iEjabvz=J}KoPLF+N)(C2LZi^Js0A6B;>pR#Gli% z+I~QFJkZSb`*=GQgfSJ?FzI){KMrU>EA&4N4GkKN5B5}4U&g*KlUvv1HSOcuo!?P` zsaILVjYSA$rynSlxqFvhVLvF|c&p9)ruX3%okS#1Z;Xe3pH(f^wPVw%N=;C4#g6Jt zEKJ={lrXa@h%|s^-Kli^KOwJ(#V)_S8IIWOxE}*kZt4xKLmGO96uhI3n*%e$nO#Ob zfvjl)=08#Erk&rX6{u5X3&^{Y?A8SxogoqK?meWVNNRi#tGSw!K+x*Y)2RE7tY7}= zk8JH*Y+y-+Si}Q^kJ0B~^$p-qx`?X^V6R~?ww8m<6|3XbBYi1LVGX7aj#^^~rcERS zbB*wX#O-ihx3zl%2F0JyRTkk7F5kNAY(0Dx7-r;xrGKgBehPM6-h2gQ&+D3biS6@r z+9f8Kz+>wS((|RybhzRC3ai+>1bi2RmIDLtLicCUOV!hd_LN2)*?$UsmbnJ=w_NO; z6xp-59m?db?f)u$KEqrL>B-2 z@o**CY(~~%@Z=@sOuwYAR0WT&)OWtD_bwzbYMj5}`}XX4O3Hkf=|RcRT*7y2lFI6y z8IO~hdbz6QLr3=y`fX}mW+aG+US0@)P?K0(crB*f;gRy z?3YfRE5TIl`+5(!?lLt zIoQJC(ZuqfTXe!yk&|*djlY-|A6thN(kQ4>8Ok{}d1ai)b6}NA-#q0DU*I8mf$3!S z58q1P{OAi`80R8cRlcx#0cgd`&I=nN^6+K7|8JG{+oz|IANul{EG^J#mdutr8&*iy`zgB3%}@PSFIR&D~e)O`fM)~Ed=|5ld*w>BVhIbf|E1*nk7Oq@eD^h^Q( zv55ffHn4UI5=1xwF^B>(90mRUcI~khAjWHXu!}XhF2K-oH{+AA~oW!J+P1K9IP@=5j*Zladvt!q*?ypJVvMHL-CO*(|O*-r}@cOJ`_uJMh&q zSX=+&ykC;DHN`}xbEp$}z(2qBr-{B~q|!OQ(Eu9q)aynlfPNSkMrHK>y0d&V%$1|k z@R2{ul%wBzg)tuB$02%;g_}sEDnTElyv#o@fz5UpTAD?Oarq16-EV@mY!O@Ht*>Mf z0|&e7IWWo|$F6Vo8#L|IKnVAEq7xV15y+Gu!!JrK9t(r_Y$1F^17IEb=XnbzuV(0l z?;fOpG$7fe^T2TroH^hzjT-6Qyas&PbV@dzwmE(0ayoI(0dL)^9|>;RrC#g37cTLx z;fk!c-8yu|&EywhX;h**7sHwe2&|P zO06wCwigP*sG~~o-+y$qb?k1GusZssP4D^4!K>-?;eJibUfO0^brGGg%*BICjVI8;?0-1y+1biZ2$64pi74>3Y~q*`f>0o_pP14OpoK z0gyD=$qhnyN$lv>{!w_SW~KWy#Sy!fo>OG;v%xM%yrG%&#_PLK7t8a+j{pU4mfAZT z_%QS+^$>1nR6eA<(fhtTgO4AK*Qpo?`OSNla0OyvD0%yGNCZT~3;+JfkY7e_@Z2LhLC3$EZAlrD)eZ0u9F#ZK1O2C!?EY&R+v z>Xvr|*>xmF-2kkUoACIcBfnI1#|8VsYq5i#?Az_FvFEtno_C)eS_W*+*;Nu8JLZZk zl}j8fcMDZGKC_%`sdl`Q`Q`a#ee==0v*18Cchyr<z9T>t94T zb=3N<^)-8Q9P@xOVM}2|0pTN&BlfMFC6KS<(*YQN4wzInlcRddrDJ?T>m^j2V@q&k z+`)nx?w&04t36E{5;IW5&oKbkg=c-e4E%=G2>>`wAq^on2ExgMFN z{(jZL_F(#Kp6K|%v%w;C`;Zg)hRy!=2bB`jZEMV<%od{O(uWW6P8+yy*zh6sb zlce+DDGq>Uvn8qp8rt)du2vL%0{T`In)%$kbuQD9TCZPt)_Y}L%HXUR63@1(@;`5M z5gfdRi^zc+=g~C@v{#2(tG+6YxL^)5%_<|dy!dosZ`I_d#@H{IAur-H)`L}lBJ{kb z{cEb9?Ui;f{HTB7zgOBGf2-RDU1G^FGr2f~oh@~bAJV)$3?N(6p!RLE+vNQYu#4`= z;3He>7x@cpmzNC;f#K*3pXV^hoF~Oh;N@?6Yo?szCsM&512|SpLAhSs_$S*AnC6O0 zJluwM@+Uxqzc)A4kGTh4dsJ9g{JD1Qg*2VT=G~3BPSkpip|wHa0pU&&F54m-L27m6 z*5Y7;EoqmJVz&ZC!C&G^qVw(IJ(5xe)EC$+sI@TWUAspQ;w z64d_jD1O9UO{7!$#&$5V=6aa$Rwq@?7o0C@M5i2K)E2zH`pa!?Kxk&F7{uDHUiGK& zNE%$LR`BjkcqYuz;N$2666i&Ea*=Mp9Kr60e`{G{Z1ec9YE5k)aQ97*05)Gbp2jHGFu|jOC4#fuNdd zb8=(Z%40kAX6tK#jWGy@5nf*=;g|~8E)N~{q~_Ey0!GyOo=<2n*(HHC$Sd47B0}Lu zY^LV{mE#9)II*bn?2r0Vp5Tl8Ex0j=XTHy2>B!*Z%OS6H`_bLq>OOsC8yn=LDb*rF z|20P7vD=Vd?fsb9cn@#Td+yu*W<>Lra%bhUwv19Sly+r@@%2NmmRWSUXG{*dQn;f@ zgF$)Fr@jDLS>(w2dT8Dhr00!>{OQ{aT8(<1z&W#+N$|)$+?fm(00ErsVXfr31t4C$ zB$&7m!hznZ<^K~FP;wcw`{tA{80t=}7y_2PIm|7-&)i%qQ@n9OHUjItyPMjSVAn`X zzkzXadvThMJGv)#zLcUV`2D(_({3a=3-`v*D|q$0CxuA5dd&m2T97AY9q!HN-K4(+ z83^y`F1bju6V_xdB^0iJTf}VfDnxEEcA}y zkNim9y`>CB{9gh~$s-1}dFArPWXQ~KsxsCoXXb$^v zet}j+Bm2=$>&nHFl~usr3hsQx7A=*!6hW2Lk|`n=j)ozti8UvKGVKB)0^uN!(Xvh8}l}O$o`0gkYn*mXPFcGX-4}EiRD>^23PN&eoGO@jdb}P1XT%$jtD>25|Li5OUXEl_`yGK( zS5^~TrG!)W7-Ov}x>AQb`p5_n`Vz2-8m{rGB;e2>Q%3u0>?7`OHxtx~oatgkojz9! z+LxJb9sXq$YI-WcW65n)sPx{vv$IC`@A1h?0yQmw+gib2UBZvi9GZBwez>fXzk=7d zxa6_eh_){Fd&Nb_;DeP49&k~s{i`8>YGC5t`eC_nso%Styvq9A@{;#9L=WHMzt1c5 zH+WF=#OCbEgKuo%*R{7U3uWKW8FKb&dN`_qj80TPE9>SAg?u*C(1C#um=U-o?vxT; z`BSLJ_ff)-*xhBsf4HTq@V{;a%+vSe{J-vVRyV6=yt{`no-a+`Qjjf$;4G7UmuH2W zXy@VmTY^J1Qca)iMdeyQ#HP}%h$fb^TdM_?oH> zs+D4>rdf7&05;SCrvcRsez-QNf+znp#AFeo$ws}lM@}>3Eqy!r+0&~?weQP^fe!e7 zr!ah*ntXEp!1JAJIN6=yJCsV`$>sf@ekfaY&IeLiyM&R$meGQagl-efA4*7fT^=X( zrSfdqtE25LVfVmo@DwmVk0S}sR!mAK@oYb6spy3a*-+@=@mc&+vidS)i={8#1P_f$ zH7%_CoIG#?OYIjOvWqZ`MuphlEphD3ixRZdZ>D_ zS`dTK)1PxYtA?3}WANk2g`Wg6U8YrQi(r<+Uh(@Mr1E>m2WyrOExC7G) z2^`^2PVxocG^hY>m~a1b2(}t5WEnINfhz!y5E|NRsDUC=rOKs>`DegF)C5vy<=z~| z3MgJ%kOzQ37kaf73t>xvlN@b5SQrV?@A z$T&;O`wfnCQyA$LLs>BNUU1Y5Hq8<+?s&v{LH?%?vU1>I%T9Y0wF7}5`&P@N)gS@1 z7~k&{Sr+w}&nZ3dFkV_5ip2|AOOVq7MuAPTlP&jsgqh_OrxG`;s+0QSv zi9Ap4k{cj$d3c2IsW70B@DA+n*67Yy$8TvcFuEBe2FVv&FX#91Iu%-W|NfW05f?Ub z>R|T0YgO>HVM6Rat#4>kLL0o;S8S6@N4F5dZHGrlSVulsUg?vjp{y&bnydi($8B8Z!M_lB`p5PeHKhHU3mMx zfZ6zFRBHByKosF55&7XUsYCdM$k9fMMsnu8W4gw$3k{FL+_?tFd_ zbKr9t4=3f`Xi*VtU~lc4kGDt4>{75)?CPw8ooAV0$C5X^ai_7~HLQz9DdZ+kR4QL1 zxpX@C1q{TPCd7QIWtzkF1v9l0?9y;4_oyM1*Yl+i2=s7e*Uuqq(nhs|8j0ER3&vBG z3Q21@#(*^mDiyCy5ycaQBBB;!GhSs?`ScQ-i!S9FHS}xpGgSHvRmu1=OiW^@*8g5; zC^j~D75FGOhrvLzVK~ANa6oS~tOm6Na3O&86SP{D8iyEbz(ZCp=GK=!-;ey3d)t5& z3XQJj|2uZi7+iy&b>of-#IaFvRm=1r9+f1Xay3@E^Tv zpj3AMQ2-24wgYa2%K?vvTvDe|aFm(@V6n1L1&sh2pt>A*Lp71xpj|GkaGD>2JD*}w z2)Ja05V7?Y)QomtcEM`+8FZ-|T%549*BziC9|iWMB>>SSzz=qHPO8}cXI*Eq!Al?v z0s&w(RBm)BSiyT1fPpHOK0CV(xP01M3+eq2s~~k$0LG@+-w0o=lSUnX3_?q+0IebF zCxRUZd8n#=R)+hqSL^TpL@_oPeK?^Uz<)C`oovF$|M^eAtAB}T{bxUX1~>MR7U8SA z2mH`CWetF)3ov60(wrW_TsfqzLw1Sclq>_d?mStpS`PyIW^<@0Eo_aPkn(+I0wWZnjp%k-0VZ=fb_xp422cy`Z|21gJbnCtbq3J3`UL&GQf2= ziq?0})X3A1m!S7OjdbB+uu+Q(3b}8X?HmEIVhARzHZE8%!Fq|hHr`hQ>jcu(Ky<9% z+g`RoOi^qwh#CU4XFqC@EDE5I@z75T%yx6L88c1y05w+|kVjq>_FB*?fnDIk5RX zTLi;QG~6MG=ahiwj_p2d#fPha)OErFZrbrX6MROJ@f{xR?LE|Gm~Q6zZT{(Q0!U_d z*t=7BKAR#4&v&PhLh)0g*OoTf5%5sjum3`tS1WAIh}M9_&KPdzscaA)Y2@fcN-+CU z#6iax1wl?ClYzGuax0e#YrOV#Aim1*8Iid20uA*XihK%OT-L@6;4O;%aM>C(-9OYa z@G2Gq3~TEfg%aEkRF~nHXfU9py{IVZ%Sy$(1EpoR4M_N{VBr@de>d#`(-j&?PrmzR z0q?I3QR~`1uU>7uM1K$t%nP^iBPfiALYZV1z(E~h`m8pcw*68y73a{gnk|917a^AY z*dP_4JBkC`kg@r=yqZ1s%PVRtO6%vmnios4)&P(e!QiCbGLc{#mzEL-J|J`Q95R{S z9pnUs8@|2fXm5UHl7RH`N^|M8aRM@WWs88+MBoxj)ES#;KfJ zsV89ANCcI@<%AGi$pwnzwdzr;Vgx(X&!#XM=&& zpiS?h7WE3!bLR-cmn0J? z`NS6EK66pYQ86{?YDnmL)ag%fAx8@gn{25c8!ukhI#lfutF+Cn~oYW|nLrFt3dDeIK5;c-N z1o?A`V$dTc^rq`1>)7@jl2>ZF@hmHwGpQIRe{b*`bB*LwK`9%)Eev`Qc$blBBK(5v zSB94b^ApUk#Sb>_WcdFjejrV~{Lyy)s|Lf%A?K7&js8O;Q&^s^J z*TP4l0~&L&OOGP_Q@T*ADpqt}Y&+@Jb)g&!^$l*2J-z^c?1uYu7#a~2-^2MPN9zls z&$SA@3fum>sV#&^I^QUyyxHh~CcievlWSt4Q-(Sh5B?$x{Z~J}$>1n+zRr5{35?+d zfI_zt1{mZ4A2HjJbj0BwIXHH<4x`AS|)tBV)U_am!#%+GNX z{n&5jm&bB-O|KF|$X*Q~j?PQvGdfTwinSjBKXk*&7ia>^B zX!#SmA6tedftyNA$`~BV@3zZ7#+s3)xcLV~KE6Tpn;Vi4i+}{b*9kH0JF!O^l0#Cc zbFwMwQVM!&j#oJAnML4@{?&^_FE7$c?4bfL=qAm7)x?X_y7r|;zvFr(~A}d$?M!54jveN;4r9a}NM5z6mm@>cS&Cv!_ zyj_Nhvr%A-(f_VUBD0yRYXu3FBp-~uQtyCA(CRQ$rF}M#)hFixANN-VTlx1A+VQ#F zX=K{QI9d2f#FKN|mi%WW9lNIlKQMC?Eq@B&T5FU9Y1{j4A$F#u3cVq&%QxGm7Y)S z!hM_DC>iS4Y?CicZ-kSDcJ;jD3)RG4iI_2&McWTNcSgc%_#-})jIwi&@Wz(uOecs7 z&7UYQdpMoxe`E`0=YUNOKLZ)?Q7@p*MlsRS*0>Pow`>Le=SN;Z>-XHJhxgKWE)cgx zwC_`j^RBoH|p1k4yX0hr|W7Gi?4! zgm?eGgp75#T8Is>4QedX0@j=8J=*A@GqZK~wCATiZ-r607VMo~LdXFmd{La#_D0D& zE;jFJ3z~wy5jLLc#Zjm3=oH^?1E=>t0i>aR*c?gU!h;?~-LtRw{?x2@h*Zm#UK`A- zT6;&mOgcw=N_Ibh*;c|-q+@x%hvuR?Z0Ap(SeTu32r#cApVQHkaDO0 zm5*vhRMKyui!OYm=Fe1`4D_Zzw|qES_u$@sxM2aVa;gaO+kvt=*(MeZ@;z*Nf#k9q-fv{0g!MYtrhpuR^P|<32Fiev zmyEKo#NNlLr8e;VDZV>JVRQd;ZeqWlOML%1ybu9A9Jls6eD{&6RbO11wi=7YJ+m=R<#s2xF6rGGhnF7pZvCPEVZJ8LGPJQfcopIoG!7)z_=(L|C5MuU zim8azNj5@d=nK%++3LLietzmn2$PA@lbc<~hRL27FE$_HJ%Mlm*WE>r2)v zL2)0WGj6%6B87X?IXnubV#`yKa{*^cGKD@ghQPpjp9{rFW2XNp?5agaXv7F{z3x+R zBBNJU|Kvx--uvAU#0BC?KmA^kUwoLpK^~$KibHttUcLW5$ij2da_ojW(j%>%I=i<< z!9#lIO`PTg+*zhj8gG#C1lhC`-;~~y-KURm9((#ClcUboY%c;&$ZUj8TKgHdFcX$e6Rz(a!N8%om!^=aW<% znujU2OE5a6y=s)KFITrk<4Ed_&2Qt6ym9i;<#+MxrybNKlpIP=N^J3CeQ`bMj6}PZ zdbkqx#-29yK-Rs>RsF>@JkVN_kz1Jh3432R5hw5WyJkojvZq=pq&yY(6f{2KeHn5g zr=G(YZmnt4W2n?-x&Og+5dz<17+@`19!tZz^LpP+|3QuQ8SG zoRU4}d-G!;IpCSoswSPdo-TAHSO%@@wu?GuVA~jdS$X?`${(Efcd`t#g?%e#;WXWC zxa9z2t+~m2y2Ut?E#H%kP$AB#uYOB@dYtG?R|QbPD50W!G}mIkG<$UF$7Iw)G91Y< zK4#NHLdB20&GOX4S(vouwVcw?IrO6AY7U8qV(ts8IwXKNuM8s)JohZatWhWl|lgDbI-mQ!3>;J52yasR?<6GJKZn(jqOI2{~7fh4w)P8!!j z6LcU}AY#M5lkxiwl3g1tBrId^EI<(A9gFLq8w}Cn^M>!|j{gYD^()}KSK50@O<{ZS zF6!YNfJ=6xwLp89KAuMWGT9;2ffYE~$-rB91LsQViXmukJy}7ApA1L&UMF}TkxBGH z`nk`8&Q=hovO?*8w4VRu>zAh4oq@dF*`F!NfPKCib)9^#41-Q7s6f^d4HFsaunNny zpqqYaT{iyuhqh__^zt8vPBB5qV-)wXnco5?G!X-mDO76(RiFe?E;N-%-1J+03L2Ya zWI}JCw9w?_-AK&n^hYeCbA13PJ~ZBYfpwYj>;XnDJem$r^C*+zVo1QGW60cbDOxb`Tz!y zAn;BE@Kcqc3p%XcgGTx_z3bR>Wf(3Wh>6&aNlFONghPhW#Vq~miq0oBFt#)D-~=MB z#JKoxX2}U?ARk`cGI9r$E%^<-`*(>QP#;l4w)7A4<3k|IQ)WS+2`ivv$xyGpSth&I zjP!OA!2MnN0XmPvn#W(jcxmDXg_7v`Ebt;Cz{R@Hb4nO^ZXAG1)*BG;TxJKTv(_4q zb|X2L=(cYUKZMZNKGsx*l<~e~$YR=I)&axOKlOVgTxE=W;AC3643XsC`{QpI_V&vm z7Vx@WhAgpSzS=|;>S6Hc8@p<1P#`WEN<-^^cYk|XX(Q2TXJuS`*$i9bWd`S`Hx~ZX z!!O>@izkEdYR5I@)f=cin_P1nQvq&%-55kS?~uB~D!U8o^vzQB_NX#+AzyL60x9X& z44WOE08T0?%az%~Ov^CFCyYFxxADmaBcEhwNSa(9EZc}}fp+ROD7mO>qU3|{q~UX1 zOCln)C6`BQaf(|NYg7|ajv(h^qzyP%fa=7#Z4#&eyd?!3;S!HT#;Vj*n3RiNc%X`^#=5)x~hJG*A z#f!K&ZbR?Vo}AU9e^q%urZJ@6HRtNoO zr;jURyZ}Mh{@$_$={Erb#u#Qb#`AA=0RCO+%?4zzxpWa|Hkf3vv}NA&gbrXUsn8wq zLYXYMBK6w!^C-9t&(^L(7cM%!ThFeukRz#6tabkhEI273b#2c;K--Yo){DhEsIJ#z zUfDS=E%`*OFYd?i!F=4qbvV8T;z7=h(o-AJTPg(6!&@~7bZ8o_p=L0?P>@3{yAP6M zTKBokbWf3lmeUZ^^R4JLzrl~o_tH+Oi%&V?IlP6g?v;zc!A7@|Hx?k&yKq6s5dWzj z@PYrKC^$0J6nK%Jy~B@Te#1z?St0YJP; z`5N5H!#huH289MsqXP&6sU_zKsn#bd7)cwcdiT_mlNfbCn(c? zfaT5C8BsmZBd)BQB^Lo1Pk&qC&i(fipg7(P@El1XZ8wG~cDYdw0sh$kDDO(+q1@Z> zgBfEVkqC!59Ub9>L?khzEay~+7!gfPj%4j1yJ1?TMZ4WtvPGhrJ+8m4UG+#AV5cC zP_~jzlN=L8VCPs53=;D8i(miYH_NYZUO3)$y4=!T03$|vkDn0{E zvlB=?B_m5Ca}4!DrUkf+%HHRinio$zoIMsZ-+^`#pPEMG0)hEP;}KWd_#*}EC|95U zd^{*?bg4+Ra8xo7JCjcAkeH-lY%c&0vL-$&1WH1bK$a&!xv&sviQw? z|A?0gZF{M@Z#EqagIvfxs7m2EY~rY|t8vOsnDE8t20%|6mY%b(Q`bgRnkv@-N`f+US%loW8nu~{h(;*aY1#5JOZHIY$ z*Mf1>c1YRL)$P$}XSM|h!Aon!XIEgy`ATn-MKtt)Q4=!O5N#K(zg|qkj!hmpCG=n;rsbOoH^5--S;L3 zNm_tAg_x7>!XhM*ig@>#eTAJ?e_Z|OxNk=+A%?L-Hm(xm9!M^69hfgWa>w{wObJDD zf*n+=%Oab;Asl8q07MDsp(B?jcWglRWqzh)m$3hg%Qh0gAG{Hol9 zM&z%@Bk9G60D2NdHwS?Q=z@w5CNq3coK^z*!SpUnLb{IG^3a{wYSS6fEwo{n4Pgnr zjKxuj{Kax{i|Z6ujw^o5qY-#Iwb;FgqbT}e`4`CIq6Q`kZO%WW(**p!b5s2rP20XX)`a7KTm26fwloCr7E@EXV zUn@*#L9;~DzPXs8SwutK=tbqsTb&6M3$oRzZ6*voBity=Tk#0(a6Si2d5?u)*`8L0 zn`~N26~PWW2Q%Ofk9;a)gp+8_=;jfNeX9Vp1%@m%{&e%g*o9VAF<4rK-wT4~ONlT$N054z93VHDo=*to-Hq9h@9~rtyLkrB= zXyf?X_$}%OBs};}Y>~X<;F2#OsefGI=z`#ye$%(ZLq6u=G7aiRV^0v=ifGWfrxf%P zw5DL4M>H)sms#VMrWpi*i(a8)?y?7>5_0gUIp( zoB%6UFDeib_(rp>KXOy|K(5NdLrcPUPbXm5)DVg2{|LaK*C(GSj>f^gF+DNn0^{t;-{6ne-dV7IGDg!f8DNUnOG5kEMvop%nzLD|51~^AqauUDNIO8Ud(o z^)*>1EY%NG#~d3Ny}0T!&$1&ADajpdgr>&n@5*FrN2)bNBJD*7JBbs( z3OvO=aNW*eS&awPdbPXH4QGU{yhrT~;l2TIrK>%&DukU~PQfgy*a5XwyX}T&S{zd9 zB`}x}JFf}o75wfSiywerA2GkD)G|rdC&o80n2hFvU>~*C=e}f=Lc{48o!hV~;&`L} z3gs6LUtcVLWKB!ic#id;f5-I=+A2Zd)`^_5qeAw4<9Vz-IoF?W5I#j=cPPpIqH*QZ zn5kW`@nGGUkmVFrSF8U zc&19>bY`la;76n0Xf9P=Kjz#ud&JfK19E0tYmt*C%ma^-VE=)X_odyMJb>kS0#zFY z=i?dW)Z79yu@_63wL}$h3AWy4(*QSi%MT_>xUF&?{HC()_3i}|aYD>V zN&TdQbo?95%y2qeD@ll>Bz8rHw3ihz%ZrfpJQRDR&nYEDBERUvCFW=CH2arTjHC7N zXsyvb<5z(R=@p5K@6T~_Uk`Z)33P20g5-Q+BzB_>hzulakL1OP9^-mDkq(9GWnNm% zCGvPz@#p7l1lJ3ku_Iv=tTa7C^yQ@H5-v{$&D@Eo;V{XT%Prd;{dVTC_f(~_2sTl~ z?N--W35~A@QY@-$yNcP&VIOz(xc2PGNK^=)_~UJN#MkW0W=e(cjzxg1zgN=w;<9rW z*{HG^7!4h03E({l(_%OWjF%kZ95!E+ z;AuV5*Co01`TyDbJJ=*EOw3C$x2XM9#x1LAIx2oDw&sWU1BMu)T(MiJI-?V-0 z6Q#}?;aV4Z8yAC5SyN606FM}>=heON#k_*kJGPvb$-8(7^4LfyCRnaxlfw01xlQeW z4wXW2Li=&3i8u*U6=Z*qO7N&_QxdaPfd-~Sq4OJ$IQ35g`E>PcOKBvnD%lgxBwZME z8rX4Lo!M{Y4QjKG(@igr@{4Xf&`nn$aF!U0FPI}fdot;9r0cB4*@k>j54a~Z<=c5V zRFkPiL)=w+*(Ys-3w@2VZdzPrakxF+Y+{H!%BjgG`M<;%kME}OqG8lp~&^2He zx>TU-_IBn4;3I(Wp0g8Sw?g+it3Ltf73*FA@XwzP?`&*!>a8VvM!fm69ot@;3yjJ7 zJ@8XW2^O)fX|O%Te-`kFZriT-$%H^pOFcVugSM~)Hkd{k76Tq8Q zD`2+#JvH!}YL^U!OmCxs$Z&@mJg_QM1!NXS#L+K;`JtGX17`e~oj9rDBq@D$Fpe4XqADdRWI|*PA zxTS z34`V4q1b099d#M8?VIBFXS{p}N6QdVToUj5-2;oMotgUaJ7lXgJ?~_I5ZxL!QNAx{ z2iF&A{a%9vuQeEI6x1y#iL|Qx$H$iYKUu8d?f>n&{{Q*srkZuU?1AjbyS|Qv=+6FX LWw-yaC58Skgq<$1 literal 0 HcmV?d00001 diff --git a/benchmarks/results/prepush_smoke_20260423_123335/charts/membench_comparison.png b/benchmarks/results/prepush_smoke_20260423_123335/charts/membench_comparison.png new file mode 100644 index 0000000000000000000000000000000000000000..30a6b2575088134a38bfbd7efd972152367f5e81 GIT binary patch literal 33079 zcmeFZ2T;`Ow=LMBpaOzQXp~?&h-45YNK!cJQ52M%5fBlHP0pYalxQMI6eZ`JGc>KB zfMm%*L}HU;1G7K8|Noo0^KQNSs-|jcs^(OkYDM}NzOeV&YpwmgyeEJ22<1si3tx;IzGwwT+3T zg%F>h0G}w&>BqLVRyJ4o`OW|R8+?}5#{4x}cFk~=!&d)j*kCYBk?8+@x9pz3!(co= z-TM2QvSZZDz~S3+EU-(VOkP`Gc1nu8wO14zfd) zZnD!|8L|VVcCv&Z);e;m_R&Oz*cjHUwCGVGr|K z>DH6($kLO}H~VpMrl4DQB-&w+)4D$`t1_ai=t(D6M~V_hdc|t*NShr0k-SIWxZ0OC zCX-@is@%70uS}=YO?9S)_4Xzy#0r@8WM5=G)#uiu@9Vuf+m|%)Bl-TP?1ctqUw{5e zQ4(pXC4nuhtlMR-gkxLsaqEZHPd9^G^G(`2f|{eneND3}gh>m8Q~qXk2T!>km6zK1 zxtxAd#3CvvICeV5W$t}5bL@C6X+nDUurHhQ<4yVW{I=VZ8V$s6SUuNcjdjeH<;Q}g zHh*^{QXDxMPi%*`XQq*uub%(?DL9d+b?dx*6jQUr(pOu(h46m2Xvfi)BX1R?I9Hq( z-v5fRT;5(A^37|$!p+Jo{mC(QL1=3xuf>mL?dMwsA6AEc!TiVH&(0Q&_%J69T_eed zjbF*B@YrmRt&K3vd|&CGmRb6w$6?5=4qljXV?)PjJY-Vjc$Q9@YR0l;@8eH_wLgqw zSl>a-I@cbXeFTax{Dq{yl%KVdK*0)j+GU-V35J?Jwp z;w>gtZc#qzObyxL+v$Dy_AaIS#(3DX6s06aTjF3vy<4hEnyLVdzhc&9leYM0nL1^m zj%ITO8^0p4q_wJDo#CD^J$JFHmKVp)QC_kAeQ$KZZmNTmNy^pn%CFS4%-iN%CRMEV zJ#h~P%blny-{0BUV*4gycS*)I z?pw2JQ)2f0IKpw_Ilt)^$+dpP*lTlku-?NqqaD{re5q75O%jd=oA&0L)OhSH#Xe(S z*&{-+x?-$aJANA1S8f33RgZm+dZ+-nnyxNNFAUvuyj z{nuFOs$rNqz7@@zr`(p$W(hs|8g5(pCMZj{!nwwC-~QVY1lS@)pC{V0_3NHZ#ClZL zk5PHVjuAh`6K&=;+*dl)sN8-fMOinV?ORNe-ZptUHoD|r72?fWwAjdvEr&(TK3JE# z3FGf~1LD#yr?dFiVGUEd zZ%!uJM(=EIVJ}(qi<$?UUtoK}Ky$H6+veLd6;+Ld;UMw33+wRfN?NB}W^$f|@@oAm zB6wclJ7rb9kNne(z}x1V>tn%GL06`dNlp|rEX7yFsq~WEwpJ%~v&!a5pYd0ba)*6S zxnE!sHdC)@fDP4Z^Y=kE#|OQ-uD@a!iXA})`t-~&*s@z`fdv=q}DgJ{}lKjR| ztHyVnR6A5UaV~-ST}~6xVYSKY*9KiC)?T*6{bd|&r}`S>Fh5*FPRAjCXM#z@ zLc0b+jYc+H2-|PdBFXN~p;_=Gq>`@BEblI5_~U`w;p z17!|1N(z!H^RNKoTev7}Ja)H+sD$B53kUZxN|vbDgiToplU?SLSw`!?~2NIacAsd^`;aLEk(m;HRcL5(^^P+0*sJ zHdyWvWCQEeV|Jqx$GZ7vSQf;Vr#iU{Mmp8>c-NQP6(AVGI-_KB9De%juE@ZpQ|=+l zl3XM=3kci2{4v}+zeb7YV}y2m7#T2?>!beHYnLWsT=?9=+;=u6mRp|2$@(&~z-&#r zX!gT3b^aO-CNvNg%XQ7ch<4x$7k8K=+hVc)d}c#{yV8JM z7bmFQq!JNA{}28skA;?bK_}(;JK^^Xv0*Bk@?@J7B!qdgK3J@F1 zd8EJZ0pFbc)y!@m? z{lWINgRJic$JSkO+J$oiW&GI+N%GM$VSGAmi&~ZyDi8a=N6{?FljBL3k#@DEFM_ewxokCJ_8+ zQS8O`aJUmj9Ua--Cj9*kck#F*{{9oqZnThOKV6SWiZ`Ys%mnDd^Lo!Q6Ep&FL5(fqryY3OIBY63Gw3`F^*^VlFYB6n;7+D0R24ij0YEk z=^nm&m{HKx$WP2~mrscKBz|)pPnTr71in@KJLiZV zBtnA$(|1(Y#@y&*?>$_fADMIumR$XZ&iCD#!$*z0NBSWus<)aTZVVVii^yMDT<-nGaLTK5|>@!42~>uzC; ztCE@>4xBO4KApJ4*-`Vr7GWP>$l%)Q?`nnY!v4Yr@xK(Eo`a|y_2_W0&~d_v2py z9kbq|-*M$yEXx;=Th#GumfD(eU3aKIb~AqNYplnP6G~g{l?!(81c|dH_X>EPE{yuA zL?9?XWj~N>)YQQ)syc-~EuhK>$mOg#_f8q_^n{9|i%G27b+=Rd9$aUK$|IE9yDpe% zhcpy#rGtI+uc-qPb`Sfp$Jo+sf@_Wo=?l=xw~cE;(R~$?uc9H z`2!hWiHhkD%OC-?EutJ5YS_cbeSJ*SccF|?FTtBI&RaO^$DMg-$bHLvefT+TEyC*^ z-r}nK4MTP;4mb$Adc)zhYibd-ePO;WR2bC*WLGf{N)(HyM-T{Br9b6C!lm>Xkarlz zx`l>PDg1pbZ0rDQh*Or})h~C=76x62-s5*&Z<57L5SktBfXjhqHb0M==7 zqv?quZdu|p*|L?CIDmWlPRI8*ir$i2Zk44SC*0xj7yYH(49Qz_${qLU7ZVwy^dlz{ zpX6$f;l_CFP(tm1iQM!c^Yd0w8w(@9F~Jy*oelP&jrT6AfWlNxN@rh^^`X8+6l5BF zl9s`hUx9hWus2rIizm{&KSo-Va+Z@8o2Uk)RVByI(SmpI>(b@-JL5l2rM>tQq_6B)fTF}6KWPccZzy@qs2q_ zccph4KaKMHE5SG|Hz9MZqQ6J=+PJ40YjvJYkw{P;NsC)}&fDGcR>vLKdt&i>Q( z&K7ZLErKe3v=T?=-^j@E<;9r9pk24lMqvZdF_=xV&DwS>P=rzJynNxYG~-W9f9W|- zCQRD+uWk35W+f1RLEfqOe8Rq|?l4ExtJ3YAmR*v9BP6tT$@&&f)vXockaQH3ejD$d z$B5@R?5_vU28~V>57_s40}Ma29Xe_`MfBKpNmUJ0`LM8iVZCo?M|EV445M*BFP9mE zxu;DE-csLi7=C&%5p$+lJ0h|o|1X&E%w=Dux`;YjX$f)@c5hqUKn{UTYcxkN+yquO zq$IDNL7O&_QM%t{jnyMco=Sew`N+Y^#_iC2;0+m463nZG=!G( z6cIT4{!xRgOlf!5pnA-faQylDrs6znnjj?O8x4iOO3ks1vaL+V8(N839^0nQ42&KM z3DHI6cpQnZ^!G=_B0^&o(?u+8Ju0*4*L55Q>`ZDPPu$M4P$Dy3UELts);F-(u~JM)-2G2)WRo6;P#<5Q+`J~mJW)F_&bi4)7;K5L|5 zz2fD(GL`V&5DKB$4m&X)tCHL0M!Zrj>Tc`6FC_SVQOBL=#!t|5pQF9<`-AIsJ)Gdz zfKIC%Zd|`ekSw-JZh>vgCXmh$Dl<-fr<}>e$V7;#@&0Qk%J7-Fja(^Brs)CVD;-L5 z5}BwlX~}|OchZF+c)Bx9ZF6SSfkR~S&h?X*9^acvWR`w=M5JH9uKk-^m|@Ylw%ur` zmW9S>{F$6w{+-ORiT=d9k+!=FHbGU-nYd8(b-h9Q0ERI`^BT)hV6c)et@Ya|h)yMc zYRmPIJcwzGq`ZoO|NfW6l@OHx`n`wY|IOe37l+iDK0}fHcxAeq*5=pO!vNt@35YR# zbw-*rFPjYS87^k?$XiDiQ%v?^WT@O}iL2}cy3xBlqt_dUFlv~R#;PxGIn?Zx&`&Si z%+M;LVfe=6iT?J#&AOq7L0=3>`tafO*GLimiv4nlP`}092s}CI=JXdL={&j4`?yK( zl5ExitJhgpZJ#GSw?0}|1xvsiNE$B~Up_Vc4}1Jx>Z^^V317>~jq~0fS2rl8g^ZiO zR>AccQq}aNZVc^>!BvJuC_=~Usu}{BeSMBF`-A{nP?Do8q^K{UtU_o$we<+l8Z+ygR=_50YJle@Z?}>3b~a_TdX2-X2;Zy#=OWLGK0hWz==+U4qU1S`3`*V3WoRvpM~I z9hZ7^j~9J)Sdcn!M3?&J#_khFO*Gb??Di$tZMu%Up4#+ zhCN)u$yyZFKjI8!qp#S>j5P$xuFVb90gH9)*SE-`o))u?(XmOpL?IH3v1{HMn@sy3(6tPn2lH|JJ3u-*~lYCW2yT^-=|_ekxYXEb|yVR zQ{}x8q8D;r{@n`9<2`XIm(^WSft;_X~ICGg;CF>pFcABs_u2HNIqIXv+sLlORHUcT@E zVEj0UCi;-M6xL4eFZm5@AuSZUu9Otr8phGtU(QE*0mKv|$EoQ$8KFR33n1x)s;<*G z{>e08C9J&k_A=r+vu6wY5=|@NUL%0?{%JWQ)jhhiT2$rjzP;&bij?!X^LAh*crkE8m=OW~5s0Z%mV-u1Wh;i06xm7u~hp!|;L zzd|?Q{o)*mgClS~sxVz=6Ut1+@iJm16@o96yB5VGK32cp8~-Xg0(srrUTzE(!*E{@ zl>Zl}DQMZE$OwoOH7gph=d|US>(a73Qq*rPCkmtowuh19yW;{L+n$qiQGg?e=+JhH zBOzyp4`>cs(&+D@Dv#9D0WFyj4@?FC1u_-(82HcE(<#JreL;0?l?Sy3%1p{DB0z`% zMRkx~Y^WGFw%F+=cqj1MBWg^fh-FOHqrb5a;Qc(wxyB5zZGcfEy-iEz`8936T~_MU zSO&gQ@+%sgKXiYM4*{P$ivwu zAr-tF(d#I+o9-yA9nn*U;=~GAJ8!s7;zWM20sF{!m|hI+?nfs12ELm6)T~rxKi|l; zxln8R!?m2JGq7EdP4N~tRX&j+ZQ{eL{&l;9thVOn#Fm%kb9 zE8DulIzKi?mbu>=aFtsEIN}ABtVjTppg~;s4!jd1H-^VNbXe5smU8jOi#|c$0UL*& zf1v;6JvkFx_jH}LiVMCI@Z!|WFP=589}y^6g~V~+0rhQGeP*YC;J z=S=q}BFwLD*Qv5(c&1@C$Hn4$lM0nQa1r0CzrOjC>*(9^j9W-|O4;u&3flf|?bt5T zuk$fc^uPYYzR$dUYd@{Tz0pSlBv?B9N;LftnSfX~sgczgaXV-K4#i*A2o z75jH6>sm`}CImcoNLJlC4k44_qQd(09MU7Z|8b{WsF;(c@*p`0N7;30T(6B+H+KLb zC21T;#>*gn7{6}elYj92!?ouEPdbvxAN>rZv+YMk{|d?Q^j^)Qv4$ju!&gVmDsrOz zKVA1{G|{dyX;0)m%2x@b`Unufv`x1y7>B9&%LN7$a1fEHP@%R%1lpF}+cQUI0jdhk zN3+_4?+>74qFy~WL8}`V)2O7zs%|miC$!su$xw)u8Y%3vkQ;bV1RF^J+tGI7$j#+7#|B7c(>ML7`(h; z4y07v&0w}T;IvG%T<26m_JEM50c=P{C{ZisfpO;~H^^z^^RWZ6@K)R~1o&v8g(f4~ zWT~tEK%g|oxGsL7ztAvM#njIm$b^wv{CqB@mxzk+btpvZAXKr&I8UX}kMh1`x-GR( zPp6-jQAjyP2v(SgfSg0Q|KO36^q~nLx2ctZLgL?_(sI|B1%Bxu|mrF@ye?sw}M=@i*p zsn+DDdD4b{#c=gNqonj6ZF<2D@k4%+rn1E)g&AeK16?o8!mR*z-T;j z@0%Z~tFtJZEmV%xN`DfNrS6U0UkwkLDI-((l8z&B;>7-T03K!aZU*z%wEa~;r@PKG z#V)|mF#xe&V%4dl<}E%~{HecBRb_8cdXh`nbv{+uOpySrssA-x5Gl=aBibX4#mVx~ zS5gczib2=O`ez-B2yM=4|4vT?=A=me5JZph6%RkN4P=^Pum>3Gb3*7*{F53;#PlsV z$l6{YarrXT0B+WJvW~@DR3n=Eb!y#u0Dr}D8$G6h>sDx#;2!ezdpV%;sRv)=xLzc{ z?)?m4qnwVfT;Cu}^zzPZ(Ga6XEc0{XD_I)O14yLHf;cZKdwc~hxr ziGuWDKTAM2NUk@o8$GrMN8h~#DV)D^mEk_ikfr`;K&Ag1{6wzAf-PFTw^RPv0-=a& z_D4Qw>JpX6$l$jG@<5)5lc@Dpk;^3=|ATAW0n;w;01uXM!kQax%*a6`FdQ!; z{P1ZK8jb|zN9OtGw%S&h+(hl(YxqA8K%WO|)r(K3H2$U7DrhMJz^k0?GyWFAfHWLR zHwy&&PKsM0_z(U8syj!m0Yip0*=ZJ8L=Q>>tAD)6vhc?v6P`-6!+@E{`dkRenWt4a zRGRE6yoKlJ(_1T3`mj^pYoI)f zts^)NaKyS^48&v8_HeFs$qnj@S*25{wp9@9s6az7cD@UkqpIbEanvy=M(IoKXHPZ+ zu`(kf5fzHd?#thAhRKO1i*^D&qvV<=y))#ru-LSOaZo>wDsY6(HRoF6Q0*Oh@Nfu8 zT#KfahI;2Ri>91-jWhe0E@d%qU^*1>kF zxnO#X>M^pb@KaCf-#_w<1-iM}1u0I$-~S)B!rUXzH|^$APE+N*6E1LuhaV+}-sMK_ zEU)oSz~ygyQ61yMpb~NzLx3>416d)7))YVkHRw%uP7UsoT6TFlo%CE6@YyKPL2>B| z8Qcfz z{|KxD8ecFbC`Mb(^yG|yfrADuz!r(6JBR|TgLHHpX|Szj4n$=fuz(z)T3cvy=ANhU z$PqC@DZh;Ky{ZvewPCQNFlpA(g9@_QUA^?F&ak;Iw0iE{yLS}n=h;im;uGBj{uxlP z-#4F1!$?C#Wc^qYE|$%MA9P<}!0%QPzkj;<7&conEaBgS6)s3tPa&WpgalrRJ7axF zw?i(LIJnD5{9AcSz^+`mM1OP2^qd_r{;;AKi4^0mRP5 zU|j&tAIT;t$hk@5{2qq^dh@O9fXEhYg~XEyUC=?7#cb#L#iDGQ#YTW!Zq_?z5jjOWi(1fG$WU>fk5xY18 z5siIE#DlsPVGAp8gCrc~qHuv=aFa6AqH22;(dQj$0vr?%k+EhE&e2Mfva8GYv-4ti zTp(!a&A6O|nw@gWz>|?^Sh3P2C&>)_X7?oz8586ZAu9tj!$lpDLF4kQe;4Z~X?Q8@ z1pf-yH};NGsqLN#Fr8FT)$7a@R>IoShovZAGlpOg#K1_~X70@5vn;l)*Vx(UsH%~_ zprfJ~fS5iMt#`~J|I6=H=1u%nQLFybtFVFdS?U^@m>AuSZeL@tgQ26Z?*pBx3A@z; zaQieXvmR2dUT#GJb84~$M0Zt(xj8UH&aecl+rXJ2O$oeKm@e*p+xSjd6@VRbpu--$ zHV@ObzSUc}+H1;hr46pI_ClbmGlr2ZCU+KG8h84x*w3gVTNh~)aU2;Qz)mktKl&D- zg3#16q?3ODxI_+>zsqZz5ZkXG^Fat;Rf4+h8C<36b0HFGGO|GbY?P$zgA!}0WeliL z&V_!oAQ3>Kg3Yb=a2q6WDjHxLZD+o4#Fkg%N^H;zvi}lZup9BlGCJwzxB)$>(03z< zMdN2N3cwjzFv2y%HP5ldgJ8vZSGg;O^qF31smQi|08l2eJ&Atn(Y*fsH_3;Ozeq^B z54Ti&Z+5Fo-7xU%79!-hV<5Wlm_aFZy(=1ma~ve65hw_F+Y&^Y#pWSRdLvo@A>5SQ ztZXF5s2p<#2sr{^>04Hd%0k6_E!BpdW%K+qpm@Uy|9JSe7{bdvIFEvcN z(%*9(!m5?8!77mWR&&oW#<#zp2G4!Y5dVmt`v9%zuqSzQ5d=1Gz{w)YmsK7y4lmL- zIoXWIC785+xaPbIMR}ub^SUhOT^{p5A?jVA;r$0x;jKTv7qQf(bTkM*K9ncita1It z*v`bWqNsJ!Xb&#Og>S{-fZjCyH*=PD)2MBYu zL1&#Pt!7V$N%i91)K>d(7sGYc0=*7-SyLp{YQps z>t#}E_YHsi>8u{QN-`9UppzwBpvcXUdAqBCDL!C5;0^_y#*|dm&Zg;W{+}P;t&VYN zvoaZ{gIi%`to-BiTl?<`lD4jJKNCG%8!WvO9%>l&b=_dZU|jQ-b?4ih(zJD(fVZmk zCycR=K=0$R)}EnY*z&E=j+{gMX{|?^u14 zST_rubY1RpNr$YQB14mIallMU>sTUpq2Qng_2wS6N3i&w<7>Duq0(#fRiz$3CRh3> zGnH>HtWVdmDpx8j!bmhR-jw%_t2>NzC2qaAU+BW!>uS5#-6!QvhdB~1Huta=$KxQFZs6t(vEc;XTVL=DZUcSi4&W>u_e zQihDC7t5V(R=cNoRZPFu>pG2+>Kpab3}u%Ir@1Q$t!eA4J(3O@Dy9t|aNq+<#ZU)B+&N&B@6vhJTcjlc{a_HyFRy*FY^x#W4ZaR9GUO`sEuuXA=c9d<5R%%r>5;tx zxP@k}wDdRDG3s&Q<6qRsJ*kd)C@&D~fpy`}&H(U`BJIG^v4pf*cekh!sOy4meQ_Xt z$L{2y`1pg-=9#D%0qmoZ&POdEca2ouzp-r_i>c@8n_H~hkLY>v1tdX)ON>f z@uFjvnYmUbWYrKvC0W1?mm=xQV*O9*`F`+UDUJ8T4rcxj7Kr5>anzq*zDZE_<-~Mf zk>yljd8evYLJoCbp*eG?xB3e^JcC;NL_TN|dm#?xOOzm~0VTLQLtfU}*fvy~0;c`k zxF~L>p=S;utBTzC#(;fH2U{cnH7$8~K!TavhSmR1yDe6}#-`nY@bv!JjChV$esNC$ z5*}!gdobe7mDMsm;0^1LzP%YiIJE(2u=36vqF@^PcGhMKWh6V!_-^m+7G~YS#dyMt zOHs2)h}yZiL77<##8Ox(p`rW?g@vG%rfEh2qo}Ztvzm^bihD`6x)*)fVcHWH|BB}J zagd|KG(4C?sdXU4aIGI4D_5eR3g%4LI&K_=yYU#3Fy&FhkO7p?{61QgjE7#j_D_Fd zy0%pf`J~1XCV|s|Ve634lUmq=zB3U8Ex36e$qH%*#+j#k1-u>qdly{HvZ~75>WtCZ zVf&E*r3Z@h#!E*ZS&RlD)xzXg1k)UB>f;9|0TSVO@aztmvQ3XpjOS!7bT{JXD=zBpG4e>n;M%M%EW(l?JIPyajL zXV0F||HKd5fboe2NS`?}^Wc#ozOtF)0Mkw&tef8gON`ZrhV`T_IOLYi{0;`Zi~dt@ zhl3`S)XL@KA(&hSOnDs&R6|mLA?SZIIKZI#!mbiwasC~*VFv?apXx3SG=h>NB7T!7 zt1nQvApfxAOb$VDrLX$w{v`rnK8F#AU|-Z)q5%kRnAiEz(gHKpr|4#plPXvVp_Lkg z?8uade}H8_JjR&%4l)ZIUwc`jmrg^1aI{!s=-L2vk(wiTbZS-9bQtD}$AYXawN%dR zEoDZ5pV>)i%SySdXRNnB2Azvv5Lr+(hb|c5%a$(+5|28Fgv}KW3r7o(UdZGu@c(dHS25GWBl? z2`^iMz=QQ75d;~n*UBe4}Uu;aF?tIzega5NlFK__$qL(_${ZX_M zz|RRcEW3Y;k<;iTdeV1_uU7%iEBEG^plT!*bm^CUa$0(WPr5Us2ge+}sM10oZXl}> zP?gYuA~m@k=gW3m=%=cWdC`Dyg*4RO>=wa!N5YJ&MW71trg)v;9fzv!lKkiY(d={K z5!fsbAugZJW(Fuu{i%+W>xGEolmJf{!)AYw*st(Q_QTLWQ498b#_-cWfi2|^o6ffw zxKiN(ObollkXhchHg#8|p1>N?duptiz`2}LT*gy<@X6*C{>W!czDvEUv+l?M3`~wK&FSfkk=;NeUC6R&LNc{ z;-b-Q_gGZQP@lYX!FIOyasaEiumljkaUd^EZUY6)7iNDCx7T2$JiNh4Q0(tt9>Qdc zu2xx!&{97*!DQXj9wXFLA!hT-t3`TOg4Y4Wi4JJ}$c;P@m%7`m$^RTmF7)94gH}p^ zWF&%q;F+)n3&IOpfJ(n^5x$6Dv77q&U2PxPRrGJv>mUZS{mp7*2lNmnChgF^)9~Y~ z6w5O|guHI$3dLEKhL{N2!h;rI%bs7YI>P$J1HB?bHpyy0Rffa?yW}^8B z#5i-rXYOSgNfyw<_u7+kBth1(7X_zLyO7C`T%%;LxI)fv#~<52}l8NAfSAK$l*ZZ9o5V(V5mI?E*)YnQsgF2 z*iU!yy7?@KJq-C1edqy5xg(#G;=O!Gu}Q7J0qN36IxinW-X784AD%1rQH&P-Y z?(D5ua!U}531oX&i!vE}tRC^hW??$^hmqNYa*_r_U@JthZfbzQD5Q^alTzZHTN6as z!GF@?=hx@zz$oyhwg!j^@4Y50pw7Kb1^+^=WO9<0)V({4VO3tJ_(zRvoUa0A18RCu z3s%qWGb+AAF#K+%&F3S{-UPjTI)Eg^-nJ@_UHqmL>i(K^0^MH{I-(>IAC+<>SYq*? z9>^56fKXLOaUNTJLyCu-Kp{#r2K$6dVv7;Ol`rjb2s82z^J>+g;)cd zeYXz{RQ7nNmJh;(oPUcW^Tos_pd!W3SZ<<0azTsK*3TmRlLGZ?4iNd0{HkDOx;`ze zZ{_vT`L`^@ieLpji#x2uk&TI11qRRWU;$Gs#%9@lyWAb`!pW!W03!S(n+OQJBplL( z)&Wh$q_P6L8=JaFska2lOfJq?Uyx@86bQ<{*C0*NUk$xZJw2KipyKZ>i9}RH<#oOh zyb##=Qfzm#yGoJ4Abr#SscE`8k6C{)ZiBM(78fu+j(`YzcOu}a;U!QaY+YUL z5!fyU23EE!`A<5*8-?kFBBO;2o-e-SswK!08<1Fa)D^Bw#^+18uFs#HC>{0Vt_2mQ z4iV1uhj%|Z27#MUh?ig!`j^iW$j!fNsrc^@1>^m@zj=GyPrnh+>C9e`kgdKQMzw2A zz)zJj4dXqt9m#dmy?EGmt_!Y{+GBU#rT#_=@;%j@XW5SHVOpp5S2MLTWc@-;kHwaJ z$8{{ge&>`(%Vy9y-!;?qVPlo}z*yoi#gyKas={qY9yCyzcVjPtUgb0x*T4bKN*cPC z@aUWJ5%u0x(LwEI(#BF9D$7C*RdjC|s11u+3_jk*l)f&wco zKYzpD3|Q_R193w;eHNE&%c08ag@kE5--tla(bHG9fX*7<{{?@n58HxG;{=T; z=2S2d5MN^~zk^3k1T57-xif=;1Qw;lC281HRBo$+GXX7Jsx%EQ>abd6L;vW>x4(lq zi@Xc}iqB8QO7H*=AC%wtqghrDR)NNfAHNLL+v^tbIlXc!^IPU$3OMHU+E%r-494|u zA#0~D`%600zkyKU%ayWc57O~6>>g?RIak1g(xmd2qU?n=%2s>@ODlcqt}KC2CT{kHHhUyxup5rP&7LogvKC8l{kF8wOr(D}lIL-qBf_ zDtP!1Z1bz+{tPzA*Ydj6)ajVNsg6bgOUGG(jamPT!`_KcO8@gm zo*7qU+5~A7%cwpCCY4Kqa_LiL$b)Aa4Vr>eiR)^U-(L8AaN>nfcUPv4IAY)Rq3XF_ zN#{*nkI^Iyu0r>>m$QB$*3f68E>;A9^jFdNqprZ&STIAro+pCnNCS*(r-6>R zLuHShs85g>*jw#Gf{5&2zLBvQU*qGn4S4J13NdJ-3gKYoaVCz-B>-RiwYfP8ks+H_ zW6#ZrQKtZnEP^&1)lSMQg&_QSBN^|T6?0u77-_BEtES-R@*yw$W9~0>5~`jW6S4s0 zv=0nfgJ{P1K5#wxQ_$_%ARz|AEtypiwJ&ArY9sTE4+LkooJ@~~6UOlmEWjqD4Z(w`EM^q^tD9{>R;eX-KmcmhOdWJk=V;&SVy3FT3*N1@aPGSgC&3xZc7Tw zA3e@-9j zUbCi6{*#P%(~&|A*WOu%A9EACL{Tz~a`}~MUeQD<%(&%eeFc|vo;8U4bjU9N8U=Fc zhLmPu1o}n~oDy{P53w)#BBj1sm1h*P^_z?uWvzer*b5=y@v4~m_fk2<%il5vM86U@ zyYFQ<5d?@^!7?i+J7rYAMlQM=iRgP@B>GzJSdIiUEdJuQzwjHs!d0wFp6}3h8=G(Z{2jSop6+di{|qZHoVO1P%=#VU8+&NL zL0aOHR?c} z_DSSM71llWGPhZY$plD1(fG}D&CE6l7WeA8I6N13QkBC@iOQzyJ~~B1=kv9V#^*;(EfbHfWH?A)C@Q?C{LR1v9pB&;60}*AbX2C8_&ur4 zRAG>=u;<|?@)$ypWD&K?>JL7%))d*-D?{#UvOU|aO0})}b#;acRYzC*+z6z?K`l$R zhVr%hlpDg+x%%#IB$t)YG_N0d6JB9{XYrD}WnP8h3Eu`u7VQ|@O;E0#q*#-~a%WO`LQ z|JbrG;59hdbPd+Ac72Y|$!n_~!wr4XcC2q`x-U;+{miB-Rv0HGRV&6_H&%Ymch+0h zuJ)N7E1dq$>pH(+%$d%ud#w1ceZz||NEVy1?to2Fq4&JFe2CLzqd;WnFz`bu^Y*IO z|Kyn4IYf&-{^cL_3mY*WcFL(!Tf~mowjuJk(_hzDit@MH`78NyJ=QFkK_{OAnrlM= z>DjUs&UQ--?t+-8jXvVY_axnarZ?yEOizl;5A3?KL|n395}qyJq1;Y?(v^N<)2yhI zY+5U`SK^X++Xa%uY?#2x#N?!9W7vOS^;yj<-Ko5nXLN_?TBzu%-YbHN@pZl6^ALB2 zvbgP#*_iUa_02xRI(f;W+YIi&@~g8l?XP~kZfco;nI9Lqk#tzrplARDZ(>jsm5VAk zM1^Ug(&EBteWF-V!ibu#&QLzl=`3{hfD=V97_R#!Z!PMlD%^slguFd&vK1RMU7p#Q zAHrpM|9Cri*rv*jKfq<%!?>GQ;_yH6ncPJGtjhO=RT2iaJyrujCLe^3SbX!y(JCLg|?s>Xli%%P?~wvPvb6{}rkCQ+4{%$dwPPGbos64gC{mxD~O zZmTNaGppDG_pi52Tl(ajay_27Bf-1OML&?*u~Cw_UUrM-RYWQJyF@pcWtC&bw-?1;E`NTl%L-Hr9U`PNqBrLo=d zpQ+n(&W|6j&osT%5A4)pd@pKfLRJaZ(vaec&L?S!Yy2GYG#;d!;lEkxbY0z2G8M-E zxOsRIrAkDR3cJjKPHeettVUXi?uG|+Y>L% z%U>HC{Zbm9MAt1oL8Tr@rV350S8NXz>{ycUjbHz7Ud}&fxd15^c`9j#rMZISb5T0; z6YU_!a!(q(2CN zUM%?U-7u_{HmaUe@1GXj+mA7#Z%t`4;0`%iMns2~trnc(xJO5T(amb|PCHQtJ3i zUY~J8uH$ei%mly7NBCVT-1R%m2pcOx{;_+@!DhC zepeQ5rMSCNK2jBda`(<1uR8Tb^FNoVxT_cT)P zkPp)-NW88sX6iz^+FnlwkgDqI;B%@4yDQU;0H$N8@$X_RF}I0NZe$3UkK>hqIBhFy zo*(puqJqQQL+~LB_+}h|B;a1IL|1;GpLz%52z*-N8frm-BNTphLspeQMphGWuUpap z7r@`XVKDxHKJ6ex4b_KzNIL+|q%2ZCa$TcbXU4<0mo10L_g?o-yYmY z%kEGEctPcq^0|=dY$fXpSOZ7ryHbU83{N@nV|`{7 zV$4o%WdXMm=dw1paqH@+sOJI^aRzX3N^#U2BE2z;dLf4}#$Ur<4^DSy=}k%DtY7^G zV=iaA*jUNv*BY$0)wN<6JEf!Hd0?0Jg*+_<&dt=X=4K!RrSv7L-xb1ZcBrEJyD^TB}Mf8Xqc zj9D+QXF|u5RkPT{&u#~x?9fpToCjQips54FJFK?)`g;kd#j|Pq{ab{~&u)!$KWnrg zyBc!ec#t+^;D9Gr+!vw|v-}~?1T}uy6Sr00FdFzVkt_p|QUc1*5 zk)k*Q_mX_Sn*N`sG)@+wne${&fT1_U_V!#BaRw3=+am>KlJdS{glw-J#VQ%0G2Y&;NPXo2Cg}YliB~p>w7dexn-emRrBNYqy zYI^6jIh~){AgFvf=!Yz*;UH_go<+?~OcIV(nscDXkq@H4iQ&->2|w@`v=3L*nj}Rt zLc8)&)T%^3)d{C!p^oP}k;1&+e!D<36;|vKhh>LExfi#F!Vbn?G2_FB z4-a0u+@CaLx&hQo^H&a_*mwJ`pY-%oGp^W@c;spHpf%n>!WcOBL2{*wOX*s`|0jV8 zzn67XUjI+4tN)5o|6deu|3wn}@6!AK|JPxU{D1dwy#MzfJ`JGGIvDF}!R&VmY21i; z^Z^5|FYrks=BU-Z4*bZq;OV3T9a8DXUDQVH$snIV;R#ZiE7I!MfkKxB41^B(dknIu z`_}rxSRLwzU<0umbt@`AQbL;518-#$9Z%_<^|5SF?0qNOlYBvG|2JrPQirTNK=tI+PXsXdAw%tB(2OW~0lIW% zgK@_joRsPPy?e?8w=#x_gM@ftY{?K@xDL#nP2&;B;Br!KK2T)vP^5XG#&5{};Sj?` zq4`M$`Xp;X!>ClB*aEj|+s3FUYLzuWveh3`C9-3KXTb}Yw0ClQT{{oFr^x`@jG@!I zc&w|CD_5n1Hv=651P*X%I1BAG>a>l6BNR-wk&~{r3iJ^#f4L*z2DWK`E_6 zkg!5j)dsg;!jn@l>6nL^R8}n9mPkF(dE5;`Vk@{<@}+`WupAu-ebr`F?VLo`(`?kB z)L8(2kjOzmTa3L$mU^R7NHhz-WTMzB;`4?|2uKg*eN5(n8#$|yE?tM#Cer>&Z6}W- z>qo}`YUjkHKEyCd!t;ZDgk}VKdvCve{5P~}YdGV7*ut8du>w~-|B78DOu+IWXab6B zFBv?&*K|6Hy#%8I24S#}?R8=oAV!pp;squGb`Q33oxpS)QR*RlIAxYdz0zxPSM3Kah=;?CA+2 zWq4aQ3d+dNb8#<*NP6m=0|&yE3qct4(^8Cltco;!e(y+j!%=>1{U#_5me5=@0t5;F zRr|2SHEWmeDL&6i0ss&D#2egMbLV>nlePZKaF;m4FMrGWarP-@p~vmT=g+|NPq}p7 zna2$2v$Q7fBj$Vr=7n=f}#IDjM}^U|i^ zv3bVES90fp8Lh1QNQxf>q)lC@{}6nVLvg=*LV3YKHDyA`1L}< zBnfqBW>oJSD2Wo?Pi2>D*t_dXGpsV)Irce*Z@@b$73_a}E2!3y!a#V6&JHHMV1nX} zsUEFQE^TXk>36%QcqgQ`^l1WZz_)MZ{ee0NcXTa4c3-VwT)!To@8N*MXRr% zYThl<_=iGGkbFIr_Y!Fmquec%-BD)Ud-R0}P&Q~yqF){90;6YXi*XsMATcu68$Lqc z0&S46(3MJusS|BPj%E88DcxP2Qx2JUOrkRhUJ-V@sX(PrKltf=w8%yAOZ3xydWBR!G^60tIXamK6 z&%$gsAJgn-1IJM7>HUs=LjmJCRgC^Qu3`nKNtF$u!o*|&3ZYATA(TB~UXAass8^j$ z-#X9Eh->330GcKi7@Ha_ho*Fqq6DLVT3vTBHE+-P%L6s9(aR?80OJa5NGOe4opvUPt^wJNR_~^xOH6YG%0+GkMF`tNfbp#@t z&|S99D0MAd-Dv$Wk_L+#!)Y&%ZVWMSa^|;6$F8KK- z`W1r+-5rb*l$PF**87sf4r#23!2iVx0%JZJLci8SKb!Yffsik8uBQKJhhWU@M<-?l*)ep zlql?E3|XX0`&<$1E@YO2E*AW`tv0KFu@(G$5SZ0Ui!*Jg{)R|_b*`Wef&LSF7Yi?6E`UKlwdV@^7i#8|Q>d7ga|@lfEx1L^nk3`g%*~^4PC< z$I$GEEU&RCAendbOL#LO@->3N!H+7+$q5X{Z6Z#z$A9=8ld$gVKt|PG05>=ofkB2T z$ov`}cfk_11g1oYXW|V#x~)9FTUaFEjtNbDa|nLZjbwO2@YES~<)#NSX?8vD4vVJg zS>n|u;8hb5wRmLuzHg@@OavHy%K79yBN^dC4=SFnzTEQNqYnnD9 zQ#QEMEk3Ww$mhQw?LAZkYq96@$w!7{nGn$2BAeLbo0q?un_Etsh-wz%%If_2^=_v~ z*PzfL%SSd8$$dk8f1BVDp;!Ch^cVq_l_?&`t-qDw+XjFUe0mFOjX zpHZq5{v8PWSO1Fl_#fK(#~QM^s{$wr(eSn+-jgke4fxl-Smld8Wf1ieTPyQ`B~z(g;|5(_p?Z#1c^tes7;FbS^R72tAv+z7Brk2l=lw5!JtQIzH=J z4|*yw5OQ;>TYp~p>wZ*8WAhOxgC+RcDB=7I&{blfQ(ltl|M14q>jRPY$PJL7{b@yq zY$$?)n-|=??HPAi!?0pi0Cb43|FZ)*<6hDZnX1OD1`55@I#0O8w$44_m(?$|egf&;-hyzSZjF+hjlYL3wF1!V_9OsU$BVpvF zvM(qseU!hpZ80S$*dUqorQk9ewWo=1i}Ic#wS%x=@!f^7u`uYRuZ;+spl zjBLi3>*ORmNP12)N2^Hl(X%7ben0x7D0L-`P+liUq}sBPf%YgJ>b!WC@JDKr zEt~;5YE~SeUo&vgHJj2rs`amO1DEz~0B1 zE}1ktR?x`vI=O6xVKBiSLEtZywlQKCmSzDQCbl`&!ja=RDyDmngmN;}&vkorQ7gHcbr5JhGnkqgr zyzJviXH3qOrdaCZx<$L-(KON?12kYa1tO zUXX4>CG4?9_$XZ*aKt)BB&&Hn`4xV6b63XpD#*gAk8(8UI7kMPd?tbIL4qR@Yf1L* za)@=-e!!QPKO`N4F42wCfElutmvlaTUeY4nliNN2aVWVx_Mj2;MHcfLoN`!EM$sKS z@hU4K%-?1;(kz=rukHcx=vgxp_rU0+Va(@%);}8t`FE<+m4GImt3z{` zY0!EEDxG{-^$X!WbI8(&{3C)@{b>z{yXl-n2O9AKv-KTEei$u^?CYpS%UtYM2T`sB zi7QMyYcj-#6&t-L7vHI+YbK@AyEGyN=jyU21g89BI{abf^7V=;CJ)7OAH`}>Ps-R;K>Z`RYkrGe3{J#wm zqk4xSpfW}4_T`l;oX&LGbBHv}<4NCDC-eZy0-j(;=i=FVQ(N*M##jGSXzQY2%C%36 zAGyaq`EC}wRrou#=nx zT7KGd=XKHQT!o&g0wDqFfMeZAtv+V6!hR3b&tc{T-*cCt-q#oa^oa61CUfJHFx%-PjP=AwZZqH8 zYX-skh7gD6PdgYtX>a+J>FCh(q7K6^JhiH~P@;eg1f*!ls`-lLLm8~RH1W?4HDrF< z!p9hI?bu!Ka|wL4BW}4hwwi~P=<(d#h#w^P;lCSjU4&l!9lX6-o5ENwk%V^XY1?Eo znOMYV&}^);o?=j`-O?-75$1n5Ck*LCCROxsm?_yvP24W$-qP`X5OwQeJo}fV7sDCf zCj+lbUp?MPP5jctb!x?K%tamyk2Q_GMY#8>J@d7fB%06(kCqg(OHg>a7~_;4SWYpF zu=HLfuxw*14(6^#Vw3w?hdys9!)USy)w|-z8lnPnjchALI+~ zgSm+u;kUU*fN`|BVYg{Upe*P#xbMh<+x+@ObeXBcMT{O15ZOX?=lQyjBRW*%TTG3} z=Ni^yKefO3I&yn%9cF!vq&PAEhGY6L?imk%2QkP}#QWZ0Cp~lWpaqFTVfK*LBX-df zd*Lxmcn3l4TXSO9@vvD|)~UOJ73YzUBRFi49vK0;UT`ARKGU7Jh-~&~!$n%>$o2UP zACnUxZJ7`xEAD@k13qwG_|CEU1uZa(V8?D`384R4d@I?sW{p?$cqS=;a_`OViY}Nw zjw9a=GBWFV4@hA7Xiwz!6J*Rv-!cN@*GBlK@Uw~*6bY}vWmvXlI%6_5Mz3My2oDn; z7BE9ui4?}<pwypTg?4n$hYM57MwIPG3gs@Dn);XTSvGNJJVtT;k(FAPvmfdsDfX5IL@y z4AZ<9(lxCvq7qyLzwn5=z(`XZ;>H*Id= zLMiwhZw#Xc@OC6lg9#Px=ig|k16HZsiuwzn=V5D{4%rDWde-`l#Gvgg+C?OfRS-qe z%$hwOk?d(R3^|PFS$b3kGk-FCEla>XG3fco`p;lbu&$rw5NMK>yJt}-tj2@iCSu@Z zse4bM-)k{+&_I3J#u1C5&(lOAMDi{LS5g+z%ErzSiXu&vc@|UWBul>^H z4XEe!rTe)UmB*t!L6@x|+XDjWSmA9hrs@tQbij)(D0gFP*klKy=fUj-6tlzh&A@ zpCQYcn+Baej|Zq{6Hp#YTsS<6*_sj>89$Z@=R2Y6WILq}6Q&!T>5zuH?ts*iC^D8< zNSb0y%R|8@_FRE4i%)%W#EF(ZXe*<-lK~?(yXpUNCy&xaate#H3bE9v3^p5%!LBbQhcS!!=r zfm~<`nz~xr2ZUaWlpt^_q;jo^nurQPqt=rg3290?rA_*t0%Wks-3iMzODEfe z4a^ziwbL?LjZft5rrbH}1IB^Dv4ISJwM{sx_YJ<>#d!kJZ|S%#lt@9cU^)M~DXhXA z(~YvL%JL2-rg%tdL^y5VC3csbwKS{X%UOJ!9^#&N*pxKK&@i%8hC*<7P=R-Yz;hy^ z5$QTrI)^;wELv&|BJa-C{Z%er5!{7m>%`!`!HW#w!8dXZ#~Bsz1)yXLsWTtoBu?7L zor9rq<-1wQzY%Q})$uN>Ew?F}Z@qYqe}u}TZY=P~_l4mfPs|D2c)uRy)>)XLGU0Yr z@i*Q>z_|D2Q^GlSCnXaH16As4^i7?tygCd!x;$__8>Vm-~6{qa^qHk)Kod(N!0#>2^o9O z94`9s6yy6!gQMigM6LQXb6GuJfM;?<&!9@8*ijXUjTPI3c!{JI^5w!wS@N6Rxq}bF zrNij1|fL9l5;yUZYfgMxV%49~{$n>*H16|XD7jiZbgL+P9+==b(bp>3a; zu#rGFpMH7p(iQ8=$Qy(9G-ZRO;v3=x0%jSC9g_Jry?hAtuqCHB2E+IKpYbSYN!wx9 z`R2l9RPE@!>ICGLMY>!sV6lI4zKAn;H_t@8k~4q8ce_l9FhoV*Jp|{1WOh9}!dRfA zGIw|IPv4r;tWqA&Z(FU^dFJwo>w8KEa@W4*2sY6a_rL!?_$bG&Bj##w4ZCX9~fY4c7;x$=#r9DOKXe`&RBpNre z3s4`to2bXyeC;C^yDzqx@~xd`yAua&arA}!WShCeqnMl1M$?TiEOQa1!BsZ-QPBKf z)5w|S*pV3)#lx^)E7dI?*5ucEI)bS#G)@UJ0gJ0)bQBXp;L1%%SEWfq*g;YsThH+i z^7+3M8s=hjq_`0q<_TrBF2_kQ_Wf97q`hb)YkW{FZ3Y#6U2O$&Zi3)M80!~_Ji?y| zcc$$}5%%%bB@U8xWT*j}gdmo0uq>Pc-E0PkAPNJVx%Jt1=i0hkUk*nL#)W{Woe81E zvmxT(f5v5Oz3mBSHvm4!?x%owI*9UkjPe;(17B{@7Ga*OPHds ziRn##-9aA1b4Y*FHcaG#SiVIy+45tKWE{v%SITMngpsBwjlzho@g)qSER4-ARgf_mY&ECC4TeR}jJdrs583-~} z&j)~_tIslCr5|Q9SDG{7tKIV>)Kr9SoLe@Ccr_H2@F-bOE)a7!mFK&sn802K#2Zl` zVyd$4>4;LLbi#zV&@ggNuhB7o|6uPz z)%!&5(-nPaZf+w|G%nya-zY}KmaOj+@tG&e)_ZoFl(G09;Zt-|9RboVK;)Tdn zmZPvC;k|6$$ic2atC`+j3;A}OU(1;*^j6~Li#uUc9`hy#DY}==ox3iuqTn-ko(4D8 zGPiKo0YRIXIXQPEZ9?UenvAm#iydD!mT6daL~axlKXCJPnuF0xcYWgzOxU&~B{jTG zSE2{TGI{SuW7ny^-cR1I`QB;X$A{q0xcLkr`YN(qtRd4;UQhbhpElt;GmNJRZWDh-Pe$3w kFf%Xy@e%(" + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "example_count": 0, + "measured": false, + "metadata": {}, + "metrics": {}, + "mode": "raw", + "notes": [ + "limit_per_category=1", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": null, + "reference_kind": "historical", + "runtime_seconds": null, + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "blocked", + "system_name": "mempalace" +} \ No newline at end of file diff --git a/benchmarks/results/prepush_smoke_20260423_123335/runs/convomem_new_brainctl_cmd.json b/benchmarks/results/prepush_smoke_20260423_123335/runs/convomem_new_brainctl_cmd.json new file mode 100644 index 0000000..9e74c7e --- /dev/null +++ b/benchmarks/results/prepush_smoke_20260423_123335/runs/convomem_new_brainctl_cmd.json @@ -0,0 +1,29 @@ +{ + "artifacts": { + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\convomem_new_brainctl_cmd.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" + }, + "benchmark": "convomem", + "caveats": [ + "Blocked while loading ConvoMem evidence data: HTTP Error 404: Not Found" + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "example_count": 0, + "measured": true, + "metadata": {}, + "metrics": {}, + "mode": "cmd", + "notes": [ + "limit_per_category=1", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": null, + "reference_kind": "measured", + "rows": [], + "runtime_seconds": null, + "series_name": "new_brainctl", + "source_path": null, + "status": "blocked", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/prepush_smoke_20260423_123335/runs/convomem_old_brainctl_cmd.json b/benchmarks/results/prepush_smoke_20260423_123335/runs/convomem_old_brainctl_cmd.json new file mode 100644 index 0000000..f239bdc --- /dev/null +++ b/benchmarks/results/prepush_smoke_20260423_123335/runs/convomem_old_brainctl_cmd.json @@ -0,0 +1,29 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\convomem_old_brainctl_cmd.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" + }, + "benchmark": "convomem", + "caveats": [ + "Blocked while loading ConvoMem evidence data: " + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "example_count": 0, + "measured": false, + "metadata": {}, + "metrics": {}, + "mode": "cmd", + "notes": [ + "limit_per_category=1", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": null, + "reference_kind": "historical", + "runtime_seconds": null, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "blocked", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/prepush_smoke_20260423_123335/runs/locomo_mempalace_raw_session.json b/benchmarks/results/prepush_smoke_20260423_123335/runs/locomo_mempalace_raw_session.json new file mode 100644 index 0000000..749316d --- /dev/null +++ b/benchmarks/results/prepush_smoke_20260423_123335/runs/locomo_mempalace_raw_session.json @@ -0,0 +1,38 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\locomo_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\locomo_mempalace_raw_session.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" + }, + "benchmark": "locomo", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "measured": false, + "metadata": {}, + "metrics": { + "avg_recall": 0.6028, + "cat_1_recall": 0.5899, + "cat_2_recall": 0.6921, + "cat_3_recall": 0.4604, + "cat_4_recall": 0.5809, + "cat_5_recall": 0.6188, + "perfect_rate": 0.5534, + "top_k": 10, + "zero_rate": 0.3499 + }, + "mode": "raw_session", + "notes": [ + "granularity=session", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": 0.6028, + "reference_kind": "historical", + "runtime_seconds": 2106.411, + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "mempalace" +} \ No newline at end of file diff --git a/benchmarks/results/prepush_smoke_20260423_123335/runs/locomo_new_brainctl_cmd_session.json b/benchmarks/results/prepush_smoke_20260423_123335/runs/locomo_new_brainctl_cmd_session.json new file mode 100644 index 0000000..e12e43a --- /dev/null +++ b/benchmarks/results/prepush_smoke_20260423_123335/runs/locomo_new_brainctl_cmd_session.json @@ -0,0 +1,4455 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\locomo_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\locomo_new_brainctl_cmd_session.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" + }, + "benchmark": "locomo", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 199, + "measured": true, + "metadata": {}, + "metrics": { + "avg_recall": 0.9413, + "cat_1_recall": 0.8641, + "cat_2_recall": 1.0, + "cat_3_recall": 0.7436, + "cat_4_recall": 0.9571, + "cat_5_recall": 0.9787, + "perfect_rate": 0.9196, + "top_k": 10, + "zero_rate": 0.0402 + }, + "mode": "cmd_session", + "notes": [ + "granularity=session", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": 0.9413, + "reference_kind": "measured", + "rows": [ + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_1" + ], + "question": "When did Caroline go to the LGBTQ support group?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_10", + "session_13", + "session_12", + "session_4", + "session_14", + "session_5", + "session_7", + "session_11", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_1" + ], + "question": "When did Melanie paint a sunrise?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_14", + "session_13", + "session_8", + "session_11", + "session_16", + "session_17", + "session_9", + "session_18", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_1" + ], + "question": "What fields would Caroline be likely to pursue in her educaton?", + "recall": 0.0, + "retrieved_ids": [ + "session_7", + "session_13", + "session_18", + "session_10", + "session_4", + "session_14", + "session_9", + "session_17", + "session_5", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_2" + ], + "question": "What did Caroline research?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_1", + "session_2", + "session_8", + "session_14", + "session_13", + "session_18", + "session_15", + "session_5", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1" + ], + "question": "What is Caroline's identity?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_11", + "session_16", + "session_17", + "session_3", + "session_1", + "session_8", + "session_9", + "session_12", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_2" + ], + "question": "When did Melanie run a charity race?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_7", + "session_8", + "session_14", + "session_13", + "session_18", + "session_15", + "session_1", + "session_5", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_2" + ], + "question": "When is Melanie planning on going camping?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_2", + "session_16", + "session_12", + "session_7", + "session_17", + "session_14", + "session_18", + "session_9", + "session_6" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_2", + "session_3" + ], + "question": "What is Caroline's relationship status?", + "recall": 0.0, + "retrieved_ids": [ + "session_16", + "session_1", + "session_8", + "session_9", + "session_12", + "session_7", + "session_5", + "session_14", + "session_18", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_3" + ], + "question": "When did Caroline give a speech at a school?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_13", + "session_18", + "session_5", + "session_14", + "session_10", + "session_7", + "session_8", + "session_19", + "session_2" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_3" + ], + "question": "When did Caroline meet up with her friends, family, and mentors?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_7", + "session_9", + "session_2", + "session_16", + "session_5", + "session_17", + "session_3", + "session_8", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_3" + ], + "question": "How long has Caroline had her current group of friends for?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_7", + "session_1", + "session_4", + "session_12", + "session_13", + "session_16", + "session_3", + "session_6", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_3", + "session_4" + ], + "question": "Where did Caroline move from 4 years ago?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_8", + "session_19", + "session_4", + "session_16", + "session_15", + "session_7", + "session_17", + "session_9", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_4" + ], + "question": "How long ago was Caroline's 18th birthday?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_11", + "session_16", + "session_15", + "session_3", + "session_17", + "session_8", + "session_9", + "session_7", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_4" + ], + "question": "What career path has Caroline decided to persue?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_7", + "session_14", + "session_1", + "session_3", + "session_16", + "session_15", + "session_8", + "session_9", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_3", + "session_4" + ], + "question": "Would Caroline still want to pursue counseling as a career if she hadn't received support growing up?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_7", + "session_17", + "session_1", + "session_18", + "session_6", + "session_2", + "session_10", + "session_5", + "session_3" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_5", + "session_9" + ], + "question": "What activities does Melanie partake in?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_15", + "session_8", + "session_18", + "session_1", + "session_4", + "session_13", + "session_9", + "session_19", + "session_17" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_5" + ], + "question": "When did Melanie sign up for a pottery class?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_16", + "session_14", + "session_10", + "session_12", + "session_17", + "session_8", + "session_18", + "session_11", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_5" + ], + "question": "When is Caroline going to the transgender conference?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_7", + "session_17", + "session_1", + "session_14", + "session_9", + "session_3", + "session_4", + "session_13", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_4", + "session_6", + "session_8" + ], + "question": "Where has Melanie camped?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_18", + "session_9", + "session_6", + "session_2", + "session_4", + "session_16", + "session_8", + "session_1", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_4", + "session_6" + ], + "question": "What do Melanie's kids like?", + "recall": 0.5, + "retrieved_ids": [ + "session_9", + "session_10", + "session_15", + "session_18", + "session_8", + "session_17", + "session_1", + "session_16", + "session_4", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_6" + ], + "question": "When did Melanie go to the museum?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_13", + "session_1", + "session_14", + "session_5", + "session_11", + "session_18", + "session_8", + "session_7", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_6" + ], + "question": "When did Caroline have a picnic?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_8", + "session_14", + "session_13", + "session_18", + "session_15", + "session_1", + "session_5", + "session_11", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_6" + ], + "question": "Would Caroline likely have Dr. Seuss books on her bookshelf?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_18", + "session_10", + "session_6", + "session_17", + "session_4", + "session_14", + "session_9", + "session_5", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_6", + "session_7" + ], + "question": "What books has Melanie read?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_7", + "session_17", + "session_2", + "session_8", + "session_18", + "session_1", + "session_4", + "session_13", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_5", + "session_7" + ], + "question": "What does Melanie do to destress?", + "recall": 0.5, + "retrieved_ids": [ + "session_7", + "session_8", + "session_18", + "session_1", + "session_4", + "session_13", + "session_9", + "session_19", + "session_17", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_7" + ], + "question": "When did Caroline go to the LGBTQ conference?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_7", + "session_14", + "session_1", + "session_13", + "session_11", + "session_10", + "session_9", + "session_4", + "session_3" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_7" + ], + "question": "When did Melanie read the book \"nothing is impossible\"?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_7", + "session_17", + "session_2", + "session_8", + "session_14", + "session_13", + "session_18", + "session_15", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_7" + ], + "question": "Would Caroline pursue writing as a career option?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_1", + "session_4", + "session_18", + "session_8", + "session_9", + "session_12", + "session_5", + "session_14", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_8" + ], + "question": "When did Caroline go to the adoption meeting?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_2", + "session_17", + "session_19", + "session_13", + "session_10", + "session_5", + "session_15", + "session_7", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_8" + ], + "question": "When did Melanie go to the pottery workshop?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_4", + "session_5", + "session_16", + "session_12", + "session_17", + "session_14", + "session_13", + "session_1", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [], + "question": "Would Melanie be considered a member of the LGBTQ community?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_5", + "session_14", + "session_11", + "session_9", + "session_3", + "session_7", + "session_10", + "session_15", + "session_16" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_4" + ], + "question": "When did Melanie go camping in June?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_18", + "session_9", + "session_6", + "session_2", + "session_4", + "session_16", + "session_8", + "session_13", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_3", + "session_5", + "session_8" + ], + "question": "What LGBTQ+ events has Caroline participated in?", + "recall": 0.75, + "retrieved_ids": [ + "session_9", + "session_10", + "session_3", + "session_2", + "session_11", + "session_7", + "session_17", + "session_5", + "session_14", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_5" + ], + "question": "When did Caroline go to a pride parade during the summer?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_10", + "session_11", + "session_5", + "session_18", + "session_16", + "session_12", + "session_2", + "session_15", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_3", + "session_9" + ], + "question": "What events has Caroline participated in to help children?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_9", + "session_10", + "session_3", + "session_2", + "session_11", + "session_7", + "session_17", + "session_19", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_9" + ], + "question": "When did Melanie go camping in July?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_18", + "session_9", + "session_6", + "session_2", + "session_4", + "session_16", + "session_8", + "session_13", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_9" + ], + "question": "When did Caroline join a mentorship program?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_10", + "session_3", + "session_8", + "session_14", + "session_13", + "session_18", + "session_15", + "session_1", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_8" + ], + "question": "What did Melanie paint recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_17", + "session_11", + "session_12", + "session_18", + "session_10", + "session_14", + "session_1", + "session_8", + "session_16" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_3", + "session_6", + "session_8", + "session_9" + ], + "question": "What activities has Melanie done with her family?", + "recall": 0.4, + "retrieved_ids": [ + "session_10", + "session_5", + "session_15", + "session_7", + "session_1", + "session_6", + "session_13", + "session_16", + "session_17", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_5", + "session_9" + ], + "question": "In what ways is Caroline participating in the LGBTQ community?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_11", + "session_9", + "session_15", + "session_14", + "session_3", + "session_7", + "session_10", + "session_16", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_6" + ], + "question": "How many times has Melanie gone to the beach in 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_3", + "session_14", + "session_6", + "session_4", + "session_7", + "session_17", + "session_8", + "session_18", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_10" + ], + "question": "When did Caroline join a new activist group?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_1", + "session_9", + "session_13", + "session_12", + "session_4", + "session_3", + "session_8", + "session_14", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_10" + ], + "question": "Would Melanie be more interested in going to a national park or a theme park?", + "recall": 0.0, + "retrieved_ids": [ + "session_11", + "session_18", + "session_15", + "session_16", + "session_7", + "session_14", + "session_5", + "session_4", + "session_3", + "session_6" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_11", + "session_9" + ], + "question": "What kind of art does Caroline make?", + "recall": 0.5, + "retrieved_ids": [ + "session_2", + "session_4", + "session_3", + "session_5", + "session_6", + "session_11", + "session_10", + "session_15", + "session_14", + "session_16" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_11" + ], + "question": "When is Melanie's daughter's birthday?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_4", + "session_8", + "session_18", + "session_1", + "session_13", + "session_9", + "session_19", + "session_17", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_11" + ], + "question": "When did Caroline attend a pride parade in August?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_5", + "session_11", + "session_10", + "session_8", + "session_9", + "session_14", + "session_12", + "session_17", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [], + "question": "Would Melanie be considered an ally to the transgender community?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_18", + "session_17", + "session_1", + "session_14", + "session_9", + "session_5", + "session_11", + "session_15", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_12", + "session_3" + ], + "question": "Who supports Caroline when she has a negative experience?", + "recall": 0.5, + "retrieved_ids": [ + "session_4", + "session_10", + "session_12", + "session_19", + "session_9", + "session_5", + "session_7", + "session_15", + "session_11", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_12", + "session_5", + "session_8" + ], + "question": "What types of pottery have Melanie and her kids made?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_10", + "session_15", + "session_7", + "session_5", + "session_16", + "session_12", + "session_17", + "session_8", + "session_2" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_12" + ], + "question": "When did Caroline and Melanie go to a pride fesetival together?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_14", + "session_5", + "session_11", + "session_12", + "session_10", + "session_17", + "session_8", + "session_18", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_12" + ], + "question": "What would Caroline's political leaning likely be?", + "recall": 0.0, + "retrieved_ids": [ + "session_18", + "session_4", + "session_14", + "session_9", + "session_17", + "session_5", + "session_10", + "session_13", + "session_7", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_13", + "session_8" + ], + "question": "What has Melanie painted?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_13", + "session_9", + "session_14", + "session_17", + "session_16", + "session_11", + "session_8", + "session_12", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_13", + "session_7" + ], + "question": "What are Melanie's pets' names?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_7", + "session_8", + "session_18", + "session_1", + "session_4", + "session_9", + "session_19", + "session_17", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_13" + ], + "question": "When did Caroline apply to adoption agencies?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_2", + "session_19", + "session_17", + "session_8", + "session_14", + "session_18", + "session_15", + "session_1", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_13" + ], + "question": "When did Caroline draw a self-portrait?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_14", + "session_16", + "session_11", + "session_17", + "session_7", + "session_8", + "session_19", + "session_2", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_14", + "session_8" + ], + "question": "What subject have Caroline and Melanie both painted?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_9", + "session_19", + "session_13", + "session_12", + "session_16", + "session_15", + "session_3", + "session_1", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_14", + "session_4" + ], + "question": "What symbols are important to Caroline?", + "recall": 0.5, + "retrieved_ids": [ + "session_14", + "session_7", + "session_2", + "session_11", + "session_6", + "session_18", + "session_15", + "session_3", + "session_8", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_14" + ], + "question": "When did Caroline encounter people on a hike and have a negative experience?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_4", + "session_16", + "session_8", + "session_14", + "session_11", + "session_5", + "session_15", + "session_18", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_14" + ], + "question": "When did Melanie make a plate in pottery class?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_5", + "session_16", + "session_12", + "session_17", + "session_8", + "session_15", + "session_7", + "session_10", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_12", + "session_14" + ], + "question": "Would Caroline be considered religious?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_18", + "session_1", + "session_8", + "session_9", + "session_7", + "session_5", + "session_14", + "session_4", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_15", + "session_2" + ], + "question": "What instruments does Melanie play?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_6", + "session_2", + "session_3", + "session_8", + "session_18", + "session_1", + "session_4", + "session_13", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_11", + "session_15" + ], + "question": "What musical artists/bands has Melanie seen?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_11", + "session_17", + "session_14", + "session_8", + "session_18", + "session_1", + "session_4", + "session_13", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_15" + ], + "question": "When did Melanie go to the park?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_16", + "session_13", + "session_1", + "session_14", + "session_5", + "session_11", + "session_18", + "session_8", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_15" + ], + "question": "When is Caroline's youth center putting on a talent show?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_9", + "session_11", + "session_14", + "session_5", + "session_19", + "session_12", + "session_10", + "session_16", + "session_17" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_15" + ], + "question": "Would Melanie likely enjoy the song \"The Four Seasons\" by Vivaldi?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_15", + "session_11", + "session_8", + "session_10", + "session_4", + "session_14", + "session_9", + "session_17", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_11", + "session_16" + ], + "question": "What are some changes Caroline has faced during her transition journey?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_16", + "session_3", + "session_8", + "session_7", + "session_18", + "session_11", + "session_14", + "session_9", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_16" + ], + "question": "What does Melanie do with her family on hikes?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_7", + "session_12", + "session_4", + "session_16", + "session_8", + "session_14", + "session_19", + "session_2", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_16" + ], + "question": "When did Caroline go biking with friends?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_6", + "session_3", + "session_8", + "session_19", + "session_2", + "session_12", + "session_4", + "session_17", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_16" + ], + "question": "How long has Melanie been practicing art?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_3", + "session_15", + "session_17", + "session_1", + "session_6", + "session_19", + "session_4", + "session_8", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_13", + "session_16", + "session_7" + ], + "question": "What personality traits might Melanie say Caroline has?", + "recall": 0.6667, + "retrieved_ids": [ + "session_10", + "session_16", + "session_15", + "session_8", + "session_14", + "session_1", + "session_18", + "session_9", + "session_4", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_15", + "session_17" + ], + "question": "What transgender-specific events has Caroline attended?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_15", + "session_9", + "session_17", + "session_3", + "session_1", + "session_14", + "session_5", + "session_10", + "session_2" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_17", + "session_7" + ], + "question": "What book did Melanie read from Caroline's suggestion?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_7", + "session_17", + "session_2", + "session_8", + "session_14", + "session_13", + "session_18", + "session_15", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_17" + ], + "question": "When did Melanie's friend adopt a child?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_2", + "session_17", + "session_19", + "session_8", + "session_6", + "session_16", + "session_3", + "session_12", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_17" + ], + "question": "When did Melanie get hurt?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_8", + "session_5", + "session_13", + "session_1", + "session_16", + "session_18", + "session_14", + "session_15", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_18" + ], + "question": "When did Melanie's family go on a roadtrip?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_10", + "session_7", + "session_17", + "session_4", + "session_3", + "session_13", + "session_2", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_18" + ], + "question": "How many children does Melanie have?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_10", + "session_3", + "session_18", + "session_1", + "session_4", + "session_13", + "session_9", + "session_19", + "session_17" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_18" + ], + "question": "When did Melanie go on a hike after the roadtrip?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_4", + "session_14", + "session_12", + "session_16", + "session_8", + "session_2", + "session_9", + "session_1", + "session_3" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_18" + ], + "question": "Would Melanie go on another roadtrip soon?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_1", + "session_9", + "session_13", + "session_11", + "session_12", + "session_16", + "session_8", + "session_4", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_19", + "session_7" + ], + "question": "What items has Melanie bought?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_4", + "session_8", + "session_18", + "session_1", + "session_13", + "session_9", + "session_17", + "session_15", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_19" + ], + "question": "When did Caroline pass the adoption interview?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_3", + "session_2", + "session_17", + "session_13", + "session_8", + "session_14", + "session_18", + "session_15", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_19" + ], + "question": "When did Melanie buy the figurines?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_8", + "session_14", + "session_13", + "session_18", + "session_15", + "session_1", + "session_5", + "session_11", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_19" + ], + "question": "Would Caroline want to move back to her home country soon?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_19", + "session_18", + "session_8", + "session_1", + "session_4", + "session_7", + "session_10", + "session_2", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What did the charity race raise awareness for?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_7", + "session_3", + "session_8", + "session_14", + "session_13", + "session_18", + "session_15", + "session_1", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What did Melanie realize after the charity race?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_7", + "session_8", + "session_18", + "session_9", + "session_1", + "session_4", + "session_3", + "session_14", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "How does Melanie prioritize self-care?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_9", + "session_13", + "session_4", + "session_1", + "session_7", + "session_11", + "session_17", + "session_19", + "session_16" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What are Caroline's plans for the summer?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_2", + "session_10", + "session_11", + "session_15", + "session_16", + "session_7", + "session_17", + "session_14", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What type of individuals does the adoption agency Caroline is considering support?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_11", + "session_19", + "session_13", + "session_17", + "session_15", + "session_14", + "session_8", + "session_1", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "Why did Caroline choose the adoption agency?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_2", + "session_13", + "session_17", + "session_16", + "session_8", + "session_14", + "session_18", + "session_15", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What is Caroline excited about in the adoption process?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_2", + "session_19", + "session_13", + "session_8", + "session_5", + "session_16", + "session_1", + "session_9", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What does Melanie think about Caroline's decision to adopt?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_2", + "session_17", + "session_19", + "session_8", + "session_12", + "session_4", + "session_5", + "session_16", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "How long have Mel and her husband been married?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_10", + "session_7", + "session_8", + "session_16", + "session_15", + "session_17", + "session_1", + "session_6", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What does Caroline's necklace symbolize?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_14", + "session_1", + "session_8", + "session_9", + "session_12", + "session_7", + "session_5", + "session_18", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What country is Caroline's grandma from?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_3", + "session_1", + "session_8", + "session_9", + "session_12", + "session_7", + "session_5", + "session_14", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What was grandma's gift to Caroline?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_19", + "session_1", + "session_8", + "session_9", + "session_12", + "session_7", + "session_5", + "session_14", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What is Melanie's hand-painted bowl a reminder of?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_8", + "session_5", + "session_12", + "session_16", + "session_11", + "session_14", + "session_1", + "session_13", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What did Melanie and her family do while camping?", + "recall": 0.0, + "retrieved_ids": [ + "session_10", + "session_7", + "session_16", + "session_8", + "session_17", + "session_13", + "session_18", + "session_9", + "session_6", + "session_2" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What kind of counseling and mental health services is Caroline interested in pursuing?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_7", + "session_11", + "session_5", + "session_6", + "session_1", + "session_2", + "session_8", + "session_3", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What workshop did Caroline attend recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_4", + "session_8", + "session_17", + "session_11", + "session_12", + "session_18", + "session_10", + "session_14", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What was discussed in the LGBTQ+ counseling workshop?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_8", + "session_7", + "session_1", + "session_5", + "session_6", + "session_9", + "session_11", + "session_14", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What motivated Caroline to pursue counseling?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_7", + "session_5", + "session_1", + "session_6", + "session_3", + "session_18", + "session_14", + "session_8", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What kind of place does Caroline want to create for people?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_14", + "session_17", + "session_2", + "session_3", + "session_5", + "session_6", + "session_11", + "session_10", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_5" + ], + "question": "Did Melanie make the black and white bowl in the photo?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_16", + "session_12", + "session_4", + "session_6", + "session_13", + "session_3", + "session_8", + "session_14", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_6" + ], + "question": "What kind of books does Caroline have in her library?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_7", + "session_10", + "session_17", + "session_2", + "session_4", + "session_3", + "session_5", + "session_11", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_6" + ], + "question": "What was Melanie's favorite book from her childhood?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_7", + "session_10", + "session_17", + "session_8", + "session_14", + "session_18", + "session_1", + "session_4", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What book did Caroline recommend to Melanie?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_17", + "session_6", + "session_8", + "session_14", + "session_13", + "session_18", + "session_15", + "session_1", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What did Caroline take away from the book \"Becoming Nicole\"?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_13", + "session_6", + "session_17", + "session_2", + "session_1", + "session_14", + "session_5", + "session_4", + "session_3" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What are the new shoes that Melanie got used for?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_17", + "session_6", + "session_15", + "session_9", + "session_4", + "session_1", + "session_16", + "session_19", + "session_2" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What is Melanie's reason for getting into running?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_2", + "session_16", + "session_18", + "session_4", + "session_10", + "session_15", + "session_14", + "session_6", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What does Melanie say running has been great for?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_10", + "session_2", + "session_6", + "session_17", + "session_15", + "session_9", + "session_8", + "session_3", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What did Mel and her kids make during the pottery workshop?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_16", + "session_10", + "session_4", + "session_7", + "session_18", + "session_5", + "session_12", + "session_17", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What kind of pot did Mel and her kids make with clay?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_10", + "session_5", + "session_7", + "session_2", + "session_4", + "session_3", + "session_6", + "session_11", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What creative project do Mel and her kids do together besides pottery?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_12", + "session_14", + "session_10", + "session_7", + "session_5", + "session_16", + "session_17", + "session_15", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What did Mel and her kids paint in their latest project in July 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_10", + "session_7", + "session_12", + "session_15", + "session_14", + "session_17", + "session_1", + "session_13", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What did Caroline see at the council meeting for adoption?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_2", + "session_17", + "session_19", + "session_13", + "session_10", + "session_5", + "session_15", + "session_7", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What do sunflowers represent according to Caroline?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_11", + "session_14", + "session_1", + "session_9", + "session_12", + "session_7", + "session_5", + "session_18", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "Why are flowers important to Melanie?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_14", + "session_7", + "session_2", + "session_11", + "session_6", + "session_18", + "session_15", + "session_3", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_9" + ], + "question": "What inspired Caroline's painting for the art show?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_16", + "session_9", + "session_17", + "session_11", + "session_8", + "session_12", + "session_13", + "session_4", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "How often does Melanie go to the beach with her kids?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_7", + "session_14", + "session_6", + "session_1", + "session_9", + "session_11", + "session_2", + "session_18", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "What did Melanie and her family see during their camping trip last year?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_18", + "session_8", + "session_7", + "session_16", + "session_12", + "session_17", + "session_4", + "session_9", + "session_6" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "How did Melanie feel while watching the meteor shower?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_8", + "session_17", + "session_13", + "session_16", + "session_14", + "session_1", + "session_5", + "session_15", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_11" + ], + "question": "Whose birthday did Melanie celebrate recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_4", + "session_17", + "session_10", + "session_14", + "session_8", + "session_13", + "session_12", + "session_18", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_11" + ], + "question": "Who performed at the concert at Melanie's daughter's birthday?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_4", + "session_8", + "session_18", + "session_1", + "session_13", + "session_9", + "session_19", + "session_17", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_12" + ], + "question": "Why did Melanie choose to use colors and patterns in her pottery project?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_16", + "session_19", + "session_10", + "session_4", + "session_14", + "session_2", + "session_8", + "session_7", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "What pet does Caroline have?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_13", + "session_1", + "session_8", + "session_9", + "session_12", + "session_5", + "session_14", + "session_18", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "What pets does Melanie have?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_13", + "session_8", + "session_18", + "session_1", + "session_4", + "session_9", + "session_19", + "session_17", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "Where did Oliver hide his bone once?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_10", + "session_6", + "session_7", + "session_11", + "session_8", + "session_14", + "session_18", + "session_15", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "What activity did Caroline used to do with her dad?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_10", + "session_5", + "session_15", + "session_7", + "session_8", + "session_14", + "session_18", + "session_1", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_14" + ], + "question": "What did Caroline make for a local church?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_8", + "session_15", + "session_5", + "session_7", + "session_10", + "session_16", + "session_18", + "session_17", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_14" + ], + "question": "What did Caroline find in her neighborhood during her walk?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_7", + "session_10", + "session_16", + "session_18", + "session_8", + "session_19", + "session_9", + "session_11", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "Which song motivates Caroline to be courageous?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_3", + "session_11", + "session_14", + "session_1", + "session_8", + "session_5", + "session_18", + "session_4", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "Which classical musicians does Melanie enjoy listening to?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_15", + "session_8", + "session_18", + "session_10", + "session_1", + "session_4", + "session_13", + "session_9", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "Who is Melanie a fan of in terms of modern music?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_5", + "session_11", + "session_8", + "session_18", + "session_1", + "session_4", + "session_13", + "session_9", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "How long has Melanie been creating art?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_3", + "session_15", + "session_17", + "session_1", + "session_6", + "session_19", + "session_4", + "session_8", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "What precautionary sign did Melanie see at the caf\u00e9?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_5", + "session_10", + "session_15", + "session_14", + "session_1", + "session_11", + "session_18", + "session_8", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What advice does Caroline give for getting started with adoption?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_13", + "session_2", + "session_19", + "session_8", + "session_15", + "session_12", + "session_18", + "session_4", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What setback did Melanie face in October 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_9", + "session_10", + "session_15", + "session_3", + "session_8", + "session_14", + "session_13", + "session_18", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What does Melanie do to keep herself busy during her pottery break?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_8", + "session_16", + "session_10", + "session_14", + "session_2", + "session_7", + "session_18", + "session_5", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What painting did Melanie show to Caroline on October 13, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_13", + "session_8", + "session_17", + "session_16", + "session_11", + "session_9", + "session_1", + "session_15", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What kind of painting did Caroline share with Melanie on October 13, 2023?", + "recall": 0.0, + "retrieved_ids": [ + "session_2", + "session_4", + "session_3", + "session_5", + "session_6", + "session_11", + "session_10", + "session_15", + "session_14", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What was the poetry reading that Caroline attended about?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_13", + "session_6", + "session_7", + "session_2", + "session_4", + "session_15", + "session_3", + "session_12", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What did the posters at the poetry reading say?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_10", + "session_6", + "session_7", + "session_2", + "session_8", + "session_14", + "session_13", + "session_18", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What does Caroline's drawing symbolize for her?", + "recall": 0.0, + "retrieved_ids": [ + "session_14", + "session_10", + "session_16", + "session_7", + "session_1", + "session_8", + "session_9", + "session_12", + "session_5", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "How do Melanie and Caroline describe their journey through life together?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_8", + "session_4", + "session_17", + "session_14", + "session_19", + "session_7", + "session_15", + "session_10", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What happened to Melanie's son on their road trip?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_10", + "session_12", + "session_17", + "session_7", + "session_6", + "session_14", + "session_2", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "How did Melanie's son handle the accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_16", + "session_8", + "session_14", + "session_13", + "session_15", + "session_1", + "session_5", + "session_11", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "How did Melanie feel about her family after the accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_10", + "session_7", + "session_2", + "session_9", + "session_1", + "session_4", + "session_3", + "session_14", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "How did Melanie's children handle the accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_16", + "session_14", + "session_13", + "session_15", + "session_1", + "session_5", + "session_11", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "How did Melanie feel after the accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_2", + "session_9", + "session_1", + "session_4", + "session_3", + "session_14", + "session_13", + "session_8", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What was Melanie's reaction to her children enjoying the Grand Canyon?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_10", + "session_3", + "session_7", + "session_1", + "session_4", + "session_13", + "session_9", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What do Melanie's family give her?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_7", + "session_3", + "session_19", + "session_2", + "session_18", + "session_6", + "session_4", + "session_8", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "How did Melanie feel about her family supporting her?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_7", + "session_8", + "session_17", + "session_16", + "session_3", + "session_2", + "session_15", + "session_18", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What did Melanie do after the road trip to relax?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_17", + "session_1", + "session_10", + "session_14", + "session_12", + "session_15", + "session_2", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_2" + ], + "question": "What did Caroline realize after her charity race?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_7", + "session_10", + "session_8", + "session_18", + "session_9", + "session_1", + "session_4", + "session_3", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_2" + ], + "question": "What are Melanie's plans for the summer with respect to adoption?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_2", + "session_12", + "session_17", + "session_10", + "session_13", + "session_8", + "session_11", + "session_15", + "session_16" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_2" + ], + "question": "What type of individuals does the adoption agency Melanie is considering support?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_11", + "session_19", + "session_13", + "session_17", + "session_15", + "session_14", + "session_8", + "session_1", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_2" + ], + "question": "Why did Melanie choose the adoption agency?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_2", + "session_13", + "session_17", + "session_16", + "session_8", + "session_14", + "session_18", + "session_15", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_2" + ], + "question": "What is Melanie excited about in her adoption process?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_2", + "session_19", + "session_10", + "session_7", + "session_13", + "session_8", + "session_5", + "session_16", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What does Melanie's necklace symbolize?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_14", + "session_8", + "session_18", + "session_1", + "session_13", + "session_9", + "session_19", + "session_17", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What country is Melanie's grandma from?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_3", + "session_8", + "session_18", + "session_1", + "session_13", + "session_9", + "session_19", + "session_17", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What was grandma's gift to Melanie?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_19", + "session_8", + "session_18", + "session_1", + "session_13", + "session_9", + "session_17", + "session_15", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What was grandpa's gift to Caroline?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_4", + "session_1", + "session_8", + "session_9", + "session_12", + "session_7", + "session_5", + "session_14", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What is Caroline's hand-painted bowl a reminder of?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_8", + "session_5", + "session_12", + "session_16", + "session_11", + "session_14", + "session_1", + "session_9", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What did Caroline and her family do while camping?", + "recall": 0.0, + "retrieved_ids": [ + "session_10", + "session_7", + "session_16", + "session_8", + "session_17", + "session_13", + "session_18", + "session_9", + "session_6", + "session_2" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What kind of counseling and mental health services is Melanie interested in pursuing?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_7", + "session_11", + "session_5", + "session_6", + "session_1", + "session_2", + "session_8", + "session_3", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What kind of counseling workshop did Melanie attend recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_13", + "session_8", + "session_5", + "session_6", + "session_7", + "session_1", + "session_11", + "session_10", + "session_17" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What motivated Melanie to pursue counseling?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_7", + "session_5", + "session_1", + "session_6", + "session_3", + "session_18", + "session_14", + "session_8", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What kind of place does Melanie want to create for people?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_14", + "session_17", + "session_2", + "session_3", + "session_5", + "session_6", + "session_11", + "session_10", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_5" + ], + "question": "Did Caroline make the black and white bowl in the photo?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_16", + "session_12", + "session_4", + "session_6", + "session_13", + "session_3", + "session_8", + "session_14", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_7" + ], + "question": "What are the new shoes that Caroline got used for?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_17", + "session_6", + "session_15", + "session_9", + "session_4", + "session_1", + "session_16", + "session_19", + "session_2" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_7" + ], + "question": "What is Caroline's reason for getting into running?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_2", + "session_16", + "session_18", + "session_4", + "session_10", + "session_15", + "session_14", + "session_5", + "session_6" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_7" + ], + "question": "What does Caroline say running has been great for?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_10", + "session_2", + "session_6", + "session_17", + "session_15", + "session_9", + "session_8", + "session_3", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_8" + ], + "question": "What did Melanie see at the council meeting for adoption?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_2", + "session_17", + "session_19", + "session_13", + "session_10", + "session_5", + "session_15", + "session_7", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_9" + ], + "question": "What inspired Melanie's painting for the art show?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_16", + "session_9", + "session_17", + "session_11", + "session_8", + "session_12", + "session_13", + "session_4", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_9" + ], + "question": "What inspired Caroline's sculpture for the art show?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_16", + "session_9", + "session_17", + "session_11", + "session_12", + "session_8", + "session_13", + "session_5", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_10" + ], + "question": "How often does Caroline go to the beach with her kids?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_7", + "session_14", + "session_6", + "session_1", + "session_9", + "session_11", + "session_2", + "session_18", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_10" + ], + "question": "What did Caroline and her family see during their camping trip last year?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_18", + "session_8", + "session_7", + "session_16", + "session_12", + "session_17", + "session_4", + "session_9", + "session_6" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_10" + ], + "question": "How did Caroline feel while watching the meteor shower?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_8", + "session_17", + "session_13", + "session_16", + "session_14", + "session_1", + "session_5", + "session_15", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_12" + ], + "question": "Why did Caroline choose to use colors and patterns in her pottery project?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_16", + "session_19", + "session_10", + "session_4", + "session_14", + "session_2", + "session_8", + "session_7", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_13" + ], + "question": "Is Oscar Melanie's pet?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_7", + "session_8", + "session_18", + "session_1", + "session_4", + "session_9", + "session_19", + "session_17", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_13" + ], + "question": "Where did Oscar hide his bone once?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_10", + "session_6", + "session_11", + "session_8", + "session_14", + "session_18", + "session_15", + "session_1", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_13" + ], + "question": "What activity did Melanie used to do with her dad?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_10", + "session_5", + "session_15", + "session_7", + "session_8", + "session_14", + "session_18", + "session_1", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_14" + ], + "question": "What did Melanie make for a local church?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_8", + "session_15", + "session_5", + "session_7", + "session_10", + "session_16", + "session_18", + "session_17", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_14" + ], + "question": "What did Melanie find in her neighborhood during her walk?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_7", + "session_10", + "session_16", + "session_18", + "session_8", + "session_19", + "session_9", + "session_11", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_15" + ], + "question": "Which song motivates Melanie to be courageous?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_3", + "session_11", + "session_14", + "session_1", + "session_8", + "session_5", + "session_18", + "session_4", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_15" + ], + "question": "What type of instrument does Caroline play?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_6", + "session_14", + "session_2", + "session_3", + "session_1", + "session_8", + "session_9", + "session_12", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_15" + ], + "question": "Which classical musicians does Caroline enjoy listening to?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_15", + "session_8", + "session_18", + "session_10", + "session_1", + "session_9", + "session_12", + "session_7", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_15" + ], + "question": "Who is Caroline a fan of in terms of modern music?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_5", + "session_11", + "session_1", + "session_8", + "session_9", + "session_12", + "session_7", + "session_14", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_16" + ], + "question": "What precautionary sign did Caroline see at the caf\u00e9?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_5", + "session_10", + "session_15", + "session_14", + "session_1", + "session_11", + "session_18", + "session_8", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_17" + ], + "question": "What setback did Caroline face recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_10", + "session_9", + "session_15", + "session_3", + "session_13", + "session_11", + "session_12", + "session_18", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_17" + ], + "question": "What does Caroline do to keep herself busy during her pottery break?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_8", + "session_16", + "session_10", + "session_14", + "session_2", + "session_7", + "session_18", + "session_5", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_17" + ], + "question": "What was the poetry reading that Melanie attended about?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_13", + "session_6", + "session_7", + "session_2", + "session_4", + "session_15", + "session_3", + "session_11", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "What happened to Caroline's son on their road trip?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_10", + "session_12", + "session_17", + "session_7", + "session_6", + "session_14", + "session_2", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "How did Caroline's son handle the accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_16", + "session_8", + "session_14", + "session_13", + "session_15", + "session_1", + "session_5", + "session_11", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "How did Caroline feel about her family after the accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_10", + "session_7", + "session_2", + "session_9", + "session_1", + "session_4", + "session_3", + "session_14", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "How did Caroline's children handle the accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_16", + "session_14", + "session_13", + "session_15", + "session_1", + "session_5", + "session_11", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "How did Caroline feel after the accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_2", + "session_9", + "session_1", + "session_4", + "session_3", + "session_14", + "session_13", + "session_8", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "What was Caroline's reaction to her children enjoying the Grand Canyon?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_10", + "session_3", + "session_7", + "session_1", + "session_9", + "session_12", + "session_5", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "What did Caroline do after the road trip to relax?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_17", + "session_1", + "session_10", + "session_14", + "session_12", + "session_15", + "session_2", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "What does Caroline love most about camping with her family?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_18", + "session_7", + "session_14", + "session_9", + "session_6", + "session_2", + "session_4", + "session_16", + "session_8" + ], + "sample_id": "conv-26" + } + ], + "runtime_seconds": 5.863, + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/prepush_smoke_20260423_123335/runs/locomo_old_brainctl_cmd_session.json b/benchmarks/results/prepush_smoke_20260423_123335/runs/locomo_old_brainctl_cmd_session.json new file mode 100644 index 0000000..50c7cab --- /dev/null +++ b/benchmarks/results/prepush_smoke_20260423_123335/runs/locomo_old_brainctl_cmd_session.json @@ -0,0 +1,38 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\locomo_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\locomo_old_brainctl_cmd_session.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" + }, + "benchmark": "locomo", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "measured": false, + "metadata": {}, + "metrics": { + "avg_recall": 0.9217, + "cat_1_recall": 0.7614, + "cat_2_recall": 0.92, + "cat_3_recall": 0.6616, + "cat_4_recall": 0.9738, + "cat_5_recall": 0.9821, + "perfect_rate": 0.8817, + "top_k": 10, + "zero_rate": 0.0438 + }, + "mode": "cmd_session", + "notes": [ + "granularity=session", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": 0.9217, + "reference_kind": "historical", + "runtime_seconds": 445.74, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_mempalace_raw_session.json b/benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_mempalace_raw_session.json new file mode 100644 index 0000000..fe13c4a --- /dev/null +++ b/benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_mempalace_raw_session.json @@ -0,0 +1,33 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\longmemeval_mempalace_raw_session.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "measured": false, + "metadata": {}, + "metrics": { + "ndcg_at_10": 0.8948, + "ndcg_at_5": 0.893, + "r_at_10": 0.983, + "r_at_5": 0.966 + }, + "mode": "raw_session", + "notes": [ + "top_k=10", + "Runs MemPalace benchmark module raw session retrieval logic directly." + ], + "primary_metric": "r_at_5", + "primary_metric_value": 0.966, + "reference_kind": "historical", + "runtime_seconds": 695.36, + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "mempalace" +} \ No newline at end of file diff --git a/benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_new_brainctl_brain.json b/benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_new_brainctl_brain.json new file mode 100644 index 0000000..540b000 --- /dev/null +++ b/benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_new_brainctl_brain.json @@ -0,0 +1,56 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\longmemeval_new_brainctl_brain.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 1, + "measured": true, + "metadata": {}, + "metrics": { + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0 + }, + "mode": "brain", + "notes": [ + "top_k=10", + "Legacy 470-question session-level slice." + ], + "primary_metric": "r_at_5", + "primary_metric_value": 1.0, + "reference_kind": "measured", + "rows": [ + { + "answer_session_ids": [ + "answer_280352e9" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e47becba", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_280352e9", + "02bd2b90_3", + "sharegpt_Cr2tc1f_0", + "f6859b48_2", + "ultrachat_214101", + "5dac7cc2_1" + ] + } + ], + "runtime_seconds": 2.792, + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_new_brainctl_cmd.json b/benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_new_brainctl_cmd.json new file mode 100644 index 0000000..3876df2 --- /dev/null +++ b/benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_new_brainctl_cmd.json @@ -0,0 +1,56 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\longmemeval_new_brainctl_cmd.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 1, + "measured": true, + "metadata": {}, + "metrics": { + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0 + }, + "mode": "cmd", + "notes": [ + "top_k=10", + "Legacy 470-question session-level slice." + ], + "primary_metric": "r_at_5", + "primary_metric_value": 1.0, + "reference_kind": "measured", + "rows": [ + { + "answer_session_ids": [ + "answer_280352e9" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e47becba", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_280352e9", + "02bd2b90_3", + "sharegpt_Cr2tc1f_0", + "f6859b48_2", + "ultrachat_214101", + "5dac7cc2_1" + ] + } + ], + "runtime_seconds": 2.587, + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_old_brainctl_brain.json b/benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_old_brainctl_brain.json new file mode 100644 index 0000000..65a3126 --- /dev/null +++ b/benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_old_brainctl_brain.json @@ -0,0 +1,32 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\longmemeval_old_brainctl_brain.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "measured": false, + "metadata": {}, + "metrics": { + "ndcg_at_10": 0.9253, + "ndcg_at_5": 0.9204, + "r_at_10": 0.9894, + "r_at_5": 0.9681 + }, + "mode": "brain", + "notes": [ + "top_k=10" + ], + "primary_metric": "r_at_5", + "primary_metric_value": 0.9681, + "reference_kind": "historical", + "runtime_seconds": 85.439, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_old_brainctl_cmd.json b/benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_old_brainctl_cmd.json new file mode 100644 index 0000000..3112e98 --- /dev/null +++ b/benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_old_brainctl_cmd.json @@ -0,0 +1,32 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\longmemeval_old_brainctl_cmd.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "measured": false, + "metadata": {}, + "metrics": { + "ndcg_at_10": 0.9247, + "ndcg_at_5": 0.9206, + "r_at_10": 0.9894, + "r_at_5": 0.9702 + }, + "mode": "cmd", + "notes": [ + "top_k=10" + ], + "primary_metric": "r_at_5", + "primary_metric_value": 0.9702, + "reference_kind": "historical", + "runtime_seconds": 130.863, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/prepush_smoke_20260423_123335/runs/membench_mempalace_raw_turn.json b/benchmarks/results/prepush_smoke_20260423_123335/runs/membench_mempalace_raw_turn.json new file mode 100644 index 0000000..564a544 --- /dev/null +++ b/benchmarks/results/prepush_smoke_20260423_123335/runs/membench_mempalace_raw_turn.json @@ -0,0 +1,35 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\membench_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\membench_mempalace_raw_turn.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" + }, + "benchmark": "membench", + "caveats": [ + "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 200, + "measured": false, + "metadata": {}, + "metrics": { + "hit_at_5": 0.885, + "simple_hit_at_5": 0.885, + "top_k": 5 + }, + "mode": "raw_turn", + "notes": [ + "FirstAgent slice only", + "turn-level retrieval", + "topic=all" + ], + "primary_metric": "hit_at_5", + "primary_metric_value": 0.885, + "reference_kind": "historical", + "runtime_seconds": 804.35, + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "partial", + "system_name": "mempalace" +} \ No newline at end of file diff --git a/benchmarks/results/prepush_smoke_20260423_123335/runs/membench_new_brainctl_cmd_turn.json b/benchmarks/results/prepush_smoke_20260423_123335/runs/membench_new_brainctl_cmd_turn.json new file mode 100644 index 0000000..0903d28 --- /dev/null +++ b/benchmarks/results/prepush_smoke_20260423_123335/runs/membench_new_brainctl_cmd_turn.json @@ -0,0 +1,54 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\membench_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\membench_new_brainctl_cmd_turn.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" + }, + "benchmark": "membench", + "caveats": [ + "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 1, + "measured": true, + "metadata": {}, + "metrics": { + "hit_at_5": 1.0, + "simple_hit_at_5": 1.0, + "top_k": 5 + }, + "mode": "cmd_turn", + "notes": [ + "FirstAgent slice only", + "turn-level retrieval", + "topic=all" + ], + "primary_metric": "hit_at_5", + "primary_metric_value": 1.0, + "reference_kind": "measured", + "rows": [ + { + "category": "simple", + "hit_at_k": true, + "question": "What is the name of my niece's company?", + "retrieved_ids": [ + "simple_roles_0|sid=119|g=119|s=5|t=18", + "simple_roles_0|sid=101|g=101|s=5|t=0", + "simple_roles_0|sid=135|g=135|s=6|t=12", + "simple_roles_0|sid=11|g=11|s=0|t=11", + "simple_roles_0|sid=25|g=25|s=1|t=5" + ], + "target_ids": [ + "119" + ], + "tid": 0, + "topic": "roles" + } + ], + "runtime_seconds": 0.558, + "series_name": "new_brainctl", + "source_path": null, + "status": "partial", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/prepush_smoke_20260423_123335/runs/membench_old_brainctl_cmd_turn.json b/benchmarks/results/prepush_smoke_20260423_123335/runs/membench_old_brainctl_cmd_turn.json new file mode 100644 index 0000000..149a1ca --- /dev/null +++ b/benchmarks/results/prepush_smoke_20260423_123335/runs/membench_old_brainctl_cmd_turn.json @@ -0,0 +1,35 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\membench_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\membench_old_brainctl_cmd_turn.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" + }, + "benchmark": "membench", + "caveats": [ + "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 200, + "measured": false, + "metadata": {}, + "metrics": { + "hit_at_5": 0.93, + "simple_hit_at_5": 0.93, + "top_k": 5 + }, + "mode": "cmd_turn", + "notes": [ + "FirstAgent slice only", + "turn-level retrieval", + "topic=all" + ], + "primary_metric": "hit_at_5", + "primary_metric_value": 0.93, + "reference_kind": "historical", + "runtime_seconds": 140.592, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "partial", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/prepush_smoke_20260423_123335/summary.csv b/benchmarks/results/prepush_smoke_20260423_123335/summary.csv new file mode 100644 index 0000000..7aad1f9 --- /dev/null +++ b/benchmarks/results/prepush_smoke_20260423_123335/summary.csv @@ -0,0 +1,15 @@ +benchmark,series_name,system_name,mode,reference_kind,status,example_count,primary_metric,primary_metric_value,runtime_seconds,dataset_path,source_path,notes,caveats,avg_recall,cat_1_recall,cat_2_recall,cat_3_recall,cat_4_recall,cat_5_recall,hit_at_5,ndcg_at_10,ndcg_at_5,perfect_rate,r_at_10,r_at_5,simple_hit_at_5,top_k,zero_rate +longmemeval,old_brainctl,brainctl,brain,historical,full_same_machine,470,r_at_5,0.9681,85.439,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,top_k=10,,,,,,,,,0.9253,0.9204,,0.9894,0.9681,,, +longmemeval,old_brainctl,brainctl,cmd,historical,full_same_machine,470,r_at_5,0.9702,130.863,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,top_k=10,,,,,,,,,0.9247,0.9206,,0.9894,0.9702,,, +longmemeval,mempalace,mempalace,raw_session,historical,full_same_machine,470,r_at_5,0.966,695.36,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,top_k=10 | Runs MemPalace benchmark module raw session retrieval logic directly.,,,,,,,,,0.8948,0.893,,0.983,0.966,,, +locomo,old_brainctl,brainctl,cmd_session,historical,full_same_machine,1986,avg_recall,0.9217,445.74,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,granularity=session | top_k=10,,0.9217,0.7614,0.92,0.6616,0.9738,0.9821,,,,0.8817,,,,10,0.0438 +locomo,mempalace,mempalace,raw_session,historical,full_same_machine,1986,avg_recall,0.6028,2106.411,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,granularity=session | top_k=10,,0.6028,0.5899,0.6921,0.4604,0.5809,0.6188,,,,0.5534,,,,10,0.3499 +membench,old_brainctl,brainctl,cmd_turn,historical,partial,200,hit_at_5,0.93,140.592,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,FirstAgent slice only | turn-level retrieval | topic=all,MemBench comparison is partial because ThirdAgent and noise-extended slices are not included.,,,,,,,0.93,,,,,,0.93,5, +membench,mempalace,mempalace,raw_turn,historical,partial,200,hit_at_5,0.885,804.35,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,FirstAgent slice only | turn-level retrieval | topic=all,MemBench comparison is partial because ThirdAgent and noise-extended slices are not included.,,,,,,,0.885,,,,,,0.885,5, +convomem,old_brainctl,brainctl,cmd,historical,blocked,0,avg_recall,,,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,limit_per_category=1 | top_k=10,Blocked while loading ConvoMem evidence data: ,,,,,,,,,,,,,,, +convomem,mempalace,mempalace,raw,historical,blocked,0,avg_recall,,,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,limit_per_category=1 | top_k=10,Blocked while loading ConvoMem evidence data: ,,,,,,,,,,,,,,, +longmemeval,new_brainctl,brainctl,brain,measured,full_same_machine,1,r_at_5,1.0,2.792,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,,top_k=10 | Legacy 470-question session-level slice.,,,,,,,,,1.0,1.0,,1.0,1.0,,, +longmemeval,new_brainctl,brainctl,cmd,measured,full_same_machine,1,r_at_5,1.0,2.587,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,,top_k=10 | Legacy 470-question session-level slice.,,,,,,,,,1.0,1.0,,1.0,1.0,,, +locomo,new_brainctl,brainctl,cmd_session,measured,full_same_machine,199,avg_recall,0.9413,5.863,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,,granularity=session | top_k=10,,0.9413,0.8641,1.0,0.7436,0.9571,0.9787,,,,0.9196,,,,10,0.0402 +membench,new_brainctl,brainctl,cmd_turn,measured,partial,1,hit_at_5,1.0,0.558,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,,FirstAgent slice only | turn-level retrieval | topic=all,MemBench comparison is partial because ThirdAgent and noise-extended slices are not included.,,,,,,,1.0,,,,,,1.0,5, +convomem,new_brainctl,brainctl,cmd,measured,blocked,0,avg_recall,,,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache,,limit_per_category=1 | top_k=10,Blocked while loading ConvoMem evidence data: HTTP Error 404: Not Found,,,,,,,,,,,,,,, diff --git a/benchmarks/results/prepush_smoke_20260423_123335/summary.json b/benchmarks/results/prepush_smoke_20260423_123335/summary.json new file mode 100644 index 0000000..691bf68 --- /dev/null +++ b/benchmarks/results/prepush_smoke_20260423_123335/summary.json @@ -0,0 +1,509 @@ +{ + "generated_at_utc": "2026-04-23T12:33:49.889512+00:00", + "metadata": { + "argv": [ + "benchmarks\\compare_memory_engines.py", + "--label", + "prepush_smoke", + "--longmemeval-limit", + "1", + "--locomo-limit", + "1", + "--membench-limit", + "1", + "--convomem-limit-per-category", + "1" + ], + "cwd": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl", + "datasets": { + "convomem_cache": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "locomo_data": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "longmemeval_data": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "membench_data": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent" + }, + "generated_at_utc": "2026-04-23T12:33:49.807677+00:00", + "git_commit": "c130fbd47174a0932822db2ef7fe64e18a74d30d", + "historical_summary_mode": "recovered", + "historical_summary_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "machine": "AMD64", + "platform": "Windows-11-10.0.26200-SP0", + "processor": "Intel64 Family 6 Model 189 Stepping 1, GenuineIntel", + "python_version": "3.14.4 (tags/v3.14.4:23116f9, Apr 7 2026, 14:10:54) [MSC v.1944 64 bit (AMD64)]" + }, + "notes": [ + "Historical old-BrainCTL and MemPalace series come from the recovered 2026-04-18 comparison bundle.", + "New BrainCTL series are rerun in the current checked-out repo using the legacy benchmark definitions.", + "MemBench remains intentionally partial because the legacy comparison only covered the FirstAgent slice.", + "ConvoMem remains a coverage/status benchmark here; it has no dedicated comparison chart in the legacy chart pack." + ], + "runs": [ + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\longmemeval_old_brainctl_brain.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "measured": false, + "metadata": {}, + "metrics": { + "ndcg_at_10": 0.9253, + "ndcg_at_5": 0.9204, + "r_at_10": 0.9894, + "r_at_5": 0.9681 + }, + "mode": "brain", + "notes": [ + "top_k=10" + ], + "primary_metric": "r_at_5", + "primary_metric_value": 0.9681, + "reference_kind": "historical", + "runtime_seconds": 85.439, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\longmemeval_old_brainctl_cmd.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "measured": false, + "metadata": {}, + "metrics": { + "ndcg_at_10": 0.9247, + "ndcg_at_5": 0.9206, + "r_at_10": 0.9894, + "r_at_5": 0.9702 + }, + "mode": "cmd", + "notes": [ + "top_k=10" + ], + "primary_metric": "r_at_5", + "primary_metric_value": 0.9702, + "reference_kind": "historical", + "runtime_seconds": 130.863, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\longmemeval_mempalace_raw_session.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "measured": false, + "metadata": {}, + "metrics": { + "ndcg_at_10": 0.8948, + "ndcg_at_5": 0.893, + "r_at_10": 0.983, + "r_at_5": 0.966 + }, + "mode": "raw_session", + "notes": [ + "top_k=10", + "Runs MemPalace benchmark module raw session retrieval logic directly." + ], + "primary_metric": "r_at_5", + "primary_metric_value": 0.966, + "reference_kind": "historical", + "runtime_seconds": 695.36, + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "mempalace" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\locomo_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\locomo_old_brainctl_cmd_session.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" + }, + "benchmark": "locomo", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "measured": false, + "metadata": {}, + "metrics": { + "avg_recall": 0.9217, + "cat_1_recall": 0.7614, + "cat_2_recall": 0.92, + "cat_3_recall": 0.6616, + "cat_4_recall": 0.9738, + "cat_5_recall": 0.9821, + "perfect_rate": 0.8817, + "top_k": 10, + "zero_rate": 0.0438 + }, + "mode": "cmd_session", + "notes": [ + "granularity=session", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": 0.9217, + "reference_kind": "historical", + "runtime_seconds": 445.74, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\locomo_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\locomo_mempalace_raw_session.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" + }, + "benchmark": "locomo", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "measured": false, + "metadata": {}, + "metrics": { + "avg_recall": 0.6028, + "cat_1_recall": 0.5899, + "cat_2_recall": 0.6921, + "cat_3_recall": 0.4604, + "cat_4_recall": 0.5809, + "cat_5_recall": 0.6188, + "perfect_rate": 0.5534, + "top_k": 10, + "zero_rate": 0.3499 + }, + "mode": "raw_session", + "notes": [ + "granularity=session", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": 0.6028, + "reference_kind": "historical", + "runtime_seconds": 2106.411, + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "mempalace" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\membench_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\membench_old_brainctl_cmd_turn.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" + }, + "benchmark": "membench", + "caveats": [ + "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 200, + "measured": false, + "metadata": {}, + "metrics": { + "hit_at_5": 0.93, + "simple_hit_at_5": 0.93, + "top_k": 5 + }, + "mode": "cmd_turn", + "notes": [ + "FirstAgent slice only", + "turn-level retrieval", + "topic=all" + ], + "primary_metric": "hit_at_5", + "primary_metric_value": 0.93, + "reference_kind": "historical", + "runtime_seconds": 140.592, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "partial", + "system_name": "brainctl" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\membench_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\membench_mempalace_raw_turn.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" + }, + "benchmark": "membench", + "caveats": [ + "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 200, + "measured": false, + "metadata": {}, + "metrics": { + "hit_at_5": 0.885, + "simple_hit_at_5": 0.885, + "top_k": 5 + }, + "mode": "raw_turn", + "notes": [ + "FirstAgent slice only", + "turn-level retrieval", + "topic=all" + ], + "primary_metric": "hit_at_5", + "primary_metric_value": 0.885, + "reference_kind": "historical", + "runtime_seconds": 804.35, + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "partial", + "system_name": "mempalace" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\convomem_old_brainctl_cmd.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" + }, + "benchmark": "convomem", + "caveats": [ + "Blocked while loading ConvoMem evidence data: " + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "example_count": 0, + "measured": false, + "metadata": {}, + "metrics": {}, + "mode": "cmd", + "notes": [ + "limit_per_category=1", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": null, + "reference_kind": "historical", + "runtime_seconds": null, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "blocked", + "system_name": "brainctl" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\convomem_mempalace_raw.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" + }, + "benchmark": "convomem", + "caveats": [ + "Blocked while loading ConvoMem evidence data: " + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "example_count": 0, + "measured": false, + "metadata": {}, + "metrics": {}, + "mode": "raw", + "notes": [ + "limit_per_category=1", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": null, + "reference_kind": "historical", + "runtime_seconds": null, + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "blocked", + "system_name": "mempalace" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\longmemeval_new_brainctl_brain.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 1, + "measured": true, + "metadata": {}, + "metrics": { + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0 + }, + "mode": "brain", + "notes": [ + "top_k=10", + "Legacy 470-question session-level slice." + ], + "primary_metric": "r_at_5", + "primary_metric_value": 1.0, + "reference_kind": "measured", + "runtime_seconds": 2.792, + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\longmemeval_new_brainctl_cmd.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 1, + "measured": true, + "metadata": {}, + "metrics": { + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0 + }, + "mode": "cmd", + "notes": [ + "top_k=10", + "Legacy 470-question session-level slice." + ], + "primary_metric": "r_at_5", + "primary_metric_value": 1.0, + "reference_kind": "measured", + "runtime_seconds": 2.587, + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\locomo_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\locomo_new_brainctl_cmd_session.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" + }, + "benchmark": "locomo", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 199, + "measured": true, + "metadata": {}, + "metrics": { + "avg_recall": 0.9413, + "cat_1_recall": 0.8641, + "cat_2_recall": 1.0, + "cat_3_recall": 0.7436, + "cat_4_recall": 0.9571, + "cat_5_recall": 0.9787, + "perfect_rate": 0.9196, + "top_k": 10, + "zero_rate": 0.0402 + }, + "mode": "cmd_session", + "notes": [ + "granularity=session", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": 0.9413, + "reference_kind": "measured", + "runtime_seconds": 5.863, + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\membench_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\membench_new_brainctl_cmd_turn.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" + }, + "benchmark": "membench", + "caveats": [ + "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 1, + "measured": true, + "metadata": {}, + "metrics": { + "hit_at_5": 1.0, + "simple_hit_at_5": 1.0, + "top_k": 5 + }, + "mode": "cmd_turn", + "notes": [ + "FirstAgent slice only", + "turn-level retrieval", + "topic=all" + ], + "primary_metric": "hit_at_5", + "primary_metric_value": 1.0, + "reference_kind": "measured", + "runtime_seconds": 0.558, + "series_name": "new_brainctl", + "source_path": null, + "status": "partial", + "system_name": "brainctl" + }, + { + "artifacts": { + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\convomem_new_brainctl_cmd.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" + }, + "benchmark": "convomem", + "caveats": [ + "Blocked while loading ConvoMem evidence data: HTTP Error 404: Not Found" + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "example_count": 0, + "measured": true, + "metadata": {}, + "metrics": {}, + "mode": "cmd", + "notes": [ + "limit_per_category=1", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": null, + "reference_kind": "measured", + "runtime_seconds": null, + "series_name": "new_brainctl", + "source_path": null, + "status": "blocked", + "system_name": "brainctl" + } + ] +} \ No newline at end of file diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/README.md b/benchmarks/results/smoke_legacy_compare_20260423_105447/README.md new file mode 100644 index 0000000..1a413ec --- /dev/null +++ b/benchmarks/results/smoke_legacy_compare_20260423_105447/README.md @@ -0,0 +1,31 @@ +# Legacy BrainCTL vs MemPalace comparison bundle + +- Current repo commit: `c130fbd47174a0932822db2ef7fe64e18a74d30d` +- Historical reference source: `C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json` +- Historical source mode: `recovered summary bundle` +- Command: `benchmarks\compare_memory_engines.py --label smoke_legacy_compare --longmemeval-limit 3 --locomo-limit 1 --membench-limit 3 --convomem-limit-per-category 1` + +## Datasets + +- LongMemEval: `C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json` +- LoCoMo: `C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json` +- MemBench FirstAgent: `C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent` +- ConvoMem cache: `C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache` + +## What is measured now + +- New BrainCTL reruns: LongMemEval `brain` and `cmd`, LoCoMo `cmd_session`, MemBench FirstAgent `cmd_turn`, and ConvoMem `cmd` coverage/status. +- Old BrainCTL and MemPalace are frozen historical reference series loaded from the recovered 2026-04-18 bundle. + +## Blocked or partial runs + +- convomem old_brainctl cmd: Blocked while loading ConvoMem evidence data: +- convomem mempalace raw: Blocked while loading ConvoMem evidence data: +- convomem new_brainctl cmd: Blocked while loading ConvoMem evidence data: HTTP Error 404: Not Found + +## Output files + +- `summary.json` and `summary.csv`: all series in one table. +- `comparison_table.json` and `comparison_table.csv`: long-form metric rows. +- `runs/*.json`: per-run payloads. +- `charts/*.png`: regenerated charts with old BrainCTL, new BrainCTL, and MemPalace together. diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/charts/aggregate_primary_metrics.png b/benchmarks/results/smoke_legacy_compare_20260423_105447/charts/aggregate_primary_metrics.png new file mode 100644 index 0000000000000000000000000000000000000000..91ee6a4b5b60cfd85a4fc065daf6c30f817c8c73 GIT binary patch literal 72024 zcmdqJby!txyDy4~poofzq?D3McdImtbc!^S2`Jqt3ML5BNQWRynn{C*N(+K8r9rxT z5|cRh5Wn?(Yp=D=S$m)T$3ACV*ZYb%V~l4!&wc;u887ar+$JNXBqbsuB9oVsRU;xg zl0Zasi0;@?_)X`hudMKgu;Wc_$9s0>jxNTJ%!rhY9qn!G9Br%~&^enua9<> zol}sV&eGA*-a(j)3;l1e;Iw;W!BwN@^c_Bh#9mIvfr#j04D#Orc_+V6q60)k^0HFu zuJ332kA#Mf5%!i|&1a{UUp^cWa)yLNs_gW&BP2)8-BCX*`&LcCT`F{bUjNjo(DNyZ zZ|D2=O6M2$q6TN@oAK8Rnh8$d-|NIxOhiQ)>@@S?s2S+!=p^Aw{~XUB+(_Q?{reZ9 zQ<8`N{;2OyYWep^U&Vj<^%Ek>zrP+jbGYpv?~9=GMaBd)yI)v(Q`J3ng0d#zLJcLuA*N7HY=uxP!dTU*h&hO}F{2XIG-d?)>p zp{!^>c<|sTey%s)XV3eR(4&;Itm<-U$}!{f#`Mw4`0Hn3(2j8-1Gd#OViP6pXShxKU zG}c{UVV*_nvz=Fg??IWz3qJfU_fb6~#dC9cxt3g_euuCgJD>CJ`;?&58DXc@$-7g( z@{&jEgJpi#1=)33Y1n4#mo~R5$MBP_ZDH*O2W(>KW&G`*5u{j#JG!nNI(p(dj`ePm z?08S*%6E}`yTK|+0=hf@D?6^ync5*v)NOPor<%Xj$nqqW;C-5$`DC1aYjO@*Z2B^c zvPw=w+`X2_8w|gmM&P{?=UBd~U*Ob#8z<(WFi_#vQaZ~*bz+9`DK6W9XJm10E$nB; zy_j%roqW?9IipgGjWl~_8 zVNvNhxT9p0^*u$X+HLP26B&vToa2M;C#V%`{mGPbHrpmWd|>p)Z=|Tk4`8dSg4P~x zFV)e~&*v=cz>QOE2w`H94`Cd~Z_VU1wMy*G7KVsV&MXc3?D6HS^_4orpdzo`W47x2 z81`Q9;gvRT#c=lE;sN(2XUy8#b@S%PQ-gzh+bcn@DftvWYUS7t#5^4N#35T<=1qK>Vgz!8su+-&&}-Tf&37Nh z^#I8k#$(uON?yyin?i;at_K8c`|RBf7W&ICycK--pkVGpyf@`3nk`SAiM}c?4@sC~ zRN`?POC!WKzjAA4c}(k1eZXOLi>wd8nsRZ?zQAR8cy+3yZmPL zBG53X+-2dUTokui?%bNwh1Dg!rYF%n7J+kH{$tz5sjb!XnL(=SF3z)0jvVh9;0h-u zr#Z~17~Y%(r{eN_ViWETx^=iEMxd7O7GeEogpv1@<7-&WCj_Aubd11EV4{P;qc^M# zq!u|U>Vaqfvfeb_5`zvBnN%KRnd!nf2>Vy>Zn4qb4y2+Flxj6@-Cgg10&u7?f@{HX z5NeQ!XwJ^e5=?;;auJ-dwL5E0?u$cGE8?3&e&p}tCMwql99-k{ifx~5iNBy=QSpU| zTf0-cQvSX1tO1*jCdNQ3y4d2sT*kVDUt>ui?!@6I)@CGzpv^PR!v#Ee+ko6Jc- z5%X0`RfxJS=OgZcxA|0r!YM8O9vLw?p3IwmQlZ$cD@*4SjK@LexgN78=KWf=sj%*1 zuB)uCudO+`QfEnWShc4dfQe30A1~qaslsjLAg4i@sqM-{t3avh%{?Ud9B(7D{0CM>oO!@W(c+wp>f%&>;z=J&-l$wo_5-USOo}#--L_Vz4t;xliNb#N*Nv6OzP8o&!=HVjOdfHZ{&|fq z>3}1Cwi}I8H(J;#!{O8-I1Ow}GVdflAz_}=nOWO>xGJns7UZ-I_ddYmHGbh5o6^q- z<+^lSqEWT?fjpBs{+UM;-_cK&67UOyb_2LFWvIl5pf;Ih#~R9by%%#|%{4F~D~+u* zf>viMtxI-mt<0#(Q)s9$jBVU2ea11SIf_T5;=H@~jNdWpSKg=cT^0;i1XsmA&}Mrs zH?WMmNVo0G4${6j!$f*f)HT;FerjV)ETQ#H5v)1Xli1$fwe-5S-r4Sg7J^<~PLA@- zp+JEF*IN962>&+sxm8i(lT^oC33J8hx`$^AXHaF1Q|$+20#1_sDtVYZ=Q0U%@8EeZ zLz_5}nOZ)w5o&|v++EI-*L>r+&0fx}hgYSo?Hnf}>I*>ntE)264wUsCtlJEx*rsXD z??fd=JMYRDhi6Q_U3vj0sj23Tvh{@u3}tx8d|*N@-0!t~epW5vEG1cZmZXwjLc%PY z`^sp8$#EH(kCa?@lb&{28=BclO-$+R@-Ivbdsy`raOMq zA)*Q=(zT-;5^1mY@#*DvnD|cStc-EeeK`HRYBvl0USl=j%dS{z=UgDo#Aw4ibg$>$ z&U#JP1vbs*tW5pVM+xMlvrzcHpAS4*`5nJ-K`dNzn>dv=RJ(VQ_=HqvmQIKBHo6xp z8^5!_LZ-pzNv=?rOsg`X}BKLz;1+kf#Ty8I;(C z(Yx>)JLtLc1}T73TYzpNJ?4}Nu(pI|H_Ony{Cc#L^^;wgn5lPE>YX1olv@o`#muS+ z^W~>6cdFsIW*EKr zO}Fihr8J9N9#|ZKix-Omu=rr-(N|nm*S$(QkyVOegBxEeq-znykB% z0=txlhS9!|+XjJZl)URH zjDN{kR(71v?2QhKLxK1m`y;WgcUVvkvNCnTPScU`J3Bjm%8nLZxykPi9gQ8GgDRPS zzvkddfl9mOk@u$)X_d5F-!Lpx5whWCoX#<-rkSOjCdeQ1v}i5g^}ecGX!)Ez+aI?@ z_xlDd$dEk1QBe zw1ykeo~C@6XDMOVunlFhW0*fcURJFlNO^AIvRAbLs;k+vK4qkfr+?oVUjaXQRM;Tv`6(^et#jM5RDi268@P7 zfTDLw7{l5%bggjaLt^((jo%pb^M0Y&H4~DZotfhn45Hr%_W-{vUsyhX2P{>+DFH zjyFdw1M2JTYKh{hW5*G3bJw6;zombV6GIV>K0ew$4aJ1{#GB@o>;o9hhJe);;?$t- zA1moI)LtX>S<>!vA|l0^115pqTw*@Eed@`rkvC*MOB<$YrfIzCc_~%<0ZL&2<){w5im6a0&Fgw+DK-_p z(H}WAsEw@0cGccHrcL1-%)USD+uUb$Pp3b@COxko>AWMKd7&sD8bK;{Y>RITnszVt zMOLnUP_CgKVdlKrG1?Gltn#Ke$N5svePPY2v65v{c`w*JPH}QjU=s*(d9Xul?KdrE1{kEvju4hMqHsP~%yfKoKeQI9c;U__cetM8muUt=68LY~sk*C=WK%_z>{(BOR;dGgob# z2_9S`RakasNR4&CDQfKQ_dSoMb;Y07FP_QnoxihK(3iNH4F9wVKaE;^6(Skbuv9ueo+WA}U} z7u%{nQvRNHD_SVP>vraS38dD8KEV_>k0~DDUO#?;llNWt&8MAv0x2lku&fuve2c~L z_218FtVyw;3nzaYoE}t zI%Vti>%PQ~58n5aXyZ!!uylD(izFU7nizG@&h8{Ot*!gBS^C(L#TK=M__qP>3@5wH zUoxr>r|kJh;7GcZnRJiPXx(Xar)=AgT>Sa+iuLnf`DTiGPMI3RA3ObF)>5eAGPSrI zax|g=&65{Vc12fkPNuw>ZckNQ=AJz4Lz#8WO9gt7(T86DpQ-}vVnI$x^nJk)k1T=9BFni)Wg(2WH`-`^LooiKI1gbbyne8h#U;fHYyFFIMe2L}h4T9ZYr&gFN`n*!ZCS$s**K4FutMq_m+Bhex6!PjQS7d-*G zgr6Y)tY1MCfam6gbN<-S+^Xs9&U)bkiT+P}D0*->-0JhLfy}!t(`%XN!&#rWSea=( zFBy8w@lEv>=WMzG%~Y%Mbc+I71MTFNZGV~BlLkN=4bTBo;hV89)}GMFpa-k^3u%Ak z8Xc0UEKREP!(WdFm_QZ6PKFY9ChV zu^w%0i4I#F4xo*0YqKsGo{ZM!y_y3u1BSF`x;)GHzIuH)M+^x?QIbwPOYEr?q|tI! zZAe3BF-19@UKO4!>iB3<8NP)fqHYBet)xfc^u1SBla8v|b7 z4~5~68kvm~@jje{^#uWwwl)voQRX1Tj4=wglMoGhtsATq7%J|(y`7UOt_qF)#?DXR zSg31v5_}cw5<3?tr2&h`Sb8a!WiC?_5pAo^?)i`dT(Xp|wII7wM5*J)h7m2gz+;*PXPauRES0Nq{sLN*B9C;ULkTEkonSzYP4puVx)*B_u99mgz+FMV5 zS~Ia)UUcPH0v-JE=gqifW=n zmnYMvPx9f&UuQY9L#rH1I*6=w%60{fx!z1K6(_Zll7}M-X-ZK_M?M^t{4U*?p7lhN z(%qVTa#H%xPw$*Pg;Kz zlPSiWL%SFsNnaYN6WEDQ4hg*3D>CU;*#E@v5XQZ0LX8az=zqj<5Eg0-x#?={<()Q0 zudUT_ujG;D5T;i;X$L+UTFf!blJ=0wvfTT5ANy#mar}nmFAVLdrBkJU!S^05S>i3q zA2@~9II$Lj3~h{m4d@`jz%zfWS~{;Dc+575(uNr_Q1tb+e_N&|#{Jp^EWpQKk0D3? z36Mza71&JqPH9k1K0CWrLl|w1A6#wi!e7shwm%8~`h)l!BE%lMfLfNCOH|r8;n5TQ zHM-(P+3HcK{JyBuh^R}~97rYfAf_!72!+1TVQv6`4)G!`wxHONYQf8hl%$Wri`yz) zmp-dwty?YThSI#eiw1cUfV^5T-+wRSlD%1sH$92|EDf)A4VdIHK~Aktnx(XZgq(wt zeRs7()%?i~*J6+;Vj5OwH@CJLJr*h_ur~l2eJW^sf=J88A)u*HaB><|>09HnTQq9- zc2sXRM}`PFm`?$f$kH!Wd+_z;*G_ng?`a0fY8euE6LLtNO-3fppW}aBd4BOLTZlbL zElKvp=v2pt5Pq~SCrnD@@V)wQZCV4EWnf>mkCFZ}B4Nt5#Z7=y^E7ZX{=7n~ekJ0q z&>6|whcw4lo*lo?Cx(c#$dU70DT=#vn`#xQoPUo0Jt}yffrGZ|OJ83ix83d1QsPrL zZanz%7Lf-%_P^yR^?_A3x!Um<0lP-1i=&|EK5dme+qNiV{P`&|((=Jqju`LD@sbx$ zNmf4qfr}p6Jnt}%Z~D&tiQOFnRu7a0CUd2-i~Caj)2BpR=Nv}9k?C!`XV>8dz2wE0 zR~Le;mFYr;S4QfB#$Y}?BH+<0G#%@@(?2sxv-i)1p3c|%h@M?KFaiZ2?0sU4Lt&={ zZ1qi!AYss~P)(WO zZcG&|s)KSUU}DpY#gqpe{BsUNYtY(dFH?R)m1GTz?w^*Zk&O#Ek>u*|!~_c>ezWW3 zmv~&=|F`Ey;q^~1)$r{9^|Rl3szqHyA%-3(Ki^l%s0|-;nVpy@c;;g2%4Nz0VK|^{G z6chwERQ1t#^ED9EBkF=^MZ7!-__=rd=pH$w&pHHin<-q=ElghpmW=QNk$`NivgYYb z>fqg;{mXL|O852G>M5`Y`k8)L##4J?Q5T7tpDWq zZ@k8z4>v?`#aC>O$BgG$(`D@f8bwrpk~FxlY)p#bp)mKO>0B3wYPenI^}CBu5h)5` ztokNm_CvmoK#J8;~*pfMV21`NTPs6PMQMAI7 z2Z4egLuy^=3?PoX34{{Hs;a7d*QE#dm<-dDF7@Imm)Aj!Ru@C~8l`|O)ub!Pzj+!0 zsK^|tvZHMo_dZ}X9FWosMcU!VlS78L+5C#0$@rhRq*Q`>*2)U6PMg-;*%%301`cp1 zNWiS|Y)8Aaj3!}G$11WP-Y@q?i|xr=tp!3xI>bl7#9@|nLGHrIT8l7n4tV(CrA$hZ zcfJMDav|~~?pj4*i2Kl|$5{9X!E6qpQ2zn^(GwT%&+|=^Q;KIEJVGkByS-T}hIBVL zT#vFQ3_E4*eg*tjRFsnYp(Y#qHdj~8(r7Z5ipx^Y=EuM{-Ahz=%c={;bjhd+X$zRx zIamw(pW|PMyUf4O@IHC$*!8BOwR*tr&3tXB)0(<-bLq4N=dKum?0aXd1UuVyN}sKM zzW^>y)BF14@}%#eq{|a{B|Duct9ppj=#|YcwlBTsAkS%YVrO3 z!;*pS<=p&lzh_H&T7^XC2dlY2-r_aR8_>$sR0j0|HS-KJQzYh}sngjW!DSTQTWnXp z?YaM{uA9eTL=j_h9tz=zeMk+|n<)HcsP;GBO+QP69z4Da6AZqjwUtY41WZly=XFIi zs@hh+2-}+u9p@RKWYu)FM)R@tg|(g@9vNYPPY8q7QV-RswlD~vrYXF6W&hhmX<@yn zAV_j;4yr;5_&<+7D`EI|VN$iU_Ln)m1J)YgE*dMz1uFmVg#|8?Z?8!TRalO_ziup@ z@-6f6hF$f}dCTtY&6SqJPmw7!av}({fT*hP#kT#V=WRiTR21Lac5u`AGmQvqGZr+e zcVOxUC*Kb0*^OmZrlgQYJ;Q@AYS<&Ve?v~)P?DZZo%R&Jo%GI3CR(;T4{Sb*gME~I z*7a~c%9lxvjEqV>*0XK;N`9!{gp=++AL!2vD}&prozbd0KNYDj%&Kwh&q&X4Q(Uy| z#VWh6PSP{n4t%}nx)upV;N5A4TXInR$Kri%HmO;v4#!K&w#l-ib zZtaKLfK9)Hafz8LQkH|EY+2FD(!O!csH%cdDY7lAW9IJ~YL!01TcHLmc=XfbL(2$v zlGMz6&BxAZSW!Zf3iHDjwSMY6EHJ*#@09u|`*PEUGZdDnwSuE{!HEY}!3I<=S^*3^ z224&ZQThn@G*KkM4AL@7Ute8N?9MkEgE1S|1}~c5col4zQL|}agd|mbrZ^PVeps4E zD#^bEaC0zBy2Cx*zrVa_&$!3(;xvOlNzw- z15it|R)_A_#Gmn=VL<-?nd~js0gL_D=Sa_Ayrllm)WZLpctXxuzzGg3jBqhLCc+Nm zum6Dsh-CjYvGB#Y#rVKeoIly+KN#|c$t_Nr{gW{yIo#$OP#?wfp*&#!H$<}6bnQRQ zfUYjD5YB!L43Q9MH;NeDB2#dRp`9D%o4x>Ka#Jl$$toN@P|BC8)V!~_E*o+z6%w= ztP7)4aI4H^fzxMaEdu<~P8+Q~%OBGP((T5>9-t||1|yHYT;q@VX%?F~W0jgEI1Yh;3j?{;nE zRbXL~ww2Q|C>+XHZ7%WKL_!I@4W^j+!2wXgkA74G?=Ooa27uL7NGOmw&%*j{oa-(A zio*%CEdv%%fc0QD*HiQ-j<7ILc?eS+2-D7LrNHaRYB+~pb1Mb2()GusW+3q{kH^?4 zG)LZ09xGhaeLS-$zTXB1!bNT!J=eBqO*!5Z(5m2PjE+d82jTu5^*jzl;eg?SMlcX_ z7j;tQf~CE;mu|1D0L1w$rirdJrWI$S_E?+l90TT%Z`F}zPYe1ma@aF5(?8Yfpz96> zbE64xh4SDS=r%|72Epn)6_oincfQTm-*8Jt!tMeQoaQlff1OL*WiX%*6i`G0A-%mVr z0|x%tsRD8$k%SYMxDOwX2Mem;3a`rkN7D&k0U&uurz>?d2)iMLKC~Y4sln=MPjDGIJ^tc*UzWs!l!nS|ArQV` z0T3GuBi;mXCKaHX`LblJ()Fae_n=grv#4? zq#_P?xDA*_1fqI8RYUj*OqiIr)8Ih8`S}o+W5iWt-5CPVK@kZof!W-iVQ8%?7jqW$$R_@lw9N;vA}+#=c}*iGtPmswO2$-YW(3I>R`ql`P+Gwf#WuivF{^oU zB;SkROrdlrhg3pB!fkQrF-VYK`(^e&N0N7l?_3-7?hsY+O<8ZylZ4z?(+i<4s@3&Y z`&4^ENahTiCc94FgJ(cVZo-mcL+FV*Mz>R4AEt*;<2Q2GgJf|1HaJj|f!h6>_X>H`6qld%lX>F9e^OhWQRSs=F_bzQ}Obq_iXs<3njunCjNh zi$ZJgQ2Mmlp(+tIu@dVt5S(w;)CjY~m=!nZO+XijW`T2M(UY5HSg8w6Ejv_uKCzlL zKXOcH&}$&RkqO#|`XKS72whL|@4-#Ckv9U(H;Bn783716l>~WT zSBHkzF~9%}S1rd-j3*pOG(r%~pv#zx7~mm*6dljUj{DTJ`?vJ9j_`>kB)7x;M{fqGRu3h}Mz{GeA0$&Qi8yC@ zF4hp&bWJgO3qMYqNA01BybN5X+P^-nG4kF_?UoQNY%SBgGgAaNeHw7w@0l64R~NYW zHGYF2nTM`+oKgd@6_nu~cf-V=7YVy8A);ZXZP#@pKgS*O$9T=+kEMTCVQIQ=AQAdT zLSg33sUM}7*M&A2%n*=KrlD)F{f9Yz;Ls{eedeEqA1-LW zUJjKzm@-H6aBo2CM{r@&ShxNoal|HIEDN8#R`~_jX5$e1ITO||xBZZ$-bNNLRk3?E zY7%Yt$@jpxq2F!-e5myT0O0@+s{M7ictTzBwy*yi&C5g zD057EAixMx@Z%JJA9qzF{W^ZzKFfK&#lq|`F?2lS%}7h#R?=n%Dto(BzD^&rKDc3 z|0ocd@quVMuB#SJwT<0^m@(m~2d?;gKwy-%zWaVDe-XqlxSlDi$1_aOj#bh!C{U=SKqjzmq4BtK!>?Gw>q2G0$=99by*>Qwbs_-uYuCPq2ZR-f|L%Gy zV?Fr$xwm~jME6#X-&%lRAu|BEq9Yk)hP@^wx&Wz~E06X=xW{>O^Y2VsJ}r4$5{ZGD zqOAdQo-ED_kAULJ-Ab;X0HtQ!8$yXESuW`pS_WHHCx*=nXBb+tYGzr^BQ8e34A}fv z_y)i<^LN#sS$Z1gU)fWwow-udWobW4+p|4NNX zGzT0HTOPxM!5I^8(Hci>vjZ!sWz@Dy)_#%w4J99UXEBII{{`u;K*TBM3%k3!+z{oI zoN;{eiG+e#*1Cf7`5t`Y~@gOwK0z! z{xeFxgddE8hb4Aa-<5`O8VGBjo3V)ZGMD}YW7_~u5*wfO&+ARJqOSK3%44nreqJ7< zP*tB;5jqpgez~2oG4LFhq3BBAlmox_wrhv~XRsm*?{z~SEaN5UFl8K6WN+|_alBR` zg*M%42xkE#8!r8fG^&B57~RVZUg_IlhTFGXE8LqQ>DKW4Gy@~F@MY)^^&n`sltETd z5mx-EUIz52MVj|mM2>6usD8tq-(VrL4QFq%vd{yNvHh<>}#cINmPAA}uZbuP=fJC3hG<|Dw5%~e6yt-yS3s;j zZPk?(g`h?otqe6L0N1Z2k|NuSfjZC#HJ5Y+d++Fp7aJY3wAlwQ)$($LEth* z5^2~ufy90tuUO=Mt>^y{Pb!XqX4nqjo&we_Xz>ZIknarv9j|u_az0U#Lg_H&rM9=X zQ_Pyev1V}Xd9N?t_^i&3a;*eK72$e1A8H-}*fj+nkVfhb4heP#&f*HF1p>X3yJ8;r zLQHS2vER>ZgWhLa$c=%+UhJSH*Vx<+Wdw&Fv&Z^;7$V`?!$yQJkGoJYZya^WPu+mz0V4<%fl0Sd z?0-##=Z$`Ql}~epss-)mFH6-Fd4}v0oYP9uUBX}yr_=^ z-d9TY-%g||3w<9_z&Z{&6uZxE|DpNC(l)ne-Z9EQm5-(qZtty6$&^~yKN>w7klYA-`AZw@ppCWzk#XEhU&^f63Ul^ z9n;UUYqzOi;nFX;gBV-V_YLN)|7xDzGj0*=I#+~6w95Lvkya3XS+-MCJ zccy-4cwCjwYs-=68x)GNe;A^MK}3|2g^9)${!NtM|N2WS;4eu=ZfCqqYZIl|R5x z5s>^4EeSo*wx$rF<5k4aZghG7;<+Hdv8VvqpZ0hl@y%_rTGK1Rz1dMK#tal8b!K2Ye4znh$(B5KOaB7#D%S>*cY`PGS4-L&w zQ^Zr1rraM+dNhIO5OQ2pjauxGD_H1DMZx~A&|%7%$A^ehpgx8GQOeNHODe5G_zsln zsF?&DUpmr`0NybJlz59scpy5D0>XN| zclXDng8gvYqOk6ZUWh?c1OlJxK3oAs_{I##p-hmP1|gxV9^&BWCmn4N^1g}Gl+P20 z-2%7`J;QJFiw?}bz@M(S!DX$jQ+ivrcDXwoqg^KF?lea?=(X}}BRqr7ZF!6wJ7`m# z!9CQwka{x)U(;pQ_+BS%uk4OzljhsV8+!|{PJu7{_JEf}@Yj`z`pIFL(M+k@&wO3T z#wp0EHW$5uy07Cr*+#zvyS_j$jA31Z2K)<)ds_|7IBnVuz*9R!oFwZ+ptWMkbZWPu zC3hS~oLBzNuKmkhv?(0&#r?20`;Y7yKw`3}`|8!7Bg#?SH$lKbN&iRdbP`l&Vq+x$ioGpI!lCnmj*GASEM!Bm-3}g;4&6fg5<`#6C^LK6solVzcE8lSvhjIae-*F!8il@I6W4&6RJd*okD8)Xx@y{;+(4z;1QR?y{e2NI7olhhWoWwHGW^6>92j-`c*MJB*@*1b(8dLNJ>LUZ%=S2myTq%;SO#pFP5aI{cp1z zcvE=HkOllpED3R}5LCqkbnT-GkirG^7i`?LbxcU6e?QyPI7LWlJ(WEo=4x+*wb8Zs z{9B%9T&`4rRq^9bE1Thh>}uz^k0sF>9A2BtW=N=_sr4^$h=8(c_~Ad0%3#Qd@cb97 zQdqyrwT8;Us7$F|?4w_q$3!5AiTY<)-Ltp4BSqds+A{Xj;9uGq+O97w>Il1~{L0UoGnJD5YeyjJUO>y_W$BJ?I%nw?0dG+jFZiuM9UlItr zZ`e0;Z`)nkc9NLt7&)|Q_U&xLd!`uuGDj)=!j8c-za^uG3!&-j@rkdUpb>HX3wdM$ zwHws0zvYoaQZkTbEsjJ zXDOy%Mw=Ji^HW^}We-452h`*jB!xv=ND?KOGV^F#a-Ax;y_a#A5Q@5>22^TO{THi( zRD%G{)57Mx>mp4h4Lqq|P^n|98SnC~cC6i&H}Xzw;5|0IM+M!YkiEpQC+?fufBLb}^bi1PTps=v;!y29*<Ti-W_{!ONdLqNG_jY?9S9ixX?Eg8ycJHD?U46oTo*}${ zMtXlxiEM9|f7GJ*lc)UioB!U%cGq0k93B}X&|TqH;MNT0$fJK4AE)I1Wt03LN+|gy zl+!>rkVcFZbe7RG`yf!(0bVBh@@OlNG>7%;`zLei2m*uFhH4HWlq;0W$PfuLAyXS9 zV+Ekwrhx7rV>`XS^h9VAD3A=q07&QZVXR?bpveM4zuJpM7of}C*R20|U*=#MqymLm zhRs=m)B*p5PqtW}zsa-H-uha5l~)E&Zs-rVCl{V=43l~}`km1(l#?A3Q^jzPd3)48 z-X}*tyvArff-=mj1&`pXqjp+hQ5`G|kmh@W-UASXSRbM6I_>&?P()sWc!lT`;0iGS zPdrw*188j&Hj}=C9e^=2SwZ)5iVW*O2YQxCybApZaY)R4#55a2ueGejLGHA6u6k>l z%|Wyhs$#U~#&7@<7v(XpTmW>~SLVb94G3xC}j47EA)j8;1RS6bRo94$T!<;8Zq;b~GVD$^%sTo5JXxT<@IG zO_C*n{iYe0f@omx8}%|6G7X3z*4kyVuiHufr1QnAHTm@3Om{a35eh$$Im0W1d>(! zk4(J7Udyj%F`B2h^=XT;Zeg&U|Mq=W9}dcj`R+;{?fW~%b^OSt)lKWR>sCGam@lt4 ztGRCVRk+ENxwxrf?fMxh>Het9Wr1J<_k;12g$>uMw^Y-K)v06nUTZ z1LjT{lq~ZT$c|y!zkUzhG~z>BfKbDns{|N=#2=Dy=Ony6GeESf^(xxx68F^9O#j%~ z2A)ISvsMK*FTSf9ls!Ei3J?fdi2n2w{@wjGz}jboyd9UhkEOUq3nW_SWY@iX?hr?e zxY?7wGB+9BLMyHoy=sf&@fxsyXN(X$((GN3rW)?+OQd(o=KI}5X$T}(9Y2j>F4pc9 zDNBQ0?QdaEErU{!axDRdBnC_C-ZDoUAJ^edQ+Tj97%l|K7lLq+uWNr5qrML>B!7Y( zkNZK>qR=aA$@P2D{%n$UCJCFpQy~QD`o?#m@z=Ohvtg#4Qh+v}F}(RQ?E1Qd=lYv0 zogyWOpk4Tl*W=G343^CNbSW8N@jBWNEf&yMvK#+iXeUS~4b9H0jeGfk%#q8eit4wJ zLIfvzp6@;G{3YEQs?%L-kDLt(qxG#WzfHjPkggT=E8KylEU>Oyxu0R$*A^TQ)gvB0 zz?XCdHc!zJ*$oCOvtJjJYb zU2pT=?I;~7=p@(cbS=+jlsi=+{L66p>@^z+RPRebFHLY0lvOP&cR-#W=fD4nh$lE1 zD|lDeslv8jn1Ua*wh5H6MY%V@(q|I%zEA3E<mV9^6~N6oGoZI1-|nbi~DF%nE_@*1E|q|WOQ&7JM_wdzZ;hs zLOSD7i6KE|5WG+<2$9Iq7Zf7tN{v30=)JS1RqC@>)%)#k-tRr;!#}a-0yj1{Z#K9& z=8MOA=BiY>v{b_udQ-jiTVs35ktNofA#(Az^`bpt>(KZFh>lN7(6hhtlqaFytzAdw z{7E_b9qm0oCy!q1WBwb_Up#>jOwi}U5WW1xG*E9w+@6n}$t(Op!y@ajrnKrVp-gV+p=(v?x%p_t0i;ZHQ_<(mjitS{V64kn;^5l_^z#9lZ)sKM314 zDmkvhHl=8ixhow98Q2|{tDr#3!%oUnpxE4Q%cj@V)8Ah5!{3?zksJ^;%O%>Ct;-pY z-@enHwi3L3>AMUgzz=I079mbVbq&`FN z2?Sn_G|X+-3}(Y2wLcC+-f;I6bRTAR7VE}}z^==C&$2qJK|xRBy9L2;qej7Kel#mQ zYJ`u*AGLPQF_@sQ(J$dWuMcxAnWFR%uS%i-(79+(^V8 zt!Ghr{}!%C9!ao3!Y*ySz`tcHk{yBikU)=|TTLUtlTI`ZeRiBEE+Tt~3rCN=f(S!Q zf(JzV@{MbcuJplkh2)SN3zF6YdGqOv`+oiSRP^BRFP|W4ng-Gpz%8eKJNKL!h8)QF z1gIbK5I#3-+!%6a;oP=+B<9itDxRx|s%Jc3w2q!=O@-zjs^(zDG-5>b%Pd}P@?RW| zXZe3b%{BLS%~y)s9T3<|{Qu)<63qc#UrU)#_$}>i|VF+yo2dh%lYm2w9t_+G7G!58TJjaJf48VhK zLzpH7p8bG`coDB@?ODj;R>qoG5CZUG|5GK?D_DJzr3tYMZQ#fBkdqEamlPky*9 zXb7OzRV*S+O|gt?9@iwaEOS4YECv~I&VJ3e%e$#epTGsX-*th9n2EoBae*stv^nbD z*7^b$AaUod4J23xUPmy(4JNL@^^Mmi{@-Y)0l2~;?F!dC2!r(;_S3`K1fOA2h#W(5 z2qf{KpmG{{EG z2bDHxljt|J+O&`BNHht7Pxqa`G0jKH?pGQhG1s-aW*jgzPQU?D;NdKfcDLQ!;5!i~ zFW5bL)rl8UY1hdJ}iob(6z8tWNW{=wvCxqRWQ`{uZ|BuRR zbMXaAew&LmZ2Rdo--xfMgOf1lV?4;pU@Ls8N~|sn1Pko>WM^SF6m}k_ zg7GkArgwb@_;n6*NUk}Y8aC@!5Uah8Ct=XqqG4IYXE5?a7n>gWV6#r_%@Q4=;&@0^H_serdQpFYD zE(Oh0iU^_PcN8B%_7E79L>}CpF#t5zr3#rq_)614LoKcq{RRX}$iM~UTX_gs z&Rv0c@Z)SP?cyKtn14j0zJI@Prk~~eo>O7&i=DlQBXW`n zYh4|2g2tW!gp6G;_sGRk*#@=tk$0+0J~h&aZmK5|f_ru0ZRXxCTb{{LaclwV1I00a zXLQzF*4pV(PUG&nAEiklH1}vf#7)2recbb=L~&Z%>MO-lya=hQMv8Ltpq59)y4Z?K z{)eHp)4!4r2eToE_#Z!KCNd6sP~8e6RgczA+1_PpdiQCPY#BJ=|Y+Rmwc39W8^x;Zi#(;7( zdlaMGd-h-a$H!X=G=yh_32T}fm4?R&{}*-d0Tk7?wTmK(3W$k~Zc&Vg3W5<(vKSFC zAVH!5k=W#%F<=%I1r-URf`kTz?k0m`Km;X&TLA zj&3@vwdR~-d}-vI^-x=4pYkEqF*TTOaH;|{p?DXsynNPa-d$2CK(i-h8O8-q43rPH zUO((giLLNv)V>aVU#vgj);Y3ZUiMmA{qZ+3P3;+zlQ)*k?(!%Y3+~wD>@jPg;mo(H z@f01!iSsiJhU@2#9T`vw>8YO|(lgRv^*yh(z1`Su{x5c-+#Upw<&=7h{M*b-s86M?(h`r=6D+$vmBAScg6kum>vrP}$v&&K% z|5&#q_B28lzgFCU=Y8N7?Y~Vb+xe7@zI-+knnfv#+nanlP)b#&#IpG-QZJp~PAXdD z`N$7^T2Rk(KhTKCIWtlI!*+S$G}CXz?zKSS>EqG>kQT#Dyu;^dq`NGf>g4CE%l>N? zz(c=`1EW)Y(`KTUMA9KfQ2|_Xl`-$3-4TwU=FX877mVq3X=o@a5&egf?2)`zCb6y( zOCpMZzhLz$5!)~F$RRu(Y<_ii(JNZR!~dWrTObT#;NR3_2Kv&UXiCk3x&u#0BQ$I` zDVNw;GEsjhbByDigA8@y#Q!f|;H&(rYr@370S;9O8}yee62OA7r~p;%?-)9Yd|Jxu=ALU(sIsum7rz;CQ zZQlZJlCC)qMD9o3@ozkXB^B?${?%!@jZ}&{tym%!-IG~#uy`_Y z+rZ`gepZ#vxD15aA=tvRQKUTk!(iB?%0w>NOEKq6~2*r=xE zky;bI*mk5;sgmYNmvI?8Ioo|S!(HC?NxF*EUDQ$1Uy?Zc9*gQF7p89zx9EiC*!0m^ zD`1|)6IL>N@5MIQh#pQu7BWX~vc*&~MJ#crBdXMN*AsuR=FeD-B52$LAS8HP&RvyQ z8UC~d{jVi^E7WnbqzoXRLB~zaFXhr1Ef*j@TWo+q2Cu=y?R@b1?wNk?`B6t{WixeR z4ro9Xq}ZX#gtn}x=o)v(aivgcf8)MOwvpZG1qJ~Jc9`GT@UX%x1;U8*zlvB^FJoLr z>rIZ1W3dfYFZ_1#?avRqf*o#^pe53;Pf5^Z{qg6;$gwX!u-=xI*KFIK9?o&awKdo& zf0j+vGp%Y-QW4=fJ$K;$M%H)(D<=kb-T%-sbqTd8*CCtNqZJK`TDCEB385$b&Cm9>Hdr z>s0SyJZ;cpW83oN(!#0r$JPzu>oo7lruXmGJMTF+;O+Y(xGP#TV`uku)5@1|0NV~b zj#@eWAxIj&|2Lu($bhH->^~v8lZYpY(10`xu%T(d)3Xj#!+aVA^cTDi2%N}6PErdL zaRE31a7ZXUN1c~|oVe=W>NtGu$A&^6CW&SZgyoeI8%X2+IPcGo{!xx}<~$RXr1gL7 zN^^YZm1L$@Fjk!^`b1i~(6K7|-7@pgJFaSkoda~f4r-Jgd60b=T8`KO9u!2RvggFG z@h0tr=V%E(SqzrzT0P1^3?cdHT3B5>ldY!lziRA^*}p^M_IT?s^mWC6@lTH;oiO`S zIb@+V|fw^9c z_)&ApUS}ZA&~~W1(ONd5)U^2UxUZ^Fb-=BjdU}l7)3bbuNH5Wye@&S9O<*=x!fBxT zw-@pyd1cJoEKCt{$=M?UBmy!DiHfu2U~olP?tY*b{Tr7f!G`?E@XnwG>PS4l<{U#> z4ImvVL$Paqfu0h?e$D77)WeyCb@F4~7XjfBG>VcZH!)4SA{LT?Pvhedg7 z@^6+jE#8ObWHn+Q9{{sWLDP&aYUn`na^beTwd3iA0y4Lo<-3g3I|*{!=$8}c0SL5KtGk|&xj_loV_GW8qj3&@u(3mQUI z^#BslG|CE+$1s>R)^p=szo)0?BB{8tktQ_V^D82Im8@WfkY0!AL{5fh???I~sp*$l z_!nO02p+>1;J44Ky#c|;5j5|p8*jW0{qy%`6Lgb51z6M8@#*@*UJvq@{PEl@KWV+t zZ70s>2FOO%qb#?aS)YT)Xb8L7iVZ)b=zgrlk_3WN~S5L^! zmm2$ZBwY^bFqjLC{HCQAbwssDp4ER-Ew1^aT4X=1Ke2t9*k_AtB^F|m3a&2i_xHB7 ziaD`*{i4GIqsQ-Uu=1>=8wTZv_TI>!qy#+K?NH26?SJfWuBq?Ti+xcQ-}9rq&gFgV zUo3rO#j@Rw3HFmii5uN7Obv7AThGD94RwbQ9T-dghk9BP$=JyA)1 zg4D-92;m!%-{I@&ss9N*v{Yc@d1Ev;Wa_w0UoN_<;Q>wbVqGajaKDc^n7x(mZ@hb7 zMC*uVoZe6SdP1TA=vDP{K+UPqbGSv4K$hUQF8~x+z3H=C6Xl{W=oSB0-XIcFEMZlDeY*kr0x&%gK&Zp=3pL zhSQNU*{GUEMVYp&EO!{INiI0@-5;9cLi|^!;77edm>GgvpPCef7lSz4je?T3BGusM zcBW3X{NU6*widE?qukx=i4SivrBhl0H%P>lh zp4yThtmRox-z&v174|IgbWA0X+;ffQRNIM~Wt(&}!$Hg|0Rl8G7m&6+nK?v@B!-7s zAAV!~<4`UdfuT{0R^OE3IqsUAgD{gBn>1q|B0)%%=q{|%fBEkIK~ftaYnVUyTMi+n zUqu_dx?D1@G)Al7=65*lwj%(il4j~(c_c6c(X~H}dGf?xF|wu+|9)~jNgy5P*Wh3v zX-OF!G4YxAAi>rl(KS(MlEPTH&lb@x;$bBEm@`n6xCBWgJv2=%@Z1sm>IqX_x|zl> z;%HP~f-kpAq-qYDJtV2>iPX5!wr76fg5Dw9j4sSuwPR0v`Q)irasLyj8EC3K9`oNg4QcmkqAdaO z41Vb^;Tk0Mu9^ANP^@~&Uptc<7s+eotS zd;wu%7DC$6*FNnD1iPz<+=M|s7rWd$fO3LPzd<=x{3N<7mAv^pZ>y7>R^0#7V#|w54za65me*9szqb_b-*1$8#$My`iZDcNg#an6s+S zVDI_S_y!Tytr);lpJ8-~TTFYg?-1;hXF45<)5G`Q@J*%^GFI{nnX^$TyJ*qc8@86V zpQ7kLeQJlgCtr%PZBZqg)XtHoe(S2VDj#`fxIOIF9^Gc1b>3r)e`gMH@m4Sds}AH; zzJAs;GIg0&kJu=)jf3b6if+YLp$xljuw~>7`ams*j8DH#*_$EE6yU1J#GA&S1ML)B zwzZn;A^rP7bo|6w48w#&WvW9Ar=y5w?B7pq40btb_AK4_m&Nc0?#Y`eReU?`}Eyr3sMn~sjC$O^<98<9MLZ0LoW8HAN?c@tzgG47% zYc3UQ;G_*E>cHL{R=)7eabzg)Hkf@%KmUbachdNVZ;{lV2GaGh>l0*Tx$N42%}MA%o zJp5>2JFZJ2H7WWj^>h0tTK&l+*UOtvC2TEBfROp~;|F;vd48HKDP-zIl{|j{i*R0W zb&?S-x=%SoQPa6VwEfqhyA0iY^402ZBap|>8Y$lgKb^{4Dqj`&9B0$!sruJ@xvn*- zYD>%cb&f}!QG>+pR8?}i62yn=r5*tVaAptQ)e9^kLNeuu&`Z9Q5&p7@nAycTwUaHm zyuBmk1_c8<)$2eZMTIcpJsH_7bfk}#tSDzJD*KUfr1<|^M;f_`-Q(M{qd%f4p%X~7HLdb}BI~B9N5xUR-K20eapv!|_*QsnXk-OqLyjE@kumi& zZnL{D@8y}8!Pp*!RMdPJIQu%wH3{9glZm{dj-FprszYb#By|NX01+xL>D7tFCXCWuo<0;r z`P0ly+|qRoMmw!|fFbclm>r+FhI$_R`D;_9LIy>bSPt@orM2V`w4OKtijI++lm5V9 z`3oQSLm_e=X3+`+0#aGsIu8g?9uS}+xwTFa_nTPRy>f2aGT+BLHr1=M=~2J7PBNqX zCbcD=a|UM5^xELYH6bx;;Iy^tuUVX<2Jj-X#W!u5WWA}kC}_c-4V%0Yhjo9Zq!`j- zyiv|1MLjOfeNXz|9m}5jeo0kvV|YT$c)s`*Y5_Nop=oazThRkuc9Sv~A4(Qzb#hj@3QI*x)3BBO}5+qWHy6g7U* z?;VF|_h|?#aNU&!p9$Cfb8X6t(_j145)UtXwB06_b&>cP@uQHQ!7ha^u5*{@LX~)f zjB@~-Nt{O%;&TzceGJI#dZ0W0sIU*iA}3%#CzeLfk)IoQg)4wfV>$`3w{2$<<`io| zlDd;tdlQuuCrR884fPm+2MFYxAb?TsiX91*yaSM^r=QDxtom8wrC>$egb1`{_ny{! zZqgDvb>H(8&yQtOszvXEIG}i(uVY6b4o%hdiALXEohgnyy6QLX6G4}EIff%za*r(< z?u*nCGc!m9cx9^75Q%`sQ@xzXI*I*9Anx^qj3do`N6Q?d%A`D+yiF};XwQ5k?vX(2 z>U%o$Jko1gIwEtWdZr1*A*?KnvA2!T!Kr=nlb_fKJKzKS@3OSFXUut)LSv%viB6tP zr(R#Ts7~MG$f^8F;}SKKf^RbzCIQw%BQBd zHQC%IcRCK?exNAv2o~TdB=FC#WJEJajhua(<$IHKQH-wg_b9~@3tH!;*?OZ>+^QLjwpDT+t~A7>E6Rpyz`aOhf`nbjmT03 z<}6Eoxr%5FWr98CB6uULp9D^#3;Kks`{nlt4QXG_Z^8i=58qksQxfy6tO&mV$7|m` zF+@}Yrc=k3`5SK;Y`GW!B}C87sh3}^W1}2S$EHt&zCRO}?{wFMzKRUZPzsi!Zw7WR zu=i46;R3}K0Gn^76x`3XDSN#GP@6yUuk%#&UdH!Au}tQ7sSH_Y9Gr%U>()l3DQ3Yd zdl*sS!IsaM_REcDtXar+{_B%vO+Cb8$aE(1ioub$hdaK#%{m?C@nw2TvT6)qlY+vz z2j075NoazXQRXzLk;l}ZqiQ)4M8{wz#-X(s$^s8?OaT9SN93HlPrqsw(`DqZn5GlD zmtem`Fb5@jcks=v&kq*^UVEgl!1@86KS<{_ROlWk=@IR<78aK+ylW9CQ%1YUxqz~~ zgl?`yf_NY8J0bG2KbiQaQXQCwVgJ=1I5~CE*p4;_u%!~uD-v>DkS&Eoh&`tLFOM`h zwtf}B0eOvVNKg54kv_H~o{k#mPrg(Ff(3RwI=#NOK9TgI88V4&X5QyP9qL(wh)G zWC5R3O=V4X_Twj8f8!beU<+y#rJYXRrq(anWz*=D@j_tuWkuwV>GyHwPnuOobi`=8 zAl21BhfL0h!|GP#DS7<4l>`>TTT{Dj-}HA^u%8!3#K|hk7f=`mInE&qJ#WYP{NL^H zxvx(u_4iIc25kFU*{Z_#5GJEW_Rpd4CT1(=n(5bSC?kbDfecsdw*3T3Zt7WKE&6j9 zUG=P4qEMgepo|Bs$vy~AdmeUhGCjj3%J3iW(#$r3ei_`s^+O2W3AW@#h{Pr&+m3-A z-r`^}{bt@KD-=$_M2G~8zuO&CZ&^;m_H1~Q#R?nXhL6L95&VnlA&uhc-(f5GjynyK zGQluseu_=AZwbY|7TP7^=K1$mCR1r_Cnr2eVx)|7H_A^aR1U)mo$moi`(G}fAkM_L z(kOowqy>^KA4UCLa2wUa!2nx@u&QEUCsKltQDl{Sw+|kq>5p>t_6&RUDk?*IS)Lx1 zNN6Id5kBQ}X3aW;lYy8=BToI%@BMB~s4yrB;`~h8tjkc*j5iPk#w4Bx#CR=2IPQ>V z8dmxmahd7YSFrDxK`HXqzW9TL(qirAQ*hsj)GXV0^%LR1O+qf)Bkysy7P#coF&F2a z%NF?jwPnh%PQUyO#EjYbbn2DTZEN@yW_pVtP4wfXJP;1eic@#WS((j)SuRTAo%!9-H@rmO`#`@|`Er&VoJ$Pk)F62811j#YA)lbMVq9wc7@5QW6}EtvQ>;i?mB z3wm(<7EnI0=m>kt9)&~wfIvuqNiyWm!lxHG_lTb?N4peBLKsf)|?d>21o$*Mmem|9ip+wPbwBWzR%K z`3AMrj#{UBae4ETaX$3%>0K$_g!?u;+K13svWp_CEfAX*5#U+m(G*<)h+F=>CjA3; z&wTSP;BiT!qtj6+0c2!C-`b>#6g%S+Ut(tTqq9*-wjj2zzK+Thc)7D9D|4>%<+;C9 z`#%8Y#ejEx_}Pl*A~+i5Wr*HN{wRg~#1ng{6}s_(#jolRzT?iT7`l91^O6yWc2YQ= z=$AqFW{S%c)@-^yMWdFS#zD+TzD^)9|MWRuz|4gu^vTZ?K$r5hTIgqs46t<(Z}@Rn zOG^J!$_&~dO>=*H#Q>e}g67c!rBJP^@WpESn(dG9y@y8e0m1$b=iGi7S#TEuy8i$H zCJYE-IXZ@QYxlPmy;DDqnOuq}wT*NwFMnc|kDubcDIo3mu)-0ymW(73DlIQxdDtxs z_OJg1_>WSj+xWuIV=BuUBFRs*7HjooaeU0fBF``S7DOxfu-}vS>b9N^K{fbG0kWyt z);h9LOT_d{vKe7B`k?f^aM>LpyH9;JT`kACys(f|;%kq}kE`QsTip3-6p=4KoV(I{ zZ*zX8J=U1HpQn%(O4qE&PBazaJhtr;}qzJl_Q?fwvKI7M1 zeg!Mcsbu7HY0`tQ#8s59gzHEm&%Wskq)-*K7s!ttgF@YSWV08N<$hcY1XB1Ylhye2LCfutZMtg`ko>wJn)W-kY~6GUZb*Ts z=q}oRAK%QXUX8uR6r-@Gp_+?yX==Tw+qdqqQ7??|Wmb%%k+-OFJ;}Rz0<;AJUL5BV zrS8u)%QdU!oQ#nZ_bv}98xi-)!#3Q4|HALy>vs-MRC^+M7e_-8R2cw;{O1}bQ^RUa z(bn>b6;--ABF~7I4^08|5FvYG;^_i8sEG5o2eWP4&XQD**sLwErC==UJDMTeWI4n0 zt-LD@Cc7SEbI}}NRq{2Zbk(F7X6iRwXU4Bu5{1(%h|6JW`*W5$v5UB$XC1y(9 z4e}f}7;UW@yjd zbtqp%z>;#(3}SZD{X}JS3j-D-i;eI8k;jo#UB$&T>J{HygcUJ$VjlN9#Fu4Nr7GeA zNc+cKl*9e!NJ%@Y>v>ImPmq zr&syjTCsckF%A2DKP!g7svCMTgnSYsHNJtnkr?-+e^G=`Mp5<$Zj^~RbJ0%sk1SFaU&A}un^{TCNw z>_55O&w6-4Y%Wtc0vnOXwH)Qdp(fas4^76R|ALLHlN?&(-4f`T`4JZ zW581rr!Y?`|Ju#8FE?f<^KZxJE}N_jKSN^#4Sp{_$|M z*yuC55T~x1ZSlkCx~3y*v-O8pERxEN)C9aK4~%82*S8n{0&nsaZm9<9u(w0f{8X(O zW%*(jhXqGIJS*G%Uvcm}L9Z;DH9io;x$>9UMpXCdC<@7B0aFs9PSaO`3=_Pxi4-6^&A5TH*YzbP3SJQNMhgAwB&Q_Doo4d1)0mS!-s`oD!hap;ke zZzQCoTFPS_MNMl@g2|1=83TYCc61qg$aM(pwterOY@~YjCBDe!c(Xit`V*CGfI};Z*Xy!@#FE4+xs?TN$7UmYnF<+Nf&md znwu5n3ZZh`gwlAsKj)H7YVXy{m;>mX@1pjFK-Xn0G4KLT<&cW-{QKz)n&we}m*Rys zu#j6Yk_>dwy>Zdxj>Nu_k8tdAx<{0GeYwrxOP?T~)O2@uj$C3~S7802Mn#U96sruC zvpM$eA-i6m%{`dmn)hbL_;b*TLV^`mqHWK;5r`K-y~QXYgypit@#J9rgE66wh2NLx zTT+a(GEq&CAj;vMye(gnHem8;!CSa9zPKHo*@oS^y3TH7WuZsk-kGdgN`JK`E{)yQ zR@(zfML91<0jcDr*Vl+T#L}8L6-Nd76J!oA9Jczp@AAQwwz3QRi-jB~2FsSae0Hlp z;rzMl+|jqpbz3J6%&7ZqWv|cuZE1gd!eG@H)qQy90-OF7`uS{2W(=@qqkl2c5jNpo z!1J2E-Z)n(a|B`6+mwLz->>) z1vQHFGuV!VaU@|3rnvWnX_NXj0TD**skFlra*is8GYk$06+9GWNjqLE+*Z#l)D}7t zY$~*73{|ZWKzfpYL$b?NAd)f=3O=MhdX9+4-IHkqJjMWysLlxpWGpI*!XLB~F1O<3 zMH7Lpyp>Z?hTcinS)xy$maTldEatMWuh$vb5HpNqd9?3qh~_@W_EIIZCg+8Mxo?on z0387Vol)<9YI?UGv5LR`(sT3E{>^n5y>^ueRMOP4@u_U+Ua2l9tDeNUjbyE@T}bJX zKRr53ou}PG)#=Z+fAYvK(D$dzgP5{=w^=LX&r>Jn7dVtk2)f!G$kGmzG`253ka2lG zDpX3@*G0Rvj*YA^dq|B}qEMSS{qIZ+jzv4OA2qeH-q>GvIstF!Z{A8mVo zqHZ)K`-CTOVuMvaza+W3j}R5^f1}3L3^kx?I(*bwN{ft7Y}RIpu22m#$A2Se4VxAB={4=5fBNIPk0R_QlrzUcQTO(u8G%=U zvy{fJqiYIrlZZ-=1+r*UoXXD{5@na%_6+xFlx4M)oidv&61fnU{8m}k7+A`ej3Zn_ zhz$4^V6M?OXO;VD&rXbLB9mD$m_#xUvjbjnHkTe;<$XM&j8_PQh|Qdx1b+? zO__34v9Z;C5}iStovHJn26Iq$<0Mqx-HMJqmTY3E)w6Hf}Zwsla)1p-zlK717o8l&3;p-7Kjc3^DlT)X|#!BTAL zW@zDoroOwB1_U|)?O4!OXB7Hk^6?m%*Njg16JA5LKVf#G4qt{%^8uwhtM??|CL7Bz z7`zu~4{gz!lI#qO!1{XWD*)ZQs6js(`Ar+b*ktrAoSvv(tS8tD;ttLcax@kIGZh+` zFX{)gfU`toTo!?|>sw8BI2w%~OYRZI6cxrZ6MKcz(3or;^a{=T`eN!HDA1Z2N^|X6 zKCQu)5{UZeGh&^eLkD3CxdqM`zo^5V=%tANNB`YFppFUvDzb=jva%AIU4x}Ya9Sb; z#HV_I!-PZyo&%mp@?3xatRAs;W1}2(+c62LS}~xl9oItZcHS>n8i&tb6abFX;KJ!= zf~`o@yAFxiQI`O;C*EAKTkY(}o0~{M3KG0-RtednVOIQvuBAp+Jza67jPq)uL;z+Q zl-W$W;eQFNV6ccGcVrvE2%zs+58jvo|0;>CUJGH^eO#YPhJJ=ConWLZ$Uu$AzjJ1WVaoK?L$U+!H-9Ng*K| z5()Cwj`Vou^U@l2txX!yq`4|_XA75?G%0*DtB`-=Aiejkgu8Bcr4RK@%jPQ{`0)m2uhrjRZp&-;R7r_Z(7#J{u2_Pden6Wa4V? zzNve(TIUAqwtxHki|Bpt;E>3JwN*9d7M@6D3^5^R50U(z9yPgaLgqsN9_jphyTLZ( z@oYWQ3sH~Cv44qO_=Gld9-|)h`D#e$im~BUZ|AvA+*D*%s0$@tKdoF{L4>6r_h1wm zI(N`rKjxGa1vtD-V>0ztXZ)UIBQ!v2X^xbo2Gh5>fsZE)vRca^^JJ;zE-f z?9|5xTD~B4C1%Sr&pWSy%qx+YgDdp@{e`*P`e*uu5&zgmcTG}P%pj6g!cJha4;ERT zx*Bu$PIX2k>l{fQ$7CYpCceliY@bAY*1@86(?DbTbB-&%Zc;^IvmLRUEI2U~n>jh* zhVUd>Ud|P*P)7POKzj?8$X$7Vd}A&Zue@?v2H4Fts}O}D zSrNIYAewKN%p}hi0f&DctRwtjR%IPFwR0FbC6=Lvicd1*873$HGkS@=;IFC09xjds zhGgKPG*d>IL?kjo9WxDt{yk{%N~REW#FoPT9`LK8a269FbD+)v?jlMeVk`yy^4`(Y z!|LBt7DXA!d$3f92%6COczm}f0n<5**8T!zIV6w5FB%s^^g@PrekNNaL6R2_V-cY7 zTX^!_X)W_AS`;EllrqN$bnQIHHNAbCbjGob9VDZ{@^Ys4taP;*wDA@*{(Ui|d2M6> zO^gc{-3p_Yq2-#QfWOypQnD484xNXHkOH*Jnog&t>{|={rsc}cYjG7&F*xR!{@K~$ zUD>GA>3>EDsRRhsKd`EE|3Ag;2Ej{K1d#j^6cR1qbg)GVT;Zkv6ud+x0_cAHbHZIc zj|1)t@bUoY<)r2XQya7^e#%aNdV;-j-v*uXsJ#`4^e3dS{0=uYDL>psWFio4+n$R z*DAWxc+1l8^+yZ;8O(X}K}mz8tm@k`@brbRKp&i%D|EsDO02R78er z{aiHO_w$5lZ7Tg=0PpADnr4#}9^lrG3<`zOw-8>+1@?+_s8@bO;3?mLcHJrGpC{Nh z@2{BF`8)k(J0vY9_%cAal#D_|Id=XZ=}!uMEdfhbd=)%&M*-BerSNmkP@nE7y@-c| z@)dg`-?}X@4p*6ocuNXuPpF2<)!uxK<@sm!!yA#DXP8iU^)ODvZH>0@1th47RHikV4k!Li zW$F@3@GHMjytB~Ua;xn`wnC~TGN!fSH&{(zDD1+AuspxRnN8@YWq_-ZKy?z61Y5eR zd5JIpI$=eG*{#DS=nuN-GqzeISV7bv`{b2lUg08!&4OpX-wDY*NS|7Z3Onb8Ih=ik z)S!#q%Pb}Tlk&9c(vv+i+fXnEb!*RThe>M|*L{+(34rE4sQ|GHkPxbfVuo2+bPwp9 z$Hy>6;m3JhYG*Bphf{XY)gLT6xN5%=o2=pISkFJ~S{~paL z8xhV#R87yAH7LBWcds<{`K%gNkX0rpY#`9j>5&KpoIFlp0sI&za_0#KT|D~lpk|$A zlp^6%)T***J*rUoFn$~6MW}b?^UH})*8&N1PM)TR(U zEh})9RR%gd z>=I`@o0GLfZs?IiFX5d(rwa&RlU+PjvTw;^9z#~aQ{F*SqdsgCstb&G!unU}cu-lp zP_LW~Fl7_KFhk)J0kO9OjVJ1RL;|<$h65zAmvceQ{sq)~<7w&8zpGIx?K;DeJw@Ir~?r&X0JTXnBy0 z=ja)FQ{jIVkyc~?md*3S%j7`{--7CvhzfW+upqZ7Hm6&6#dpi8( zV8k74ceMMgPw5+`^8LZ7Sc;h{Tj_Q>9pTNE+V+5R>W$4Yv2MnnTFBQsP;!{ z`k1+0K%bvxxKZJuX;&-p27pT-o6H)`d2=+T1*I1K4&@UCOL(5q*+iu2Nmgo_a1OFk;O(*2T5<<3{hog@fri`&=0A*4EjTv#x@zh)d6l%VVok2SiYkX?l;&L@ki|wMjVkzsQzqkcI*Q%Jmol?AuxSyU#yXj2S zWEI4Y(k3PEZSi`G3|1nkIpbHieTeBzAu)D&sDgRaZnP`g6VQH-RZv!qU3F2 z+I6F7+>KxnPz}2ifHxV1XZ(Pp+{)F-Jcf6?J8xw5w74UVZ(!NC@S&JKXxMFXqI?b& zRX|&6ZbbSJ!SaQUB;gw13-1MnNm2HgH0r0f1M~$CF3!%^p~q09)X{>xfD?7mN<-#A z@VUOBsE8_;=2Q*arlD;vRQqL*c2O8KX5o6p9E(v;<7JXA5rCyN31kG;)gPt9zYnn> zwIj)b1vv2Kc?AFeKwBDb!hvmRVuTqVv3RG9hrQ3s#X1E|Xi25@BQu_ekCo~=n{4GT zb|Li6-PgI3-mF%YM~R2NRP|@m>SJA$-_dG=%FLAl2Sf$pO0bXX9Sn9ZEoyld04w?9 zpdILne=J#2IW$PXRoczed|yu)tEzNMW|zEA%?M5DilhT8pG6I6@5 za;GOt{=7%@(LZg_Sk^%2EWM9Clg&d~jtL^toAw=_d)Gr*TR35ZvJpIA`?!T4NT~m{ zUrDEfZ!H%_S>0G)x$Z@W@~6p>n(?mTFWSNWAD0=|#WNq9|Hzl=&EH|St3W2S)y>O_ zEf-^97k(_ASv#og_g6b-{Gc8{qvv(gEvw#K(Ot9WuV{Z>&<&J7E7 zZsJ@jNCtp^E?`?2e4!QRVei|ie;1tDN@bVI4E1?#jc$dg%2c21Dk5;ipdA`6)?@Ry z*XX&lQ5DY2i(d5$OO^#!4uYMED^DDiHA2%&2V0Rx=BJ=ery(DwOmq}#qiXIg>8DZS zCg4cbK~vO$zjMm_a2`8Ioan7MsLZ}+SHO;4(V5DO^XktWm`KezFn^U9cjjaBtg(a= zvBNfZjlKkY5lb86iZOV0ySDa)+s)RoRO6(K@gGda>ADr8My*GbAN8yiao>O3ye9Sq zbdzskOB{(LnrH-~Zykj!hyvkRr|~VzjS&A~5Mv2i=Vs3VO#VIx2}11X7#V}741M2s z>;RZpYKoGV4mIJvpEIZq>X?9Ad)pE?PiOBKhz|-Xg8+=S6T!$&2T02vOIocobQ#ZB zw)M2P>~M=eMDmh@y&FQ^&*F;2kO8iiX)KAs(fr5a%&d|0sR#Pb`x*B3*r8tmJk%la zR88M!R0F=OwJ%W}KsZ72b=`4Y_0_576=I`UffT^i$JtG+@VO=684&@9S4P#Xz}IV- zkYzI1ivX%sdkoeZ14^VVw@NR9w8s!(Y&K`2SBfA$_kO?X!>w;G*Af)qOl)}!C9O5Y zO>bk!$t>1*+2frrS&D7x!#%oX?ext^xa(}~ulF*pj~|sdz?R|u^K9B+h&1bs&v-zV z!k!qo$hgZZ;J zd|an?qqBadh0V=3sk)Jlj@=D2n3MV3H**Y79X~Ug_^}4HVVuevmOdvXJTN zW#u%9?YMRzs{Q0!D}*G4$J(LAiCbJ%;s!ua_!kO{{n`WQ-xx>(n3W-^vANzRN1$`? znF9tBNp?^4Js#0Vs&dukNys~H{&JTs#~?j7P{X*9G@oNSeC<`UHL{uKrnk5W=}AG= zR!lLj+zcdI=~uD02l6AuUgi65@ZP*MkIE*FIg2G&5wsL&Zk>j z{bx*=`d?MEPrT>^tg3gKe}&wQ@}~idul0{wxz*BzL!S+5YdLiBXha`O=?YG&e+BmS zj1w)bS|=$(WTBig-jmWT@~*b`V@9v$>EdG?Z#%`KHE%7FQ$`E~9J8%~eykn97BWy# zi0kq{YBwdhW@a4xIum~voUew#x5ksrBM%0%M}DKvY|pR zwtOjf=daRd+NDBGO*gh2DYY}?K3-;)Jtyw9tO3pBxD&N5=dHV!p+^752(T3aCoAqx zL>BZZ7x2)ucgWM%9h{$&Ha3=WYAmt#7Jtr&bEk|8^;lyb3!#bC<)T5g9%hD5PwVr2 z;IaqaE6`=G{_@yceUT?xH|A+dJ^&(qer$5Nhbr71IlU z9=l2Zr?NXv{dOE*F`|o+Vy-$6coHDn&3?~S=W+&;;}@6j^LSf2Q=4n4*b1gpbE>>t zQf*zfrD;tmV=TSW)!xkNdt7D>e;L(md{pk7@)FUjug7k)Cf%1%t*lOe;A*+r-&69D z$MTW<0L!jIOV8$VM$c!FxEBY}{MMTI=82`{)8n1tIMknPI}N0ZfetWPF0Va}npp%q z&WWnLxXN?guGP2kMIcIWs`6rR4%eExwbajUo_dPrCax@O7l>(yhTbD?0~3T-C;Mie z96wdPw~MSZIg(xuN%~!RY%uxZjFnDcXNE;x;Hht)zvo$$UAM9rb2F@Yp&1<1(Vz4& zmFs-{0$ocp>l-hQu1b{Jb+X)|l0B)65suz@0E3j1Lkd$#?saDm*wfT&E6SSosXL5) zSb8e?m3CPx7$V1|^gYMD+Q|8#p>{<_ZPIe;1Fk)OZ&qREh2yeDWplIwlV27vp~m{! zT)rlycf(HPM}$7FzKe=Ci3cGhxB7o2cOJdE_QUnGJXw9$Qupaqi@9m>M7LLJA&iS&_i$hDPf=9BustUL#|3{i(g*L`ip%DlvMgT zPDKfCyp$-bTP`k;dj-#yVC5Cvj1hTW_Grp}3wWbOL(^{`X)@_~dmyQ<({tk2l`eKa zjhNwrOV@VD2RVL?e-|CES+Rgd9Q)H`&8=p0A0Q1f2*oj#A@%T9-Pd$CEL#)g1-5ayuIq<$hxBF zzgJ3FoV>|7Sm)e)YhS$G=l2U(Y4mOHYCzkeY`#&$N*|tZQfmzns};2i*er3r)Y@(I zONC*d)qqEMr0T@3?q1NNrXUrpWy3B;6^|gY&o0TqHuuEkUSFQczPt00{wm2Xz=kWW15>X4{x z_nom=D=+~8K*{BQ#b|C$+&gE1&TA3nlfC6mvjYl(H-z4DHh<#KRZ|z@HNxq}yro5L zd+>o(tZY|_jnSev;ugcBdPy21-@(z`|3~L;Y@c+bN2P2{MubnN#|mT+bG&=pa6EoD zSl&10AQsVpFn{QDwT=IDA!->nKUV{wb~Q>qmuO?h=*dU;0pJ#3fSIh*+B4AIOL$GV zSCl`4Qq3n|^`B>GR^in{(a9S9)Mz^-vbIIv(HIZHM6C+`XTbSdjiiZ<4wfUQva~8+ z6!Fy$vw>rx=ody^o@qLDX3+;9+AK@!@<-i+!pC13?-(W^6im)e8& z>6LKm)Ei90tgL6Gn3RIy8wc{C($*>Z*}L$tjga$-QI`QgGilLB>C3;R*hMZ^i*GpW zf7f*Tj?3z=6dpzvY?L#*){Pv|lxhV3yoS5F_t$#fAjgRvDF$jHkLA<$r&^DBm3L(L zE}1V>Gklag;JKsyv6b9&mW^~CX;)Ray3@69om}fBg6BmDo*G zk2!)3cWb=7wBzlZRX2YnyMBruq^@*pkBUSHb++N()BiQw>PE2y!B@_|&WeD;k8Mn> zfUAR)jkt(y>6X)`5Vi*P`C;PM0<*s-9zR4i`qOJI4H4uF!}e+k1ZEf07R`NvNPK!v zy2T%cW%aUIg(%$^V0)_23N-5-@H1ss6#e5LNbVNQ6}*Icga(XW7f~>==nu-_NEE^7 z0XGZurUQ7GD>00+15I8ML`CaA2>sz?#NM-!SQwvbyyNw>_vb&|S<|_AGj1PXu&6s$ z>!!b81N*$Nf+xHE1q=S@5d?eLMlh7%$o0T~{LP@!+&|xmbY0WGx?1p`*-LBou<;1; ziiBV5nGJ;Qp2$_HoGN2`TKU{|n zr+Tv=+U{Mi1dunh_kIq;s=l*xGHFI<$D>f~JfVK7$~8sJ26VXRokgvqx^M?T2>;ci6EbPLtSb;M zl)DagqS~Qwr~9s4TH7kmY^a|m2o-Oz@h@WGxToQ~N~y*p2D56@BtNI3Bj#nY z8bUD#+`nybr(@qCW?fY9xI@#WvRM6sxQ(taJQD5iI+#@yf4seHS!?fkPl+#`7M?d) z-!jWAU#iuD$T$Pzl7k<0veC_L?5+XBx>0fIUy^2}pDqueH)kNQP8)nH1&_wHx^f_P zC-Gfb!2TJa;gCI1RwEVa@?)7fnv~cA7cAIVWF*989E7U91ILd;HJJ+L_Upq^(n#Cp z{mjHn_}W^{lW}jPhNjknci}URIcuwsYHNYB$`M8HoK7a3&pdx`eM7ik@Q=3>+~Ky+ z6OU(aM__vO<-9sbe`6s6DMwK0C?HEwT)?=(($0I8K*YHJJ%=Vy$x$J8dc zc4Urrjvi%ibE@}zXRuHe*BR zN(&VA(jN_>cYWm~_t3&gBCAX+wau%QT3WCDpq}d@0UAh&&P%7_h;ra79ZBzZxt2Ww z%(x0$aBWuv$V8v-@~|blp}NyA>qTQ_jx)(GK~Ce|X!{Rrm^mJv(bffzc0|T7rqmK| zhB4%1ZPgo}na?g2^uXUmCK6kqtv{|~wsE88ohOo32Fo#j;(oVwXV#>r*8U9B5}_cJ zEe)Ql)_VQ=FsXBTvkvVOmd<(7+%x0SzFw{}W32(?_8J<5?F$VyXtI`D-CiuGd{LLb z`j(`mef%OVKKl=!H4J$|*E@x{#|caAat{Ei@(ml;X83z0&SYQOXBAU73Iv32ZYW{V z;?^Iuc+ePqa;e6lg9}2xN5{k^EM9-smpQfb9O743SQj4kV4uBNqzw{7(lq(W-pbCw zl{{LuRHv~Ue3}K)+pj0Kz7FW6$XuA&2KVOdfZn&!RW4&VC(#4i0?K{Fl+Qr$@F@O9 zfAluwV(Uj_#VxD0e;qacGE|zWov^Vy$12(@PrHIT_`-2n`*CW!vB%1A&A&-;F^&=x zYnL(0Dn{C}tW>HGB9sKTmr&LK45`rTTX(wf$M zTUw8K33oBHB$ZD_0s^91&F^M??W&hTlZA#$E?u&PN9v6S4)DY@O(u_i)3dD-L*k{n zIuD94l;stUk>c^{Li3j+I;>qG9s}A=q3msTrdex-xtClYpJ1P?$ub_?@>(t=UZI9} zW#{O)Bvx$(RWEaAT?O$phO@W5cc0+OSPwaMoZ>Vd*0?2{-GqMVSb;X3`z6MY=)3>= z5LfngouABE!WWa+Qdl)6$|dz`wca-&S!XW ze3^KDtLmOf&ztuX)kpJNr=Uo0?xXK#6+SLhX-rgXMcJi`ZJ>6Xn9sPMiLa@_tOLs} znL|xkI$j8i)2p7PsumfNe8|&XWIo5o_>2La@Q|3T+LK~`iHXE&JY>y0Pah*$TOR#o zTG-#^5M$%R?TAri6Y&hY)&YuUQ}n%Uo%QtIP_CvLmmVQA4|oM$51;aA%s#6zYz*A@ zbMwL5D{NTX6u)RGi?qpEH?V0GPCb-My{~SPC(fTV1X3Pd?T)}KJJPe%QZ8TTVk|ZX zT0ekOx_agUnbUB33@u3FZ`_v=`rZGi+_hr@Mv(~-L$x4QMTdbPHhc)q@i=tf3oSs&2DY*xlRVr6F`nw z!ts1{xFS+}3XWTu{rbamG>l!p%BI8n$n>5hBoPibb2L1n+co=}G%g#TeR)AgqoAEB zjaqurB508(KCdD6!Zg;m*YAjfh_raGZ)O_?-#jMnzLg`XLf1n!nB$;H76i3Dh$~Aw|D9XZ|;|n@hjH zMqOKlTtc(f8#?z7VI-mju=;H)CmXMe7fVOBW#zhWZg$zhWm#6HxFU?oP#i(5^^0ot)~uk~KTs;-BgtLS_2s^2Ke2AHRXc zq~Kude11{bw%_V7-y5#y;}Cr9DP!(_sAv>C_jtd(b${Z5B#0;*b*zVsCaI224vaae zPPA>$e5;Kv$G>7#GrlFoXr<BI+hXGoULFW(8PgFFT06r+G4!WgwaJC-O-eK1 zW3>Q?WJB4DHun5el&^+vKOPx(yv|vf2;?kv;i;^P7@kxKz_$gc`)#5Z3(ykrC&x2V zC)-?|nCDJ_n5;3h%IMeplr)9$uP)xdkE0JeP+Tk%=(rzFSDh5jB}O!oF@e z@BSwLpjfhvhzV-Wjajn!alYD`HUXy^#DX_n7da~0B8(7Ari=N za$-DPcOcEIbxYPm(;$i1B)H)!gA70FUb(G5SteCoU-H2>t3RaH`W+7g11BU+BM;2m zc#{Mmv)tROlj6BE9l|k;VsB@QpKG#+1<9wF2|No)=sKV`YK>zjLZLUd ztf>8rz$%T3=vDNW;}^to_uIN|nf#brvSi8H@xj&6p5<-q9!nnUsZr(cRdq!U>g?K8 zUH`K8d)w|Q2UzI7~Xm00$7E8FCT`h)5n&%8hVB~~vm6tp4) zpM7ug*y!m~E)$niN3_?jRZZ&r~i=`CwK4L$EF{lvdk$1+pc8p>Bj7BqZi|lRjQ9k*%EJ={RP)RHV6cDz$FxulNtuH@VV@HG~Z^omqXF^q7%YsT(7v)blt%{gxxc&b$ zIoEi%&FG^f)t<+L9U5fu3Td#87}=zYa3w07{R|WhF?uNpvi+qej)&MGyp0S3r_`nI z&SfmWtLk6-bGixBySi4$I{;ut73A&bAe>A9w_JU5*$zFo`5OC)@FYOaW8`_=zJ@1s zQ%8(~z4I^UC*Q6cl!R6HMPePZs zT^eZ*aFG?qrFBXXP(ru!<8mweu4&%#?wTeYYc?WEO8XfR)PY?5S11&dl{WHi#C*cM z%ejvqLX=RxO)eHf_O!xOxOv1*mJ2=r17w|4cF-MqCCdb5Rr{z{4mhj{Xs}d`xlPq| znFZLrY54$?1*uxqjs>!V-(U?4eR)drtFjpVqpEIIMQ;|Z-mBVNBc{T3(1l}5 z+L-+4cS4u?E?8wLSz~>XyBXYr@lqKP`vJ_)Yq&6mSi#>Mj4?WpQ^^J9kfxFjW74>K z>H+lb-iJHWi}%rmKvQoh_Kq`DEsi+UX04R>9K%Cr*juc;r4q~p6$Q1==t}qQJ#Kh1 zShzy}4}0Go)b!f@8Bhd4u^>f}VntEFLKCH^*hN4I9fBymDosiN3s?{=C?G8qrGzFW z^sb1C5I}0^B2q$A2mu0Q&m-J--`~#e?9S|OXJ>Zy{&(+iA^Dc`obzetHVY8k*+y(R z`*iIa;{1U8{}S!9*KWfXu^m*$+ zn2>{-W53GPSa#{1K_1cp`nIYY2;{xFCJONV)A8pFpYcoRHyJU3P}CgP@q6D1 ztWIXBHk|(l4fj8KrVbuh!+_u+P~47r+yF0cC3y1rD5j?!l5-@2n$*=(~qVzKHP0a zhnd%5c#9M+)9|=SP=m2DcyVy~<@9wrck#ia{*+f>bzBC_OZV~6&+xY{4vM>sNRQ70 z$eO8CYBL!0yjXJif7uD4w>yI-)D~dYYy(y6OWWfhC>>9SO$Cd(B1q6vrk~+}zXeE+ zt_b<{@n=QdnfMfV>~M#L3G{(cA6nt^FH&&^I8#2a^L7B5@YOimK-DD>+?Oo(8&3Sw zr9$uwAh~NZru<7E%ea_K#@4z2>;uP{)Hhwxm8yoEsn>vZGal zOU=}kD%Y?{W!H}+<==42N53A(a6Xx>e;b8V>gU;y5nrOL6sXDT=Uji*Mzq4oWd;~? z4x~&{WLgb*+WcXWj~>k4OU)h*sXd>KHV8NMD7abL;YA9`^E2aCDcW8@=__(3Ey@lYzdGH#5vNOMqi|DrhY3*ns$aO484eAn`lE+@R_GCqGn3>zqvt#83 z>Ji^|V?>ja7OQ}S9+k9+e=e_N5pie|F8eJdZBREc7N*Gk`0j)#A6-*v2baX6kBPA2 zdVF(4;PIKNBE?$;aKhFVUQXJpv+}6U#{O&h?8eRi+6wFYTy$k49R~AY_5To>TmDoS zjAa&*AVtD|0UM{h!XzyKgzD1$+%3( zBe3Q#MI-henmkd7)P;`D77o5~l)K-CY&$S54=-RExJ7e;JCtulzAGy13o^P@ z#Tx<&^BRz91NjNYGutLPy^c7guSiz`)Q2u0><2-nrCs9Tg5&_JY9wc^(@A#j5w8!p z7rZXbiXl04eJsV%?T{iN6GpGk*Ab?PAjG6iA5y)?1XHzMcxm^A$F*crjdJG_12gT( zFb7b2E#^DM)>GTon_X&a4js(BX!gWlkUPx|KIh}5R9Ykm5cA0z1RR*VBtSEcU?s@H za4#kstw4N0QjRLsK(+56wHn(XIuRb%D%~aIfMXZFQw~GH^{q#4Iufp_KxfL)Xy(B;B?G}mz08FA~;2n(&WIW zc@Wq4vNl4M(T6cF6?k$84n79J+v_e+#Jj?nS5+VZ6Kx)xbQCD=UGo83_M84aaCOpV zKLH6H;$q#?xTJ8a60kn2g4=_+RXB2$~WH0-MZ znjSjyjok-$kHeAMpmx(RLt^)8-vj+K& zu_;PQfJ4KE`)&}-{}cw!wd`Wp95xw$wU7H^>H?E~;Yp&2ODU&JcjCJ?bG zO|~I{4fw#`r+8@h_JBWSA9)dGZ&u#XEWonp?h`#_J<$A&wrS8LGca9-O3tW=4p#ih@Hp z8e;08XRs9dI}N6}d^#Iz?>Ku9`gx~T16k(HwDDq)Is_0ii<~D0?4|T_MC<5xCh!~9 zR9Y~L&&{uYZ&)yu@2k(^`hG-)`Vlb0Z!QCH2kg_I+<`YNL?f@u)!^x5drt;NpXJuH z^)L96M<(B#U4{~G+sPL1IPITLvt&#&e)nZ3MLoW^2;D}J_IB%(707t%KBuLsG`9P! z4w5=ViDSie)(f@vkI0QK!p)ZjLyen3N#Kom?d>B+VNa-a>&{cnB^Digv=iFCL`RZG zn!;gWDXrPtZYA_q#apG-&|H4XSLY?u1Z8p&h5-C-s%C$JrZ5xwZQJh7_FStE9>*O` z%A)3(Exct*f7BgDK}oG^I+T4iU=uc2MBY)f{@EOmbQ@C1Wj@%W86eO0`wZu|+r5?y zmuh-@ioQXa9QKL>B?o0Jg(QuwxEjjPWS|Y5jR1ZBZax}bn?opS{!GBBo9ZmT!6n-@ z2C!))&4uJO&BvE%7tkXtK@f9T(iLV{7&K)ZlYNtY^HylvPbRaqBb(uKvd~M~J5}*` z?_nTj&PR4k^w^wQ2QWJJNOh~vu{(Hg*7NwK5#Fec09>kXu%*>2MZnf%IFtWY{oWHh ze9x+K**8BAtB$+ImIFa@McI-B*`&eSr*Kk-JAwr&B=(mY6n%;wn)0>z-cU3Vtet&D zu_i*Q?43HHf@Zj6{YkYIcP!TKAmotRg4fAyMNEFJsOl;$y*g07$UZaS);jm6+IAOc zP;a3!7H7r^Y7R3Jr7#$ zg>%tfe=dOcE06{othzzEw$JWZF*u*@RVP;&P7I!KpVBLd z5M&cBiuN~0?#ozza=C}KFPZ!@f z2pXxZbx=TxYHmDJG}ke5d2Mw2%bxVjXlE2)uUBkfzLBwN{8?-pJTHec8qv2gEK9&9 z>9%K?*Vt1_jqEz|>@r~%q+H4f%E=jSWD6-PakmjXKCi~;DwPo2@}D)|xd z&UIn-Ftt59alMd&bVUaA%=8Wzuh3#OR~W1eVCd<(`2Z}+d5DBCa_i!?dF=!Yh*1YO z-hYH@SbXP}4sRNtnDXLuFjq|V)2%!l`S8;Ys98khlCa0n4nZ(jp)*D8X3}$m>ohn{ z#-N|b1103bIl@X8U+2$h*{j?H@(%_8ziGfGrSEhD$zA$Aox4^L=QUa|80}OsG8gxF z$v;TIuFF%=76`E{DRZ^F`|L9yRSC5Hk(g~%zWs7ZoR}KHCat5}43btd*`|I`MqbyX z-6n05b@`;D4Pq_T5^n%O?cjiStrl|27iEAJBpYgL3v|~5<2n<+@-6RG0`^d328O<$ z;??|+9t5ySE)3XhYL)^Z4!0i9MgXgkC|WTbap?NJG$=4YrwlIpGoFS0cV1HNB6CM)PR8bF@|F?m+HqOfD= zgU`{54{1KX{B%W9u8|^B#;<&{h$HVxZ{bHTGN_A8^+O*N!Xaq~aY*AUb@=T$w*L{` zy!@m=PwSzw>Wjt_D;t?2xS~?#kP%-=gu!4CGySh0h4y!_3&02o#Dlk+h5jC1KEHO~ z__vPQ1PpzHMsRZemH)!QF!k?0Rd<|?3|De6c)7lw|7BTm_tD#>4~4J|`$j5s1tDnN zD`B1ojoC=1n};TxihFDlfB}cVTZ35YlDS_IGk^)4(;EaV8j*{LN)mX6hg)#BH4;>@ zud0CpMjhXKqz#7FW^>NsLvG<5oi)lg)>E}IZ31}Tr)0pU$@HSH`m&|%{-v|2QGkhl za^fECaX5|hsIyPj>H4}w%CdD!?L@S)*H8c?*Ea~%P!+vqmx}D8T8)g_(#0cdBul2R z7J&*h4XBR0L#c^H zWbjvhKOEu%=E=@^3d9EhKaz>sQAl$9;cWr+3z4SnQ0X^Mw=dt+-m=p`L582I1B$$3 zy zSP!lPmBLYs#__mLNLlpmwMe^E;RM)^1b%+nqRZT`DlNPzFHq1sXY|-@{dZrR590mT z&4AEyY_K*0gmnCAr>i|BTtW`s9bCw{RnmpGaOzluCwsclP#ZR@UWyVYr7gO?Uhn=} zFG6POs385fSjc{-U*%4qVzvra5_SNLEt@KC=+K@$m=j#Rp@=*L{ku~|)`sUOAPE%7 z*=X;(A5E^yIhZSmHU-U>E&SJ+`qLZs47tUT)yYgCu{sNpwi&n=UwDhid_Wk$l#;2g zH1`qM5T5=5{VQcPV9+)It9BaDqvLd>jfhb80ej4N#c{02DHZm|$g{TgC%hI!41%cS|G9ieik9Pm{Mu1F!jb@AzJir_wbIcFZ z&|-YYW**>Z{j%nteAwOvQQshqWzi7_7x8PLgjWbCfT}&p?)`*brzpk=V8FwyLly*l z3$TVh+?5;n{>L)D{YunytNn7`5w=%j7BvAHZjvhcIG7IFJP z>3$yN93X#mYZYjtOWjP{;7}#}8qblrP`fX2Y_fKLs#^wsCmy`ilA0c_pDyXHChEnl z9jE$upE7mvk$5NwgBxVU<*#tyWv%gXpFcE`E_*jL|<(|4(x>kw%XYDl@@Bx#R?~aj*q_wc|jwoR)cAQ zl@j|%drj1>_MFJh>$y`aJls4+__Dx>vp1_Tav(Vrw_7)t?EeZrJ*GugdADLw;U%x7 z*?u=S{O_taqXnVHB&7{A?{$m?eoBrA85R6lpZ*o;Ia~q4Wsi1)N>)fm@q-;mkh}Lr z@dz=c(<1gcPU6t!O7@QcZ2?FG1c}s0R zam0+~qiwSmtLQdHX-4Jn-etcOFmB}__8v+Wyw{xJyq`vH!<$g}ya2H5yCH&{8hAEm z(+-C_B_jAnebTTAu5PYV+dKQ4cAD-l4_Rs)=P+F<+I;Jgh41ltEO}aKnjN=)74Pnb z7DPwXvB-GQVANk6mLl>WwX2f5)MaDIs$!(IDV?53-@{o9`u#yl&}C2!(wbY$K0ews z-UvAm^b>g3*jM!%MV61rL3Z%faDha1vH*0J2)GV(E3UebyH*Z8BA}Z-X&`prh5rM5{iSZ^|{{+aCGB-UXoLkn=><-4^xG zqJdI(u;d$93=9C&5HN@TB`I(DW$FKdgMy*cOwP9OHm?blNF?hf0LR+Rl4KZ8~z=dm#8YMe=5SQkIutEj1d2jr;aG!sh9=v{VUN1k14bd;25d+(3u65oG+cgd}Vw*2`; zts3sFdKCX;4&f(hdz}ql7l3QZ4OTu?uK&oU!l*D*5sdi&=$t8?5UxiSEcmG<~#!^o~%ovz4Qlfpm! z*-a^@PDsC?Wjkkh#zAk=0AM~KDY7=mv(2rIkkt>04Al*2vFmbzDE8|mgWWIM5*mYmu{B+{3;z5h6&-iWE0mMrIEX8> zJ?1&oAZ9z4wFe=KGyQJVgJ#56+V53ZpnylDsI=^U8Z90yQ}6>+eRj|ar29Q)^`za` z=HjM#*McD#HZA1L6yz5U?2}%sIDv2Yb>`UKH!TcSBmNTHvv4C#%J;0J}k_A6E z+rZ{CD5YJBuD`x*^&BI$rb4HW3~@Ce*CWB|%9-&l!NEd^v$sSV;REUUmEe%J0QF2J zkk|#Sj|1oT+88vQbBA}MCFbNRh(CDUR4@y^^?>Rd~L-mZ?^fkF9JPiBc*$8-KQ?e7Y18$@}~ zB(5cVlby=8YRS1SWKeR%9JBDJH_>kTb88pX5HJmTBp^%_9VhbwyR8b&mT^k`u%siO z+#eSpnFSxr03cJNeM~sb{*Fb!7eAr7OBGMvJAN3}j_K0%x;?0AB|}Oa@>wL~R&!*K7X9i3Ox;^PS7hg%C2F1xl*? zCxdV+KedH~8hd>4OM?k_Z6_VVgeWyqWwy_hnm){ox7)?{5G>`ilE;M+`OX!Jw3TOr z1kB0ngg4)aDsmlnoVeaNU7%h3vH9-c`fc^(Zo#8pk!aSu*1+7VF=73QL9kVf`t8_d zXd-m^x=VF#t&wO?j@31I#&GwPZunKrAIvtZ;Tu>2L_;4EXbnmE_u!s>XBTFG-)R)k zY*#Fqa{;2Mga-Q;2gxGwN#nCho>&_y zJ>lyx2qYkduWB=Carv9fx^cjr;p96g(68k8F|L%%85@7C%u@5w=zdX=4)?K+X*Y0;m{J_-DT9KL?Oxu8Ynd-e@s{+P#fywmnbY)7N4}JFO8OehEggLwr zkDVBco@$R^t7QJgU$gU4>=|TrOfnf)636pnkiZ1)x0ZqLYiNd7hNyeYJvZ;l-&5sg)7r*^7k0E$@Sx)xty_{U+@}WeriYK_ zN7vwL!-nP$h~)%UK|_<~!(5V7)nNo6@CXg*DLiQzu-!XdYN{Z&YLZY!2}5hfy3Mef z+<&iy@6@gzU~Ab<2N<#gbelF15Ese|acgS3k!7WImuzg)**+i}@?hUe8`6FN-1m^H z9l5ljh~^;kh&4X{eX?B6DJ*3AUZ77q$;;pA^)33S%~oPl&FvO-T)n#-e>73u>AGXY zj8NK*R)*<`b7KcK_XB|8+Di0oVLpD4O4hjt&Z-U)_569>9AX_Q{hYA-N0P#Fd!=&c zQPKqtFqe4C#kRxpJS63J{jMfg(6_{yru8Oy^3aZ^GVr%%rF%>P)SU^s5DL~<+zaif zMbiO@IZVN@zhTbv)E$i~M}UswAEfu0j{JOP+L;(EbyO+phdmCLe7kpP6_e>8i(3k_ zQ@yfX*&ZtC#U>l-$ET~+e-44Kp%sSz00BC1QXOjRl8L_{14C@c#lCdz*ZYOh<&V14 z2z1BDdI-9eLh%8Jj&BF-#4s#@_ysV&cJLaHL7i5goJT??e%D2q7iXZ!rl5dRIDJVe zPqioX*}iW$F{yICho`azWWHA5gUEzv=?>su1>Z?LOtg0YzYEewJhG}d3~S)rz|StC zQ!<*2twh!Er`d>R%Eo!y(@Qdkk^0yafBE_h_|3tE=PyM_8kx*>MC9dW#`olilp5HO?DyFWh-*1F zcO59zvxZ|X8}uWEJjWhybwLMX#NiwMMi4y!+qKWgZAEL+!DP4YFLkXy0JOx7*4)#C z*t7gwq>MYUXKY?o{#Duz!TGhCbsT;gDVRnmzfI7C6!GOKu zMlyi@dz*}PByG-`?S%$hHu!hAGL(cU=JeH|+D}4^I{nNZQuV6wmm2{oeZNF5-blUS zLJVt$?dPc6tNe;36uBh?ps@^*1ftx?Dnw(jM1IZ#6OYb;)aD&*sPzQTYQ-0}=v5>!aERKZ!N^Y=|4N2$Zb1h<&_QX20oG#^SUw z7kgjh)Y4Fm993}rH4o?F`ckZTFATBXGK=rTAq@4}FW z44e;Y$~Mfr9>gW35V~x1SJ&^hjJvR=|A)v_&0aK}r$P6Uesd9}+^ud{xlJ3t?A+-q z3DF-#-FXhU$l!JV5(8^eqXFn8*|s;*?h8STm&<6|Z{;@Ae{S#O8C6UgA0Kkq=RqY9 z>Hqy7^uRpD-SBodL72$+-aXf5p)(s~Iu|Y1bWHClzW=EhnDv?3nTOGLr+M*A^f&FVO@AAhIHB|lf)D%<)5s=P-M`6W0P*<|;2|j;qY`kMx;?H%(iJ$T z*A08F$b5Vqw;X24Stq2>#8wL{sS3HCf^)|Tg%Z$Uv84xX#0(CX3-ullVPOrnWyd^- zCbI$>*gmobw&W20iSJQSeGE=h;Uo{`tHs)1M6dLLQP?0X-Jq z=>z0g0fXmuLl%cJ#Bw|G!)17i#sU(q0l!2q0YwW^*P#GW&=#C*-38&osYuu62{YOl zJlsK9SFnwVm)}6Pa1CF7CQ#e*li}7qn=gwUg-uE(P`k2W2&pRSI__rcmL`{Qd7*x> zbFe~q+`?z;%%3UEkQBXS=SXVf4gIq=IX;PO9#v0hhn-M?1Y7%NJbH4xl%>1VPcoWijhOf!3cOk!BKG zIX;**km07xxb!&X;*v#KZ^JHJp65itcWx0STdYk}E+_ASB45QeIR1x~*c(9!bmhRy6C378mz|x_@cWocuSGry_~b#!U_Z(mLqzF!bP7nWjbPEDYNe-2dD} z6qu;*YZNu%tG~kuGKyH~)Io%OSq~1esjPHj^e#p8Q-be7r$$T+lr3YT^e zdf79;J`tF?$>CWxMXq~$eum}JsAh=l<@YM(kA3rHS!H;q2h7#~5&%oCLqH5>~d|U-a6sD1V=OLf_eGz25KiKzdZsQz0svMnkXe>Jbd`pGNJ3ZP^P%A z5o{Yo*z1AMdl&KkJ(z4=8_x8}i0;5GWwTd@ zM;?|9f35+UVM_#ul}=dW!^JCAXpa3g|Kg8PGD10<)aXO3_hb zS3f0SAjqwl{+9y?IiMq{v(+%)u}=l-d^NX6V^WCVJp6;OQL|D)dYNNTxwAu&)ie|K zf3zm9VDy%)8x+9Nh!7C^EC|1TEOdH>r8?@_?L&I!Jti$)g?3W!)JMk2?& zL-BZDit z_34?im&Ga(pWO`^E1K-xd_ZEH45u*Ky2@xA!%dEvgRYuL?3Y0xuz8cTm>zjTpqm){9!u8vEo1VhQit{ELs2xR# zwNZ*k-(Sw?#vAcYSNB&_YGn>k@J0%{0Uh>PAZFBsc9uu_i|F!~b(DlFDGw>I6~=e^ z*oq)I#tu+0#A3I;ya8cM$bImUQE+BrZhLD#@qzXkXH&3|kxJ%&9oL>O{4+B)@>Y0H0wTAxGyz$&NIt^5H28&_N3xku?;O-NqDg!)r3K82)SXj*(c zw*Wd8x0aO{Akn$_|8&K%7}~PO(WwObxtI1_&kfm_J)BF*%TKj$7)MO-3#3my!@oFn zKoJ`Is}9`H56&?S=F1Y8E12LTK^EsDan|1uirR$|9noO}85688c^3(dOo;~OzqB7e zLPrih>t_uHJ;reW15z3x@wf;Exms|ss%yC3WiTeb^Km22$sw2U(ETXQNHA+DZSWial%@{ayPQsAEWT*s!BqDK09hajruEP zdgHD5xVx!36+2&eDB4t>^iW!nr*OlUPAor&75A>?FWe!JvyG>PCUv&9+G| z_=NNs&r4SN0Zjfzut6%ijA$YI6EZmf)^#KTL@ub8MJbAq?3e}P39C7PZX+5umU+qbI8b2Q%li+YI@5=n4sqQMvsZBBTM2%%|RC zXdoFzl2v*A`WB`wD13xak^x%5_?KdD)OLE5|XEY!A12y_$ zc6?HKG`@1G=(O&2oO}0vCs*oLT?1B#qeeF_`iQQ}Zk3SG3^b42v`T znNAweA3q|uAmP|}Apo!zMg(Fw7(F%*xDRIi1Ff7-mwfV|Pw2lHy`K!MSaErB>VV^{ zE>I%%%);)%t}%#okL+(*nPAKP4!~Dxgkj%pvD1m%4t5e&dW!fwBpA8m!Sxb;N0@@c zKNYf-k$X>oS_BJlP-})Ucfj%oYO66Y~)_g||mj9q<{a>l*r@{zwA(>Fi(`^WP!Z`BFifOrpx$3U{3DUdXIXt{Z^92aKej+~yYa`utP z27Q;nBDw>9pXek1BS#~*Vm1bCQPP1#M-;^b`5L(ZiOVF*t;K9S=^WVv309t4;pVMIzT7Yb> z*tq_8@UZB^tI|nEBxY-&y4ZQ%vj<+alztI0VDSZM{POQnaAKmW3DBrH5FYI8({Noc zo_^y}FWm9m+~>Ad-xOBI$PB@`z<{1}Ux)G4_$_wap>TMw+aiiiq{423HU!5Y(W#Ng z@H==o?WnK?<+vs%Mwms~_mL~S=BS2uxpvrvz`8|*`~ZRV4KXVy6BPw$NL1^2VN{a1 zhw#0rBZs3lHoF+&W#HahwHkY-p(+X$ZZMq%!_my4sm@5~#dv@|Y<#}mfWQ(PKCcQeoIPv3EqlK^j$J&ZSb@b2ltz%hox z7@jUcn$h3KaVHuQ2N&@e%u9>j?S`4#JLfOpqe$dAcrYc9qZf2fDY&vyWb;)Vr`th> zG^s->4Yom{WZ%3AR!lF5kA~&HptGY$ou}`~**;4Vc-fWPV@}?;1?am4bX!F*pNzq~ z;P#pf-k2=tijn?)?8Q0ZKK@nZS0iJsw)@^h=+`*cHL&ZfVjqt#m9{#P2pNrSphzF$ z{zHmH>j#kiiUII>S7-w8aJpbmdpQ{eAAJ|CMr-HfP5vg{H z`uqhAUG$(k?y3MY$e58}5a9f}Ko#9s-GRX@2yWAxYSzIO+>3h9Iv2Pz;~^WJ^UpcQ4Y7=xhZ_U)B@Cg$+FeL1WP_F<2%5$a}d_56BPh zw$}HTHtO+M!t3|RkJY-01*2k{gS+tAkcFZQh^gdz<_i>MhS49B)?({|dde>e0Vthp z^?NS9a;!m_2X_Z8>+YfEsHWi(7TIcbzcGsvg~iRFN@+<(>*F<0X34!gx9d`Wcq>NbCpcjpPflLXE>%h1 z&<-ZNR6umnEA?=%77l(>c;9-H9|D2Gk$hP4M9b>?BQMbQ%?>#CSaDCC>nBcAO_pxa zI-A~jwnKK}t7;5}quXJ2@fCOO;*xmSjEsT?8TP0i-?t%sCsKnUOayz(#%Bsxz@i!T z(pvwszfdq1>FJ-Q`$u6>n*GjG%GV|1oiUlFk#7^5^A-?qUZqm*yG;J!lehPd)-12lhZE;4(R*ekA7-xG!oQDJ*`()=y^q|9>o;8c+!=eCoj!1peVj8#y_q;HKAdq1Yr9bxZv4)k7G}qZ0~AO3;_Sx~?Y zbaj>bwj`cdZ)QJTin!%_Au!`(SKbnL&inBjJKD)_k+xyxfMJAgIDwMuASa`MCvBVX zdW`SP{)%d8YsTB)=#h%lhA4&McBuM!uCPF;48XBGyk)_LpPI8^$m&1Kmg?i)E9HHa0{Oh z$j4C|ywgGV?+Ux}3NNgcajO8Rwu@?sODxwfi6)V!v|hV{oQneQ&epI*UrhzfMm^rT ziX8Q;0*2uE&6B!Xm3<}1dJZ}e1LzYv&u_0)tcS9hr_5zv7K?x6A;-W!&WH}G>ltLI zKnt(p~m0MsN8X?%8JIX)Sz0tZCd*ox&ZRLus@%-=}z$QrN?D8|aj$ z!uz1MDhIxD2R&_9sVaMu!t>UOBJQ7O7>`ks+<5WSmDXo^V@DTmsx7Ieu-^2mQV=Nn z`SMaHUa7q0E)!1hE^nkdwBKwvMp!81)jjaGrQV9UeJnA@p>Zvit)4T51@lFH@~3yh zfyn#&F)fjD;fEX{lF!~b>54tzFzqP7$gMcOWt(^#T9RmJmIdni`Esu{g+d}ePa(fl z#!V42Tk`^!yc-I&isMxb^01Tr^oN+8JQp~9r@FOI)2WxfA}|+ZIrg8k%ryXPjX)&D z44hF#qU}D9FN8`)q4=Wl+~~v^!NM5}ri`;*whXoArT81H86RHIxjch-)Q>~q5r0X7 zf6Nyi^%I)}WYd()ttVm+ovcYIc}VEmig|uw^~AX!QwRpV{%IsI**X)>tHRi?+wk!h z!eBMKfUfpyb4uv2i4UY;jw^|5nHAL&)92mB=p$2lYQqkHkeC9MRN zeb+Iv^JbZR4}G3fRy}mc645YPgZc7`(5XHajn&lJA6L+AO(Y@rS-k%R_ig~KWMkRa;9Vnu7| zqjL=MgUc;kqH{Za*Gv)``MH<_J9Bp!%>RgExbPWT4D@(SskMH%ihD3!bvk?cg{oUri$lSZnM*G?{X{nfg-NYEkazow-VpQ`1 z{pM0QnoSGK|*Acn7cYR0c`Hnr(ot!?{|OajZ)!oGuDoRQmdd+>?tp)#Db8 zAxXlv+?VC<1t%6v-)Wy#nFiofZgi2Edml9aMvF2H2eN(@XeOm@)Q>;rQbvqky!C=| zbgo`UF-&H&^qmLEGZrW~GY{+7pRDc-O<32hSNd+V_;<=I;vVbAYP3fC>2@G~yWL7r ze$qpjt9H4Z_}q|3aK69=qq-ODWz79T(u}WWSi519cDm+FU$(gp?3c@AWJ$wtjj)0e z7;Dmf+CV4LTe=TXjcVb2J1IjB*lp|f2bJ?QBm?p*&^t!#r<#QuiTZxdpRcRhdx>!P9H zxqCKIZ<|T}aw!V>)P~Z}dA}!fgsfX*8&o~EV>TK)A9Mz}R#Cm~G`)v>AG#1OweCcB z7UyqoHKoF1w9g!|PKKQ5^{p3AS4n&OByMaMIXV{{Ztb_}GPY6wikQ~@K|YP^L6$v9 zJZaw3K{FBhWt20}$1Nrf(sAL^tx{4;0+$)}y)D}@J6n}~gfP9^XHE2Qx1IMI1+$Fw z>ha7CkLFQsTn?AsKk28+KE3S}3ipvqj=?=FZ5v80y=SE+pLCIBJ@qO!^pq9$yiHQJ zzdzaup_zN~WLAg}1YDZLRrGRNMcvX-i2}_nJ-_sum2S8QYnrt!ZTtjuw<*#AL64&_ zy_FQP;msW7rsJ?>3vw9S}QxR_|eSiixh6G%4e| z_ZdbxD8UP=VPoe2_a_-G{p-^x;$|0S5?k+mKBv-5Eh{BBHkues^YZK27IiJ(@p_&} zz!mBbj0mRkS&+KcJ|IWA-5U0V^=^_^XiqNdL5Jp_6GHKpTce&fmHq^29CwcEAE>K` zUr_9lXu*58Y6XZvh4n^zLERdU%ojG7YA_VB5>n8UV6_8gl*2FUl*GxDU4d!Xh{4^J z*e^x$yy}RDv4*YX9saF&4#|hrcNXODk3uI}xA)vl+hF06Ylwg4;+Ps;tKirYDoQhM z(U33-@UWPdIv6Kzv0i2d_ffhp=9De=2-Vk+C5QEd ze%R(754jO^W1Zgv>LDY(`V0>2ljKvjuy-l_o&S}b`6pJS=ZzZX>)mELVh`zF&z?VN z;{#;7fZX-;o}QszqkHre;>{Mz?SZA5k7CSZob^i*7fZkE{ z$+9ND?bRKVmfndybHY0+`d61GCJi6{|HYH>B-npx6L#mdro19jtfgK91%yrC07`)DYfRxw6ETdoKx(i;YpSpWeo_yPUYH_ zBg_oP_;N)s1vBp7!FpNk4yZI6JM9&~Mbf$IU{0#3znjaQihpKSdoqS6QlTIjQwYQT zxHqk!X|)eg{`;ReQN6K9t%d~r6ML$ta{i)UkA?MCq0cF&)Oh&jXM{YAZJix6Fh6XA zyN~Q-$5aL=yFO7m7pQ|40jp3|6M`zX{Ft}49J2_K!Ub>4O{V#n+!pbG zE4J3FgF>*L8+iXDRqBZ;$!tEyk`r{-`Hz-Ff>wPzMq%D{I84neWqsL1*741D~5i1%>LVHNmHhg?6*E z7kL<3{HBG?2eYX^`1z2HVMe3O|A`~W96ac;C$zM9nClhyAG`s#$Yq<^@KWDoeG>qM z?_wy*=~9OCI}h+MlQM$>G$TP$7=EBT0*_@mmi7-DMA->&iyQK6dP80!PWAQhZXfKoQRI$4}@ zZ$D-de?+mX#d|8+bNm+Le2?%5jBXbAp3+bY!|Q7>qQh+bl^=E9$Gh^!34aamT>#JI zc-RHHjmTT^n`*Wx7tNy%Vk7o-mBE;nXG|)ue0wzRv^Dw;xH4mE4T zx<}XGKf@}=L8n^ZhRJ9CbGu+%3(m3bs#QV(lVh|5UR-fG?4 zS2j6Yu6NR9dX)`!6yA8I?7M}X*rzlfp1z9u_;CG^eaYcFLY;11NpkpPE%D%;k3^CtFaZ5Mi`ivXObgi-i zAhfhottqijC)`MnXV-rDB?N^!(awC@fHQ`r?nT-znZ%kO)((BG85%0oMjM+jJx5Y# zOLA-TXdUM({2u+u(ESY>R%~4rIMy?~mW2ZTt^VzO&W!tWE;fX>i%H@Q(t2AYV&K$U7acF+mXsEIwqxk# zJE9*+N~4H#EYWLIV06ZNo#3p1Utct7nX8}uZ0hICkU7;f;h@zPfAJ>UM+(!5Fa`Lo z9*A_h4y!xP7DIAG`O>8=S5gwQIlY_wKa?3CvP+!TJbG$N`5eFys{53;TuzgpHv8zq z(#Zb`^sz-3&9GZjc3wUzgp89yZPaz8#JGp+1^EP6e{D#>cJVF-s2yUmU9y+}Xr{k3 zY&gdF!#ff;Hg~ue_u*}GzusM6@vrxWFGYy2%aPdraaFlrm-a5D@OYJYW9RxbZS@hq zB zkL9G^ zjGLEXI2OPVhq8YS;BYiLl!_0Yxh4Knb5Hi{VwA`&V!u$-p=PS3Pal1&^`4|n2^6zttSTOt{y=%kkWWVW@ zgt`2=#@?f#nJj(->zOQ);J;rTR1QLsHaquf(%ukDwp^sj+@lK?ri~mGYD%$XJnc{SvHNC=4bPwp`H1CeAHIjI<{&zGfZIE)lyD&Z;sy z#aI&<)}VBrXAV|0wW{47XNK4?-}1brB<}Oo4mhQbtya)(EM6_7;LMRa0eYhiTH*(! zXU5jtx7rAsO)5N6yUz*s6~R;Iy+phtsrx>LE3`R7uUcHmt3|;@QzbPxI(&;!HMTVDtYVDI zPAvw1*~FjfOBK~>;j`KdY2GDEeE5NL&CQ%8v=c@M(8n6t7z?vez`x}(c5I6V`!26F z8?S)Wh2xjl?`x#c*;wPa3y~j1v=}yQwVaQeb*_~hqxoHI{!R>Q34M!lA+XQqR?$HZ zS#;lJy{)iU37`qISxnk7%O z*IQqYeIZSuuV$vTuD#+{5EPX_~Ee@961Oft?Kxe28nVrEo_bT+f}`%yn%?wy_ha ztL={%lG!mM!>JlOO7SmyCaPq2@M1)ZS3gT=8mfgT$NYN&LUhY5iINn;9R+Po&y%%s zQ%E#duEP);vyr~7C3aj!q#RDg?eZ+!7WCu1zfl~9Kc<|G^w?$c*kO}m3DbKvBkP@ac({fJbq&T)_nue^EeFRjZj(wh zje0uGK0xff+8hF!!rP+&!R8DSe3TsHuJz^kAEMq`zh@@6ZhprH&UYf7qla6$u4Tl# zuayozTj9%!@vE(H;@&6S;(eL7&iYv_{#lQ4kB>fY8)uBn^=rU*u#UZ3xS(0m-G1l@ zw8#&aN~`7ae8*c=hbhV>K#kZF;##&UTp&KzEqMvu*7(_hA^CGTga!^lzwZidFXy@Pe z8QF@(H;Xp=!&>^q+cd*cHySJ~U8q(b=}Qgq2S1;y*kw{iBR>482IDMAs%c(p4}P-u zq@4QQndH(lJ2AWEo(its8Oe-$Cu~|;_Re3l23#y%^_JKpDRGzk?p30PhIvJel~)kI zqv?@^byLv|KnE%b4*1iMpBmU+-RIiMVlKdm6NCRh%taMK4dUbZZKlz8vSjv1~3;KYa`=>E|i!8ZhH`i$d`e?GX#+Gw=K5;@$>!#TkjAue&e0khw50^irL0 z*>viOeMgbESM*FIuk}`IJYAtB-{W9INP+E3)2!#eRy4f@XKpp`sLoi9$;y4Q!XE7X zgk{vNWtL&FgB2Y)R4veeR7(xE6B$O8qNxlKH=WyBu@u^b^KDDMcfRhgX^%2zJm1~- z^ZOPz4X2>$&EdNzRkDk0{Nfo5IV?@bspYQf<>1V`To0%-oZGNRRJ*spFHYsI>1{9b zTN3`nxz)3k*&~VTAWrK|s7CnY1s5>(M5c>4Gxwx6tdGsm$s18_LA{Z0yXZfbv-gcy z@b)pckpo10+Ck5Nu?@}IfNZXrxc2E7_MIi`kJSr1#8)Y;^ESLWldvoLnMC66fj{53 z=Yv<`C5V*&k{%XE4}UBq&>n44cqOhV(sF~wJjF~|P z8!oEqDD)g*9i@p4!nR?k9!IDW6+d_JUV={P7$hQG46kEYoL3velJo#{MY(sg_I8jV zUoO!1=>UxlQ{xRa>8IQy_*$A-3zTHbkf#82J-wWprsqnOGsx?;T+x_$K6g&HUTot<35>!xkc^T-|T`T>f6bu!Lrk9&OYV?{i+3cI7+K}NVD zY*lhqct^+7>##B~rd*Qn6WsvE3uyGm!R zRS>iKq*TTjbXV_mzFC1iTMa&afYP$fMkeYGDG~j#6TOOmSoDywkudtJa#9kKy3mbf z;1#Z#^a?Ri`);IOjB+i2sMhpa3B=x-6?KzK`0RB?F`_vFZqI@yDgr}fb~q^}WnMgQ zTgYjSQyC{m&H>oT3nfH;BkC5tV25cDb=ceB2x2#O9Y{oj_M#ez=y0gN)m({EyTI2i zflo@Yod0rkm8MaWx{e0jQuNA_=+Ij`x# zyGRqn{=Ly6P8Pzca*Dmc$!2~jsgJez151r6B7GfdEeM6lJC$<-rR6zO5m-O;nT6n` zi^k~l!JhgiCiw~{_av%jp->wG_W+HrdFE6m5+VeeO8YiF>RXw3~t z6d(Vh#CVbOBRQHzrf^r8!Y^7Rv#m<)iSHRQIY)&KJI>r7@^JcZpo9=1uU(uE`6@!y zmRdKrFL_F;6izGvtnBm*91Hy(TWWO53oHVq8#meFXF2liOX`$o3&ObY95GMHO`ZK( zv)U$&UVd?$Lep6&LW^waz=#J{BU7TmHL`0 zbBIJRY!Jxx0q3lWK85dU*vpp-(rOHQSN_7hG^9P23GTi3pQ9A`gB4%z-wGu>`ubgM zw_6S^q#%b%+Ro?(YfNfXB$y9;z>vA)f#sxZ&mBEU`UlR&borblVTeF9Dchi90wGTC z*veX;vdbh~aw4bLv?3NZiV+8)Z}Og13ULK>J&JQ>vPY_pLWOD(Yzp^GA-v{dC% zFsKddLyuGIjxr;X%KJ zYY%t-w(fo}{BTy%2@q^-dAt`tz;TyuG@hM2=Bvd^mbb%yYOU}=mYHv~;QYc$h6$IG z^;i>I@x(o0)abUr@)E23Cj)DNrO?4$p}Ez@FF}-5Yx%+yEa4ms@`A=QxXE`T8CrWFZ@^MVTkPB7=O!Xa` ztxg!0;KH4@p6zQ?I#XwTVhX=7Z`x7z8HAgJnqSJC-Rhh|XmHK`*|lo>+dXjDzcR(f z#Q{Pvj|2CS*?3jzqj|DGDRM5kr6<)}_TyN;8c9Y9TBsbbYB9_Op$Ezh0Q+OmW9S#4 z*^8pVQ+?#V3uiU7i1*fe2mpuYKJ@(cZ#;2a+Tv00g=ZHeBpz4}i{4Aq%ZL0PSqJ-) zOX9;^b()6EuSe>Z;usQvTEgpW2K`gd5i%9Z>Pg=dP0H z&W#@yFkhHr*u!DC<&5u58`d9v^kz!6>sxYI{J{socT*;C=AekHXEmd+2W}ep9N^Rq zn2)I0l^Ei|0=n);e@?KvQ_V+}BH}EE;5bJj{`u)4ShA~=6?_5Nj7aG|f=+N;hkTjr zjs5dA;aUgY4W-rHlWW!~lKrxJUSgZ~=*S36MN*;LL6nDh98L(O;;)E!WWE)i=Scwi z9q>^JBBx2}JlCM8xubY8&lN?oCc#m=9 z;xnqCCKbkq-L9A!ViVz(U{gXTXb(@!1ia4b&}L4%dIs=%ykb8SJt@7MMxlXQ`i^0V ziK7>o2YX866Mqa z(JuK6lS`T+taF&v%J#kG-qcxuPf9@%q&Pm4SFV!5>{=S_!uvf)66dj*Mm})9)_|{) z%bXm2?yu0a`xSKhzxgV9`qaVl1a#+|Ck|PUv?U=Phf_cU)*)d-U9RFU@XJG8a%6fU zUK^SW(FdSF_0wr6^r4{Hii1iP4~t!Qif3MQ{tZGu*5btQEQWJC-sY%N{$2GoecYO; z$`RaO@FWidX660QP9RiMDdB=Lt+ex=zF*r%c0=BSiNaxuE&c-f+4 z-u?$YJ?~H$64%zFs6<_RUe_E%pP^f=6W(Uk7=N>M{s~;;XLw4>3d~OST{xVXU6$7j z*x-}Ao$f@Rd|Rb8Ak~VV9f#}fg9Og^0-Ye$r_1qBA+>SS!>}0RRPbvdV$ZGSFnF&* zw>AKrsTy}8;S~a@oPV!kCu->j1(_J+*B{;xa)~Zn6+Y(%#3}Ya8wK;CqVhnz%14A% zDd04_sSJUFwwfjMW`Wdcd>(Wq)k;cma${;+e2-b_wbgDeol}Kt+afBem4?gG%OE1@ zf|x6xj)ysp!;q5~&NH$2`Z2veyvi_Y9<)gzz|LGu8zE&eqlaKP?xHzDCy*#NjTJ0bXLCGDzXho<~N>qVXHpz1$Q z_zCh2cik$-OVAH(QuZ;T>IWh==8BYm24j*q7{!#JB~Bb- zct~WYx{Tw?_olaKB9U@^Lfm4`>9bdW#u+NTs%17%O8!s+n3rQVy?nW0lmY+=9Cz+^ zEhd=KfMSXsLf&xykQOu$*!@gMeruL6VY` zoC`rf36eo_hMucD|G96xd%NH3?l;D}J^G$8&M2wcwQKLS*P3&FVXkL4m1L<7Fde{P zFjR8auie66_QF5+uv6@V7x&1PO?VNvm%eSUYGY#WbjQ{hqj<;uzNL-5rI``Cqp_`> znT@poFP5KI^c=gXz5RVVaXvn)e~#d_vAxIlNyniX&a(ghbuBv#h9L(1Pa@~=_!S28 zN>1+DWi{v5)BSs2v8ZocoMk?7-8*6b5sD9ANCQ4L%$^J&U#;nPcyRl+8k*@vEL*UAxV9O-}wb5fwcqc28;5UeiglZzfB*Y<0jfOg{A`{4U1( zmbt+k9r_ohmW_lRy}pXX97eD53K$CXdWsW61~1-6XfdSldg9kgJ#Mb^L;(LWj}?EA0n0bRC)LanT#gQxWy%A7?%^ zw786hY38nG;m_#P!)^P(?P@=}Q{|Id?y;G6I^O;KarfV!%pUGNcHO3IS))voGQs-t zvgP}xaJH#PT<|UMUzg>%Drq}LHS4CdMj=1FJ^m{Yk^N?BDa(BZjv)=2N z#ad>Yl$;N}Tn=*sl8O@Z)n>neqriD z+cnzMT)mIK>OQUw1;ls+F$G7gY%R8=!|BgAiBG-BR83b}cL~f*yz2XFwIouIGj6bl z;r!j}CBHvYb}(7Id13PULG4zw?D7N*T3lVcJKT75=?u*L7}u5QsKMl-e!&if&~QD+ zZ}z=r=|US?+dm#3%8D;&;QjdK+p_;^ma~CKzkP>H#Ix1kY`bf++Lrk>3-^S&Pv|$S zZI!~W43BvDE&FYX5BYH{*adub8IlaC(=mE>s=ZB?E!K1Gr>`IvHtk+%9Ly1?9qU?) z(r#^w)XlkyiZ%!H`kBJPIDvr^n=)9|l~fCT)YZ z9j3Z-7GP|JT{n%QZtxbTw7>N|_T#O0p7hl& znp%GI*>{lP=K7SuZggJ!-IsD)h2L#&7+Y2^zQzTMMSDmUOsS#SVjRHp!F0tD7Msq_ zv@pK&){W6AO)cXv6T(b>9c%UgLC>jQsC`-F`~9n_c4N&=xm6G9j<0?k^zF0$X!UAt z{h&bO^NXsAE^}q8Tmnsks-cA!S9)+U9u-x}7q?$kcs1qJvF-f!t%s}BWn|vn9NguC zuw=QiGX|?mrY|pT2cBYPT6EU8(c~G?`D)r2_loGvfH}{q(HK`AW(yG}@#Ric+4V?^ zyt0+sW!)J5a)`YNbo;zvB6&F!!)eOfjD=yIx_eAl&cQr!DW0p?Enk2o{wv|`fXi5p zk2fsFuIq(-vll-nP+X-J8+l$IV^gj*(N}0QT0qdxwtV|(Uzag&$#{}V?wC;kk4Ev% z#uPVPpG$d&(|}95?L=yX23pbebeg)@&oln9%P@n)e7(4Hi;PAelCx&GZ_X5k@s+P7 z*+MALxcbRBTr2I{@%599z3`mv*cYn~T^w27?Hk>y-Wy0 z>+SsdNa&bxxQ=zP4BP2PkB&R`oFEUQNvqvJ(I_m!z_Zx4Q$=?4`FZP#ZzeH9uA4)} zwVQ|M)FtBG=E_6{WlJ@jzCWltb2V_Td^7i5^^qHDRrM70wK947Wn3K!0*XS{0)MZh zM(AgGTC>8+tcQz^WXn!;_6Q)@H3<>kNWkx}JHo(ujzWCTw4dxeYar5a?tS1pLgD>(bCs zC-4mr1{bBStrg9dO#N!vR#lb^+w;LW7v+<3%h}?w=2am`7n=L1m;~E#+Y1eRVa30q zx%p`GZn*nMt^IiH&Ng3?lV8iUUVJpZzxGPD^Ui!!;1tJF;N}^r!Tj=hmH_VuA4#R?gC_1y6dTDnamACisX zm#ULKQwqU&?67FJfPM%}aJ+4cfF2H%R|cRXX^WWwR+47bR2VXoAU zT0WXG%o?(Xc|!aBy5bM6XR;m}6-%yuw@aC_$kxs+-kt`lTM50fa;yp-Xm z`+4;$&G^w(+7p#MYMs*e4?k4Gh21V_5h4xkZB5EFwzwKdal*gyEEfaEcNQI*?{tZ3 zWGaegYo67(V=X7H#Qd!C4Y7}bbx~Z>NJ7rS|Jh-_L|L%y+IorC_LM=8lnI#>jdksL zm~N{S1nDowKI!Sw|5U|=u$^oTG%j(R5x>PR!Z}x!QN1ekc_i&~ZLj>hv~nGboC1wd zb-SN|0?ipdUW)2WGKWQBt&WaDI>%#OscD>#h3sxj<%P6a$%HsKgba5_G6e9|%2N{b-DW6oKfC<%bBxa6eR>OR;_Jo_od>NZY7*}|5WH_m&$djt3$eP)@=#CWI6f5qe5~M^99(sXub&-aN zne$(w-~V_-`D=dP@UjP_kZ3*O+OGl2OnTT4MuC^?(Mm`VL!n(U^`~=J(f#oRz zUp$|b(^Dv?cu_!bk@&FKd?Bx`?hsbkAP~0-calcV>TLMB>=0GWz4A3nNmxG|dTfKf zZ%1C<5MJ;1*D8;p9SnF2(cu?XGG=3Xu6QhNeM^X-DV?Y~@BC93GH;DD`^rNw9Q(#?&N#lr-4lEBq1W83(wjtarzVA^ z<}O*h`+{6@h&q`-5JX3p1v3TPw)qzEH#iuTvGKJV)hQjN26MP_BCofH;$NyT)&Mh&2KR8d3 zPz6g@N}G1&sL_>}lh^w_H)j@c!F+sWOSe!ySj5#neCB*8fRJyRJhW_W-5x)mL!srh zvpyz4>87Ahipi1d@a-ex(u!m|2mu8^BKjPep(WBcZ)AS z-WO!ASIS01S~DfWCB0Wi_0HJ1>uSG~ogt&ZVc9QkQ#pmdxY`DjDKz~VqwB6=X{Vt4 zt22q<{`~qdi3XX}Q-^M?)0!Gk-YOR#c)Map8^4((o=IorKIA+j7tpQmo+J@ASG6ELQcz4Fd^S5~&ovys2*U~+*wey`hfzN|Pz#15!7 z!ghzYgPuFp&}lY)a-|8}WmS3V-Y$1qNb8|Q$zc4vRhYWQ*Z4M5o8(7AS)%k>cVFn9 z{n#$gJ<3=yY)FE^t8qbj5awt7uuXlz#%oi@6{>Q(>$G%IDzO5VJ#PogLRs$I!bNAr zS!6hpOJ&_XP;okvDW;CKZ<8T*lu1fSqGg$2aM4$e#^>7ec_<&#q&zl8*ts9Iu0e7h zU09&{C?`Uf7A#K|SF=f%SoK$^!b{J*AxY+#RqnXq@$Vm2Him?sK06&JYE?R4yS}vx z(~PEs!oX{LS*$bI|1IDK^OViwF)gsTS7C^fbj+W4wT+_sskz_XZFw>b70#1xo)Arh zH{$A?K77c&Zx>zSsq0l2W4rS0-q;Cp{~y9!^n6gRe76!KT^wg+i(k&Co120kjjYh5 zIVU4BS~%!=!7F}fks%LfZ^tLNx*c`cx@00vcY-N(~zf}0Jj=zh+r>+J8~A3qyPL1r{vOmQ#SNZ?95 zLpt{Ji}8AfIq9vSDFS;*rHrwL-(-cN@V6B0<#o5g_xW0clReo~g-oh$9=JPqRQl=r zxdV7d3i`c<1(=3=h1A@C;l7u(ZI>DVKsB?x_S8yk(1&4;zji2KGLIobpd^)k&P7!{ zvtY30;U03~&D)A)aRsaM0KWoP?1CLwXlaUMoCxFL+v{WTW0kzMN2lA8WvPdUb{QU@ z|2&&dGc4p&6avNnIjj7dW5xCS)b`b59C2fF+qwj)h_*J}FARE{hoIEhV)VlbBvY^ezmv7PT^h0tKRwoZLGdwVlnna%*XN% zF$+&Nf@Oa*UAB7?N8>GBVBu%@HnQ`sr9nf{U{VuPKjDD_o%*{Vhhe^JJ!{*mgCX79 z(t8~hWN2xBxX;(nuW#N~@H^W7gd`!A268EP*GH#08SY0(3vVK;*-d0~A{aQ{WL#nC zJCGP|{pI8EAS3L5P zL)WX4gnXo9oJm)0Qt(vCzTm7QW|jz5c3B6uDdrYiKgcO)&{cUxcUcjq?unJsCOb9U zflz3jS-|geoaTPkZ*jCn*Q7goE%IVp;vIX^=^`KtC6GH2elvcj`Xdwr?6vCQSH znARf{5FDTS=bFq!hB5Bi#sCLD|4NGE9Myr7IVyV@c+^Xqu`bui zjOLkS4e35zm#CxQGm@yJU@>aM&z~ht2e>NeYwk4Y;nZvRn5vfRhd=gP(iI+S=JyS( zjy-pZZ{QkPnZrC(x{gKAg+?Derfr%iL2`xWdW~f!Dx%8u$W9~V-)rBYn5`#=`$b4= zgfhD8(xx9raE7C-b=Ia0meZtkU|t_p=`J2g+2(3{YNy!Vuhc%FqIp%jdGyX$PjEvK ze5m<2o1z$Z#E-(uXvm*|ncw6|klf|y7#gwKHg1#e!TY7*p%k&S)=Vuo3bd%*iwXM5 z8GYokg@gv#JsR&?0|{&QouN&-nX8Sj-CHk{k8X-BHVV{@yU5U)+G_@nF~nyURTK?U z>`SKd_iuBaUTc)5AMnXj@yq=@A6@5mc<{wqHkC|3%QXe_qr9w;Mzb+Zj z;#Jz*=2N@qzsh5>mp4oGg468l!HXEv2V9s-0S|y2lVxl)-L)$E=@SOIU&z&-unhHIjJ3p#nMKg zrP@aBC5r*m=4$-mn~BBBwm%^`@x)ahwGTZ;0VX!|tD8W*1=}8P28c{!EmCz*Kn=>L zLY0Bp+>f(a3P3R=Z-JpTGmW#W(6W`vCnR?W%_Ssa_N1o~SQp+1gTwJ>q`5{(aRJSsf zDb7q?xc{TF{>Y7UZ6n+8rL8c;GHiI^MS{wL=OryBh2AWG;Rq2dwgftcD@1bTcIe>p z&|=$n19wgY^suh6B!*X15N&zQeluY5rOU6V(}$#Xq4rNf086~c12=q2nywSd#5nSd zkOx&pX@rcI{#+5#$F%-(Vhfs#m=_AZtwt{G)SfZ@%sejA^ki0n(e&GlC2fGu3k(K| zK{MS<*Pg1YS>%*VX00#B0+Y0O?U)zvD|SNKYWtPWs3;x_)0t z?~eikM7EeZ#_EHXgJq%7yC7$!4E*bJf&k~ov-i0)$kWIOYAB zH@yW(H|ztm*G58>f3@hirw@IkYY{6+lqn=+91Z+%vfLMEH0ZI~w0b-&cD|f`?jHU! z_NOBLtV(7f%$gAUK4)H)u0p8pM^~~kg}DYIGI|5#N7i(!XN(SZn8qXx@#+s7olpFw zUzZ^q#8o}p|1souvDq0ZT38CY);2?)!vf6e58J3~pZukw^6gcqvXoe%dYgya&!_aw z>V$c5Sh-WoGnWFl>Giru5>A=ZpEWGg1csVH#ojphInN_J1uqGvHvGUP;JWW5y_^Rm z+Bin9W0~)oM~!K%N1A~lX~6!A(MM=+U+Xfk?|e4Y+eq!Ew7Bxets^6kf90tH}Zy6;%jAJegX5iCA= zCQlUe`{SSA=yM(p>S#^8x_Gb*RHXr-@EgDmX(kgKfL45zcL;qHsBFcH0j}w|oe(?v z8^70t1ZhG4ri~O7^HzpPZJ4?Owo={Q+3o-mGbUWu_A4Iozfh$X4P>KlV1{*LC^303 zfnV^(Jg8;tFpkg*@4k%-DtSYTJ`UuY;R=&KRX}p@^#hr>QKy0~jT`Tq#VwcwO>epa zan^yYV+oCewFN8%Z7Sej_MFm%`sC6)c(5Wa{Xm*Ui91e5B5oo$9?3Nr<3&KqzjV!i zR6ZP@!S*U(dR{qiFk8O8T(Z5=*KviC=M}I81%q^$m5Gki1vZ0HK=E3!Um<*Ufb3Swn!|l`4PXsWOWh~_)K!PXgGKY;#tuB_1{0M zI)LSiMzq(~?tt4|>^(hnsQ0_^$X;(qqJk|;7l2PqN1{Ya>t=`Y#ni$9S8nAE8sS0r zMSB381UI0-F&M+w=CmX~(2%V3I}I+i-{4CHaK|SJpfB@Q<_P=Sdo6Kh5SWz_!&9b7 zd|smT?92bSOR#>$Ki)ozLX#7i`mfplnILD}5>@-ZA+0gh&%l33A@ zpVXE)ei9De3ZAW~*j~{o*f^9BSF%;%c&0biH2&{1=waS{k z2?6r$ssD6-+d1kttkbm9v!>OlHpbDky$m3xB^~^}S!b`*gS0V=Ce6Eij|! zJUz?@L$mYB#Wm{`rPfEx;4+jUm%9l4&Qar<_-@yth}6c(@5Oe}fV3Bi0teyluCFrE zB|R_z;=q`?m$-N*QbN;EHGpKe33yhPwjM(d;AX>@3M;mGoVY{AnA72HcX437Iv^f1 zmBH%sh=3;6`8-l4)bp?&P3CV*Uo=;f^})LAz~;GjXXi~$LASQp zVJQf?9Y}>eU1FAFkJ2S9gGcFW0aoi65wF$3JgwUjh;*h{K1zs ze}JZ1+F?}gbnGi&sNz`;cl&$uW$h-B^3_J4>ppc&dVnKD;^m$iAKs7C@3yp%cH9BX ztuEDG-1d-1#Y9?+tY&nZ?>t7vk1{6Pefg+&cjQ6X5n^6>yTVHU(e2Y!*@l`UPf}aMr zfty;~xVGF6Df5+QUWV%%s_#h+)+#d$pSs~*^{95Q*pmzZ+mogD%0har0On=z;hm@B zo*po~D4Co^E4Wxb@37vpWTQN14+}!7vz4ZfRRIgH@KKUEKp&cLj<&s`5=*UDaa*p} zQjKVKo@O|o<1BLl?ou{`E0F&;@XO6%K{Qv6{WKWw$;ahl^b8;FwWTwA;{VTm`_GS^ z3(t~Udg{rPlMuC{x%>Tjb3wsWFJO9$rQwum?YOBDOF#UzX3axU z&)|ZvzJw#(a_+EW(AnCzNJnX(qw80X(axjDPhz@n-~F*0VU?;+6jJ2WERk26lDK%A zWkr5u1Rtacu9;$KDaah_dquI2Ii^j7bf7g|IUZY5;anc%U=fdi#$F@)XTjKQ|C_}b zcLQ@99`q4)m_2(*o`7L_3Z5v9q5m#z|T9_(DX7{I=+EJvDz((fq!~&Ooc* zSptH`;zE;R@8RJMp|m?fcM_cg({lN^YW<&Gxnj>j-PZ4_Mw?zu#!#V&o6>mWRB95m zS0^aP$?q*+7As7uZWEvZ9M09&F~@v!iE-7#Exf4hwow~emnkq(U0ojzxhTA$FD-4N za!OokFxwRszEMwzBhtTup@E0f(eJC2zkGh=I7e+o zK$H$n_aj3H@1wUFL#{s_k}r@kV1Mc0jG}%8PHxod=VhD^sIyeEuiGw*)+xkl%^k)u z)=n|s8K~G<&XYJH^9I*;${9j#)9@+bW%1xltX{=>#C9ujA<9_l~K%W81D<2sG@v<$;5yOoH-0V-y*3JY5%P zk+3Vca2IQ$pW`D?-hJV%804bGbpTm8t}cV+9*br(cj=#IXnb7hwK&}aXR@0sT~u;~ zWU6Y>{(Xvp)}*?~p9-N=v1^_XPCB^rN?&S+BIPN**7D16R#rYDBjKmSrk}~3fRvT~ zB5TaZv2Wb^LSoG|CY7hh)IF%A3k<5BXGIF0G)d&)+@h%Rp=y}A$jjNspoiVpk4l_9 zsTtSZ0g=jRb9PU?kY$xakj`ArpMW)K&!G!VpN4D^G{DIzd)9;w2+bhbXdV%LU+3KI zjpZ9WO|DZp5qtWefIkt!_N_ zl^NsyUuh6Ww0}chx^TEjxbGa%)r68{m={fDDTFDpK!5-Ds!C$1C0k(E3B8vP#D$0T zc_9biE}scriV2{tTLk)9ZNHoda-?E4NOD^Mo*_;NQWgI50q&9cVHB#j+{?j@pj*c9 zaB45T?(@^aW@k?p1-4u+9cw!-+Pw*Ne-nt9&&g_xMyEuL7PVtNKL{H$MIi=_=GL4 z9`|E(@4JOzcu!h)5TDF#l1zs=Uqa>$M^uv02rRT0Mbh+0EE;jSeh?9!fbMrXiC=7bXilxhCSwHP3d zrplOFvnEIn!m(D@>)3(;GsL`v{^Rf&%Lk=_o1aP zRlCmJ&9~z}zn)}qe~YcqZ2T#`{&jLT_xswmUr%))%Mu}5r_kCAFoE15$)#3N_t69g zRp9I3IvaO3=i-|s78_Z{a{wAo0Ewdj$k)adYCOAy!cY5{1K(~{>~7E5Luvg>z7!TE zKe-RWJV>tXK=|Qt)$KIwWAa1ICk&tXA&(Fc*-sI&e~uP39*ER5#F0|5i1J(E+hmAV z$N^r`G8HjcB8%dUP)2s~FG5OEf<>me6EIc;5G3%i=0R-{5_Y&oJQhOLo!sX-4qfWm z5wWh5nZ?o85OG^R83Jg|CRjnb6SIr~T1}=X9u9O6WVFq@H_gxy?-4SUL9rk;F;UaQ%B4j{*-2Ud|xA*sQ zwV!-Ypf`joF~uRAK=Zlc@SSg{zC{L8VlkDg14vWm!0u-l$k1Skg)i<>T8qH)WG(or6n*yZD;Ex z?tXc04@TE9IhiE+2f$mn=pmPx34yZr2yt1p)-HkFl?Mu94!s@ZA?QG)eRblzL7#)f4z~BVui&Gnh-K4jRA=@rAxXlF_^7W z{Ky5UJXpXREACLA4je@vLw|1ZGBVh%QCTh!Zk=GQf*Al+ zuB4_I>8|YG6T@h_6!4u;3zTh*0%MH!uu_{k!K-Qy6DF0>Br>sN8f8UQ!_}`gP4;izFt4l-2Dh%P<6mT<(8+ZcdO9eEY{8}d`~2}7rEe% z8^yhR&4Z8yTs%%*LuOm1*aPuNzS?I6KK{w5NE7l=fl_$O8i$d?ny zuQoOYp#c$@if`+zt-vuFp)5_(YTKK#|HUcBJ^!&8U4W7wCMKSycIWX2IpU)EU*o?BjU+cB0*j|_!awl`3!2rz^ zIsr8kTTb0YQW?kxP`Nqc-qO4JI1 zhS-+?^I{b-ND_vW zj!gquXwrk4RYFv9;R{ET)GX+I(T^)Ea*H{6;F(W=lqcU`;=r6(G7laiWu!5nz|;}H zvy?vRfEMHicxA&K`yB|n9t>UnHN)26;4lN)-VEtK>R#YI{X!@M7QzgP0)wEGGnCf1 z+Ge1F$y=B+YuCK0;yz-8_;U;@JSSib7)L#>8KUZJbP*9Kfc|L}1y#f$K-irjWNYT> zs%RDg#Owf#`W5%o$=BHk*PjTU{i3)Ilicn;!_Ps05HUz@gkVR(5F+*~(tQWsVm`4& ziE_L++^L$Wr)n8|gS4PHTATpPrdsyKxjDx2og6j4?5fb??n{41WmM>y@XhsqTs z-m?4r>ljxl?APmolnit)TYHK5#G6KqE9}_1za>1C32p zM5x#)__i4k<(bGsgxHXb^~`WS13pr~X}b49&xp!^Y-6d;cTH7)^HQU4hXodVA{p)~ zNQ-=@*gi=S$mj}`buSc6%t3Dz0)Nt{#X9!d1EvjJ@t~i&UAm77PNST)Yg!QacQuEY zN)tZ5+AkJL3JgbZYs)izA_&WK=v_oPcE9Xx<1qECh&*I8UV?)S;vN9G$KZioF+flW zrFw2s3uJpJe#d9Rh_qEY?EpGpHOvWY-epRCT@p zhd{1MC~YLbQhui^xB4d(weqK5-~W8ZfYvngZ!Xd#LeEhS2>{wqLD`Pekq{vFM~T~y zKSzX}gy+DxC_?>~l~BXpkG3gaw@o!5^Mj()lDa*Kc{WI7AX406Qmsr;ZB}}(L0}90 zWdr;D`T+MA-(j`BAm%77+B}Q%AAQ-`-b@EG{$B?40%Uc=@er)m z#hRh?HOnpjRsZnLlb1-Gm~CgiJPzVhtPL%prsok7o5bM}6CrH&evX8|w}K|(h*yxM z5Kk=lJJBnsZAy22hns%=IG3Uz$zqmPSps{dU?4*Qnsp|iHf&feKzaf)i2AyzJS9rT z{5B9`m?s>eg!=O7Cony)tBNTO+5k0Q%GyRJ!pnC222X(+^%zPS)DSZ5t7&A_@-Cph zAyCm`KR{>~gcV#G2W+2Hu0$YjbA+y~-0T%yZq`i%^l1aP)`mmYi>s$A>yA6=mcWCT z-E?>uo~DXQ1c~W?vIDI(3nZ0X*YaDLozP6hB{Xx>YEw;_cn0kP>f_=mD>J(`tB$P+ zmzVXACj>Ds?e->`MkINd?)3=Eob>4&7HzIf^fYxXZ6G0d$8Xj{8vTGPQmOor@+B^YeflAW|?E@t2 zaPk~|wO;$FZXTk~c}z#s5X3cyDUBQj>pwsK>NDzEg6Z}Ofeq+!N#N$^zl_9MUr&{F z!pqMU>?jK)2pXkd*h#uD08Db@frAQ%vEMH#thL?pJx+`}H^uX52lA8k^t}y_8N#pr zj|a};J;@KCKVIK^>R&;tB$_yojhG?Kuyo8pOgt2!_P|_?{U%D}^>3Q?dZXH;7ZQIv zXrYm)LO><}_%Bcs^mB>nw^i-HibR4cvYW!ISJLU`fIc)VHCHA10rW{Y^1tALR!V`A zDEnW(r5CdwLw=Cq`3{&$)2;hUQX$g6|NBGQYjvG6EMS4j)XdXI1&AVq1BWtAa&%r! zKTu|+epl|9NDAPUN2x^{45?nL+t9*R_H_zc5VCcRd%#gMxGf6Grj*`ko3T?*#PgNO z_+8Os?JZ=hh_$~lxVH$>DY3&3=DY{%F7bAIU$$#vfCMPdbebEO08FObp(xTXTHI1p zp`uQ%1eTrQk$iATxD8TgIIkxtP1BMD#?9+(gD_mlu$YId3!XrhNv{G#p|~Tomj(>V znILl`kREl=}BCQ1e^t2w- z^BlS#fH6%$g={7y(_{y>{*O*J3njpNae+UH2P)cCr#!H2RVWj__POj&{1af$+_D(- z+;VKUvLpEcB~Q$*5B2eADWLuk&z24u>;%#!iHQQimom+K>5bVh98+ZnT1^xI?{>Oxrp5jzF+TX>{@@^2vK_wEGPE~J2y zvZ?FOrbNT+z==^f>jBq;yfZC)U!WA$n_ETm2U8ohVrBem5Y0I%Rte{nDb#C-2Pl~i zF4Po|%*AxRSgLJ#86OZkRHrzAa z1g1KUEicI6sQ_2rv>78`%o5yiI@%ooxEN#7oD=BqS1n|e8018mTnAW~O0s9)NJSdz z8_fad#{{VRuP<33LLD)K$P^*X0ApH1VweMVm{Qh5#K&r-6cD?Yf%r)(op&PaJAf(&WuWkHPJ&qy z5uC{lHBU|e?K@9qa1kAN<}oNMKyVvt14`&XhN;tg&6LQz(Px*q4`2Y%{6^hdX^4zD z4%oyDJx@^Wj3qVDP8S&Y9A1(i&`S{yRcJVJNFy@J^$mI?82o;RD6$doq${9S*}scF z223;1+GfD^E<2l|qfAvNk6@6^L=tLdL}DyEI}_g*fcFtnD?V^PR8d$C979%;8Ylo@ zdY_aikiG*gKMmHEKtJe{2o=dgTyu^FYD;x zpS?-;v_loyJ@St95;~u$G0Bh72m}2PUvy92!s&2NU>Vc-x4%QZ(Qz1Bv=apUC;x_5 z&l~kFg{-_I`nJG$CrWwW&Y$rScbNa5W9rXS{zisFG^hxh5>0}@R*3L`TZ`6x8=WWN zoc74MN}?MLmRSdKpa5GPw*P}Z`t+tSkK>W&2%Vvx&Iz!3WUXeQ4=!bQ8SSYIK7Jt$ z85I#h^>=7BEzq{aFhI}nD^DoxT=R%w7_ZGEjB$qswhq)}48bSdHIKNW@Y?A401;un zPy%ps>^G`PIdr!e24Qon_)?xg^VMmoSY#CQBbH4W|kCUiLo zy6~i|Awwzd^9D{`9*N@Gu%J8MHSk{eM4&>))Th9hLJ6tz&zROIk^Y$_?FTA-5YX=g zJE*%<$u1(>m-SDfZ=HEsqJe?!FH+5bnrccMymf;oY`4S-HlF|lj;Qi3yDoHG!qLfN zm{~B=nt@8D1Xixgam!A?)-`)Q*n!E}LZ~B}MO*7hE0I{Bxr@ZL2{`@+f@m9{X)O+- zHLnCKmSvv|-4CeH)u2}sPVIdw;|2!tR;g&bK_%m#0jCd2HCNfC%D( z&{n~$Aq6H(L;EaBy^6f=znlyhF(y9O5^Hp|H#r>usDM!fd2j1SjiPQJLvNc(ll(y9 zTQ)}aRQN=9P9K@pY4rY?WSF_xZ{EqrO{^O1ZfJ%!yat}#B?{WetsdnvqeEWF)wxPQ zOB`!-E`*n*5U(u`;;H~BBL`hPZsoJ!)|LY$#F%x{=A|5`oG***bx93k9+yvqCxdh- zYp}85p(iN~>{VFA*mLL_WU4J#Rw6^R{=Y8K@HX(q9WV~hiQo)Yx5P5ofeoY(^%|zB zF9xEdQnHW=F}Sso?x{RXGDsuK@&`J2zf}11&|isS(53(#Ks806*v}d9Ozx6V6jBoZ zxg|Z)O0WxG^mT)1s~6f(?En$i617;J{%>ZcJJMqjF^U*w!Q#d@@0+7hXUq~X!^*7= zDCm2i&xDi!lMC4yAtl^Si#+}(N}eM50fcfmz_;bAb>~5njANf7WJ84zjl; z>+*(2fqQ`Si#TyPFm|27=z_A?fy@Y}_n6fq+X3PSOV6Pv(dtiH35kmjxh|25aeWI~ zltX=hblM$q*F8q4IghyUe}*0>$+~pNYwcE0-qOszm6?L38$>kGCOEM>{ygmGI5U4?eIpT8iZdWlU0^|Ru8~eP$NwRGL zt@GdI9y}PE^Bc>9sY?HkD6|Zmpc1#z&q*VU-SZT4{}VKgI_!P|8js9=UTk z5k1cYMVK5#0Px3VE1DE{i0K92j^XDdLHl2bgIq8<{~KLws|h;Y={V<;i2a{{{Z6h! zNHx$#s{u_qXq}e``3_sFaIT3H3lhfvCT`l&|M=IUYU1U?5aWu*QH)FiJqbHH`OVLm z-~QaW8Ff{}ug0iG$X!D3d+)#c2a})Dh~TZ%)N|-!;ITPEOhKkk-(y1L8;`Iw{Ox-n zdyhH9CMAJ{y9ro_a(l?B6@XjxL`dPE`6k^=x&@Mb3ecS59#DKC(K5<#iWs?!J5HV^ zfw-en0y?~{&=24dU_PJ`@z%(e-sgP>nR<~5gZzzCWnAb2Tg?l4F#PuLbRY<%4`dU^ zFGnMQ3WGevad0t$U23Xb|0|u?fN1pohKUObq~{YQch-g{TD*4bq}UP% zkwz#EopGo+%pS7nuL@}77t?e8`4rh&mi1v*DJb~w^)EL?&1{YXGS=6v@Z)Qwuz ziw>+@>j3IY1&mPA3lp$xROX1^_kJLciM+J|aeXlW3w3#LpC*ymntTHdCg9a`C_xsN z0GeSk;u?y=KhyG#SpelP6>Ua^Hiw7u+zczecn|`GaP27p@&goVj6+2$2UHNAWSOpaAvJDjT-|)Nr@!c( z86a82!0y1F2kF3>j{LR(J5k;$7jaCTJJLuiuO5;VM1B~ttpMu^w330CM%P=LEli7H zg@O*P#^!#{r(Wn?Je`%%k>{w-eH&=lMxq3757R@anoth!vBkHKzI&aTSCqe4Dpfu=iN3A z-N^rE_q_YBi~ob!{y(=_;s3%1_oj)=8)YML=w@jL4{O9R-TR+9^Ko^Zuvdfq=l!QU z-H4T+Vd+F87k~yL#!0w#zas(_BT#rm_tW^ z;aBwYE^KZXHH{#*x3IziA^`n&Rb)XF7E_lBUM}cEgx%Z15O)9_X)*P8EAbs&KzB89 z>A1Du1ilXa@i+G+;N1Sr!#P1|UxXzaq7Q~Wat{z)Vz9p?7pCsY|BJ9XyiG(x-X(vm z*8kp-V`XyWIiROgGe8w~?E+>Tc6UXP5E5Uu>pB!u(P%fpk~bW1+n7KLD^3imemelZ z42}nu7rft~2D1xk9wYJNi}0~7pnTQTTbcjr0yI8&34QiYM*pv1^KbGK=bw7{KmX>U z@BsfSOZ=zb6}DFw10FV!G{jY)zBklsK$KR}U)#!~F3vJdV$_3nT{@y}YlhfIC|vA` zdJEb*l&BM+Zb@X}gbn(@+fAf9hSfmuuz{Um?7Ow@rGiO9f@mfNdP!Mi(A^Hzt{smk zc_R_EY(Mfsw%bkU@5)$!c{CHA;246f>$?XtC;#=aOr63c8We1%3@@W2|}=G-o5~6`1!dtFEl1F5DPI zN{QA?GY>(k9fl$l_%eQdwq!*zZ(77FF2o!=VHkyvH_w9Oa1+pyRi6&K-6j*BA2XXg z%bByv#UGC6!|&^j?7qQ#H9%y2X<2_y?aP&?;heT9Ah4L0O35H~0D76-#bI*_&Im^E zLQA3cU|AP0uuZZ0T96NGs^m}FQJNZfcUKETh5hhBx~{RgVkEQ*6LCh{$oU$Fns+D; z-`oeizOuz` z_to}Ff%VP|_#qfYRAz&GmU}Bxt#(H556F~`$@&vuKgj&eHT>TRxcrwvCK5$xK0Z&4 zOWy;-&_d=7KSD;6I0%M?SfT?B=pDB%5<^mAdZn!fgG)5pdVm{)HuNGIC<-jby2xd8 z@XSySIEX`j&EDOUjIKlGaR0ph*FCkq6uW4T1fl^N(W~IHs->g~e92=w^YF9s7<^VrYPv#7R)s1|;JzH!f?D#9-n{wX(!{Uo>=?_Q3+{?i6;L>@=66Op1S}eEye$N3ni$iLR zEBJI}il`IBYId4YOi`YLhGF*&Bqaei#{(pvQ6{*Mhz?FbTB=g57~%z-2Lj7n8{FK) z9Y~y7m0`DNJZzDp8RAJ2mUwpnn6U*F0p(=oeLl`jEu6#kw^EQ z;3KUNWoIO(9l~c0>_;psu@eJ!0BA=BH)WFOso0x-A5uX*0B(7~Y?`)aN5!(EdgTU7 zYbk*3Tgc=J4{Wq%FO1B&8lMH|rijmlW;Q%*+M0qK;gYZ#))$bPcpNP9Vwh6k;%r9; zfxeGJDn+p5O3$QMAdv7|;<5pnj z=U=>KsQI=RX4c|9d#r%IyCM!J?%0fr8sE`bL##ye2F#^cqB99tps`VhRFVVOJ`?>x1h6LY ztOMbl-2gl9cx5Y{Q&Tu_gq+>K&d(|#L)a6xPg!)N!8;tEDkE*xxV9R8=FS`tqs#9o z=8dk3W^Eu@lGwRoyE0BK0M`2v*wWY}s`P3(XcQ?(OT+S9X|z>;U8|CVoco$oG9ocB znE6UAMP1+F|;Mtn>r5pVicEEQUqEhc7;S3ijz_Z8Oz_*qbc6 zXIPUnmI5y0A>{o+eAC!|Kj>zsu{rojP9cP8VLFtu zc1oeFS(`bem^76M*^0!>Fm0r?h>{Sd82eIYMj@rLWXUo}*+RC6@Ad5c`6IqRx!um` z+&X4nujli+9@pb>zZCj0LnzI(-rFia7RxJ{4fXR6`+ZzNClSxFA0KX&=k>7H9}&C& zZ@uqV@d(for?zp{_rmC^Eo`GBUMIJ}Ol*aqUePYj$nXL#n(1kDLHCiV8N*BoMQgU2 zU_kJ0J26gTNLV#d_nycQ!X~))$;mfxY=UtSUJ9ak#KMG$)}5TQ>GgOqizQGQkt5i= z=uf=joZ60DM|Wjm>Hgu6VDLCtmxyUG<~`*|Hs?D@0d8s?@$IsA;7FN197K|7LQ-07 zb^CHWs~@F)D>qV4SJ^u*!K3l5PiPJEsIssXQSBPxg=(k%hu=Hztv2aM_9Lo&rk$?j z%v#h=geep(!Ap0ilRo?rfslskt3+0)i(AB`*<*Wy0t!<21% zNayN~ZG0+pA}5hduah(GBHs#wSNaf=+yz~T=9f;qwNccXAH9c5QL4z_OPwaic%GmE~y&E@Gxdtdj7%xg~2o+KXEly{%iAL zzQGi17VBMn^fg?B)2Lk?>p<;@Iqsyjf`wK`Vmc}AKx)YZobbBJF#xzjw(1K4iD~*B z$!NyW!xEQ?K}h>O`z5*;qt0si^6Rq;_{bkrxV^LotKcVf6Lqp_Es&S$%{N-6GQp<- z@AOKhAMMobt^8H#*ZKkG^yaChytA(S?kDME)LG3kNpm+1Ow43KZ3IAO23@lV0ja}h zFCdlr{y5kG^wzt}TNfb`&yVJD@o2BB$kF}K@ZMAe6y_>Z&r8J zcRq3)yVZ6&*RVCuVkXexBYV?c95IkCwbA%XbCpH81n-KK-31i>jTW<)QIIjRbdwyj zcfdx%WTjNzCIwM~MT))~%b zCWi};N?^GzksQk5xaHYd*dh96_Wk9$5 zxy@v{;>N{;MINCUi-CoFOs}Jbt1M@{%{e|Cd=yTVFg959a#bDcFrClEn6jnNl7BB& zfZ6;Tyf}ZEE2aJ(3YMqlqoqy#m;y@jAK?onKj>9Yq+Z27)Zz9~m8<42(d-KBmVzy> z*b)h?IX@9p_){$L7n|WW`pf#?%8cO;zbW-J!y&K=ZG2RpC-O7h--9lK8byV*fnaSysUntdW?u*cqqq$W2ZNLt`ehTc(PMJLThwZ31$vasjBjRI z{OtC;Qpf_FO_e4ta|F$rS<-WgV4RO(-apc%(UIck-0y867$qUgzxC+Yz{YhtOQ$$h z&0u=Jp*b4&;NRx$s$yc`THlIHwhf&dn~RKQ-B5Vb{f={7_>*9I`|K9(&eI2&;V+>S z8u|o-@UX{Z8(tOFEFP~cY)qRK?NEdFJahP?{N@1=ir{im6Qtv|P!HVg*K%T(;xejW4|z5FDKuV@Q%H~W2ygYp@?`?a`*FM>={$- z=nGI4b&cuyGHI&(>AUfLF;Zu*ITxpQAC*x zqHq-+Dp)$?Z(!oS(MB0G;*3QCFi8D)nnI|zzp#d@&)KD^{X78^w4J+ zA;x<}I2l1&qF2G7m2ek2v+TK|zF7GoI|DJp&$6bUXZ@%L`t3>1mh#e;a%6Ayj(hEy z+IsoBw~BdNEw&ZhmMPpKleP_pG)vl|@`kz4`7j(3L9eS{t$EK56@62KKx`%Ypmylj z=$Dsg!9(JR=27+VExVnBe6SBREBRq?Z?rg1LZi`i8`b4#d4XC_8+^WQ-+?bu6$T!P zXSz_LFZ_nh7Mn~dlk;%2+j zcSO2xJwivV^nlS$8@O4@>+_`z47mdrmZ6S%->HN&fdi{#qT^V68=?ASa@qrf=09-=|^-JX7c(Ad)rzm7I| z*2qC$pcc;U&;ZMa(!vYyBht@AGJ7Q3^nv7{?v5f--x|=jb!t&^(WGeXD`++j3-F<5 zuP6gSd%=*o^{B2gwNn%bdASd^-%7w-Gn{}NWs|8Gb6*9#>W+~g`U9crd3Fyz>Lf@P zNQ$k6X+%wF=+&*7)L`e9n^!&<6a8!X0#DwL1bQ%TE74%Pl(54<1cyVt0er*Mm{Em7Ma+aV}<-x!s1XG($DYEoYbGSHc#+Nn-r*(q;G_@?Jm-H8lGaZkEo&qR@L6Z)d(*MpN9! zK?wp12t(|H=6*|%VXrq{Kw8k(GpB>SYUbs7N;4_Pkl%6Xj|mtXrY_abFB?hUM(M3x zjgdfQ*;x4ypSFSTc-V?RhqYfJF0Qx0uNrYoyO!B3`$sO-aE)_X`7j&(z66 zSLW^@=n0&WA4@^WB2~l&Klw+$>fx342tRD@&OyZASlmWdW+t3LH+T451nh-uX3jtZ zFU_Y}3AR%h6dxil06pQ;aJW^J+;Oh}VM}k9_P;^Tbnkxf9xC-2l2?$4QLiR0on}^O zL{5dFb9k<&vY(D%RH5r+t@JC;B^S1RMmr`^{geHIoT-+z0@-3`4eiw3_IXl?s;5IE z`y|;xM>{kvv8igK!7=^SqR0E~4%*=;}(8fLIpGZJj_( z3HZeROoaUbr}f9-P^+NBxu2jXy=OE`Tg=*B|25yMdh+KQz6^f{;{kHPB$$9|MTECP z8g~J(Vih;?^*gSka>}b1ZkNRK%7~YhCvq3iT+?XD#D&wwILmJ#l52QG)r?$_1_N7l56=K1 zd)dCZ_SvIhOL50Ai2+L$mpK0T;e9hGjM%&=hUBQK9+0?Lp{AWy2fzRcr(^3hgkrY(eov^LH{}+@o$OzW z`w(RFTY}fhR$0W2TXxyTd?WAS3ew5v2?{k>EmJNlj{KXTK*dh0ajTKnOx`2kB> z=_~UlV&LJ=`^V5b_d@Q`=U8=J?ivA&TtY=vEb&~cwuT^L`c30Q(Pkv#ET{1{&dG^h zDaV`QoQ#_+HfwB@CoJ7dNhtQqAiQRz*a49cyLb-Hp!Whpb(qCIltX)36n&;{-ks@Q zco6z6-v?xqjs!x*J(oIY6n6J$jBjt%rvqe*lM+?lAR(>GYe3glBZ^FJgTKpC2oiO6XOSmy)SLa>CeS?8O4V zIawF)OFyPb`1oVz#&muzvhlB)eG!B-M2(KzVkkwuP_B4$mc7T;rbCMO2l-J$jMLf! zskjwnrm3hrgEqOG?_s`^m{a79F(TJP00`DM^sprHJN2ML7a-mD(GRAu; ze&4VWyu20aoo8k{V;^}*EHe&_hSfDvaQ+u;`X#oH9tK~nde@1#gzp4_;J-}w_T&0` zjM>dXbqv-fN*66*D5Y|{#-I2oeKWRS^hTjD34SX#shbz1`MjAgD?z9q+N(PLG}tDs ziaK#(G^Q@blvyd>=NYy_+c$awt`Zb+cnq3=Zs2=%BANMIJo#(Ud&Rct9&5yNa&aeFMTHfqX0Zs3`Bc@xLMfblV4*&L+5#J=xH<7qilJ5&bG2ycyG z6KaioAJdeR`s-e{Z%J$bp~p2*xl9eCOZh--?Jn7qSYa*V(l{V*nvKXadydI&|IU%f zZZAkwX18Bo0W0KD=ys<9UXvf2M2OOq;mOBhR|rHS-=V|_Ux(MyGFz{7e9*sK=x0iJ zYGRnC{vS=TyTf}c6w}|y-N=(^USg);pmmwu&m;;MF0a51WtANp&(bwaCt6qHwd@Ox zVZuQCD#B2o1?}wYPK<6IBVoKfg>fnkZy<4s!Ln9yZHyJRDY-voSGZ+ejyJ)#9#Sbf z$mAa)uW`8=?x3A$3)q6O66pg>f|82K(41%ot1Z#8Q|4*W{P5(T-18G_JYb2zcJ7+I z&mcV#S)Eb0ukaB0jH>qKQyj6dfR)cIOY)u$SXN@vQ9L{;F0@|Br2d;nW8oJEEFWFI zNSH6x*v0kNj!;n0J^}jYju7~Ui=F$Y{LItW1cJrrZBiJypzoTw*xTEvL0yQbMrp-n z`75&I0(c(&f-{wM;{<5-mM)F6@+6ype0R72TUZ&#-NZ@u|6_$3>Wkfz?X;LoVeUTa zR>QZnXJsW>xKNGxQYq=&q=2K2;RAICn%y|6hpqDo8Agf6>aI`Ye}_Y~+~GFk{>c2R zHIucU>;(BIYh_HYO=d`+GHUqdDBXBVvBJKOR? zRUmDds`w>cF-~ALbE|!4hDz0lWzkPK$nND_w5W1SX!IT<{`9olbn!G={|cCos!C6nNEdtUPv9+Lle?saOfd zJz|aNWXEP4DezM-+F+oRK%npSyH2oM!eR}DI`r8!ZNQRf*nUD*V5p7yWvt&2kyr=e z0>}}&p}5_LJBaJ_ArcHaQ&t6{zT;?<9CcqlQKJd$8Q=I=C}98NPb>8mg;+pJ^NHun z<}I$E8OCG2d?L3#MOWST$L`^@Xl>+ldk}L1`SgqLKsTs^QYAAKTvUX61YAWs|M_lP zyBG<-=Cn;eZZD0yQX+X3d6s_Jo1Jiv;DSFKwwnzW{tYIe+^J9*u-ub ze%xPST$=XRtS$j(>eQ@7STXU%L-C%86a!s$KuWWxMWRsvg zH|+5HAX3}|k%#P?LY#rioVm*RraD#~0&d=pUle7oNG|0xI6`$o|Gp_tM6kWK!3p9OFpjr)y_YD(+(EG|F17He7V*vDkbV$MTHJ zID(tCaOjgsiYEW34Fc*P%bm6kYYiRf({jZ$>fMeR5h(di?Aa#UO-`3TL$1COyrku& zpI9TX)w)?xql{l}%+g_q?XnZWr}942w9AK`=H{wbb~}HzY`cS}DviBD6wWcBqhS*n zW>05_Ddv_kWQBdG`8ylrMVG^REK`|g8U6UO-`7KsK@yUy3bQo)R%i;0C9HJ8T%Q@O1X4N&5k&%#wK?y-L$7NZG zCP_1n8cW-Ys!LX8q>sgawa10oJN)Eavm(0&T-?zr#5NFWp4KiCKm%I!Uzt===5SEU znOvgx>*=u!%lLH-T3Z;(v07e%T6&CxZz8`5zVEc`^&>u0m0zWVY?tQw%b5D%3`plM z#xu{UWzkk+%mV(i4ha=aQ8xa=`6H7)awgz>9g&;1c%?r=y{i-6m~(CYT%k%QuGm)R z(SSPc$pE{$`FotXon;(0_grJ)29GEjMYV(BCzXDmYPh+q%Az>g?tDBgVTwR`cZ~VU zU0#&SN5@WYFB?Qa=TiB@G2zTf1&^CVVii7NCcBapu)l8fmC)NC(HsP*@|~(Zo~LcZ8NPf{1B5!vv3A^%Ex8(0T4qSw;D_r{gXFMDUEB zAcz}$n@J=U_yrPn6La^HL~XE>;fbtArP}=|&y2)oIRkqnmdD^=-<3j}#Ao#DYpBuB z=u^cB#>ZjCnD_O<5{l+5r0ug2aT=Yoke4B1xR+x5l&P$z?M~0Sed9gtp`zahTmr3% zxF4_aSrWyxPa^6WKYl{;to{;VbS*+gwWt)>{p#ZXVT*YqC}Z3}j@d=YhN`_R9l7M+ zv;=cvPl)CzciW}Y4lbohNBgu1T_rnY!z8;!yGndcPQI1dtkAOK(fskPhV?vaUADs% z$Gekh?;c_wml9z)`a-iaALE5G+iikW7aBmj}tU2@a^LAow%sT7y(>wq^{JQ+=!&G zY`V_jF&@9v>8L+k?Ikr6nkQeghY=+zV=c+1N@C1vY;FhZf?5D=O;5+E6mek;bykD@ z*xOP+Rs_SVAqFtme3$Hu8!m_{iYGPLha7bn`kd#V>)a@<LEE?R7$*3QmvPk-JLarrsLCw=Fox9TAqKuRl23N zxgjm9#NS22-C>Qji(G$1e;Fb533f{BaydNXvzu8Ugd4xM zZnn=O?9t%w6=Ch@NtHw;&U(iflEe^4x6(dS$+|$TfcE7vLEXsk7H@y(sEBp1Yy%fB zN&cmCpgrT_0UXgI-~%_7a;vKDaOh;TjQqQ1NBy$zT<-ZLW%;6_Zb#~xDy>I>B{&87 z@yXha-56r`DNE<_d4YOOZ7=GqY6smfx*ua6C$Q!bg=lD)n=wLS_9@BNSzhr}xWh0U z06nA8!v{{fS^dke{Za{ak&uFJl-c`;HF8LSuEqVsuR)(5mo?@T^~|(4#f>Fp-G~?0 z7c=0lG|%Xo?)hLRugi5gCNae!a^X%r>aNAQ3R4D=Rd@YusfIca^6v*>S5E0H zWPGbZCeHOjTtn+$Dsk0*>(?;vi<{HfF4efEV8PQc7IKrQlqHyIa;NEfB^&aWp%8|;a z9?;8Ww1HTEEk6}>2yIX=DODyB3jR+s^LK(D&E$DFC>&kDM(`d4=tBM|sMHIS>X-^B z7_EX8q7ZMOSRP?ZBXHoVcwP_h%sqYD$i`4`BIq1FwA1l4s;uv8c-6r8B4eWpztq>^ zrPjZMwDM($8aW`qQpL*(v|1Q>BI;3!K0g(HFCE0W`8u>3tWD1!$Qpl_Xp~Vi(52+H zL2=!c*ad;)(%pW#o3cFKLvN-t5d)C3rX)zjIO*%NUXFtT6NX7QJQJYR+CUNeypm>i z99Ej{`j19A4!vLQ5|og*A4$wM zE`hkitnf+lF>Pg4sxo8#RosEwS!uKkw<7s$q=tHGMELdB<>SqqmN1;AbM8F=>$TD$ zQngd@|H#a3ND7^dBsqGI@i>e=i|>V<);2XmNsA1q&iD~i-*@W&E&o+L!zOs; zRnVU}p>c=8XPKTJl6@@{@ zQ5&2kGLgI$mXG>PGko-{XT~ou{i(gQo(0*0h$^{LMV`&GpV2NJm?r=rLnJBGUQlAu zb=^fnne=zmPPG0+-QClKixxHJGqMNl9M&lVBXnU3gbwu;Wns;YZu)8Nf>E$5%G(%S zJ>2OQSpUG)lh6s+CDbWLCi? z*-^N^p8@Muy_AqQ8}*;V=oP{VBQY`Ao=MQ+i(pIq#m%wnwuH*=ki}SNVbLI<#c4@4 z>5_zQ>*<~tzY|I@kgC^64Y;x1iOX^%84ZXi!g8^8D}g#f_r{SBKDrw$AJd+vO`HgO zpD~=~XTA(zK_}!Jx_-HK_`7tQ7w8`9-S0tAa0ai;xoq=QVK5A789}s(um$AheH|*@ zvqIP(1zoA{;~6nnW_+}2&*j%04ZNuROEM!Oh3hoj9RUGGO+@EBHueEU z`GtGDfctT_lL9izQ)CHSzwBk~azQj#@ns+1R@(qucH=EvO}MCwGPKI2MjB$YiJ-^H})Z$qE*pK=S?;Kp}AJC#WhU8q2D!nZU>x%mVyVHxCviT{Ugs$Njw@x3RAe06hB%nm;)(ye*Tw?xE`}i#?dE|Qs*t?%I5X-VMv?=G_c*a^-{cb*2NtqCfT9>!k_cU ztk4`k;Y^7AW*3`Adr{Mm1vy=C#NWxpJj{_?SaGyLk>@CS%Rz4Z~*dQM{SboP;&Ie^0-NNlnQMm{ojZ zX=;aWWZBomNOj8EUqJ*RQYMVI$a_L1*_78{gl_R`Iq=J&1iU{VVQ7-1GxZm+my~dIx@Fp#a1mKKucV)P9$KoWaQl!R*Cp2Xig)6Q z91ZP*;9?if1cveUo=69NOMU9Rd{X}|0ja~V5$hvBmG0?8qn)XRNDx^m2KRB7p5+xU zhb{zRM_YgnK^gH~r1Eb1U89aPG!>f5L`=x38D@aPGJ?L2!Iy9nCNPI-TUjor*QWm{ zb{ch-A?^S5n(9NW)}H-zs@OjLIepDQ);O)x;tFo>OREhMM?%s5@ICx8GLbk>&!eGp z;~swlvhC8i%)_-CF4n>|cw4hGj{M?OT}j*^?~Q|v!pBi$5NgY&YG7i_d$dGqzNmS$X ztfXpbR3Ct1lB}va3F~RBPNQG35%fMm^+S zUx=k2;=>Hu|5ETnd-^^Crst^QOYJ6v9y|%7>L%Wrx><9Fo&b2?OJ8S-x8UGcBnBM@ zcEQcQHl3crt(gDdOpbQ zyr=rG{B9wPuIL`ONT+P#s-Vf-G(M^*y*pl;H!KqbzLde)T=AATdmn|xqqFf*J?5Bp zE=Pzgy2I^OW*cP$=BU@VMEI#Kp#=idLeuDIF`FwXIkaTQ6`Di->j@+N>V9jS(A|X7 z9MnXhjRo1aul6s)E~WXfFECj#edQ&L@g7C%qyB;{3N@;=j7^~l$imnwICM>ES_Akbrimnn7Bx+UwRk3At9ia{_ zQhr~oqp^C6^oYA?q+~bUUtb>-(Jdj0wAK6@j|RB$ZFOeqPl0K&^mn*kM=t)2osFz? z_iuC3HRW#^{(;R+()+w@2*KcN8SJe4W9{4rO^=Zx7&uA~p0N=_b>aYJv&L@v2PWR7 zfR|`Iq{nF%I%)cUpOh7a{#433he|$sw?m7eoC}6s^#w?o*YlQS`s=h3OX{pb6kZ)W z^J>-fXl2oyoM|j)G_1%pnKLt4D}pK9GwHUB%y92L<&S%tLl0QJ$aD|5V38%qTVJ=Y zmY`XTPl%4^lpM2{kb2b*U6*5bUL1+uzE4a7y23XEdUbp9Jbc864zlOtKkURHhaWP$ z30Tmgl`TMIHG)95u5CAavdSP)wV;?Fo5M5*!hECZYYRd_JMDA68y;pc`)Z{*R zUMz>@`n;Bg*+SDUv;!p@Ew% z9K3H=@+%iZif{}bW|^M!LR_Lgamqhh&FAVUX+YVoEGIST^^q4pjq|SLwjbMn+?uJf zrTafsV~*Lss~f)mc{cIs4sqOO`#I4Ei=i#w4d)*0n%nlN^!@a5M8vO${=&37Uv5X& zIyC;5>U%8oX=dh+k(1E}u_@OI99ss5c5M^(56kq-J9O6kL8;62k?R^SOV6HrVE#6s zUt#?q;KH2luPJ>H^0RlJvZ&STT9a-5{yQ)VNfX`LVwd{rSrg6F-e$Gjm8JYg}ey3Zo zuWx3V^pUX(5bV9n3|g9>yz75&L)5e0w?T(JjURc-hIN&YGrZKc>9tvI2;moXlQ;js zU(p<;e+<(uW@4oiIsM?qlYN~mQqJ5nr!S&EGyWty)|mdj+FMs|IRQkn>{XgiM|Vxu zZwWtkd3$+3$Rj~F9(*xoD4UMU>U(xQta@6wTl3hi@BEmd{H_6CA+NmEACt?&7N&lO z7#Wocn%P zezYy$tBr3c+cBq1PT%wM%)bL6tNA{8%7vU3)1IF2J$EJ|nZ6x1IUIJJc)Yc=(s%LP zKxxNM@8tuQXSHWqb6t;)m1RswTwi?bFgaMDn!&eCS-SVcQ~qmfrnb2oNBezhy-{Da zBzbUcBP{p)vB^a92mCF$UNJMKO_x{A&jDueP>n52ZNT&hf?2OfK{K%9qJv2f=@R}$@+%FTA1*Lx~R`9-J4Rb{h|T5pN{C4wBa+MzKcE)S51~c_oNq z=D&oMSJ6X8AUIeQq+NSz`pxS3p?9hNhL#->b)|SAjFnHOV{C_JVdu*ewn1wibBsFf z^-=zO;IC|y^em^4ihd`ao`9jHzoN;|-AxUa9U~SwLN+D03nqc)v-4a|CB>|wQKzZq z&m^gG1M!BBN=QWNr&R`e>H^!^Ob!}DjpdTmknz0gi<#I@9SO|GhG$NPzfEKH&k1AZ zjbZ!ulCPfE+VLa*O7Y}(emZpRKt)XGS~d~Ecsg4Nm-mA&JxUw7`URm_gWC9H~#zi41-&82@aeP0Czg*q`4;*=zVP zL&kh6z|5PTv?6Z+3f1N)KI=Ogeew!CXT{jL z+~y@(;ey7}L6sCzH7kH06Bda!Sfyw^ufSkTeU-Zg^o$AEz1&A%6ZV6X;!D(I)^{{{Gk{tr4K@PC;~zW+gN|Nr{WKdft;sMR^J U*Qm4bFT?C#%fl9VX5N?oAKPLa+yDRo literal 0 HcmV?d00001 diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/charts/locomo_comparison.png b/benchmarks/results/smoke_legacy_compare_20260423_105447/charts/locomo_comparison.png new file mode 100644 index 0000000000000000000000000000000000000000..7f5f7c91e81ca4dce1506f440423e88b57b953ca GIT binary patch literal 43742 zcmc$`by$^ayDvIWRCFqeiXb2eC@M%zK|%y2rKP(?3F&TB1ZfLIQex1lbW05-C5MvI zC7pxZ_rv_wxA$55T6?c^{yFoy<~2az9dA6(egEnn{tx9OPMx4Sfx%!-Nj?x )rV zmxnKt9EBJ6h_-Eb;kOr4w^y=yZ0}@XYmAXKu(y6{W&hOF@Uo+^t(~cr{Vp^~^sOJ!A6)o^%IjJL3vHvSm;3L|sp&^`2e zorCBydOaV3IfGtL$Y4m&>k%dlF}!#mI)^y~ulJrE#}L8mr)pwvc)>i>KXeaX{{Q^* zPlJ5p@0Ao2#e+L`R@(INeur+opC2fjT!_(e8YsvX`W{p5to%Q(&ikUGhK5Gz?s{LRFSTXFM1)aDSEj}nuI)cxQeyAAC7WlLFa=+{ z^*EuAQqbL*f1!NEcGM^2(w*dwPh$C6?r!v(hN-AZro_6MQE}*3D-Stw-S%fX)joBl z-)*77G^?PeF~@44)M2J4OKVX4PvWD+-(RPv5<}$zSqs(8etkC9b(+m8%3j)J-de0Z z-6^>J=iLR3M{jm|Z(H;xUHzc__a&P;GSeTQ?wguzPN&2z6EgEVq|!wEND2I{cNWSC zrLkUngy|@&%CvQdM2S#2H5~_)nf62p!Ia_wEe|R?~U- z<;wP%?NIZs47Hrq&CyW##Vys{$db{JJ01M9neTE`-Z9BCbMHi`pL?BCxy5|R@uz_) z9J>`qhj-5|nUHE?D|^}tw@|6dbC=1P_FAm7;T~1+N~ql{qdu3(m`?MYirZh)>pC3J z`*yw}W9%?Zi3tzaG)wDBSIOA%thZ6~sKR)srJlrmisd7Wy<;TYfZIe!*AcXz`th0e zuIreCS*gQZZ^0k9`V1SoHh~qHU$8&~zglFLOhhVq4B5!#$wiNPZqKQW-d^uD&ne&Q zv#Qd$JsEAMS|-<5XwkH{J+$BB973VzZleL0W7aPbc!9NXzG7qW^9>ug?coKM1Fp~U zMjvz>FD^L1owfM&chk*VU-4OATKSD@eNSaLR$(2oE7p5I2lEUTH&{eF^k}&3;^@4} zpN4BZPIHWLnfQ2(uFYrNu7w-_9u|}4xgAZy?#58YK*Q%IwmV-}g8C}A=e0(;**sdJ z?K>om@66?Q66aXaNXX)Mem#j}HSbRECJto`SDP%EjAbz&s@x^kb8F_rb!}FtYEF)S z&|}kbU#d6nZ-B+6JL-VJFvUSdh!{Rpy@uDT65x1sZw$U*--h3`rF8s8@AsbDtm``a zEG*|=dS1KSgR@4vuoFrIMDL6S-D(_!CsrN|w>kOG8wuL)!`j5sN^V0~jwfG_P~1H2 z$uJgZQNGf8$C%n8UxBdB7#+Aqp=lET8Sd*{zknCVPP5H+rpUWY70#q*^iVd36@HmY z2n-*vYr9J*cyaVpV-EYv0NJS9tZw6BYD}{prTcr^O>I1dPT#M7c<-^Zgfp(%+cN)E z{p|3{5{|p~Nt_3Bg=JT&g0W7i{r4R6{$iV7@qVDB>B{CbYk^)2q(D z&KJLXGq?H>VbA+8>9>V7_;>B~nT(w9wVCb=_n64Syy@hq@Jk-6?Z&6?u0H$)3r%Ys z-X`2*cMZ>{+-DSecbWSYUeBRhrMt3nmAkXVZgSb0ef<*d4F1mZ3jZVCMz|2xD5pV} zmCsOe!gV~>x{VJXJ^5`x#a*XjvF0zr_T_8^l{k-e4FXzJU845%2!unaYR!D4;WHvIis+)XXBG;HHg&S{~Y+}>Ge4T40MF&rXC zyf+4%^jspWe;W9yY;XO=R{Y65Mw-1MzvHvbXhRrl$=F+$UzuN8HP_oE=(?S0pyDYNN4o@iD^Y-l$1SmD`3T)u6?GD7wRu;agg1uv`rPxG@>6_;uCp$v7|F zo@|}6@>bg>R<+w{;z5*eUPN~YGAUbVtnrH*8&3aZlu2*Bf9#TSrJRqWib^I`*;Pn##k` zW^iTNDauS&viS0m{egy&+-pGtr8~^GSoNAO!-w<*%hKs&sit|@uFLq1xxS)?6{xOu zjd(Yx|BXyy^_ahX@p1u*1#10j6wYD7SP~PX3fbGg9+i({J#xyI8@cD7G_e-l)MHP_ zWGHb`y6q7r&HLN5?OIeU#GbsWW~s?j6q_9LN#lN5LSFUQki=3}&*|J>n9*I(Cyt$4M|8wXXyw>M8V^LwN?B zNBey~@wY#{9LeIdG3dsA>f+x??!Vs23w~U95^0*8UrTyN2JfMk54CafC~3Dp6ZzHI z>@v%O*=DG6B{t%ld)o^vtZu_EDV9CBMt?qfCu=5oK}9oK4J#t4fMk-~|G|-`?&vzPJKapax^?EG6Cm%3r2qj7g({;u^o|dE8mOTiyN* z*RXRRD#g(X@Y;8agqp>)RlpU$!4i_!wR#j2QA+HMsXl4+GuMFc z-U;cOd3D72V7Vgf@A=uBs_d@gx8IkJ$hCQ>osUs7KRb+1%H**f`?kbi5nhuuwXFGW z>bK^JxP&{iN!N97Y&xYf9kU~@?@PCP*7^6gW^)KzYALtz*PeGo)|3U{OJ^K1O_u5@ z;(q6aw_;0YouRr-WS3Fs*ekfMwD8zn*H*&+ju-LkDGohCq0J$iy{Rb##ddjAP1ohB z({AwGTw$MOntYrfx1$-G2dix3Hg|ju_IQOJb^#}}w`s)izZm6A>BGD{;d1Z zd06h!8T1hyqj@bJN*e=B>flRhBHmhUc^M$%BIxZ#$b{I1BgVs zKVj3+DYE&t%E|ra=3c*B@qV2zZ#1*GHHr3#NO1`Fr6!+nwNoc@ibn!yOjw}~PeYf= zzFo(n5-xXI%d*6{aukul-$jI=jpkfLNm~gmtK7avcwA)AAOkullZsw=U&W?WNmP zP^R-4*1eL-Nz3YR$eh!*y=rq;YGto`oxc?z&iFGUoixphYTa;K)aM+6%&Kys8}z7u zrLYr|v$}N0u^DGS=rTEOR5c#1wY+^+l?j@4Dw^H_7I!%^Zi-&J~S zhipXlGV9>1s&)&@=^t2#zAEiflpe=z^xMe?&(^!UV~yB#HcD52zc@XS^KoVeAoVl= zg{Dxzk=;^FQhL|8TL=`Em4pdpd;-0kX>4t%&|dKlK}A&q>;BVndbX4wZiOqs{n@*% z0%DAe+o5|Hec#WMv2GLFah_W@VytV9cW2w7{;b9_2f!#SeC^M#F9P2D0JLqxv}My< zaksH>*0%wvcBLt$EZb#fpR;M+cU=GVWD*AjPkW}fVD`$K5!4Cx1>th&_>+8dOl$mm z8&&(VAM`w33Uc`7i$^=yt9BGq@OcRfmD|}nt}5gM9@1gq&~V+b#@P6L0I})?@k>hb zcf+y)BXDJjJ-icd(5!GuYv#EC0M;-ex&@oGcdOiZ?RSGvla_S1p+_YH8sHTQk*h_h zii`I!Mqi2qL*#u?%7?5y^KvvG?~*`Md(~jMJWk@J+*;D6XEIYgS$)pK3d+l3qc0+w zZXQZDd3EM1v@lKzE#j+i@0|IkKE3F;p!JkfW#evz=;TlbM=J?$V!$l)g(i9(Vht#} zM%n3sD;B-^ror_6K6TbTP*mCU+I~v-yh0t^ljfO`{kF&NXUCSg`4_7Xvs5?&JZD{? zrTMDBmGWwq4$qlA3EUu>t#Wgu+RE*dX1U({BjcestT_!CBrOcpFBGFD}ltTn?4(OBGT0@_e%_Tls;{`^ddP+FImNK2hEN7n86vIVn# zT&KK{&TT|0tf_&FO#sD;R?0j#ss0WMaizP~xxe!)bl{AQWh1%LmUy-DO%xm#DmKDp z{Q*^0WEwYxpDU*=1CHXHL(OIK@IMeZfbM@I@DWYfdAx!}p^B#CTDRH_^y%wtXdv$by&K`E|q!an>N16@B~AS!O2vTpbfr>oxsL#smVt$Gb!S4 z4)v_v2LZxE`NUwHRgsQThPA1?suS+lk14fL6# zXg`KxtdBlS-$9E_hYpEK7$t?t)|k%x$E5ercmJ=om&l$qB*I`iSg7sK=EI9-k z0hUQNa%_9+!GUyhgu-A59!H0f1fkR<#1RIe6uVELyx%=V;tY_G2|pFEFrk9*I8 zsd?BuprG>Y98R|QAqYKexeqM}DxPe*FAE2f&!=in;3&(BITn3|X+XJ5Cq6#OR_m60 z=(A`AjLLL#aoBX~M?Aret#SkQ^y`_!M3>U_@^lOA3MhO*!i6~QOC6a&L%)RHy{y4^=rwc3n z!G7$GL^5bG>F#uS0(Ev+7xHUQJnIwwx$9icsLyL!RjhTu7m*pgs3LQLl0x&eVzon> z&)N+dlH!-gEoKf`TkCUCfJVw_XxIw66!{)(LwWCrb({Yv2o(B8>EH)#Tgg%&_O_pj z+r_D|z?ua>N3$bbXkw_#oQ}wbJDC5|Fc1rJ2TPnevF$K`c3BV>oJfxXCT`E~w{60L zn8JF@=1Gi$Dipo44wjtqx4`ELCHZ@REnmB2`;8tgc~jsG&D+o%C^}K|$q~`Urz6?S z9u$xyptIPj-8n1MPFWsVMu`v^#H0B^1M7S%&GxajisT-kzzzGhyBlM775h%f|9|*OW=;lIwtGMO&7Pha)2OUdRaqqlxMzPyyxlH*rF?Q|jHIv%O)1 z^^y;rD75Hiq=^9NScC$Fi|S`rVEYbvx@1?qljFZp<*B$jKX6CwOhbWrN0OxHJm7SB zmaKfmQb0qtVPlOfIog$;KA94i9F()S%9a|aJE21Bn7YwZ=U+7B)K5>7h%v9?Ot&WORZ?U{+i81Jn)or{wU(XX7S&;5f;^)h1@S z(h}Y-u6D|^pWYYtdB=H$S9D6f9RZaEMw2Y6y_Z+R)frd;%Di)@ITdr_pWDyUqE`f@ zC;YU3*7fHFlr@@EXQ@{X^*nY{5h;Be(Z|L-%A{6anLL?Dl|AF`ezo8t19feTQEW9V zU4FHFXba<$MpG-$J>Tct*ADCrFvHDEjIz9RMiL#i@jK<;H1v;@AO5`tXb#=dO6g(n=gZ|lgwJw|M%L$kk|8V+S^Zm zUOD2;?QlF~;%dC1n6XcYK`Y;6^C|TR*ONrZSh)ULv0GP#RWm$y4Xw~E*E8n>ut6)rh$QO2a=cV3C7{4e;MdoOOoM)|`&35kIlHo!&Yt;?`0X;~zF z2}U+2n=J6z1I|r}ql-zZd$Le$A{ZT}JFcScvNqkBqMGAha$JNb(lp}GZg=Zhss_Xv@s?7aY+v1Rbrc86b2^-$yZZty$Iyq{d7{`=84=iw1l0YU&laaFu8#Xy^0BDkj?Nb0~b zmG>54ao+XwR*2sO&?r|l8r^Pk2zB(N;Lr*?-?9^@V=G#ef63YJ1!FAWf~A338`B;w z>yTo>4%hmnq{Mmhs~yP({4+k>Ro4tG;h+-VnJ+ftAA2p*QMJFjkdKJfQHFMdjUOVX zWl^DeL}&HX4nVOq5?eGMy%FmGP7$jMB3}6)*g^75H@`GrvB;5p$26VvPEN}2lB0Sr zlD2`x#$M32{@VYZvGfqmMF2|DbVg34RA(;~(%4pkm1g^zyd|$eS(J1_R$sy z*R~jEXxSf}$U+Pnqn>`2^xO~Ear~9vYnFixXVsc8wizG@G5@omtzMF=J^(92j{mp! zVcW|WR2<2AKnmyfdyaHUf^}!_PsctBy8=C8KeFfnD1&^U995CbCyNq$I}JFfWv`=h z-?6<=6^H2N0ew7UWh;8!1?nLd$l=vflpL3>%mFFzoh(Q7X4Ssm1~>kjQ@FMGF|Y?K z%ZFzTh7=?&g}JUyDVXlU~no$)SQ<*HJ%K z%CCT$eFr%@CES}xl7OaI(*JyWrA)dm-MPL&kR*yN9$?A4qF-Z+})Dh7(I+G`6$mZ_$8>oNq{UE zg>*qz%h0(N8%n?!rJ~M&QNp2hi1hKULJO@A`QgOg?+kudKO4=6ETI$(K4rktAXEimVY2hm}ckrj^j_>+h006p!s#PJXS`D!`!Tj0}W ztW%;3wge2pB@`YG!3h< z$wjt&!KU1%wSa7r%7bxa+v?kEdcwAqFc4Ut`U++y@dI}8T5$`3IB{x=_b;kUmP;C$ zGsb=}AIbrtHf3YTt4aWK(*@vT2RN2*kq1Y^h6;fE@xnpZSuC&-fdhdMBob+_{q3Pn zfUz-y9vcHRY+~M~#YHmshtIj055Sxc?|GDv7BVh7D+N zYI+`FZhUofs5&D9vLGjTV*&v*knh{>>z_9iyL>TNf3z3p5TS4!b4|qb=K{1|Gwn&Z zC2bq<*rl@yFAkZU%OFL@-nFF%fF?}7F+B7?k@_Rww9}pCZJLW9871mCd{Y_#W;*~t zysgc6OH##<$~^Z)S@vSsW1azfCDJ*m105Bl^ki84cl^}Jh{b0 zPz@f&-F05Soks7r`1M#P5TiE}(RQsg_F>8E%>MLg)K7E2I_LDkwq}0mMK7F0vY602 z_l+_1CO}GJY1la0gAzWjE^;g3in45Eq!iDL>YCbv&VQPIw0VR7Qu3h5VQIww4 zI1SF7zsSVJi(P*if53too}-9S|BRRl$X68|4+*J%f-FE&RdFeHvN%+|Wzs7VbmaWi z&k84KMrz~vv6JRhZKvTvyXc^6cr8O5!AeMXlZU?hr7Guhc3)C_zWW=o3*R?5_jSeE z1h=17=2*&Atl96C5Yq{^UtihMuTP{hZ0nd^02JnjLVNwhe_A zzQnT_Ip4=Ek>i}p!@%jNtz6z$H&8Aqnw!DNe)0?)8W(A{;khvp<1}a%$NutFH!FMa zm7vDbndDvTXgciDqD3~Q9t;-(QfXd!H1>hXkRkyup2q@Te*q^>lpNn1A0Qp5xrnK&q~iAfB8dpL68nGqMMf5T<^(wn%w zL>8NqUxFPNKLu@1h0_oj5C?Q>j=HYv%F04}cL#W|OkwjK;lc%=-ZM~}PKOB%<$(1* zC{!n!#;Ge*c_t&^5kJ#x&^(7iEA=S;j^m$*rUyuzjDCvSv__9%k441`+20P1Cjl=w z4eo<3t}(Y985>7KRukqs)9|{_q@T=g6B(i|y0u=Q?VCN6Ux)8BDZ6xH)F zX(TH;6K~Y~fpUXWfH%_OJ&1wIqzzfh7EWNe8;3c8gbEAc#8yH*Ta3 ze;;Yw4~J4Jk0YUxuGheNbX^fHO}(uB_@lAyRl_m1b4n*pGxBE2#R+;Wdz`$*rYZ5q zA0UsuJDDi;g#Q4u3Ri$ zVV;Sp?hL%MU0wac4%U~lQ%dH>g_mOBGuY0jYbiWBr(~35puAacf=fSonpS)82UqEK zc(nWS_Yw->=1b0?yNB?2={W<}LU={X)P5Xbp3q<^kYP5fLPZ-o1)WV0hC+)5Qhj%T z|KhZ&nEYsLACRsmL@tGES*GRx0VLO!c&`=#$*vz?;kE5# zX;rC%|2FJv|2+cqrm-(HzVfc?>Nf+E`WrtOWCB}DZ2}u;!1okI8kC*W5TJJU(Ci?# zuSkFLlx-fzL-`c=`)}dyh`%k6HWj9(`#t!+Vd^}u1Tdll|$Aa1===tXxaz{xF zr7t}*GO?w>)!hWV=CW5aWnJDWcW2z3yl-d5t3@;M8%sb1py`;d9|x=(v!`Z!LB(3( zDn+nR;@JgY?u`{hhR*Y-Ft+4$BDV|l_-K*5pI*(TYBRTT;r!%nFwO|Lf#b&OXJbRn zZ*M|6wFgd73WCZOMFWnbr$U1RzEM_EtN3%0YjRD#nn;P^caTDa0zsygGb3+FPB7L5 zVio|NH$T&dl#g9175(FA3F3^aWW5MAx$qaL^Z4rHxI>3=dGpCpmYTCGfoo;<({D3Q z80~Y~7lUG2;w9pfj-_uoC7by9l~5Xfgu2S`4%(`z!IZWHKqTWhvaV*(k=bavz5suD zv&|L3_fOJ}{w&gHL8Se781V`Hqvv&O+aM{Rnd1eb)mSnmm%yQW@B{CQKYo9G(FwUB z_1U#VK0i#Z!9Rc5>!0Ex=DWx$b#=wwJZv_fGQ_};)zvJ)8R%MDYtGSr%^)^0+TMDWw2_v zTcC}YK09*Z0SIGy2&^(^K{{)XXW_|5hBEyne&m#=pEEbuHRj819yG@Z<)ZR5CE~t9ly~Em4yj z@zNjIrU9kM0E1Q`2pYP)nv}wqj=1Ulqe^@4qtu#Gj&8;B=&9R3FZ*e+SF9?6O&Vzo}00MB1fTJsHNyxtrGDSUo`ywMFyObGKt_pw;ZMgiu=A*(wd{KF(9vcC;o10W!YvU{YoP-9y1fp3gifcNv! z6xdnTIyV$@T}^01D~lt5DzLB;xagXu+7mkgZ^fbCE7uh$0z-TPN2V&SufC*eV%+71 znp!1b=jtRR-!vlX=N^dby2>^Y#R>Um*Kx^c35S9)$5a3e(+M>Y9O& z5x9e?xS7k~q?lRDAiuc0au7iSkCB;G3tYCQ92_ii-MPXlU|FB9#kqfe`sZg}SS%f< zq3ADU#OjqH{dS|Yqc9FGP^xVKqI=`p0y|{ApN6r z3n-4Jw|^BZ#O7ICLE3j{7;$)iqe$8zr%uKV&SRHfyAG7U$#^KgIxrnyDhSzKlOJ;6 zR~~r>8Mg*hMjO$Z~azVfe;f2NRB}EqbZ76LQT}7)zoT-TwG+V^N7Dd7s_?D zbUV?fPmeg{J%!!W)>Z+BFdh5$#uQZW##!8%UzD2tz+bKp9JYb7mot+6&i~F@V_qN>(}uQpup${jC@Ja}REwpd_yJ z-&px|KQ>&G`SkISfW}lGZI|&d7L}*Isb~6KXR~m9y2^JFAf^NbvV^ixQ{|+aj`i0g zD-Q=zz(D{L^ogye1n!Yn`C?8I7hczt(x%>^9Q5`sXAyU!h&f0shUt3kuC0PWGeI~u z1g63&6#B8@TId3iV@?^#!&yu2#NH~Cqb&X$wuU3+hQ*f63X}sL`#BMRkH;4a?XD_q z!QxWfUY|4f$g~Mgj|#Oqz{)ne(f|XZ{Z8WgeGr_^+Gsz6VkH>NW?W?CQ_+>6ML>Ds zLKJqRFOi`?#B`-5H(>i!$q_haCl{FQ`OumtFdXY--F}>EL(n8*;sP<&QWN`sVH+s8 z?+St2WcC3T>khZ&tqJxZ!m1%@_3?}U=*;hIcNy(Z0!Mpd1}zJ{_dmhyy=&>;&5%D# z5R6n6#r-;-#-GVsQ%-_0qO=CMdiYJMXCQqQ*;au`I|F+leYO%6MouQpEGM1UH}Ve< z2ELuZ;bP=v2=0??J%-R%xqz>j#7anH;a%=*p#qdo0j26BHoltdCzLkhcdNZ5D~8$S zBpVDj13B`}FV=$xtWG^(p7?Ga;7{C(f~kNzd`C%X)Gh9TaIH77Ju|T!^_q-Pe9WMQ ztf$_i%yF(ki?-G-r9iPjpEwxg5|tyoM20HxHs0UQ*zLbOM(g~k<40HTtE1FppTxdx z{4@#;Yui^lYoO0XgaUQ8p9+y`3!6%|XWJg=o7(mcR2gMK`QfBjxfZZ9UDPeR7ujLC zvA)wL@PLA36hiY<+_^VJK2EbD8KQ&)!E6f=hJRfeNLno`zr~*XW-a`gP@a&TK(b&7 ze7(OeF@dBtVbr4e4aI$JP%2=ElXnK0xsHKs=RQ=RwhHJ$29U+Y+Q1 zxg`4n>6ga@*lK8ffX^Z zMw*Kr(I$QlCXeO(3^l!SH)$o=j9Af7Ds86vuG$HTC!mT65Etjqj~%GgAnAT!D_^YX zcOQdi(RRtV78PBT;gamth5T}d(p(!}wKbE$ zm1O~8plOJD%An3RuSr91R(|Y1+%r-P;Iyq}wi?d&uJ2@Tez*_2e(x*CyFO>I9qrqhBkdx9@k@CL)tpT)Lz=DR%CS(SN_*-VhBWZvARMl& zxeqm>b=b_Okc=4g!O}${{lw4HoY$jnC`JZ-NiOXFab{maPkC+l7?{BUjL(BppZ}vv z`}HL3=>DF3@_6+?n2tDrasP9`0zv>Ybq%<|=|((b%ZJoiMtHKAWOAa*0Qs6W^8cI_&_CJ^+Jw9LH_PH-U-wgv0}l97!S@$^|6_>b9D=33yN7N zLNS=#dia>~(7k_tK(=#ypDCEW8hy9F0>WHUAbTY*GNBLbQUK3#Nb%?e`qBw_S=BQJ zfyv2uH;7|(B8%lVC~#b0{bJF40{9@ePz-r|c5hN>UlK{!edZvdYJ%z1sby86GRo}* zySx(=zBtfcKi~l)j4J^xJTGAq_YXJA%)8AQIG?$N511=5&{@~ zSGmgyO?6gY1JyJlMUK1RQYItXk7VC8l?+uEA?U$sNX70%>lb2)I;OiwM@8`_lvhfF zEq!3ILzY>;7q)#TXw5>Bc9=@gl1+hZwr&SnK*oc!%1d5gNwi=V=V<7bH}LH6vE&%X z-Wx3t?jb~sx{nHgdmi@q(Se16QIr~TUy(iV!+~u?Xu=AwO($8a1ms0C1$@uV6gp>) zeHJa?4;o7GAAr%ki4J$58c2r<{xmcZxnp$h*0i5X0YW7q*ZMA8INy~ZW5hZQh0@P_ESBtqBlvn*pxF_2M%72Q8b+a_=BrLRf%KdVf*}vG{1EUwR)Daq z>r}ko^4xnRIZ1#Wwg;l=YZ21#_btkWpgNel{@afCfCN!TkkkQH;Bp2b6hcPOaXzsz z_c4?+`!BLQ&rWA6{{7q@9D3{}r~bU1666=^z;H=PBb#t05|;h+>1#4m?aKh)cU0}I z<^T-t-PZ__8^p~Y2Jpk?kcQHHCHy_HWEUD zFk1*;wt^(Fxq=3T$_ui5z_$l#^qyEMMYr3n7A__1ERBH~apqOBT(M0GygM8^HGiLk zN(l5$%@{43S-{zGjg5`VZf9{i%P`@S3@GU4G4G7b&r+l&88eW9t7JMz1&E16OZXIl zeV~`!9As@Pe&wH%2({4chuSbm)-qP*OX88x%hX%OjUf3I=t42tO_hGIV83fgLYh>vy{CnNs~Cd);kFfcq#bji6BOMt6)fm|M%a zg8>ZDfx|ffL_D$ZJGsNW6KIx42keV+B7IXjvd=0f_k|&EV&|Y^5qiUy3b&RTTzuQ8 zA?W+R1Wq*EG7b1u8UgtNqTjHcKFGm>rg01xC4c;M7Uqf?!=i4PVN4FfPM^L8x@-xb zTmmCs_JKouHw+F$`u$tX1hY^FJ82~N^_?DD`8cw1qoy<$ye=#}!JBHgd#ZrXOXL_(+{DrUCCK?-;SP%h>`N`jfFUo^6v_zp4=^`iqX^G zPQ4ez+?1YB^Vx7y?K1$_$?epMXprd-9DR1*FTN3@mWs^~2I&^mJ3*GAMH?(E+}&`N z^R(Wp37YP-X@VpfxxaZvPA`YrRlO6VT>`7AU|+li?Z|P&wqa zf7kD2p69FKY7oZlw?MbCGx-IDN?=I;q<;wg*E~xxh)X)6Auv*Ea23|vWjOuxxrS2b z7KZ|ebY$_SKouJ|$dtoF3T`hL)5uRd++r5-hEv5bDG}={eRg*ZEZIn)w)F8(Sff0} zH*%pi&giFYbG#Fr%cKL#QHO#w!JzDts{nxHEE~u~G7{6-BTehHpV_l_!3715ilHe-ac=c3M6Npte9t zNSESCzf<5;xlsY1iqS4ov;FRbcY(fot={YrID+!z|>KZ5-gSI8t z63N|UcML`+f=641C{T{v9)(f&%k-)|r{O~BtOkG!ec;hbD-KO%+kpy%>$6lA@!XnD z8O_prCql}1<(Qj7i!&sjl{MF)8mAy@TnP2ok5rMqoViHHaOZz8Lbb0Mc$K6Zl_Kx$O%9)uEd&2_2 z>}UKP$XuNI_8|Ombr6xbDCN4verTbdk$Nf6ByT&+_w)G~=Iuvxvhh!euML4takrl6 zHwt|nbl`@13rH*SecA>P8VjwlC?6VRFpt1(W(K28OXaKLhu@gAa>9Hg%An^1q)9n^ zT7cs;J)8e04&ic=MI<@UkQ73`&o7r8OAnlZ-bF9;e_yGcoy8ToC1xceiuk%#>XHa-o_Vo^w5#C%sTewVJb37wcx`Fh(sfMmKL=uwW*A|Z z?3gMP$*zExfaEKC`_&Dw%F@=ha5Q!@mI3!`8UGnOH7o zPCcFe8sv(qYTMLzK{V)KFc-34Y-ong1p=`eX#TVVT2ho|fms;Jq9Rzl3Q?*a=Tkxe zsq(=uOZtzI)`N&9fJqmjZhx1R5p_e_3S{$J0G>82{PkEGQ4!G3AG?Cp+KAt6^V)mo zy8h=Q8kG%2a7Pa1p`82B(CNx;=%agFFzR9t^dcDz>mnK*SBuimf}rl0g-(JDFO5=n zl}vWcoDya|nHn>&!VcgyxDI^Z_3kW1z98klu2=hW0~$&+lC5Tma9nWFp3{)W2ZYt~p~I!1n2&ieh;d^Z9N8iw^aoFR zGY-ZKZ=(?*zU!Kxeb|{GVwwbaf)%-pNSA9d=!7-$u?Gy&t|ZBK_TXJi{GJ0UN43BY zd3K&-%E&JuZAI%iA1;0JwPi|-qZCRO1mpK0%OwXz%5@76(8@YworYIIBA&z{Nvjlo z+m3e=Zd5XIz|p)kpLXEuMOXwGHf3b_Xg=gXhXSUmL3(ABmyyy9S{hDkzj!o6nnK(9 z{$Lw|7<%WP4WRLtP%H#--mR{E0nR%y+YVw~K8%Vc2jA|!PFD-3AEv5d9EKvB%G-!c zvirS8_IYJEq#PH9^ap%wKL~N!f??!Y7JEu*RY_ zC_&?|*_js@x_S)*SsOoy6=*8|(18a~GGYbbiFQGq!9Fn89G=BZulZMuArDYWjFhwr z9txV5o;*>_0riMxzP=Jrt=G6+8uR)cNNT%>fEF%8m_Kp^9QV+F_^xePa_dOzB+ z*K~YX@6{P*-`C-XDu75Q3GDpE>Ei%b(G(izI+G@aUqV^|jO$B1iMA0(sPi_;(e3aS zYQ&IqGy8fC!^O63kag61O2d)N>QZ|$=GB=4W~vl7)m(Ly@nYT-LHATXtp>lf4}xYuM_?W^z5t497AsryS2I&0_xy8 z3xokqriAD_Kxn5?X*Wj_MdY#K=|p^3~n z{G8``C+6IM)2*E8PPL5snO7I>M5ZrR%pGr(<$P16p2k&0tX_}ii84h){|gvA@xYGf zRZ+g(fju!9{{tKU1x1HkF7w?O*)()hkmZBq^fF-+1)`Vd!dltW2@t-Bp&6Nj>~KM^ z=TkRfyeAZuRy;HFu~8Z%YGWDxDX=?jXn=aK;MHq3QF3h`ZZClD4J8ehjL$_vSYB>U_F z8;EC2NL3zhsZz6X!ydh9vul)H-3v8^ec7Wqo|h94{<{N+^eH}-SJ_@MT!oz6`&@c) zM*2v!Tfx9k5=~oDD({usrSlCtE}SwzD}r49Vss9gxXwAL%eC@qyx@{Jh^-sRD0}1= z!MRfib3P>(B~KQF%+;Tni1=V76+Gt%tMB(FGZQ=a#>ACu zKl^mT4e|c;j6(+n*u#F4)xUv>tU(OlZ&IV6VoG9dA(k?9hTI#a#j^_4((QvRzj108 z-&XNfaJbmV4_s~K;jFLRfllI`c9VIcN7rpmDqhWl?2<VS|bs7~@RjLcxXJxq!)&H#81J0V<4X?8i!)iD0QiR!|IYq)}#U}&2dyq6$JF&kG zY=R3WB_(^EOx-x`d4+LrQ~w0Uci8B~C{p~MdakWAMcE=C2$!<#1tugj&cw_Xrt6h^ z)g)=QQKzIj^slq{k)>6%?2U4dEh<)MGPf5pw03@dBz@r!h@95MUZ^!y19#2+X?wdf ziSDUIB0>>6XH z8$1)+BZJTv!)akSzn_b-rq(&zYk%(@FnTgZ8G&J+a~+__Bc76R)% zo4qDy&dQ36Xg1hLj6br!8)D5qaV^N0{6L;U>LUFRighB3q5o0|oERTTFh{OofzcZX zvvw3R%h+MR^JEkb$G)*}|Fhr1a&PyCyZ_Rp*C$KxZ|Fb7UVPiOwpIJYb$7(>QmQ#X zaXB=T;7Mc92@?j&qos$c4FiwTkyVOeb<%zvbJ+Lbsj#K{hbR8;fJ^ostVPTTOXyDW zz4y4Z{}HbK_uo5MEts{&{{@)o;{6m!4&8%42V|ymz(Z5j5CU6;f{4<@;M$%SevyIs z10pONoyAb4hgo;J%1j4VnjaWuCm4FyT(lU)gB~E-izc%>A?y4W83A45+MQR^{owmMbzIGuThH51;p6y{CFa-uhRC*^~(~BsANN!7}*lmn5 z&;&s`h&)O)Ayx2jE_inGimD3j$P%^lM`I-}ck(Kr8efBH!do!_5xOfU05C~|9^aMpkoWQkY3n-od;-8 z2R?(N(|x7o1CnIjTdoB_XFd$eb!WJ2+xaA@g;1o2NN^v#}O}d29TIGH{I=&xbA-1>?>b1iN z@Y?Wgn*q$!xa1oWyOJr36Dj+6FD)SP6b&^^$L;St!QxS19z_tPq3df!B0&P>O%GS% zb=_F?-XZz)F1P|Nt46Qx-O6r5Bu+vs8^TuvcSh~NPwdaX0DDn z48X2TeJ7DPc8co##xDGL;$F_}ghdD(7D~bKpA4#=e}rbu`4(KDzIUR+feA4+H;eKz z?)P7Ukpsgto5{ge9q407yhskBF{ulhv>P5ebix(ZAdHTmNi7{3hax{f^OhCnt)R2( zvwlV$kms0ph3gu#USW@>`8b9{xH6sSH`uXZB8344`lcGw1@A^7EA$qRL0}MnZHimC5!F1Yau!B6S7g zSOpC=TPWNUiwi-wS|320bjLa{{s}TyrbeHb(hG>#)|#R(;LS(uN}xwJeEJOx{P4wM?nkya)R7yg>M8D#WogFT=mAo>1DqLA|?t1r(_+o>p|Q|rB7l2t4C zQ^@ot8ZS;91dFXYh@N@WjCssIK1-#x>dbxJRf_9tw5h4*N5tMdlvE}sg;_YYU-hYV zaEDgqWaNil zPfZ8d31T8%^=@)NSjbg{@nQ%ybj|dm&+S1IaM~Id?;F;`b%oc|r{2qyL$jVJ5C8}3 zIP?1#cvL`t-prS;_5Ay&lMi5m`N+dfoSJVUQ^xN>I)VV0)oS49N zsDSc>PLxrAH>dBE_>rNpS0zNXY-N8G=a$hp@(|8OocxgrI6IkzU#y$m)R`Gr5{_~b z^D3@!#J6%{KX-MMo;xSvWklod-rnHzZt2vep=i{z_QZp|aLy~eC$AlZ>ii=1vzd_H zN%}n@-bB_g!dy(j^aA7Q#6yb^{r*<_it$fiV*$xN`XCs5;H066xeVWhmD&0(Lvo0$ z){RaA=tb+-wV-Bd13x>el{7h~_>VumqE3z^^-kkfx;1j+_eC}YB5h*xB40QhS z`J7dHCtrjdrM{Y{dy+b2C8)73_4*&^RoOdzXw1dI*y8=6(#R9#3{n1vakg?>@7~+0~u?BUEPzin$clNoK`{~=_S&~m#&hAs1djj|xyPSJPN_#Yl z_blWd&vr$M1`L9Zw8VVs$)ihJYHatDExFA*+O;0apP$~2_N||E-5tJF;kks=9f;^m z%x95d>OH!LKLxQrPn+Q%6kpSA-lq`JNZh(P@m&kt4m<1f_ze&YB!OW)qGixpTO%tL zyG-%xzCg%pcHgrSo@a_kzA#V*;_J2D=nqxTgd`Rshp^v&1NyZrBI1=g5TqbnAkt}e zpuk;{x)i{8Gc%h%JK?%L^CdOqmHPf?Z$j(J{qsU$ripP#PLMDrz;BBN)@H>iDRF0& z^6UA;#UAw=4pv6cairgXLv~2%mBp9NYG?}P&7NzLo(;f6$Raw4^^Ae3ztl-9%QoQ`7fRm1#-*jbW{*Qt+bTb$UCPcO7#ZRolyg6 zAgdaZ*rbqUN6KYrk$^aHPm;0?h3qZEnLHSAO^fr$I4~$=b9A_U;GtXwLg>; z-*vhLx2KNxtKR?LSHjfh7TAN6I7k<@@Gxa2o1g!x_{t!=JIfAeC%!oN+lhn@_DBzu zfF+brp3v50=f{gqG2b(_^&h#EigNh<2MmFEAGTiek0nCs?o z&L;-tYFC5z)eI;$x-TioZb#9O6^0j{6M37MZZDkdu%Z2g$3X%@y5P_hT0QzFilf2@08u&TA^fLDmqDIQoF06HVG*Rd&tGpq0 zM0bqg4YsaQJcQ=sixTt+d6NvFxoKiwucFTy`zZ7DjxNP1F+&rFa<)Sq5Lr%A6$wc! zk>t9__dnWu^JuE~|6hDl8l;d6A%rL@oXm4YY$0Pwrf4#hOv%g`P_Yn&%NKzUF+WUTX%ie_qV?1pL0&_={>z(&*x)$0`fScH+Wse z7l=+2^$nb@kw|;hx7ea8Lg?R8B3ZujN67D*5=vh={NXv!!rHE0tfbqh@922KV)H|| zJr-B(ghDlcD6NLl)!EG@FK%x6z+YN_%q6Oo`?^#%$A~feWNcEnR^`4#irCh<<*S`y_8zqGby66Av+`fv0sx=eBS<$U;dM>!@<9-mi)SPqP}NB0kchF z@h6;wht!re<$M}B-?X^?twqr}3)C!t48nper9a4d{*8Xp8RN7%X6|D<9d7s+Hg4M` z|I<&ynumV@5ygyea#US#;8w!XiLNr2kL)a0%F%g$#~f@A&bD^9&O{Mq?L}*IHRML& zg5AZ=)@-Y+U3-?|6J`0uG4NKQ*dZxn2wi>zmNsazm{QCx2bMtTDAjGZ-KGb1#@_h1 z315AoYXlPPh4sOAfD|`!9j@9&)hA!r0sPE- zB~JM1kFNYv8enf$Z3Kl>r%sTf~puEg@qw4Ilpdzh2lX{OJD;GxJ{{DfYjAJRh8b z|NFac>p?24GXiOncK;06@p=Hykl$D6R9Z1o2oql9!IMN~ULY?@r{j-vn%MEbpg7vN z(oz*uu#YE30)p8N2$v)Xafn~3r0&a?(0rwXX6P?WnMAR%*6{JEZl7vSSjvQUJo`QBPQ9=&)@HG(W)$lfaGuz00wL^x^Yegu)2S&)ql zk+2g(J$3`d+!^rOK|S$7_Vro$Doyv>YH~FGpbKzjDtFj3(e(U-YuhMa#^Li*i2eYB z**?EFptDIsv%d`EX@|lZ=lO2bQ!?)9g42ZZzgSE%4!!amG!*FNvHVsZFL^`kwg7`$;FSa@%M;|LV#Hf;Lx+rnDzf`}kH6 zm}KYY<>jqRVO9|jh^!TF$R4SNbj36f1@p;l49%$o(Nv~x6f2wq|KR+?MuZo(Z9?5q z91@Rf-L#HgM=OY_qu3ZIr98SS!tP7jMuV;~nVR7Px81XoBv}wyho0P6%N?vf)jvT| zQA__N#k*`{&Pz*ayYjO-bGOx!?deC&KWrQSrH9+s+Uy8HeG>V(lxbyyb>aX$4J}`q z){kj@2A%OYi_~KjnHz4x_gqTH*IOi*c@}eR1xuDCP%ot zOAIGNg_BH1!8Qc{B(Py>145EVUxT$n4tmYwC7xoz8#Ro3VB=3TExIH@{B{OKrr2zO z>^vR=kX(^a0Mc4M#24fK5U|YnQi30-dRbD(4Q|}CC?Ty2_>RRx$TuF42zz{i>mH3x zCaP0K>vi^7sF9jpe*Fwx*39K2ROXEupU2IGpR_}S;0|#^M@Wa2*UKp^|gh;v=w8s;GuvmXIQq-;gL{+bWm{NIMn% z^#uOYxhe{?o&E)UH=LpVumt^el_)pH+b|JuZoJKHAwVT7_b7h3|GS*$yDPE(<&?c@ zWj@Y>pU-&%4tNL?;X!p>`c(=4OmdSO8v&uelKu&xIYvjQlIus4Z~_&ao`lG zHR!1j>NN{Gc~M6dNrxg#L%?7$DmMYCiCWO```NWTuDL#1tJe!%QlV}n3x3Omb2_8X zXrD@M>;VB@GvowvjcTBA=AXDb z)x!_`@zn&nCxkpZOriqfcage!4nAzVO1&= zoDPebTz4?T&S=!~a&I`s+5;ED{8R|_2OKJbbBxyKc98PT!s?X<3>qe>z3~2moDu3A z-sC|>j)(c5ub?Nu&N^NMkq{ZM=Xbo)?R;WBBagD2f4ogK07!(YM7CwlI?;r>=q3cv zR$^SpDH>hvyf%R6$j;jMiF%T7LD%|uDen_~u>s=jlmnf^j67hT51){*4<@L$6g|1- zjhSIUP#xL@^hY+x`D{VOp^JO9#c$xK(jb*UqItCw3F}HL6cP`TQSEAows^R; zjB}6e#$TnX#6UQ@-WzQgXpV!+Y)j2FBJ7YQtU#bEH4q=eOgTG4h4=~~S>I35JPLDk z2w{)mS7C({;bV6JWjZ>3?sJm?M_k>Uec zM`p%oA%3T-e*0+xZkk0se{a!sEi=h%-^GF7kZ_o`5*?bg5yIumCPTPL93-Jxh@fr= zPdURX%D03|E%FZGfyUe5$?|`k{haU=gaKrL=9(H+?T~ZDL^_cI48NkTEzk+aVa2%( zq!TNcp@Z~q2MfUVy!`%AV1rfFzY#C-^c3|z};IQlPNk!Vba za~=J3U=EV}Py@dTvarPxsABU^?)n?gkH04mAx^DF2JWaK63CLwJmIEkMb<8NT~GiX zl(@%8-Un30zVGmZJ-uLd1zGIZYsdPdFrfhM4$l&aP1IMeqw`0yj9L$4y8GI9xaaO5 z@`WY|#f?2!7bU^fL4jBMKi+^)u&YpGa$t2l=y&k%a!ike+EVTZJkGclH=MNs66POH z1&x*D$}sf9=|ebRg6h9TJv+qIxBewuYWo~+B~r7LuiCErE!H`>2{}2lbz8U#PI|@86hu zgYMBZAkf`&{_KG86aInLC4v`9)UO;&jAY=$-$#i-juSG5#I9@Te8H6;wVP(Asn7SE zI#lY|1)}(Khy*+5>h%WH=SV`BRvMI<10Im=hbw9B8YbFfII~W%4en6>*(rzD`(tTW zj;ekX21EuT*q*oxc9=OcQ*`If&FBV?<_UEYY;n8OoL|wtJd@)U8dT< zrFlWGCy&AfDOPpTr_w#0!j1P`sbxxVQC7Luz0#?R0m|-~QpuxpAkyS{!4by zn`b-q+n!17`>tv_?AXmaj zuC4iMrxk4_i1t;w{h%(EdSYmL($xJ3HcS(#&#yxKRPcrnaGE{$KK+4J7WLI{cjDQq z{AmmA8kQxiSiL>)HtO8G06`7!=iz?Hox66tAiD|FB(1RJF&q@ev`=)i&haK!-+=KX*%3E0oQKS4;9>j=jLzFnXr4d&+Voz|3-+j9LG#B@7xF~>MT_A^9{Ac%81=y{3otrYXlWW{ddQt z5P`be(*BQ4lzZeOP$E?@A>h$R$6}y;zHDMk_d_+s_0RHPt(1L^bZ&3HY>PG!^ z13LjF`EC%qQVjeeKBXACiY=|}>~Kp=Q6e-93W<=}gIUDT=ICNdVf{CZ%tBd;a@{w? z9Q|!mZ0%IHfe?y{qC)S5U@R3)#fLUEN?BeB)&nwzNkTAM!tgZ&6rj+G<0VO-G&vi@ zO}h)Pywd__#%n5 zs9cc(3>2_0D+Ao}AUHP6squxo`v5>m0uMou*-lw-vt^)YYZsVk+h!l)hgw+!Hgp|O z(t%~j^2&JQDF*xLZVa|soC2dnTrjX^Etl2lda*P{5CY**tp(nM(!Bow0WoKyE z3JNBBQ_MFrzX&?Ef}&q{Z=S750j$O&8GohTsecB*Izwn@EF(=9#FEr<@guNDXfq5P zrd9k88>?Ya0m#y%+-W1n69LjJ_BH>vZFT4%hGg1{ghl3aLAbr%;palT%%eQ5HN!`8 z?`CuPu?@Spo6eM2_$_D`SpGp$1NNV2s6!rP z@}b$c1rdDq=ute5S^d^82t54|CtAlX4n{jXHBmYPVZ;HyuB$<jiC+DvTVa?l$cM@c@leniu2Z#US`4;e>MOZ^NZ zsL!fUYdVp7XaewU<27$1x?jLnFHT|4_VvZ(Gv#>*4B)H;fDgDGy%6#TbTr?2!M(%y z)Mg6)`Yb@_S#Cc?t^*{)vS_{qB=ux{iUmZYwxS#nv67Pz7Kmc;Q=W}LKu|j&I5nrq ziy(WEK%+DnBe6PysNx>uA)q==Kr5e+F+|-?vr{(ur$8YqH~bnMI94){F-nEvFaQr{ z?V%~Y_-hk<*c}Xo?<$*z85em@KR}Sf>ZaUs-$^cLpzX_zSrh+msISj> z+d{1y+B>WU|7^^`e+CfzHzNq^%71c^@}FBG{pWo9r=lm=!1*uP{HvRCV{=#S93dlt zUKJm=z2a0fRPV&Um@3zXj-C2@=fC|W4IjSf|5erazn&6KBf{sGj@|9hnzX|zilZv2 zqZa_^%K%^SiBG?~035=ZnP3ylL|CV&l6FfPS_DyFY`FTFTIV|h4>S1@V#Z+=a--%c z-~c;RJIw;TNZq~XhFb(8AR0XSg*&tiNL23YZiZ%sDv1ZIN*8AdQNcaX_)7mPwVHc) zJ$28xnsScW6tmRjb;=aWs-)2 zAaHopKr>?fsANRyii)Iy&@ zBcxn+yeiB3>pP%()BOl#(??N}{tWHjb(L5+bt4U2Ley}J180ENsP21UKO2VTMa!;e zy`a|rZ3gg4$Oai)hJ$=r>-L!TX-FP9l@0%$3d&$iD5DATtUc|y!Yd$ikjw~@{J<^X+!L2;S$%gvBE$3?%|D6u~ z)0obFow;XGv0zr9;+@ecroLoGfz#`38ETt7Nn^z<3CTEqyVW+C>=^%6lm**x^iIku zj^UeG1%u~l7?!&g9VXQm<;Bm#`(+OO9x6?4JKM)Ua}TP-)*_5*Z{H9Wd24wV#7wBvj;+ab^{0Hs=~EBLdIW zN*Xd$X$O*psSWCgLFzO3M_vibJD8?p`)6GweEgTFu>YMg=szrt)CKpEJg^`d7%_n3 zY1%+s_5j>HLhLy2NGOnA^^t`qMl~Z`k2NM5+E&cKtZmMV=+lR*>-|*|F;vJ96eewO ze;BV}AXhvT_KDnUz~!7-EP=YDqZe`GJs3zLVYLAwxklPCK?eiV5jc~C?1kweb`z@r zs{Ix~*#o#VMF8Kg8^z*Z$I|f)s<`Y~N5X~g9}8yh@fgk#4sGi$R87Q0>$FF4kxl%Hf4Ev8% zv|jXFx#f#94biu<3RRe!&!tMWO@sh>mI{s+U7D|gp6@g|c22)eK) zu!;fpIQ6XvQ0u|uAq4&$8QX1WRUb9sb76}r670q>2eUpI@BbOGTM{37aI6qMcZod0 zsz*<8{=&J4cU9PFEZlZeL$7hT3!~OvL8|ip#DvwRovJ@Vn*Lc zg8^zd|Ar@2yxNDRI;&b&PDmMFzx(%F*mYI9;f}1rp;!YAU@Dp7-d4-V2jrEVZxdYT zQsy?v>Wf!DG{}SAX~pbA_Yy=49Ky)F#qOzwejwYEaDd7RB>?qLyAZ_2yzk^iYF<8K zx1#Hi(C>*m5rEIY&bAp{qTo`pL;fOlU~I_)B23iO~3m>LK03025&Nw1no^G z$OV*;W|b3Ukoe#Z5how2=#u~p!76K-mj`$Cl|bT(9`j5%2)PwKmPKf2G72trF;~^} zvAGSMPSgMWA9^B@mZR!OP*?(D+6l4W36*R-6b_bdRm%R z`)E3|T*vCz8)O&(bSPZuh8xHVu=&L4A$O3C)+dXVNsG{+5Ja(EX6-YJOGy)$hp7`Q zFmO3}RKIs?y9%wURoLL1DBX3|8Zlp-Yl$V;+8!tDWT|6xQMVy)kcGxnH zKX4wh;XBTst8LAd4g={UZH%h7hiO8Q2bU^wrh6g+nFR0(tPynY#bEDv)J zMqn^9DxggKqPKCn2Tgx`)k4i*gMndwBwp9P_{Lv9UJup3&6?g|^>r=J;)jvqlm~A^ zALHcIJs!VHQOVD3mJ+l2L@x_mnOeHk#7^Zp69GY#I|FY`v_*l1IYm{7cVnQ?-h|yB5Y#FZ16&mgwQqEOGz^i`Iy&g#IEbhIYU5t1}vq51qhh^lzVLb0G0?`QZb4| zl(xSZYK>Ei(r|O#_Zz;sih%%&a$p0_Fv~Wb01;7>-^0-QsI2yIm3pj`fv4DFNj< z3tyJu!yDSh)PlM&T#v%Rl04455SwAQ*QZ=VoFl+urFRg(n!MSKkC|NDNP8bL?FvXu zG3=Oh1bWD$G=u|YenII1&UUZkDLl*s+G2wx8oqZgIg+?4)+2NxR~=4ii9b@-YPrd# zwP&FQ$B)P{0FkXU^U#lK$$iRRwZ*#WHfRm=S{V<5b0V0r`gh;P_~?ZILEr0(u=37W z3LRG|BIqeFH`Rz;HbxR2rJv15ogwOWcx9p~QDI1=eBoB4C7Zx-?%uV%EN$}DVxjtE zeFJ*h{zJZi9yL)WZa%q#2HW&bjpvz{xSzpD&NRS1a%**`2-+({GhxQprcU#^T)^cP z6N>7R2U@a~);?};8>@c*@Ksdva+r#}MkTx7D+(Z(BCBSW+rOcGvK1Yderzh>s%ZtV z>Y<-dRMXq3kfem|eONqhef&7=X&?9Bs!Dnr#F^1*Igj?aFHi>Pe*!JhGq6f$je$*b zpBJ3SdK}&L37b2AgJl8PrDu z4Y+x{$pbw3Jk&mJi!)gJec2ttZGdsfD{{Qa^TzXXAi;YEx+u{KUnp@!a8IqrE$tc9 zrm+b4o^O00BX$PuhbUQicD5?mW0@etuWTvm+Z7C2k-Hd5d>&X?%SD%|L+&8HXh8F^ zR`Pr=l)n2$fs+4%UCgyV*;qJ3rx~N4=KCwI`3KtyE z(TP3nM%0<-<=ap%)M4yIXC^ zU~@2w$uIrZ*^TLP8P>}vv>Cw>W(OU>>uym{#O)uN`}Kg5$nq0NMqK&AZTwyT9=ox( zgy8-`=Q60wM5G9zDP@=`i8YbS+#I@}LT2J#d;tic2or2d($IITC}WR7Wyi)2-S<~J z{~q8zmvl{|95rKi?cIg2_8Teq?I5tQn}12&E;cx@1;x8-B?=-7)^>>ohoTmCEy=33m5e)1g<6%C^(iaa?#`UJy>ma{nj$NBj%bwgQQDCV( z!eh=O&=W4V#y=ZgtCimT3kC$FzFBtkzDCk}vM?a-OT4A>@vb6dV|rXSSur@xxq76uI4gzc4OjY&#=WgxW-N$$g2y2%l1!-8en&*dqSr`-tY$9}odl=f(2Xi!W zHO}_Alu344YWFiG*QHp`L?o08`wB%p8G``7=C&z{KL%h^4lDcCiqQp4ZM>II-V0A` zlg?p&!YzJ(EKnSU*DkRb&Z(`t!!g3eubKCG^`&$Wo$8NCE=ftNDV4#6WY-lvCYHH1 zxOL>R&H#YxVysaUd8onGT zuiL=z-p}J!2~kT$QEk2_fj39!or-^GxX1kCXaV`>8pY;DlQn{q5{q6n7r3W9cJ;X^ z`QCd2wQ3upErF%5ILct&Fc}^JZLv3#M{xREy-HhZord-%-eyaXRC;t(LP@E7DvTtS z24WzyB*8UGvAgiclW`sWBug%}<5!a|ncMCZ+1AUnEsH7P^y9tjZCmSU-nPEm$^DUu zNsnQtpyAVKRVS^8r%X~VCj4RhooFTR6{*rmsJFK_`T5LK=DEAQWZ4fsa#8|_VDyQX z+`RUC=j8b>&Az`cye$Q-P&IoU=|rv4{R!;G0z}hEC*HhY zRZ3EV7anZ{WPLmPR-?z0_qmyStb?j8b#!zjHx1IJ?wI&?kS;o>F_U%v8ceE7_%U->&)GT(_$(=?gZ!vZdg>(@Z7of_87y1Lzwf(JppR^Ho84v6`cjCDsfa>w{Z8~GL0oaNw^ZTbd&CF^N1-nPTew^McrqUD)6 z_N&7r%j*)cDrL>R&SlNbwgNY|dYVq@F;(hYNA8SK{HZA}!MpF;P{o6V7*1tTLOMNZ zqjOF|!>Mtnyj&n|TKdVjPy0>=@}2{+OxUg(*CKy<<%686DvWF{MnShf*wCE*YAHh8 zLostTgZra}5>+Fp;dlDnv?8dZZmA0>grNlB%lwt}j^jZ z252<0zua3|Z&!k}tDsN$X_~#p9MRq$l4io2HTGn zFG^ewRR2CpZnJsG$W3RHqQQ6@wm8`<{rtOQSA@9*gq$|9MenC4u9vp|v>zv=n)!bR z_k$Dn?gM%r-sI)DG&{BZf+@#3=rk+SMSc}$GicB6iOaNkS$*;D=!}--Nw#1HOa8@p z&o|G8QzzOU%DMDy66@$&d3fTdlb>|-=doUeI+gxXK_BVi0xgx3X%S1#9qtZap9eJC zJ}@fLSrJ|&)iULC%H|)9cAWlMS1j@(bOOcCpFoYVz}@q_&fbkY)4cwX(b-n32=#Eb znq*?RQ(AZpUx$wsDM~8}0#|BV0u+}NT4ro}-vvz#?K8}%c4xcIJ<(UlEnua8-%}=C zBpM1oWd)dxA<&y}b>&ue%>E0DtFYs+6g;qeG`R7Orh!x{ujuN%rTGb7zl@jLjduys z+-pD6t!AawRLFcZeUsmyOTdL+9=7X3f_SoMck3tg$;a@Cfn?X@YHN7_odBsulP#`l zh;gF%)j?u%b&dD5gU>g`+m-h4s0q%<607vP+oQxNE7flF1Rk&#e4gCMCC@NqBhI4q zGJ9C4+_1y^R0H4VZzpmXcxd887#e-aqol_e7;%%n9BTQ!3g88QoVigpgHCIeJWjeA zg$l(S-PVb{^O>_102&smrk&t5_7a>Prcpn!cj@NUS^peW&P_2!f`{L)FH}6JX2AG% z<(zm_0I^m0W^uI{n?sSUl|V3kB#5Wft#5zYYu~X`sWLyNvF&qykB1p-hZ>zt$ayj{# zXAHC^M}2OZlTk8331Z*Wj~$(c$rE6AsFu8@OS#JXWSO`oF&SEaC0LD^vyF>&?5UXX zj>v9<{FlxH^{+Pd9Ib$W(c)iKv{661A#Y*3B{^#+@u;WTR80`g>wt822^9K3xc^fJ zNDRz$n<9&Z5E$BL^sj+52?bU<1DhJ>iaK_^a1?F713=tGhgg+C0J$^T(VJu zJayHrH5R43ZG7ZqHLsgD0{AdsZg%cWyn2qHu|lw$xKI4aE!Ahf3c~K8%zS2sot9>gA|n)~wkMy-uizL^=H@!y5!f@_$-6cg!m;5<5PgS;cKpwT zdxExet@=+}?Mh%mxq7;k(*s#;+S_M!s;_SM2X?|@h*HGh{+RQjdN&$TxJUHuruu3M zHPgxoYwD8Gg+>`y)M!xeS=69IIv~5%{KW-SI^DZAApTaLO6}1*zf^#@C@A?(# z>#dU+J5py+@9J2&XJ)%Uyy zuhppc!E5`!l%z#2ukSdosD6ZHgO0pDwdNb|Z%Zk;@+Gh(UW_EV&v7MXZ1_kZlR9&tzQ ztdt{u$g+jcu3Gi6hilx`{naPS@e6Ragpmi1!{x5zRugZ7otjeh2$4JNu>4gv^?G*M zdx9Ar4g-gW5;n&qt%mlpmCavzP}z5TghAxh&bVrVuy%s@?sCHe&y)*eM}A=(?U|=m zhXr?yh7A#Y-@SG+kG5+domjf)k{9>03j`Nz9p8bVw~z+<=M~$KLfP|0rNUSm9>I-r zK}WfGp44>ih-v6udbqE==bUa>=Rn#})4bqaBOh&o78gfoNzr>P-VfU5+HLN~S={Fm zw|xSXX5pTig=g1_yU6;}ms-Ck`uYQtervE2(2%G#!)b4VTkzi|>;2-k*~ETJ z!Nuv-gz1~j`vq;^szzm@#-%CM-2M>5jcQ(>On6Ha+08ugkT@Ayy@=2e;113()IN}P zvYyq^IXWpjxSTlQEo}eg+4-Wuln7n23x#~bFRYO_lA#-5wb~2BWI3oFwhA8Dj?=q8 zm!$GX9p*%c@p*yY>ZsF>{W@RG<3&`XZyMcuntC13hw97er0c{5Cx(c+a@56NgKF{| zLd$^s7pON^++ArnwO7laI3ubk28W*t&(36O?$U}FDk?xEGL`EvHU zirMQM{Kc}$Nr1Q3edxVMAub&hZ#iL_(PA@UUZG5?!_E5$3yj3()RvK@8>TzlBV1)v+T|;O(5J= zw(;hhp=s2$@(qbp=WqOBTi;Ye7wbPu*iEB3z+-atuio(RP`B()Rx*F-aHS3z`iN+> zj!u^AtPpdKg|mxA)`dyB481zIYC!xNTdUSLgI7BbKsx;zc&;XfQ|r~b0Ynm8QrQDz zxWi8}YM;-PT9-^`A65vG<>S|YJ*^$R`jY{A)+v@+o7rwg zo{1Tc4*)I1uK+AwpNZJInBm^)^_Ce=8F`fzUxRhUuK2trDKvC$F8`A(bfEpqVA8d(fGZ`Sv}${JZ+6u zokTK#S1gokPL`|x2B8!Sa=r42uXO2pV0?ywy-g6*5JkUxp9gMT1s3@NI)SQvpEN+X zwl)@Dv2vb!73TQae|dX+K-&4{vXL;_FYcuMbfMS%9cv^;M;sjkKgB_BAVqS^n!$6i z1t&jXqG!!F3y76_hpT;7cJ{sx$tpk-d*F6X!eJ<*b7GQ!VZQ@%pwOauH?gA+xS#zD zT+;jd#HJK>M`D&ljS8+3PRUi<6!In^a4q$dOP*QDGbFJWR&Q)(>2t9rx26Zd9;0yO zfuya$RB`mm#X3|%jV_`P`7==B{e_sdn*!pipimZJw&eZ_n2j#t7jo~=#oD3w0DK#M z1`MLV5HsL=0lcHMubm+p9My1WB$>Ve@p};TxGz7=xdS!Jue;q55Y47kD?gasu$w09O+QJo*=EE<j~ggS`^YqoSZIK zfTyt51`Gjx#yqDu+q&UrzJsGT;{fYg@3-!?Ybj75PCW#jg9bQDNst#NtERZ<#y&Mie_vt@Kz$Gp*F z+Q1Qp4HB2(hw86t3~cD|Hww}FL2|N;Y{gi}mWpg>O~1&LPeqtElg1TEp!Z^p<<$N= ztyy$k^6(xh;5HZJbSyHR1&kf9`fZ35-}#qRP{ps((Z_y##=S;vegmSG`^rz2N8c78 zaWf9O1b$*PN}t?`hBgG~YJP8zHmbWh+aPn`L^Rd3Zv_T1FuHkw0nz4^_g;S4{kZ3q+|#Lt~u)yZDc7#NcXJw z6BpiJEBo%Xbxl|vp!O~AWqybq7?E8IhvmnzhBOYMk}pvBF!7FY@9OzGbq70_nH47g zIlVpNm|aZ%cJ~BZoYzEj+6n9Prh=`Gj(ydkQ!|hZxg%V0X|iyiOEt4b_4xh|c;h?3 z#&oMoXLFr`P}ZTMb=-`(@DQx6WvZ;&a&ujfc;C{C11X`p`_VUJwKPj^>5L+^x#WQD zx`s5hmLPvGO~B#J#f&X3)$<_cyP3utdgP<2H9S{NMxRs9M(1pnIAp#qMpmF?o}{uS zVST{B>&6Y6VrXw{f^YDgd}6keAg?n z2ddsD#fJlRe73ur4Gm4iROkxp%K{g;pv@;&=i>Z5zWMfD@w;hmXKD3JBig+Ly4z(O z;ePu9m1t__#@D{_X3oU#VoE~EPqBgZ9+J-Z+vyH8=2{W$T=e^#-qXft9Zp-{bDzzL zJg2pMGhrnr%~+7yb^#1QdcqQn~=`^9jv?H zSve!vf1P{vp~+;6tS>-=VVylvp{s%f59OTAr*F6JO^V>lSzFkD*2x`6KsJ>_S~lL~ zb}Q*<-`~%Z)mc2FLyrbC)al9%+%=sz1!t?Xnc)(@ga&Os2u=$_k8ZyLwvI1jU!Dsb z6J}_8BqJKzZM3lZ%e^A}mF|c1>4|p36A>hr$3n*5FnAX{!Z6j(yxRH65RBE_{$MG+ z!@K8DpvB+bu~&ZHwa7dq9@faMOW)#tJK~|MwPy6E5UA#F&Av@M@z}}CKYSgViP4U+C|Y2H7MQRjahwr?^1nt;dFoD&I? zb3h`KbZy8IJ?*nfQr;VztAf@U@5TUk?dZFL9Zwhz+hweQX-r*FSb{XTt%F;gXDybg*9tQgKF!2!+e#BLOp4;Qlm+iIJplYsBpk@M zc3i8Kk}~B5X}$D7OotDSAt!nk>c+jdb{-P4_6;kP^l=1z8G}QgZ#qZy;aiv6GIV;u zscD*S>wUy#0k*~`&I$!pUiXx|t}$$_6J_t-L!6s|PW9M#U$~<7KYVLgPpN&splXoq z`r|u5Rqr5l^Q1|kwfdJKp$muhibRnnKLdX7W#K^f;pen^A~vvdeF5%<%hKq|)8p@% z37iwHQT@=`85>QtRE4HiKvZU!^C}#!czIT9YFMC!fOnyNg4WuXss4FxjaLve|4i9u zNo~d|M<%Pie-6#4^j zpe#IT%XwmdX6fzDo;hC_*v#&AJUxdchcP{dK1#c8 zFtED`nt9W_u>@!$a6=Yb`vrM?BHp8Cx`TDl5!%MM3BXhZ2yeWDm?UR^Z$8Bn@c z5krT|otZG*p%YkDr1OfDPisULT3#QKE;V25Ar-&nY;{h0}MYy zR%5E^GVB?-a|%hxCXFtbX~mq%e!$e|I@Cj{B5we%)mJJ9wF(=AR-DW|B1rbTX_72b z`j*VuC)GKyXp;162Kp9OG?l+P0qDHI6_z&2bbQ4Ql(k&~Aj>S17cww{U{6aMvyZdr zKW=c;hVGG3EjAJnUu~~pCwzJn#Al`$=fOrxbj$x zPifuZz~9bFD#HrzA_dE6ZDmhPWZmd81R;r|QCM0=l~Jyq@sSvJ&)a%#NBm-74)Ou$ z+0rRV(K!Y=_XXArq;yWGEXhi=2lDvM{#ZA4Hz{}4)f4n%;XX3}X1MZxJQPN1IUc5* zm(93iK3)DCgi|udkO#CFGIp^L~{Wm=OJh8sn&30h} zo6Aa}?yOB{^2H8s*2=RigXby3b<`hO^KHufyWXjGe2p zzvcN&lv6ziPL4vM*jCJ@Yc3T&O*=d_!dZLVp!gB%5jgJx*PiM1kHT+^4F2VPXGhu| z+NrqRJPeNfGz&0Gt9E=qD6_3E;KCFLDcKU-@2qv@v<#7PkF219Xf0hd&qg zZ&>2eY|SLW;ckfKvf8b&r^nHE=E3lcS~BPb?};2sX7UX9SP`|JFH{cixI#V$*qz4s zo^2Z75A@aBo7mS-qcL(=%Bio~M70SxPPUBfX1?%-l^XgOp~dJlJH0C@ag<#?XTZzH zOMHb{csY;B-5OF|co@SW^%J2a?PC{N(7hf-Uk>~7F>fq;?UfwEIzZw9z6TWB(NX`Rqwxo>nDC4UELk=x2;4Jh*i7X2($B@xX8-9-pyl5kSkT;ZCi z!_Z8ckeW+~g$p{orz0Rpph^ibE6o+BmH7Tw`+2hIciWPJ2ps~F41<6B2km9&lJvY- zsR4aps;xMQ-e8YrN%#h~+hc!CuZ+nXn7_A<9Z_M+vh~+A-(p3~k<7MT9@Q~Tcb&Q^vct4q9v52ckDTpq_VYJdCkOl(fs_$3FFzt*ar#KQ6xW{WUdhYxpWpio*J(Kl3ynmNHF!b% z0Kc3jt4y4jC6D1JGR^y11}^z($DCq0wv+es`lYU0e# zNUVh{w#N5`+mz0dmj`7k?lmSUDR}h+Y~-g&up}64KGJ+UZI)LcAP#o;aXEtz1Go~#w{gRMU!E&hzZ0s^>&WwU9ksvZ0 zPQSn5kiKmra{;9D)LP$uwN_l(Dx`xgM9l=~X9BtA_J> z`VM(C9>{L>SZ|WEj;K%CDSyv#037#xmHy}z`?-$a5kM=6 z{5ux;Mw^_!hpSomifh2kI_viv$Jsu=CmP=*YH9uzlo-TcSFzHrNi?X6lkT@|t{Ng- z_PRL_C7kIJ5FDSgdSdg8*eLcN_g{ywKPNT+pglfEWtD%!;w=0c6qWt&4;qWWl?b7* z?y*(6Vn#`2w;|5pk|#^WeRXqR|%ILS&jg6k2Y%(#OsmL;Th9JeGzRg5))4 z7aLBoK6`Id6)e5Q$adyp8NjtQMOhm_2FqeNbqs2*EMvxxN4Nm7)H^pcpmW0L4tsG8ilnMDYD3i@MX7X`XSt`Ivu?B;9b zyqZ#3tsz3`DWoATIWXDFy;z2Ih;^gqU26^v*j}H5%ji5#cKrlqEzLe0IpqbOCMsm> zC+xX=XC+l0IPqisdd0Gzkakbm*BMODBax` zA{i&FV4+F9AJ}gOGMKXt?PH+Y{{nhhR;6|E2hBN2KElw@lqEwx()Ji%+~qXIKe2w> zLyS(-I#2gVkZ32)QA;B<-}*pIrFb4kHsCC(1D55A(SBfxR(DF+xcLqE5iX)UO#6JY zg8P;4$E?Mhee>Zre&hB{D;iYq5 zGPFDWC-3GX+x?ze=EA|`+}W&3jBbBq2I*c?KCoMV0i*SH6##vfzW}$=cT2hIy#OUgLf8A2F3h{I2R#ZR z@D3myn(8LvFWu-x{iQ*6WvCY5f)M*p;O}&R{r*+}kI_74*f^<~z$*<_fnX&@b3UZo z!gdZZH3qtS@@{rc1kFBiM);`gHNfiOn5nbB(NOLH4CHB`&2dh+I;$1=Bfg55JH*5U z2grw7c}&L`ulc6EhJRt-Opiy-ho?B$-vJuUFVTCBOV5701E)p#B{wZUe7Rr-!WI%s6Yc(cj@NN1sS)+M`gE(a3*9Qt0O)C{z`*l-OMr z=a}C^B<^x+yNwI-3|GFqmOI13V_}nifqOOOR#t}6HEF!Wri4XLinL$YN2Ta~w=O*% z^B!phZ9@jN9sfD?Ije?BuGi5q6>>qC*|rKR@vfb$b8o`gLNF{j-{yBPgQ%(*yMXyIn|o^uo$m-laF)?-Dy{o=;{{ z|At&CQT(JwMR0KNcx$x!;S-d!ckkZK5y0bknB<}u7-hmjLRggFL`O%%hq&kZFA|r9 z(fVu5r+drL7?*+ix91soc{L7`Qe8cC?A&Q}b@lDIi?=MVlAOBy?#W=0)xfgrsZ*zP zdjwn-zy7R!&d9*P&>tT_FY*2>bG-XUf5jEmoM2%u&zYg};)90|H>|s}Yvzk(sA5vu z!dX+D7e)y&J~oW#Z322@v4xXSzsh}dboAqz)o?{=jF1O^zw2 zpE-TnpzvK!<%2-x3;ZsNNjsI!YePd(7X;MxNnA z^&GuI=L%N!KZ$y&aIV&JRVhhQc*2-Q2yC&#qk* zQT%7Hl$%XGTZf{>*3RzOz^BK-LZ<8UzaBa|I#R4*^eSC@m!{g-yH{X_Jk)nzc!@Q_ z(wj7gJjAHyykB)k&vewAUx?*LcWlxKx%0AFbZ1hmNd+^>_5Mhd$kN%!&4s>wY4SDT zS7^^_u)NSwx7FX8FUG327G;(;o!93Ru#4DRh5r4;^6^4sMSe8GJE1w4%PvckC(E4| zKGx@Z5oVY%J$F(#^sA`UK8pE1!atn3h*?|8FzQH^r&)y;(QqGH{WE~~U{%dHx7xy4 z{en?GZqV>ThBZw8hs8D{q}SB5W9mJ++WLO?6(oAZ!u;^*8yL`LqetVq)bNC|9+s_b zWc@t+{{6dJ+Z-l4z3jHLo127o_{-zxnZCMFg0&?>R;QGt1CasDgl}FyVL{xo+}+vc z4^XI8YKs$Wy{itZ=}Vk|i{Xm@l+%bu50y@2Gq1~{ac^q8r#Z*2*Y0L~J^v@`q5cZ_ zmN53#Mw&a0vb%S)aRJ*0zEMX+pg*);PbiZQ3} z3fc@;xb)5rmA9g|=f9k4ok$a@Kcmc>g6qoGy+fTpRAPHiSY=v0MfO7FP<=h)@AZYT zA90*$PnEB!xF_RZ19QvJ_$Yj?anqT~YS+2y9Rh~*dbWP`xE`lbuKh(yG%`+ZjrEhNzm0H!pFx)=)p^&1v1odrCZMycQCw1?)TU`_|vkwsa`u9 z5+wCFr!Orn*P8R_#ot}8E-H(|cV~4rrZ1%ubCahTBUrM!_4yL``_rC;F+C|xCodyxog92^1Ght@v18tpB^c` z%bro!_b?e=?K6vi`DdV*^LKxt1;GOoFM+nKbX{4V6BDiQAwmV5MhtntiwahhPLXBg z#wl%KVd1xp{uj6#D%z5yf-`p(GB4kIbs|sW72%7nl{%e{G47OA=K|P=y!x>(BwRlZ zd9zCLp>Zl99x5LU`x|R(@9rwvNP(eS5t^&s^$f8B^Lx|hXvY1IjI0J%^{Xm|C0@GZ z@ay06A@$1_ym|8`o5XvnNmtXBFgX&guxFQ)`BLTM-{)6?fpd*M7B4C)%GQl;4q1?YRYxGZ~N_@tpjHv=|qH%Jv>)uP{Ky8gzrz){LYNyC0#Zx{rceX{B z4d=MVK09<=%Q7~_b5_p8;ZHx$Z#jMD&h7hxDh4id!-5!-N?cUNlLuKgU1^G$xbV}n z4^b!vm4n{!f4ZDwD7mawmD`FkcYV_D81HOtWE~bv5sp(;FkzWiP{B}l()4C%n>w?G zDBxA6KWe3Fer#5qXkK2u#Jf-ZUf#q-0Qn+C|DrSF`f{%*p5J$JxP^9Hh8Fp|{ zwovTtpAEv+Qd^>xZ(5mP)U#MV`$-J}8uQIaZnMm8<=C^znAD?ZnZ$Z;IsVqD)I(oW z%_5~&&MJ`AeV{6nSMsPkGZJgD%1`ux|8~mh9B{>X{A!iwJMkG$`Y$SDuA@m+-rljO zjY)o@rdev2>_S6+Sj*^GgAidl@#?_m>hItVGsAwA*cw&(nDJe z-9$$VvwWObQHkak; zK=1i ze3Ge}T7rX4`ba8-Lo=I7;1-f(!Z}?-^Y;&EdH)NI&t|hN<{SjaS;^igZ6<) zvGZqMj$Ga;nb#v1d6Bw7lRNHh+ zvMV?;HBQxQ;>u}hF#;}1q}2RL@x>NB9L=^FsyZ+|&}m6psV6C~i@=2`&(D+hrp42( zC}f=oHEWA&icWe)7aVc;L!ejH$mE=uPPVOc-p0^T{hr${v7bqdZ4z za>`9xFY2bQjd(3O|S;Ul3b=-h1w|=2=fS%LAd)X-`BCsb$1yHs*)%%lg9XqN)dBW%t`6xg|`$ zMI@9bVY0E%%^kY^sx7kE*i8s2I?A2quFf&u(B(v-hAFg=n4LN(qG9lk?P%S#w%05S z3cB+Z6&1tNeQr~+PKGy02cs`Da};$&JcSU$zoY&p9T#+;mSei@kumAQIxat&)CQAH zCoVoD9MwzJ^4B!qK#zYVt5gxz?-ph3bhccmy=kqtURI|9$%N6$9kLxtZN(vHRSO}D z9Ri;7ExK?8o!u3?=u5Wuu`B#tG=ye&y)4r2IXge~3(Z4YbF};}@+{jm1u@=;N~%$l zJu8}K9Rp#vS6064khkh_HlDk$${wWHF0!F_Rb_=`a*ic;ji0=6!HI*TtF15{t3Pg? z_#<5*;hA1WS!5ZXwqX?IJ@h4rE;DFpIRbM`1vtywi_F7fi2aHkUdqSys)}v!JY`y~7u~FU- zcbp@XmNkTlGGYbMgyk7RDULXcsV0|arQ9eTlZ_K9cbHMKkBEk(=%9V9;ZnJwIT}OU zGtceJ{Nmrd6RszR6AW=jn}q*mb{fz1gv;Es5U!N9#KgzP^DQflHPtuLv_aSxM!~K% zJ=K{a*Gcp)g8%+yq~&a1Yk zXf^p!@hM4%T&0tXzEvnCsM>NByC{clQ_U*wk}dh-b1GbTcS~7+K)cv_29Igbwt%#2 z_LWtZwtc>FQxY2AJ*j4s5~gAv+i&WNdtiR-ao52!IJAnl8{}&q72r8Mx2H`#J*mwr zL|3>IwY)s$Yf1RODj7AWuVsJ!DCo92J)2~Hcxax>q?zG4$*Jx(NTAa1eLs2W*7TT4 zOCzKMGu|t(D9VoGAT(w!O8KmTy)+v9V&aPyWV}zCWtVmSD2wZbP}s!k0XmarlS(k4M`kjTh|4Bl}Aa$N61ss zM5F{W6{lL&RHjq4&^#91vOfH*B{yG&hgIPEGs27##r@v2-yhIg)4%^xnw3`dVYaz$ zZmG>hl zb>Fay_TNoSTUh978$632;uiHYr0J;tn)#%J)?BlaFLF(8~RFj={Lii-Ec0lF1E$ZRsbinvX`rH%M!JU5>(gUl_=qulXN2RpSY zUxs#WSrx9*YkQ@~Zt~lOrS;IGcT*q4F3$x-mGa4iu@T%cVX>^|REG4k{FEb3X9s^2 z6({Gy;6ot1_sGT;D)q=7%FSpgEIXC%#k}8Ffw~cdHz!djS~A74y!7h|Cr7=Orxmr; zNpkskJX3Hu-E<{J>K#B^UPvHbGaxOk>6XfQ@%w&RBq$W>Cl&#!M4o*#J9bkwP2sM) zhliD}F8qeJb`S45`aUXMO)fj(a#)%a>(=)6%0Xx%3Z>_VSZao=8;cX2+~{oGa<%s~ z!W_44hC{$i%=DpE=X*XqR^EVQZ{fYZCz4cDRCV9KKX`TG!lKrM^^H$S5bor;wa`em6U)|z zH;G5$h1}fhh~-IE%hH-$;o#ulvY(W7K`_f}HOi?EW~^)Xf9uW85J}-}(*X~6zz8Qp zMyyy=z4ZN#Qnn7?wmT#_HkF=#uRKa*Ko!GhFMe6jH7P!vO|xTEC}kxBbAA77gKG!9 zZJ+%5e5lgF-ac3*T}gA0S-Z%RY!%^8;IO1?dPlU*bclompA2>j>!Fk)n;s4b-woDI zIB-RPvd&z1H8)s#7LVk9aM1p8g-)|*QzfG&7OC+*ocYHH&q~?f9+haNd+2A@PrWnM zCE>IzzdYeHo;H2|Z^}#1%!smWA@}umoY1Mu0_`nb>B^01-@LuOA8C`MKo%Cpb6*LK z-#DqQUfaH|E2dOi$>3?rc~|N4Hhc=-{rh)b?7H=Ksy_X&eyw-DSzC5E)*+pLk6eueWwPpQz}JD_ST=uqzt0Eh)X=Dnwo*Ur|} z0Z!dA2E>WBUE(occ2)noJe*G+^v<~sfFm-`J|P)gk>@tSH#RnQYq?WC-|>&u5&;~G zRO5T<3CKb_lBCXh2<$(^;F^@_nE=3)q#(_2-G!%A=sk$OCFi-j?X*OIxKtWqjFlyb z9-APD115n@u~}Kyt?}Ij`bxL8Fu=xSlBGi#lv8B$jT%Wy9sfklG6?{fLR5VcCL1f9ER9dSVe88k(cWkH5v4+iD+-6!MjMw{>z&{{7H*?A;(Z8vggrJ zw0T!h2fa01Z?ai@;(*@P$AQZzRM8%ef%MUTX%8X8ygyLA-H8u1&te&$%y_kS zt4gZulbEz*WB!gP5r=8w6Z@C_#{t%Vp;!9vNB;-$f)oczr{qb(`ue(L|EI_GSgb1W zi&r483j(~np)Awq6ixH|uP@2QO+i8YXmhDJfl@fVkL%95c1voojo}oI|@v*2k7LbD-QQCsVul0R^WX8-Q$~qA!m%0a`GD zWI>oVfLhhsiF;llr;k_Ji6lX_qI7;Eo&Zrc{O8F z?YW%eEeMEwhD0ldq=Ny&=J%r z8llYajAXs{RD5i+iA;+}s%;$1u^GwXKpThLahetDO%bgFlxX_sU4Te-qY9BM431F_ zDOqAWBp6s;T+A=?Hz%U>9GOnbt_D=D(`dKyZ-BAHKx&VxFk#pujnbYcfU!55Ma{VQP3`KRUL$KKSpX(n)`LNA4;B>i+znZ3HcZF?$qzd`d6wDkBygP0R^2T@emHYF~yC;-^xCYRjnF-*| zvK}nj{;VOYWYeo~{xqRm<=(e7sP*=4=Qxe{#o|-@Bjzg)vqkm17M4dHuu;#DKQKAE z*ip(!l5~X@J@1Pu=7ZgS50wKQ0qpUQ7An3oJ>C`jKfNBtK!##9FVZuqdrf9H-A=dU zie=4(XprCnxZ^kWwGbqJsdC>)f|*Cwytuf?Me+s^tB|+nImLU0oMv?v*4BF3vCK(ZITMJy7MEv~ni;x~$6urISPr~tGc2Gt^0y;T$1;$kSXo2fKEB?QaF1j=m&tzeTbWFr>)(|D@zI3? zfI1+D^ z(c_hWzD07|EsWkz*U0BvyMR_eBv{o>_IB@wk2_PyL5iX4qnIe(1oEdbofPu~jcQL1prGE8 zvU3BEfC#~Csx~J5u$TTeLWSr<)I7a>oWPISwcmZ*E+A6607SMs;MZKP z%hQzMed)@~Lmw3mdBY2ZgN7}lxYBt+6qvTP{*}GN7xGe*<09lW)B?^5#We+!pwV&N zss{YEBJc)DC}iuAUxe?!bc8(mM;j3Inaqs7zR=VrfK41Msz*FG*+th=8Dzp_!5!(B z|4W-spoD=m6Fe#j4V{4KQ?UpW=3-96!=0;DMT+fs0x~6n3q~#Lqa`OONQ^-%B6{zh z$qEj8gVi1)l1b}&aN}NChzb!oUg3!|8=67K6*(&!&29F81>{qDISo_FXU#!Jp$hO7 zZI*U1g}(ksk82iSMR*g?(=dPzg<_2bC@P&&yZdt((I$15ovl)9qBKO0KoW7Bf-L~L zMFzs|B;aNc^kHZSgjL8P8ASAf%WKEIDfC+SI~rj<`Hu-Pf^KTUU6y4K*_6JcyfIp+ zKlS2?Jq=|`B;=Bm}w(tGMV z(ue|QoDM3;(iwyVTpvoGLtb(Xkfg#Fh3svWXnu;wKOa#V?^u;G9^J@U-MdCK;|xW zQ|!OBRAei04X|4{3Ok#7AWB?b1R!5+?FRan97y!uwJ3w))PQvNbFt;`!O{YUx#6L~ zLOy;XBBCEfq$);~7wi2`1W7U6Ca5BCL?2a)=`6jpbN6eeyJSv02 zJH>Wc<;c|B9E}N0p>Au*CKGYZtCN>-*R8=kjsE$=4oMecFsUI5c9KWvf3OBtFN_e= z4ZRi?5D;*6BK^WU)mUKdIpaME>9SEbrH0++1o;5WqY`r0m`lV_byB?i`_Kg9n%&Ln zT{nwh?+8Gi?~-_oc!I0DxEOixby`o@NDVxm!I;|)+u37dfRFS|Am!rB$K z&#cS%zJgTHZ_y7AAOE}FerluoMFEIjp=W|g_JJk`w78cyHm2f76#ndAf_g4X{CCi6 zYS>Zq<(}JR2M->co-*(7%hW9BFO%NCPUNH65`{OAhav78i!<#xvXNYbUo$i5%v})> zU}s_eW{C{uw@#w&pm7ZG?u)(%vfQ?c$S1cI+OWUN&CCA@c_odBeuC{l&c{)ya?wnrq@?)|zZ^yod1QDi z*a6fWotjE52K^}hpDdf|u2It_0AdkI@ zFxrm7Aa_0g_f@YPLlqb|ebgorm-rJ5YP4b{=KXJK9$8YL)F7E?TfSphcLcL}f<{PZ zP8nnb#ag%jUUHSna^*9){AY{xjPi^Va2 z;bzbQF&ra!0SWQ?D&2~^a`g3ebaWi1Q!j&L zx#Pp#1AT=S;h(Gqy2Die7Gj*?i7rIlMCKzhDIoQx10{4f9??uEUU>Gmvl-@5?P7;K zL3rHNR%|uE1U@{_Y1l=gn3Unf-}{Uh;sv*6(mpa{0?|0D&JmBGL11#nj&EtLUj z5(9{BCef_Z1qB7c5WU@pnujY0-t+J$*ak~gx&4lyRV?VE^fv!|%Eo5y2p%924j@Lt z5L4-827nzaux1cBeNtp6LcxIf#_}@B(5*VI-pXp^ZoC=Pi}2m=nt5JYd0jpeJ;nK% zVR`2G(E{{+6iVbXDGE#Sb>3+6<*QfC2%-WG;O+GppD0Q(xnnSD^ZWua1ybKtTo|HT z?u;6kZMw8)8$V|7l>=I?E=?=E#T?Jdz%L)~(=;*QKk=kSbqYrrt2?=qzty*5V zVfdmeLsbS!Ino+;l?!&nFJEg1j!X>;V|w_}M(XG46WJF4qSOp3=Y3t7ScyAssoWPQ z5LVY9WY2LUqdI4(08M@-JomcG&Tn7dmjdUe&iN<13k7~?YUH>3@6puXXC3!6EEjI zj29@SRLwVjM?{IYH=Uy;egFN6os1H{aPXj}4w8yVegNw?*&HeZ?v>G~;awo2O8`1C z*=D4QY!w!I@HrMGF{qg^LgtdUn9whui%at8O;z^YGt%oMlrIUfc2Dfc!vjvcFh4J_ z)@Z4t1*wI=Eo!22k*CC5pn1m(fOhL64a_Rr1k+8`vz}=;b{&fKQ8qD^&l4G(BvZ5t zQzYYLMdP@t7-4i;y*mHEB%pj(In>fFe|sx&f#$UQ&gMkEd;{mjf9eBJ-X-))AWGVo zeIOI|e?lPYnjmplVatWF$`ewfK0lxt3G%gU|57M+U^r(Lg$)u4Dsy2F^DwK(9e z+sL^sRDf0m$%o%*M1Y9$RjCPW;UC0od4jIx5WJKDX;C)vr~nPWi^530!!Ua}t|%Lz zEN|6=-tdJ8710gFzZs`rY$1uIKt>H(M z)-a%q$F94nbLa;N2?~ei*N+4f^A$*A?HG**(-IaLZg$}|?|3f1cK_9*ni0azbjZF` zCjA*-vYy`?F1xQHM^BU(dY&xQRm6AUUwXkiMzkZUQdIu#SKIVX0(YtXT9vblap>Wy z&PuDpN${szA`lGTQ>YvX?A8q_{hN>iW%ne-cXV8!jg zF!%6}L8VWK?O4N%NFKxv2$hb|>Yiy<5RVmWrGgRK!x*&+&5;B_D)7q7VMkof*~OJ` z&mazu!%Yj;94rcHuMZAYGdKEgvnVtB2lG^OhL5Oa27}kC&&X!p<>$C}cqRPih#^MSYZZe2LD64Jy zAeyaoX*{%{Ra?rpV|a1;$34wr*W{9tk~cPgTx|ssfmoZ~Lpg#<+S{2sMuV&PxA246 zk?(d_k0X4?@akGJD*eaSSLagHY@|)VKImqVn z$sJlm&hk`^)a-Z|_g(RdyzW|Z;vzve6UXm1d2^MA~#rI9Fq5ZGwO@*#E_GKN!RX5cN;Sl#{qi#BkVSK z;-jRb6o_Z}6|Yoit~xVVx@3VZzWe+z*=OMiP^Sk#Le>K7)!~|m3}jb^H@1}XjaS+q zYOT-9+d4W1)_xIx`~3OyOx?mO-Dq}=wAahHff1@57MI0J=93q1ceXkG>7{tM>uV%| zOCC9Y;NuG~>1=@w2KCvByln6RlKqc}fCUJsvl>8)a7N!wV*w-gXLKOVQB?{brZqc! zJP6d4(l9giG~+H>$oo=tM1BHRfJrvu^b{duEGrAN19=-O^cFyJeKMip&y^?rJ;1Jh z-~hw;f)Q}$o%mp8r;t0vHci`ZYYcwT+VAi<{xJf! zvOWQ=TAzFuETA97d=RM`t?%&)Pf+U{FXfi#9?~$6K88AQ;6NffCoSzy6Bj-^jX`TG z583KP;TSpFB*VsEpka~WqTrAUr#gJ-(Nnnx!h~FDlU)LBts>LUoPtEfI6EdB|gl{Hlb`MBg}_W zuOi_7PFS0+aY}%aoF`a)gOfzGV93w(e6mI_EPiEG&lw33dM+VxB$z{oG^FZzK7!7LzyU!+!dlGHIdNL}O|$3YnPvta*BawGApgQ{>}0fYt~E2DACy1@0s;!>+`E zAb;B9TL(*QZL@P&3m|afg4&#>*gM5B!d`l-vqP9-zsrJJNGTI$@%NuU69sx6CcQt2 z7Vqa|bJ?uQHcf`=X6u~{_(HNOwcXG${uuSWk+Gua#$?XvjPtDT$A!u8XjGq)A0FH^>$OX z1DKJIe z+FiXV+suwKY8J8}yOB{q-VW48MLB z?6Z(1;gomQ5)Q!>M~?rLqIFO8AVM|x7AGMZ1gKLUWD&sd^_Q{29epnn(ZUBD2LD7$ z22DU)j)R(M^w+Q2qp-vf`4n6r830=3$9sw)N@87a#3I$cb9tNm$wo#hwo|)f>KZ3{ zz@5pB^s=90OcgSEp{%M5fS$V!A%UF`BHu+SR)9!ngmh{YDu$5+1&N`)2GZ|UP3M1E zT3EOd?gf<}j|WTgjK2olh3u6Mu;_3=BP2C6DAmcqnCb7e=lZ65`=gn~`qSWEf(TVY zL9K#sGl#Z()^8Fd0m3q0kxg&U0*e0(6%`xMgCWwPtiF|%mBU3)b1?%L4V9p)%CO56 zb)`N;&l8YVcI4;_Q(t`b>Qw`QHO;OigoPe(vmhK6(*mS4r`h+IKp_vp>-cz@{YE$t z0dno#=WkEZ#J~GkDcSw;QrQn>&oIw>4J|%2o>xJ;9t`-*eLf>Z#Aru5)bNRY6uJ}w~F1G)c@97;Dk21%A``3C}o(k^lL=~TfP}9=ZL*(gB#gVb z*q&m#z_AslDR9Xus=Jfu!$qeyB~&kjXX1ha)M+8c-r7be;T|kLDpy`eYPe~ zHAhbn^E7eK7aHk8zV1!7*YX-RpKzc0g23&O6~I@+ip|=rSJ=R~sX?I<15m=bW%3FN z3Yj_2!o$PsSInQ0)%TBIftE;}4EBZ(+J?;5Y>eq@p?D#w$}jz)GPDU~!*=*?c9+F* zUyf@W!CnyzG7S=%@W6-TDs3iAD*p)ZRvfGkzhKlTCkhpd#{V_Gk{r& zS|l*lX+0MTS>O@hCOPy>7A(K%Zk6j1*4OKBmP4iX5}c5!Ccf}0gV2s|6Ruj=-_pUf z)08tLt9(_0mwwD^>;5W@5C`JwAT(^iSxz(sOCYwlH_yoROu#(ARxH3*kNt+^w~l*H z=~a`dJiw$>*~hF@C2PCzHAwM6j1XtzgE%bRFyt1%X|Uz-Ya`+mh_0Jn96gJ$^-2k% zhlVaHv=ZEa*5ANka$u>@>A;fob{0D>0!@qXqaIXeIyJ{xTYJiQqd&`z z*tf1=m%5kc6G%+QqLvv!1&oVYjvhNgY(c5W2FPLmz8r-&+$g|I-9MsDbjq=HnVyM5 zuY&TA5LCY}B4E6Pr71;`n&WtOUEs-!HzkGP>iIKBNygss|Ne#+K{}|CC%;4&78U|w zX1)O}ERqQ$gaH)Ld;H0Vx_@?PJ3$fZ)Nd>4?>F-&njsyP3tx3qxY%z1ikIXt)1g#g zDhCxQK=e(7!`=ntE?euiF)dX3Jwv`KoAju+eJo&0qXwqo#ym3i>+fp z&!O=>PVgEiHiKgCI5opw71UFUux0d|f2{WlSpAd@bC>Ak{;qe(x_bnq`jZdfJ;6IFNC{IfVWfuAd?wnsNL+xk*QHk_p{lAdREo5`YJOOq{n}N55BGd z98OMrF^Hj?R{_D6fu+xI^QMab*!fE!66wg=K{U9atZKc4;f5xksZtzE{Tjtyp8Vv; zZj}L8K*>izVkp?H;9;d~I7cj$(|;Ki0v+tTTpL*^<-G5Mpw;If{JO264ZmlgfD}B4 zTm3Q^?K~uTM+G^eoc{+sQ-nsY!BaM^Pm=z-u=zsiZ3ShxZs2GM!GD?Ry69pkhB=#0 zTBfXGB3X2Mb#aXI?KpUdJPsQ`t(7op?6&syonzNAoH}N$Q7huOj8_JVmAwC&m}}vi z5{lCE4qzi*-f{dL0Q`zn!@CQNn)$|y0OsR8E(;u?*xj%nFL>}L$@Qg@5`+A$?KJJI zig(m}V$U*9oHepKA;L2prTs8ckXMXYDU;Kx-s#Sq09*hY9k{H zi4x<-tFx7oEpMhJ48)?av#nR6WiuFa3l=1f%P2j@m#RFLw3m6-j zB{pRjv^a787RZPZy!;>O5uqRyGQ;dIrOM3Xc+PDG55kEcdNU3dm6F#mmF3^lY%8ZcxXf&! z&K}Nqtw#zn*efAcQH^zUk&fQgIQ!9~M+0Tf?}kNoGRaiAyD)B&WF}i0I8}HK*`B!L#|}aH?WODeO~Be`i-|I>Z_+}YxNkOV`58~in+(P8R*6eq(0Z0 z1@BRrww!s?D_&IuLoOjkU(|IWcPQqxTbtDt49Q}L8o(N>06lGB!M$Nhmx6`_lgEGZk+HNZFn9&Qb+H}D_C8d z?k8dGZzQB5BKq?mF=05_EF;G#Yuo2I>4+Ci+%8ccv{3!k&R34scaUSom1HjA6;JSJ@+VGM7l8P_^zhZO3nQ0Y8b?F zV$N9*aYzDqk3KHMxMh1InYa#6)%zN$pKHK`O>>&1%0_+VTkO4QCki9%fEX_t$$CFe6VV_u(QQxS7@ zAJ*4<9b2*Pv{_y{5`5S%x+%xrE9}oekyL`=?ji3nasMuv>Z${E#Sgw5F=6eqpX`+B zB+L)c&si*P)qA1yi`wuC6B$eMze~Sn`8-^_e)C(vTJVc^ZA?c=;^p?_mIzZ09J4-X z{1MPJJY&F@C&enoBk_{GxFYv)2|(~~4>tp0)H=|m@f2*Gi=Pu1)ze-@fA~Lyw_fTB zImhzRQO;VWJR=D$!|pV9Xr_(#ho47gL%%X;^Q>&$V1Bg{8R(<|l%h`b0dS1}5J|CL z`!(J~J@NgIw%zWqpw!qT#aJ0ZWr>F*;Mc^rSP9PEjj1S?q1H>gUz^T0F4GYGD~`ep z!qoaVl~hz4APaE~IwiwRM8iA3_}m66Rg849XDDR=d`)dkQ0$6ehn62z{kamJRbcn; z5f`Jug4zzVZ1pVv^y%ip?t)QO{pH~*v~rn8$6eCdktL|k41QV1gmQ2y7Wi)89Kz|A z_TtrMOq!?!qfyGYAlAc`urC?PkoDK{b6<&!<5uZPoaJiY;)R8qSHY3vD7;yek^%s? zlh7UpiSKf8ZP;bu?d9NMiIZ$rTIYk+Uc#@Pxn;9fuh=^}7P)@&uBvocSI-n<*?2sB zrO#tMBxZfHET+^>TNtD*Tg~}dQep3W+hE1~0b8LkGg4ix_nz{GC~P%gte5kbL09K) zNQN?DQFZ-JHe0yA_(b5u#EC2~{URZzh5s1CkzSd0RxM3g2sV=TJ6mbha(HOPkznb? zqMnzZ)E4kVfUG%D;?%@s>pIPs??YcY2g!MPuU;M22cU?7T8ar}*_)fN3@EAmYQo{9 zY*kFyWfCTE#CBlg1;gaVHUIp>t%G$R$V4KUg)F7MAlK3-_55H^{qjBfqFT!~bD}h| zfcN+juAUyG_dWV{+vqCzu?jU0F9@sD!!8X-^>>*G@fg`QtE4laOE{m)sr@V1GufiA zfN*nfx7g#$zsk~0z_cDMb%YT8h71?chmVJ&SJ&3Ukv$U-9t1XA>K-|bQF|JF+s4zR z7=gGG&9#cl5NsRMVO2RD^hmAIvOUl9?8k?Hs?Yo@9>fkH_Pbm;x%M02 zwR$pgaLEZwn#XN^e7x^Ud>YwvzbCsQEtWH@thj&S>k7 z>&E2hy}!hx(w(J8X0lnzojN|)*E#8xDieP<=nj8#R`l)h;h}PytT=cBTd6a_-Y|VcnvWP9#LqdU@Zb%bwa&n3k@+gC_SKM{=Hw)Ai2Y{FYcT~*rcP|}m%M%3n ze0?eG3kEoTGo&JnK(Ce+H$auqN)c>EB?H@V7^%iVcI*!CapIDINYyk}b+R>%rLQ8t z1&blFTdxsHA1R5X>MhqMK9DWxj#0#9uY24OY4Ex&TjsQ*Fx(ody;!NPAbRg|3&`ci z2O~SV48Jinh2F_2J!F8z@tFL{An*8K%E+SFtR1tnt>o70WE=nZ0^pf4LFNdwi)xW3 zXTM6)vnkUzgr_eI;sfFJ3j*&?Opom9Q+{LeC6qZ-r+CmYt0H`i10_@t6LQiJv=_qdx<5_|Y>Q z!xOCHy}QxBjFRt1le!j}V$GlYcvgyN!wd}1uiiR*v)1CqNn`5M1|9~pbt%KUR~&O{ zqbEW}7JDU~xlQ%?BGMit%j{Yhs(=oD&#SFwcYMPXI;S)-Wk>2VL{_fM2?g67uZhTS z(-M1!v#g=kR;T*`m3pNDWp28})(4v^gtMEzv`@XNh-WZ%za_=HZiy7?)p5=;HhlNk zhuWYc|DUM4q5Y9tP|%4;r|YPQXyF5n{}uXF07V-uFJ|nM4WE++{i-}R8dMolp`8(S z^HI;e$+mc#;kL#`3IFZ3H@OJG#;5?*C|oZDzap0ix&<@7MER)PU#h7zp5Bs+fnv-GH8Yo zEedH>;DX1SYV1 zuFbTu?X9hE0}z!n0O91Gv2Gs1+$IGyoDu(ZpePF!CDks;5Z~ zdcz|E(ArKj1??XnvP;e2&Wv#zvYbOQz{{uF`&R)W!B1c@>hziFV@Fvb?k`%`f`W%B zX;>_EL`1aka|qp9u5;*T|IXvT1S%DZ9!%4nOe^)XFRw!33|Th`sGm%b>X@*{R)TQ( z{+0kJThKn__w<7}&3nC(nG3#n9WJ5fLGd8A{82%{uV23s!q0PRWj`-mfv=o-h((Yq za^y8VD7M0L0)6~14~)N4$xxBvoNA3Wvn`LZLd0@;5_ zr`h|w*XZaHki83?1(lJ`HX@lFuGD6F5JPG5ECAq3yJYzNXbl|)3A|a2Mwf=iTT{Q# ze)jU_{#Gx^_ewm;Ix~hrLRJ@HCMv(*L2C_mr1%=-C+uoe z)2DTn#NSh0m@~_`|CXYxFdlF`oiYchFiw3AWB^x2U`mxDp^dG`K0K7{fDWvR_ei4& z>@#CG^vo75Q}@kIMzTDt6+pM)P+QVK;DsM(i0}OX&i+e_5B~=dx;GujTXlc`{yjcg z|29bB(_=OLMPnIGAYDRUm^axhp{c8vH@|hO!l2k3A)-m)4mdPMh(Us&%t^NgmXO4q2O~R`<;hDv zSxK^i0zp03o?oVscbTWv*qUk!Y~wEs#*s@)1q$>nCt4RSaygElK3e7_+@JqNdF)U6 zO2*c;a89ZCxOnNl(K)H7F`CKdrDwQ+x&nH{d}W(j*j21*9?z=UOcDI7bTC<^HbyB& z{Qd<#;|~W9>)fb*)t{Y)#;tq?Y|qFNs6=sBl5AszQ16q)LxBN1{ z*jqN(CN)DnhXX`$2px+v$N}Xx|3TOkEaTiYHCa*`AcPs_5@|*`rxW2 z06N4b=z&6gxr#Lg5$&G4svt6lDO#ESG3Iv=w#J05RYCzpjP8MQM;TDLC&R98$Zn}p zza}qFlx*0WHbHokgx-9wSma{b0>YCrv5FS80OO{e%?G-EbGLK7Xs8?HC#+k1SzY0Jm^x%$)r#Y57#ag5^gPir!AzX9SkGC&g;Bg2@lKjO$-{`?v zi065yFbol|*ze!VOOxyaI9mH5TxifOUH(d#HAK}bh{zoZ>y7kmjxE5xl?HnM2Xk*7Q04lqi%vvZ zK|x9-6i`A!B&1PLN~A%Ugi3>SH)0}4he!)ZOqxZv3P>v*!XyRh?l|LPt$pvgd;j)6 z``ml}xbq)aESZxzzxR9JF`n_nHuxKT2Rd9D?3k*bq>1R*vvM2cVY>ojqruDcjhY~M zzxvYW9WjdrMso9F)~h5IDAyjHIgYrYE#~-vO8h`AE?&=QCI2U)O@(>oTPHmdD)y_@ zds~8UeB={6m*rkGwr{NKiIm(M@R=A0b^o*7;LAFm7{;L_4ppp8HX=w3e073}^I*^^ zPa}$e$&ykvKh5?gI1xFjo1-{mK@9LiZCn=s=8ZD z)?yUqIS%>SPJuXs{MuBI{md3;xzBP!s+~iCot{_L?p)HX!npGF*=exGx|OG<_d+S* z_GhW@CY(dxcK88$>XJG+>27fcXH#|F&r;8IrzIzUs=p0=)O_099)>>89(c;f`d?cW zfUJ{)#!mp53?RT5TuJ^rGCm>?>o*_6ez3`4sg;|Uj~lwbLO_El--d(i6mcKkXsffhucsDJv7+t@b+4$Hzl*7IktugtfBmb{V>_o8R8#|0 zC!3~pD%PS_+KHY@{SrG-y^_QsGeSZ~be+wj>RM?2^4AtP>8I)ux3uFDdkk#U@~{^g124;c4Z)!2X3!wJEVBg0@z6H7srV z3wFk+wXql|g6jCBv@9u&ilL>M3frR-wa?!7SpATu7k6d6)OW-M1SCp=H&b1i3NEW$ zS~tHOVb3sda1J^Cn0ZuuQTMOMx!!QAOaj&R5^ls;@dy0Rq=bY#Z86!~wQom^S4@Pa zOGW$6j|L8EkzUJE`_py&kp@xj7{B!J52(iQty^#=m>2q6no!G`*c?_W>7ed+iqB(A z_Y~WncAq+T9%Nv;^dr?=ID&pgqYV91Pi7sYMwQ5U#bcQ-dkXj){4E(boDu(*P8p{p z;ctV-f3Gxs`@%!zP*xT!XSiSth$se6sy#*(CZuaa9%tI>`zVobZXi5bgb@wSh_~;+ z5AvR$xc3vO0*4noa^NRYNhCb*&J5=xc-+$fQ*AN}c9MgDlfTC#k?bO;UJcDvSR<;~ewo5@)6nAB<8z7k z8JeN=%t>IQtl)n(fC#$Mpou|*7|`GK4@1Rp9Z`CNG`>Y-xhiFw0|bKLHm?I3xpb+o z;5qZi08hgjHvB1Q)m5wAo!r2%7CEyKnmm*$|NIdrmGM>HPg`ZB?(~HcZV`-~`~Pb~ z#1gX8a-v|k#bAu~CucE&I~|lfbWaX}s+9&EOriIo`^epVU^EwG&4bsR5zv{t${kGC zE81fP6+q}=1oyu%;l{>B;6D_Fzq=YIi(zKya5w%8-I1qi<^~h^Xy8CR5=A3tD$Nn1 zJlBT}Q!Z8@2N3~D7?jyQ0h+-`dC;^y&R}#e(Vy0l^#A99*bx z&eiS=U4TVx;y|5O=qm7=>qw8=%hU6zQwhu(Q=*|xQ~4sH2j9neK||nr|8HW5c@cF> zpWZ%GT1a z(FF=NgU@BIM$N3(A0|c{UhkYcX9E==u!Eu=#pfV*0gR&&ZxMujiukEopUp5i6my&n z@!af%djI=5gz7CR`U#H_zP2@zE^&mv*^V9?BPjG-)I=Dro*dEPoC+p$&7u=`i`Qa> z96q1)B=n-JfF4s(Q&Vfh)Y)@?vGkuece13u7hz=DOp_fdP4>=HKzS1A%pf8nvOAj+ zyul=Jiv-OfVy`vv9slgbR0eoiSO1gx5Wf5WiOFBo)Tugf>VFp;9GnUn5W0GLNNI%_ zWw`Bam}j>jS|da~g}`ozrP@?;_&hJ-uLhbouk2Q2G94TFe{;sJeNY?ztnJkt&RM-M zQmsaw#Zuee?{v_$eCuQE)zu`N zvohGW1v-nXVGmGc&jxwvjxy`4L4N!j1!_aI@EP^w5$KyZVRH*oabuB>_)-4^q+l$F zie?j*B?a8(gQYf!8%rZ-*c2&kmWkjM{97|t)uvV_WhR!!%@OZLUuNcJM;v;AXCL&p z=KDK=tNiaBVr0-!7I$31!NVL@>dO@+1XNs7&A#y{-#51oo{~qc-k?L*?c3nst0LI5 zM?Zoe2R=T23&3le>j)e!#$IEwMXyr7lGyBUL75+ z6*r27BVyCE26lLavhApsz6^x12;l5GUOr91^r74b6?03~|0|usq&zuD+ViDU7-W0js3^!s-JpGf?YG-n z(5`lLbnMB4D;DzT%%1f-&lNd?!z#_lwJ?!4(TH{dL7d;1g@PTP0wTPDQMmoe*d4ls znAfr~fI}QTNJX>bB#f1@w`=WpmIqbb%NBp|QwUi-;L30|kQ+vcjJAE9u*-sdE?+iz#CToiWsSE-DABpXLsOBJ`E%>k3d zgNq#1;`;*=Cu#<;0BkM1cjW%GBCFRDq1LVeP#7a*Rk$#+$y>##GKBWD?C1B!x5 zH`RH1xL3=@%O)mtH{;=HicUz3`uUoVd-c=xXZ6;vwMC4-^KsY|~ zE)OVZYhb!Gr`*{iBK69_;=%7zc*L__y(IvkSB0XKdkx``=&EqCTo(obrukrD8hsJK zU@c)^+QnMz)@Rzno=e@Ao8O{Kd4^CFypNpxU88dto5K#b<0~#=Z5|(s3kSyjO9h0u zK}(F<^)xqJgTXTE;>+S?2Ja$SjYoaD_xO1&W*Ox|zsTJW+Sj+}x1>p~axi`9KQQ=b zrX5|d{16QwGIb{_UIwLZQq@RFrQ{88I7StV`r?Xow-Fzw? z7`(pmm-x92IKnq>e05p+c7&dK$*$HC%^-bV;_~A0{v?UbsF-Ohr$0s3IouJEtCL_G z6llwBSLi}TP&xJVy7L5MGmG}Axq}tam=tE|l7ez56^lICk`FuYd3~m9>H*S}cVA27Lfa?Yrdw#_PfA$3w>3>& z)>7*h18et=u;&GS5Nf!&&&n7zt5tl@Kc6zaZ5&`cl10huDE$rfL%l?bH*|`7U93Ig z3KhGq^xoEd9u_0_{M52s%I9{Y(YQq8PKIIN%pISqlLPIBaiTYF>YcA|PspabSNl3g z*MPPVeyDZba{uM_1a^D+F?U#f??c`2VcP%5bsp#dV)9kbWEuwGlHN3t#(DE5&3$$Y4xb@A=-tj~h_I$PH344A$ z7ZCk;3(P#*_JKd&T!ol2xqm5!?iEprOaUmq1}5UN;P$*3{i&e0#0rg>*Fa@_kx{Vd z&1A3|gx;uo5;B|cWdVC4zxKr43jtmB|7XwzqrMI^l$Z>EsR*B3gA>fI6;}k6FkSQ9_$jlu3)JJyGxY`M&z(DGH808Z z{<`aAeJfehTZV_7OhK711zQ(a=g(5El$8IQ*T6}0Rn}6JpDrBXc1?*wokVh!T^pj?D-40M3`Dujmc|?AQC{J9rR9S z=M(l74=8*C!79D-&FrA56p;64VsCAIVY2DU@5N}Y z+Jl1+H`Pq+oE5E#bH`s{n}&`$4Gy%@KSX41sV&bBI)tLhzD_+c`*3CXIuUK6A~~se zEHd3(4e;<^{N^%W3&ckg0RFj|I`2{^n8%u92b8gwF{_s9;q6e4$Gnh)QGV@|i0!(jm&K{`{6R^?bLxb%3k%t$XD{DgDQ-LtAnJYh zlN=Pen5Hwc!wPSFEJSPQt7}1NPa;3ZXYEc-n&#<$;C}aDxWi(eOkvVyU`CXWzSG`4^rk_xTNUhEv=dEV&@lZ229A1Y2^>a3wChIEuyZgWN9 z#M?bIe(VqU@H{QZi{WIR=lU|Y9y|95R-Z4J2tDo zp<-DgzI^!tbLPZAI5ej;24DUVPz3k9sZdSRCyf$?=8<){H_#yqFKI7d8~B4X(Lmgj z7qcy%BTf;ob6)zj`@RnZ`HWWpgOAeBeN=E)Y2f@Nu<-kd^O1PB$B6MWSIsr{GwO*2 zJ*gHJG}Isj(|S0&#a?@`7SOvIOUb~H_$V^{i@@fz4UjoNyw&oKg~V`Vjhjoo-=KF* z`Bi$r!A!#}jT%YAz{6Cig~&@0Z!&14`V}^>tihD`_vRdW8k(q&rdbLUOH#j}G(lwE zzn^D)MOV8Q^H~k4`?!7-c48lgLR;2eotuO|Hh1r;M*FCxg$i{7@iMg5>DN~YJKNh8 z5%M9pmI%L9GZ@I~)Vub%&-ZiGtGw<*Ew9h${Kt1*Iw4|g7^cp1H9^N2tl@?n*B~zh z)uoqq$$-R{5~ep>tsl-~TZX%Gv<#{*3tD7kqpPpBY{7xjTwavke*7?6`~|a%V+2~z z=6iLe3sy%Agrrt1{#?U6yg9(b$dJAHuC?U1VU5DEw^ud zx2m5Sz`y&zSc?P;q1mKmP+9ap9|07P1+*537z1zy3K^Lun#lFppG-t&&+>O(6a*DF zW}oyT-&+L#pwWc76*(j)mY2B^llgGB%R=_jowHy!r)fS?QHIYuVe93>z?3eJ5(S5YCX8CsD>*Q>h2ST-)n}6$39z(xrUKkCH=V+)pQP&y zO@EI;BS0GGus5qlOU>mMdsed`%B(WT^=pga*V|eiVL{-u+*~%6fXX+Hb6t8>tr~&( z$AVv*-l2ya3%y$Jk60*sUDHHHnLJ#T`K31{iSz@^A6rlgT)bC(d~bp4gp(X1K15X;ZQgK9-#lLe*{1Krx&hU z_?gng`(#fyb0F!3k!zk`nGrY3a#R+}*`du4AaKWSgZSqsB#(fT5)k2$2)aCfJHh+9%kd;QbaV zrAr3ji*JE0Q5Bbyj!R^1#{nl7;AJK1DsqW^il_3hA@Td9&8tO+h)ehRAZ^HfLCltb zJe2yP8rEYgFcLS{p|Wa)*#~lKBepaA7Ck2!kFa>0WHU;Tq>8(4|45}<0CG+Pm0ra! z06f0K=;{WjT3lOA;m@d}IT>X$B1%RRhV8y_gH8G6*0^RV7Q;Qq-?u2p$=Qb=j=r=1 z44iZ(K!HPn*72d=3noN}xDNpy5Q~w%JcB$mxET)YkG&v64A6P)r(_WZ*lro1Y6{Tg zZi>&jLgdGWc(gRCV5qH6$ zoo>1e#9+wS4hhLXDHY={fWfBFff`6!(!l3H7^R-9?~3@*0C{~xhsSd#J%bmNOE)O4 zJ#dv;266@CpT~&u6!pC#r*cEv>}S4a#LBY9jaPmky6T?9(HTwX zZJ|tW8hCS%w2C^+$)uXG1Hhv#oncr5=Z<%z5flLgqFJ3^G99R`oCN^c-B_Er*XopC z0<9A&?{6?`b(0O#>36)wzqazzFg7j}CO&_rAwcVfwN}wtSn7;he_y^$Pp{NR^`7>d zdy2bb;o|oD9o2-UpkY%7B!D8%Gz$WA+!yNLaA^f+i}Gt!J+q{D_xQ4IS^2O);B3;rlcFO7#z1`&D zgJQFvcT2aRWR8RhPcsxBNXlvH(IpY*8(PHhR!RRE+(zVbb!``sZ8GuhXWwh<)R5g~ z|5<$D3<*h5MGn{yfq(kYBL?`~QLX<+DQNJ7|3?Bc>MoVIY(GRWE1-!aIsmTwEa(J0 zhv)w<3=w^Amr?hU8IMkAOz2e>r#5fa7_mLl)^YiBJsLOnj}>Io@`9s0o6cVu&_c>X ztU)Tc#6z!t7Gt`K;N$>TBaWqDNGJymys0pJ;{`|6fc^cQows0}h4{t&Z|-u+5xlT# zWV8b9zY??<4-jP&5=aXH3WxOW&bzW5i6Vvt;y(1^tSPTa!>&1D8`k7z z$k5FUA|;J`=`wWd?fnMwURZh63R`QokXsmaRt;r$=Rd46!O}S%uK&O^@?bZS`k2dw zO@k-B2cx2LpOlvMIgZw+h2zdmOWtQITB=2br58FD*&5c|JG6<*$@12cMXVXhFiL{@XSYSSC?Uzq`&Cv1>(BD;RW6=DOAG5Q!_YT;n@XpW# zHeh4uZ+{J73|2twLV;n25I@skl_7Hyx~^h~WrN&6CXA^MFeDHq9Ac}_B>@a&1PDL$ zMEqi*VV<|;VP=-L=*hT3PfuS6Fb^F$>@^)0=09me_)(2Q8)Wco$9Cv)@e>~=8FWuN$aTI zF^)TYOo!yG*lXbc)w*W!3}9=|F@BRh;3V3 zsq|KE%=LT3Q@zLD2e+jyN*1pAEQn;kD2C&y8OSnI@h+o3hXKE00TDzEqIG@a1w1@~ zXeVHC)2(Gjxj?HO1&3E`DM)nX0DQcTFxjnF;nS;Rb%L?UerKqg7r>B~)*s{gRFhN)K_s*FPj4f)<*hcqf!Ci)M>qvazp z0pUeOZo{89Egd<&U0{ndnkrjPbFO}UK1d9v%Wy~SVi5=}(W^_}&u^(UDH$slAVB9h zmI08POl=1U5{!nnClTfT%`8FaAgcZ^R!SwfaDwHJ+<|Z@KY=^&!1>;5IgG|-wiZTdpA3I1VA|{gDL#x{ShNy6!#K8m`WI(n)SQHJhFu~w7sYvjyfXCH zAB4YLK{@^_(R9{&1Wv$#f8L#rMyrg_rE+`M>VvV_v82kO`%(WKw^?o@umFtOKSn2i zu!uDEpCit0Toi$Ez)~Zok>HGy6Lk>SV>!Bu)&W5l{mVL&E40`KE8Zg9JCWyHeSfWc zgRRh|lR-64r5m!oFy%S1eXWDt=~G9C9%~EO+@ff%qnv2!qmt3-0djr;Y+w7OsbvRK zIALTHjz#FP+Pz^)kuFDZJL;L%*6l89-)_=P@Pjt~T(QaU=d06(6tR!zk@>B{B!;Zh`5 zgXi|&d)}*M_KUGrp_x>^c@}om+V5-)ml`%UpQI>L(yBfOj6sq(G_N0y*D>qrXpoLGDoRWk|^RJ-w`inb+p}-=p5c--Zd+yX!3Ah z{EOu)hU>PwVJ|5=;BVWxL;V;&iEjabvz=J}KoPLF+N)(C2LZi^Js0A6B;>pR#Gli% z+I~QFJkZSb`*=GQgfSJ?FzI){KMrU>EA&4N4GkKN5B5}4U&g*KlUvv1HSOcuo!?P` zsaILVjYSA$rynSlxqFvhVLvF|c&p9)ruX3%okS#1Z;Xe3pH(f^wPVw%N=;C4#g6Jt zEKJ={lrXa@h%|s^-Kli^KOwJ(#V)_S8IIWOxE}*kZt4xKLmGO96uhI3n*%e$nO#Ob zfvjl)=08#Erk&rX6{u5X3&^{Y?A8SxogoqK?meWVNNRi#tGSw!K+x*Y)2RE7tY7}= zk8JH*Y+y-+Si}Q^kJ0B~^$p-qx`?X^V6R~?ww8m<6|3XbBYi1LVGX7aj#^^~rcERS zbB*wX#O-ihx3zl%2F0JyRTkk7F5kNAY(0Dx7-r;xrGKgBehPM6-h2gQ&+D3biS6@r z+9f8Kz+>wS((|RybhzRC3ai+>1bi2RmIDLtLicCUOV!hd_LN2)*?$UsmbnJ=w_NO; z6xp-59m?db?f)u$KEqrL>B-2 z@o**CY(~~%@Z=@sOuwYAR0WT&)OWtD_bwzbYMj5}`}XX4O3Hkf=|RcRT*7y2lFI6y z8IO~hdbz6QLr3=y`fX}mW+aG+US0@)P?K0(crB*f;gRy z?3YfRE5TIl`+5(!?lLt zIoQJC(ZuqfTXe!yk&|*djlY-|A6thN(kQ4>8Ok{}d1ai)b6}NA-#q0DU*I8mf$3!S z58q1P{OAi`80R8cRlcx#0cgd`&I=nN^6+K7|8JG{+oz|IANul{EG^J#mdutr8&*iy`zgB3%}@PSFIR&D~e)O`fM)~Ed=|5ld*w>BVhIbf|E1*nk7Oq@eD^h^Q( zv55ffHn4UI5=1xwF^B>(90mRUcI~khAjWHXu!}XhF2K-oH{+AA~oW!J+P1K9IP@=5j*Zladvt!q*?ypJVvMHL-CO*(|O*-r}@cOJ`_uJMh&q zSX=+&ykC;DHN`}xbEp$}z(2qBr-{B~q|!OQ(Eu9q)aynlfPNSkMrHK>y0d&V%$1|k z@R2{ul%wBzg)tuB$02%;g_}sEDnTElyv#o@fz5UpTAD?Oarq16-EV@mY!O@Ht*>Mf z0|&e7IWWo|$F6Vo8#L|IKnVAEq7xV15y+Gu!!JrK9t(r_Y$1F^17IEb=XnbzuV(0l z?;fOpG$7fe^T2TroH^hzjT-6Qyas&PbV@dzwmE(0ayoI(0dL)^9|>;RrC#g37cTLx z;fk!c-8yu|&EywhX;h**7sHwe2&|P zO06wCwigP*sG~~o-+y$qb?k1GusZssP4D^4!K>-?;eJibUfO0^brGGg%*BICjVI8;?0-1y+1biZ2$64pi74>3Y~q*`f>0o_pP14OpoK z0gyD=$qhnyN$lv>{!w_SW~KWy#Sy!fo>OG;v%xM%yrG%&#_PLK7t8a+j{pU4mfAZT z_%QS+^$>1nR6eA<(fhtTgO4AK*Qpo?`OSNla0OyvD0%yGNCZT~3;+JfkY7e_@Z2LhLC3$EZAlrD)eZ0u9F#ZK1O2C!?EY&R+v z>Xvr|*>xmF-2kkUoACIcBfnI1#|8VsYq5i#?Az_FvFEtno_C)eS_W*+*;Nu8JLZZk zl}j8fcMDZGKC_%`sdl`Q`Q`a#ee==0v*18Cchyr<z9T>t94T zb=3N<^)-8Q9P@xOVM}2|0pTN&BlfMFC6KS<(*YQN4wzInlcRddrDJ?T>m^j2V@q&k z+`)nx?w&04t36E{5;IW5&oKbkg=c-e4E%=G2>>`wAq^on2ExgMFN z{(jZL_F(#Kp6K|%v%w;C`;Zg)hRy!=2bB`jZEMV<%od{O(uWW6P8+yy*zh6sb zlce+DDGq>Uvn8qp8rt)du2vL%0{T`In)%$kbuQD9TCZPt)_Y}L%HXUR63@1(@;`5M z5gfdRi^zc+=g~C@v{#2(tG+6YxL^)5%_<|dy!dosZ`I_d#@H{IAur-H)`L}lBJ{kb z{cEb9?Ui;f{HTB7zgOBGf2-RDU1G^FGr2f~oh@~bAJV)$3?N(6p!RLE+vNQYu#4`= z;3He>7x@cpmzNC;f#K*3pXV^hoF~Oh;N@?6Yo?szCsM&512|SpLAhSs_$S*AnC6O0 zJluwM@+Uxqzc)A4kGTh4dsJ9g{JD1Qg*2VT=G~3BPSkpip|wHa0pU&&F54m-L27m6 z*5Y7;EoqmJVz&ZC!C&G^qVw(IJ(5xe)EC$+sI@TWUAspQ;w z64d_jD1O9UO{7!$#&$5V=6aa$Rwq@?7o0C@M5i2K)E2zH`pa!?Kxk&F7{uDHUiGK& zNE%$LR`BjkcqYuz;N$2666i&Ea*=Mp9Kr60e`{G{Z1ec9YE5k)aQ97*05)Gbp2jHGFu|jOC4#fuNdd zb8=(Z%40kAX6tK#jWGy@5nf*=;g|~8E)N~{q~_Ey0!GyOo=<2n*(HHC$Sd47B0}Lu zY^LV{mE#9)II*bn?2r0Vp5Tl8Ex0j=XTHy2>B!*Z%OS6H`_bLq>OOsC8yn=LDb*rF z|20P7vD=Vd?fsb9cn@#Td+yu*W<>Lra%bhUwv19Sly+r@@%2NmmRWSUXG{*dQn;f@ zgF$)Fr@jDLS>(w2dT8Dhr00!>{OQ{aT8(<1z&W#+N$|)$+?fm(00ErsVXfr31t4C$ zB$&7m!hznZ<^K~FP;wcw`{tA{80t=}7y_2PIm|7-&)i%qQ@n9OHUjItyPMjSVAn`X zzkzXadvThMJGv)#zLcUV`2D(_({3a=3-`v*D|q$0CxuA5dd&m2T97AY9q!HN-K4(+ z83^y`F1bju6V_xdB^0iJTf}VfDnxEEcA}y zkNim9y`>CB{9gh~$s-1}dFArPWXQ~KsxsCoXXb$^v zet}j+Bm2=$>&nHFl~usr3hsQx7A=*!6hW2Lk|`n=j)ozti8UvKGVKB)0^uN!(Xvh8}l}O$o`0gkYn*mXPFcGX-4}EiRD>^23PN&eoGO@jdb}P1XT%$jtD>25|Li5OUXEl_`yGK( zS5^~TrG!)W7-Ov}x>AQb`p5_n`Vz2-8m{rGB;e2>Q%3u0>?7`OHxtx~oatgkojz9! z+LxJb9sXq$YI-WcW65n)sPx{vv$IC`@A1h?0yQmw+gib2UBZvi9GZBwez>fXzk=7d zxa6_eh_){Fd&Nb_;DeP49&k~s{i`8>YGC5t`eC_nso%Styvq9A@{;#9L=WHMzt1c5 zH+WF=#OCbEgKuo%*R{7U3uWKW8FKb&dN`_qj80TPE9>SAg?u*C(1C#um=U-o?vxT; z`BSLJ_ff)-*xhBsf4HTq@V{;a%+vSe{J-vVRyV6=yt{`no-a+`Qjjf$;4G7UmuH2W zXy@VmTY^J1Qca)iMdeyQ#HP}%h$fb^TdM_?oH> zs+D4>rdf7&05;SCrvcRsez-QNf+znp#AFeo$ws}lM@}>3Eqy!r+0&~?weQP^fe!e7 zr!ah*ntXEp!1JAJIN6=yJCsV`$>sf@ekfaY&IeLiyM&R$meGQagl-efA4*7fT^=X( zrSfdqtE25LVfVmo@DwmVk0S}sR!mAK@oYb6spy3a*-+@=@mc&+vidS)i={8#1P_f$ zH7%_CoIG#?OYIjOvWqZ`MuphlEphD3ixRZdZ>D_ zS`dTK)1PxYtA?3}WANk2g`Wg6U8YrQi(r<+Uh(@Mr1E>m2WyrOExC7G) z2^`^2PVxocG^hY>m~a1b2(}t5WEnINfhz!y5E|NRsDUC=rOKs>`DegF)C5vy<=z~| z3MgJ%kOzQ37kaf73t>xvlN@b5SQrV?@A z$T&;O`wfnCQyA$LLs>BNUU1Y5Hq8<+?s&v{LH?%?vU1>I%T9Y0wF7}5`&P@N)gS@1 z7~k&{Sr+w}&nZ3dFkV_5ip2|AOOVq7MuAPTlP&jsgqh_OrxG`;s+0QSv zi9Ap4k{cj$d3c2IsW70B@DA+n*67Yy$8TvcFuEBe2FVv&FX#91Iu%-W|NfW05f?Ub z>R|T0YgO>HVM6Rat#4>kLL0o;S8S6@N4F5dZHGrlSVulsUg?vjp{y&bnydi($8B8Z!M_lB`p5PeHKhHU3mMx zfZ6zFRBHByKosF55&7XUsYCdM$k9fMMsnu8W4gw$3k{FL+_?tFd_ zbKr9t4=3f`Xi*VtU~lc4kGDt4>{75)?CPw8ooAV0$C5X^ai_7~HLQz9DdZ+kR4QL1 zxpX@C1q{TPCd7QIWtzkF1v9l0?9y;4_oyM1*Yl+i2=s7e*Uuqq(nhs|8j0ER3&vBG z3Q21@#(*^mDiyCy5ycaQBBB;!GhSs?`ScQ-i!S9FHS}xpGgSHvRmu1=OiW^@*8g5; zC^j~D75FGOhrvLzVK~ANa6oS~tOm6Na3O&86SP{D8iyEbz(ZCp=GK=!-;ey3d)t5& z3XQJj|2uZi7+iy&b>of-#IaFvRm=1r9+f1Xay3@E^Tv zpj3AMQ2-24wgYa2%K?vvTvDe|aFm(@V6n1L1&sh2pt>A*Lp71xpj|GkaGD>2JD*}w z2)Ja05V7?Y)QomtcEM`+8FZ-|T%549*BziC9|iWMB>>SSzz=qHPO8}cXI*Eq!Al?v z0s&w(RBm)BSiyT1fPpHOK0CV(xP01M3+eq2s~~k$0LG@+-w0o=lSUnX3_?q+0IebF zCxRUZd8n#=R)+hqSL^TpL@_oPeK?^Uz<)C`oovF$|M^eAtAB}T{bxUX1~>MR7U8SA z2mH`CWetF)3ov60(wrW_TsfqzLw1Sclq>_d?mStpS`PyIW^<@0Eo_aPkn(+I0wWZnjp%k-0VZ=fb_xp422cy`Z|21gJbnCtbq3J3`UL&GQf2= ziq?0})X3A1m!S7OjdbB+uu+Q(3b}8X?HmEIVhARzHZE8%!Fq|hHr`hQ>jcu(Ky<9% z+g`RoOi^qwh#CU4XFqC@EDE5I@z75T%yx6L88c1y05w+|kVjq>_FB*?fnDIk5RX zTLi;QG~6MG=ahiwj_p2d#fPha)OErFZrbrX6MROJ@f{xR?LE|Gm~Q6zZT{(Q0!U_d z*t=7BKAR#4&v&PhLh)0g*OoTf5%5sjum3`tS1WAIh}M9_&KPdzscaA)Y2@fcN-+CU z#6iax1wl?ClYzGuax0e#YrOV#Aim1*8Iid20uA*XihK%OT-L@6;4O;%aM>C(-9OYa z@G2Gq3~TEfg%aEkRF~nHXfU9py{IVZ%Sy$(1EpoR4M_N{VBr@de>d#`(-j&?PrmzR z0q?I3QR~`1uU>7uM1K$t%nP^iBPfiALYZV1z(E~h`m8pcw*68y73a{gnk|917a^AY z*dP_4JBkC`kg@r=yqZ1s%PVRtO6%vmnios4)&P(e!QiCbGLc{#mzEL-J|J`Q95R{S z9pnUs8@|2fXm5UHl7RH`N^|M8aRM@WWs88+MBoxj)ES#;KfJ zsV89ANCcI@<%AGi$pwnzwdzr;Vgx(X&!#XM=&& zpiS?h7WE3!bLR-cmn0J? z`NS6EK66pYQ86{?YDnmL)ag%fAx8@gn{25c8!ukhI#lfutF+Cn~oYW|nLrFt3dDeIK5;c-N z1o?A`V$dTc^rq`1>)7@jl2>ZF@hmHwGpQIRe{b*`bB*LwK`9%)Eev`Qc$blBBK(5v zSB94b^ApUk#Sb>_WcdFjejrV~{Lyy)s|Lf%A?K7&js8O;Q&^s^J z*TP4l0~&L&OOGP_Q@T*ADpqt}Y&+@Jb)g&!^$l*2J-z^c?1uYu7#a~2-^2MPN9zls z&$SA@3fum>sV#&^I^QUyyxHh~CcievlWSt4Q-(Sh5B?$x{Z~J}$>1n+zRr5{35?+d zfI_zt1{mZ4A2HjJbj0BwIXHH<4x`AS|)tBV)U_am!#%+GNX z{n&5jm&bB-O|KF|$X*Q~j?PQvGdfTwinSjBKXk*&7ia>^B zX!#SmA6tedftyNA$`~BV@3zZ7#+s3)xcLV~KE6Tpn;Vi4i+}{b*9kH0JF!O^l0#Cc zbFwMwQVM!&j#oJAnML4@{?&^_FE7$c?4bfL=qAm7)x?X_y7r|;zvFr(~A}d$?M!54jveN;4r9a}NM5z6mm@>cS&Cv!_ zyj_Nhvr%A-(f_VUBD0yRYXu3FBp-~uQtyCA(CRQ$rF}M#)hFixANN-VTlx1A+VQ#F zX=K{QI9d2f#FKN|mi%WW9lNIlKQMC?Eq@B&T5FU9Y1{j4A$F#u3cVq&%QxGm7Y)S z!hM_DC>iS4Y?CicZ-kSDcJ;jD3)RG4iI_2&McWTNcSgc%_#-})jIwi&@Wz(uOecs7 z&7UYQdpMoxe`E`0=YUNOKLZ)?Q7@p*MlsRS*0>Pow`>Le=SN;Z>-XHJhxgKWE)cgx zwC_`j^RBoH|p1k4yX0hr|W7Gi?4! zgm?eGgp75#T8Is>4QedX0@j=8J=*A@GqZK~wCATiZ-r607VMo~LdXFmd{La#_D0D& zE;jFJ3z~wy5jLLc#Zjm3=oH^?1E=>t0i>aR*c?gU!h;?~-LtRw{?x2@h*Zm#UK`A- zT6;&mOgcw=N_Ibh*;c|-q+@x%hvuR?Z0Ap(SeTu32r#cApVQHkaDO0 zm5*vhRMKyui!OYm=Fe1`4D_Zzw|qES_u$@sxM2aVa;gaO+kvt=*(MeZ@;z*Nf#k9q-fv{0g!MYtrhpuR^P|<32Fiev zmyEKo#NNlLr8e;VDZV>JVRQd;ZeqWlOML%1ybu9A9Jls6eD{&6RbO11wi=7YJ+m=R<#s2xF6rGGhnF7pZvCPEVZJ8LGPJQfcopIoG!7)z_=(L|C5MuU zim8azNj5@d=nK%++3LLietzmn2$PA@lbc<~hRL27FE$_HJ%Mlm*WE>r2)v zL2)0WGj6%6B87X?IXnubV#`yKa{*^cGKD@ghQPpjp9{rFW2XNp?5agaXv7F{z3x+R zBBNJU|Kvx--uvAU#0BC?KmA^kUwoLpK^~$KibHttUcLW5$ij2da_ojW(j%>%I=i<< z!9#lIO`PTg+*zhj8gG#C1lhC`-;~~y-KURm9((#ClcUboY%c;&$ZUj8TKgHdFcX$e6Rz(a!N8%om!^=aW<% znujU2OE5a6y=s)KFITrk<4Ed_&2Qt6ym9i;<#+MxrybNKlpIP=N^J3CeQ`bMj6}PZ zdbkqx#-29yK-Rs>RsF>@JkVN_kz1Jh3432R5hw5WyJkojvZq=pq&yY(6f{2KeHn5g zr=G(YZmnt4W2n?-x&Og+5dz<17+@`19!tZz^LpP+|3QuQ8SG zoRU4}d-G!;IpCSoswSPdo-TAHSO%@@wu?GuVA~jdS$X?`${(Efcd`t#g?%e#;WXWC zxa9z2t+~m2y2Ut?E#H%kP$AB#uYOB@dYtG?R|QbPD50W!G}mIkG<$UF$7Iw)G91Y< zK4#NHLdB20&GOX4S(vouwVcw?IrO6AY7U8qV(ts8IwXKNuM8s)JohZatWhWl|lgDbI-mQ!3>;J52yasR?<6GJKZn(jqOI2{~7fh4w)P8!!j z6LcU}AY#M5lkxiwl3g1tBrId^EI<(A9gFLq8w}Cn^M>!|j{gYD^()}KSK50@O<{ZS zF6!YNfJ=6xwLp89KAuMWGT9;2ffYE~$-rB91LsQViXmukJy}7ApA1L&UMF}TkxBGH z`nk`8&Q=hovO?*8w4VRu>zAh4oq@dF*`F!NfPKCib)9^#41-Q7s6f^d4HFsaunNny zpqqYaT{iyuhqh__^zt8vPBB5qV-)wXnco5?G!X-mDO76(RiFe?E;N-%-1J+03L2Ya zWI}JCw9w?_-AK&n^hYeCbA13PJ~ZBYfpwYj>;XnDJem$r^C*+zVo1QGW60cbDOxb`Tz!y zAn;BE@Kcqc3p%XcgGTx_z3bR>Wf(3Wh>6&aNlFONghPhW#Vq~miq0oBFt#)D-~=MB z#JKoxX2}U?ARk`cGI9r$E%^<-`*(>QP#;l4w)7A4<3k|IQ)WS+2`ivv$xyGpSth&I zjP!OA!2MnN0XmPvn#W(jcxmDXg_7v`Ebt;Cz{R@Hb4nO^ZXAG1)*BG;TxJKTv(_4q zb|X2L=(cYUKZMZNKGsx*l<~e~$YR=I)&axOKlOVgTxE=W;AC3643XsC`{QpI_V&vm z7Vx@WhAgpSzS=|;>S6Hc8@p<1P#`WEN<-^^cYk|XX(Q2TXJuS`*$i9bWd`S`Hx~ZX z!!O>@izkEdYR5I@)f=cin_P1nQvq&%-55kS?~uB~D!U8o^vzQB_NX#+AzyL60x9X& z44WOE08T0?%az%~Ov^CFCyYFxxADmaBcEhwNSa(9EZc}}fp+ROD7mO>qU3|{q~UX1 zOCln)C6`BQaf(|NYg7|ajv(h^qzyP%fa=7#Z4#&eyd?!3;S!HT#;Vj*n3RiNc%X`^#=5)x~hJG*A z#f!K&ZbR?Vo}AU9e^q%urZJ@6HRtNoO zr;jURyZ}Mh{@$_$={Erb#u#Qb#`AA=0RCO+%?4zzxpWa|Hkf3vv}NA&gbrXUsn8wq zLYXYMBK6w!^C-9t&(^L(7cM%!ThFeukRz#6tabkhEI273b#2c;K--Yo){DhEsIJ#z zUfDS=E%`*OFYd?i!F=4qbvV8T;z7=h(o-AJTPg(6!&@~7bZ8o_p=L0?P>@3{yAP6M zTKBokbWf3lmeUZ^^R4JLzrl~o_tH+Oi%&V?IlP6g?v;zc!A7@|Hx?k&yKq6s5dWzj z@PYrKC^$0J6nK%Jy~B@Te#1z?St0YJP; z`5N5H!#huH289MsqXP&6sU_zKsn#bd7)cwcdiT_mlNfbCn(c? zfaT5C8BsmZBd)BQB^Lo1Pk&qC&i(fipg7(P@El1XZ8wG~cDYdw0sh$kDDO(+q1@Z> zgBfEVkqC!59Ub9>L?khzEay~+7!gfPj%4j1yJ1?TMZ4WtvPGhrJ+8m4UG+#AV5cC zP_~jzlN=L8VCPs53=;D8i(miYH_NYZUO3)$y4=!T03$|vkDn0{E zvlB=?B_m5Ca}4!DrUkf+%HHRinio$zoIMsZ-+^`#pPEMG0)hEP;}KWd_#*}EC|95U zd^{*?bg4+Ra8xo7JCjcAkeH-lY%c&0vL-$&1WH1bK$a&!xv&sviQw? z|A?0gZF{M@Z#EqagIvfxs7m2EY~rY|t8vOsnDE8t20%|6mY%b(Q`bgRnkv@-N`f+US%loW8nu~{h(;*aY1#5JOZHIY$ z*Mf1>c1YRL)$P$}XSM|h!Aon!XIEgy`ATn-MKtt)Q4=!O5N#K(zg|qkj!hmpCG=n;rsbOoH^5--S;L3 zNm_tAg_x7>!XhM*ig@>#eTAJ?e_Z|OxNk=+A%?L-Hm(xm9!M^69hfgWa>w{wObJDD zf*n+=%Oab;Asl8q07MDsp(B?jcWglRWqzh)m$3hg%Qh0gAG{Hol9 zM&z%@Bk9G60D2NdHwS?Q=z@w5CNq3coK^z*!SpUnLb{IG^3a{wYSS6fEwo{n4Pgnr zjKxuj{Kax{i|Z6ujw^o5qY-#Iwb;FgqbT}e`4`CIq6Q`kZO%WW(**p!b5s2rP20XX)`a7KTm26fwloCr7E@EXV zUn@*#L9;~DzPXs8SwutK=tbqsTb&6M3$oRzZ6*voBity=Tk#0(a6Si2d5?u)*`8L0 zn`~N26~PWW2Q%Ofk9;a)gp+8_=;jfNeX9Vp1%@m%{&e%g*o9VAF<4rK-wT4~ONlT$N054z93VHDo=*to-Hq9h@9~rtyLkrB= zXyf?X_$}%OBs};}Y>~X<;F2#OsefGI=z`#ye$%(ZLq6u=G7aiRV^0v=ifGWfrxf%P zw5DL4M>H)sms#VMrWpi*i(a8)?y?7>5_0gUIp( zoB%6UFDeib_(rp>KXOy|K(5NdLrcPUPbXm5)DVg2{|LaK*C(GSj>f^gF+DNn0^{t;-{6ne-dV7IGDg!f8DNUnOG5kEMvop%nzLD|51~^AqauUDNIO8Ud(o z^)*>1EY%NG#~d3Ny}0T!&$1&ADajpdgr>&n@5*FrN2)bNBJD*7JBbs( z3OvO=aNW*eS&awPdbPXH4QGU{yhrT~;l2TIrK>%&DukU~PQfgy*a5XwyX}T&S{zd9 zB`}x}JFf}o75wfSiywerA2GkD)G|rdC&o80n2hFvU>~*C=e}f=Lc{48o!hV~;&`L} z3gs6LUtcVLWKB!ic#id;f5-I=+A2Zd)`^_5qeAw4<9Vz-IoF?W5I#j=cPPpIqH*QZ zn5kW`@nGGUkmVFrSF8U zc&19>bY`la;76n0Xf9P=Kjz#ud&JfK19E0tYmt*C%ma^-VE=)X_odyMJb>kS0#zFY z=i?dW)Z79yu@_63wL}$h3AWy4(*QSi%MT_>xUF&?{HC()_3i}|aYD>V zN&TdQbo?95%y2qeD@ll>Bz8rHw3ihz%ZrfpJQRDR&nYEDBERUvCFW=CH2arTjHC7N zXsyvb<5z(R=@p5K@6T~_Uk`Z)33P20g5-Q+BzB_>hzulakL1OP9^-mDkq(9GWnNm% zCGvPz@#p7l1lJ3ku_Iv=tTa7C^yQ@H5-v{$&D@Eo;V{XT%Prd;{dVTC_f(~_2sTl~ z?N--W35~A@QY@-$yNcP&VIOz(xc2PGNK^=)_~UJN#MkW0W=e(cjzxg1zgN=w;<9rW z*{HG^7!4h03E({l(_%OWjF%kZ95!E+ z;AuV5*Co01`TyDbJJ=*EOw3C$x2XM9#x1LAIx2oDw&sWU1BMu)T(MiJI-?V-0 z6Q#}?;aV4Z8yAC5SyN606FM}>=heON#k_*kJGPvb$-8(7^4LfyCRnaxlfw01xlQeW z4wXW2Li=&3i8u*U6=Z*qO7N&_QxdaPfd-~Sq4OJ$IQ35g`E>PcOKBvnD%lgxBwZME z8rX4Lo!M{Y4QjKG(@igr@{4Xf&`nn$aF!U0FPI}fdot;9r0cB4*@k>j54a~Z<=c5V zRFkPiL)=w+*(Ys-3w@2VZdzPrakxF+Y+{H!%BjgG`M<;%kME}OqG8lp~&^2He zx>TU-_IBn4;3I(Wp0g8Sw?g+it3Ltf73*FA@XwzP?`&*!>a8VvM!fm69ot@;3yjJ7 zJ@8XW2^O)fX|O%Te-`kFZriT-$%H^pOFcVugSM~)Hkd{k76Tq8Q zD`2+#JvH!}YL^U!OmCxs$Z&@mJg_QM1!NXS#L+K;`JtGX17`e~oj9rDBq@D$Fpe4XqADdRWI|*PA zxTS z34`V4q1b099d#M8?VIBFXS{p}N6QdVToUj5-2;oMotgUaJ7lXgJ?~_I5ZxL!QNAx{ z2iF&A{a%9vuQeEI6x1y#iL|Qx$H$iYKUu8d?f>n&{{Q*srkZuU?1AjbyS|Qv=+6FX LWw-yaC58Skgq<$1 literal 0 HcmV?d00001 diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/charts/membench_comparison.png b/benchmarks/results/smoke_legacy_compare_20260423_105447/charts/membench_comparison.png new file mode 100644 index 0000000000000000000000000000000000000000..30a6b2575088134a38bfbd7efd972152367f5e81 GIT binary patch literal 33079 zcmeFZ2T;`Ow=LMBpaOzQXp~?&h-45YNK!cJQ52M%5fBlHP0pYalxQMI6eZ`JGc>KB zfMm%*L}HU;1G7K8|Noo0^KQNSs-|jcs^(OkYDM}NzOeV&YpwmgyeEJ22<1si3tx;IzGwwT+3T zg%F>h0G}w&>BqLVRyJ4o`OW|R8+?}5#{4x}cFk~=!&d)j*kCYBk?8+@x9pz3!(co= z-TM2QvSZZDz~S3+EU-(VOkP`Gc1nu8wO14zfd) zZnD!|8L|VVcCv&Z);e;m_R&Oz*cjHUwCGVGr|K z>DH6($kLO}H~VpMrl4DQB-&w+)4D$`t1_ai=t(D6M~V_hdc|t*NShr0k-SIWxZ0OC zCX-@is@%70uS}=YO?9S)_4Xzy#0r@8WM5=G)#uiu@9Vuf+m|%)Bl-TP?1ctqUw{5e zQ4(pXC4nuhtlMR-gkxLsaqEZHPd9^G^G(`2f|{eneND3}gh>m8Q~qXk2T!>km6zK1 zxtxAd#3CvvICeV5W$t}5bL@C6X+nDUurHhQ<4yVW{I=VZ8V$s6SUuNcjdjeH<;Q}g zHh*^{QXDxMPi%*`XQq*uub%(?DL9d+b?dx*6jQUr(pOu(h46m2Xvfi)BX1R?I9Hq( z-v5fRT;5(A^37|$!p+Jo{mC(QL1=3xuf>mL?dMwsA6AEc!TiVH&(0Q&_%J69T_eed zjbF*B@YrmRt&K3vd|&CGmRb6w$6?5=4qljXV?)PjJY-Vjc$Q9@YR0l;@8eH_wLgqw zSl>a-I@cbXeFTax{Dq{yl%KVdK*0)j+GU-V35J?Jwp z;w>gtZc#qzObyxL+v$Dy_AaIS#(3DX6s06aTjF3vy<4hEnyLVdzhc&9leYM0nL1^m zj%ITO8^0p4q_wJDo#CD^J$JFHmKVp)QC_kAeQ$KZZmNTmNy^pn%CFS4%-iN%CRMEV zJ#h~P%blny-{0BUV*4gycS*)I z?pw2JQ)2f0IKpw_Ilt)^$+dpP*lTlku-?NqqaD{re5q75O%jd=oA&0L)OhSH#Xe(S z*&{-+x?-$aJANA1S8f33RgZm+dZ+-nnyxNNFAUvuyj z{nuFOs$rNqz7@@zr`(p$W(hs|8g5(pCMZj{!nwwC-~QVY1lS@)pC{V0_3NHZ#ClZL zk5PHVjuAh`6K&=;+*dl)sN8-fMOinV?ORNe-ZptUHoD|r72?fWwAjdvEr&(TK3JE# z3FGf~1LD#yr?dFiVGUEd zZ%!uJM(=EIVJ}(qi<$?UUtoK}Ky$H6+veLd6;+Ld;UMw33+wRfN?NB}W^$f|@@oAm zB6wclJ7rb9kNne(z}x1V>tn%GL06`dNlp|rEX7yFsq~WEwpJ%~v&!a5pYd0ba)*6S zxnE!sHdC)@fDP4Z^Y=kE#|OQ-uD@a!iXA})`t-~&*s@z`fdv=q}DgJ{}lKjR| ztHyVnR6A5UaV~-ST}~6xVYSKY*9KiC)?T*6{bd|&r}`S>Fh5*FPRAjCXM#z@ zLc0b+jYc+H2-|PdBFXN~p;_=Gq>`@BEblI5_~U`w;p z17!|1N(z!H^RNKoTev7}Ja)H+sD$B53kUZxN|vbDgiToplU?SLSw`!?~2NIacAsd^`;aLEk(m;HRcL5(^^P+0*sJ zHdyWvWCQEeV|Jqx$GZ7vSQf;Vr#iU{Mmp8>c-NQP6(AVGI-_KB9De%juE@ZpQ|=+l zl3XM=3kci2{4v}+zeb7YV}y2m7#T2?>!beHYnLWsT=?9=+;=u6mRp|2$@(&~z-&#r zX!gT3b^aO-CNvNg%XQ7ch<4x$7k8K=+hVc)d}c#{yV8JM z7bmFQq!JNA{}28skA;?bK_}(;JK^^Xv0*Bk@?@J7B!qdgK3J@F1 zd8EJZ0pFbc)y!@m? z{lWINgRJic$JSkO+J$oiW&GI+N%GM$VSGAmi&~ZyDi8a=N6{?FljBL3k#@DEFM_ewxokCJ_8+ zQS8O`aJUmj9Ua--Cj9*kck#F*{{9oqZnThOKV6SWiZ`Ys%mnDd^Lo!Q6Ep&FL5(fqryY3OIBY63Gw3`F^*^VlFYB6n;7+D0R24ij0YEk z=^nm&m{HKx$WP2~mrscKBz|)pPnTr71in@KJLiZV zBtnA$(|1(Y#@y&*?>$_fADMIumR$XZ&iCD#!$*z0NBSWus<)aTZVVVii^yMDT<-nGaLTK5|>@!42~>uzC; ztCE@>4xBO4KApJ4*-`Vr7GWP>$l%)Q?`nnY!v4Yr@xK(Eo`a|y_2_W0&~d_v2py z9kbq|-*M$yEXx;=Th#GumfD(eU3aKIb~AqNYplnP6G~g{l?!(81c|dH_X>EPE{yuA zL?9?XWj~N>)YQQ)syc-~EuhK>$mOg#_f8q_^n{9|i%G27b+=Rd9$aUK$|IE9yDpe% zhcpy#rGtI+uc-qPb`Sfp$Jo+sf@_Wo=?l=xw~cE;(R~$?uc9H z`2!hWiHhkD%OC-?EutJ5YS_cbeSJ*SccF|?FTtBI&RaO^$DMg-$bHLvefT+TEyC*^ z-r}nK4MTP;4mb$Adc)zhYibd-ePO;WR2bC*WLGf{N)(HyM-T{Br9b6C!lm>Xkarlz zx`l>PDg1pbZ0rDQh*Or})h~C=76x62-s5*&Z<57L5SktBfXjhqHb0M==7 zqv?quZdu|p*|L?CIDmWlPRI8*ir$i2Zk44SC*0xj7yYH(49Qz_${qLU7ZVwy^dlz{ zpX6$f;l_CFP(tm1iQM!c^Yd0w8w(@9F~Jy*oelP&jrT6AfWlNxN@rh^^`X8+6l5BF zl9s`hUx9hWus2rIizm{&KSo-Va+Z@8o2Uk)RVByI(SmpI>(b@-JL5l2rM>tQq_6B)fTF}6KWPccZzy@qs2q_ zccph4KaKMHE5SG|Hz9MZqQ6J=+PJ40YjvJYkw{P;NsC)}&fDGcR>vLKdt&i>Q( z&K7ZLErKe3v=T?=-^j@E<;9r9pk24lMqvZdF_=xV&DwS>P=rzJynNxYG~-W9f9W|- zCQRD+uWk35W+f1RLEfqOe8Rq|?l4ExtJ3YAmR*v9BP6tT$@&&f)vXockaQH3ejD$d z$B5@R?5_vU28~V>57_s40}Ma29Xe_`MfBKpNmUJ0`LM8iVZCo?M|EV445M*BFP9mE zxu;DE-csLi7=C&%5p$+lJ0h|o|1X&E%w=Dux`;YjX$f)@c5hqUKn{UTYcxkN+yquO zq$IDNL7O&_QM%t{jnyMco=Sew`N+Y^#_iC2;0+m463nZG=!G( z6cIT4{!xRgOlf!5pnA-faQylDrs6znnjj?O8x4iOO3ks1vaL+V8(N839^0nQ42&KM z3DHI6cpQnZ^!G=_B0^&o(?u+8Ju0*4*L55Q>`ZDPPu$M4P$Dy3UELts);F-(u~JM)-2G2)WRo6;P#<5Q+`J~mJW)F_&bi4)7;K5L|5 zz2fD(GL`V&5DKB$4m&X)tCHL0M!Zrj>Tc`6FC_SVQOBL=#!t|5pQF9<`-AIsJ)Gdz zfKIC%Zd|`ekSw-JZh>vgCXmh$Dl<-fr<}>e$V7;#@&0Qk%J7-Fja(^Brs)CVD;-L5 z5}BwlX~}|OchZF+c)Bx9ZF6SSfkR~S&h?X*9^acvWR`w=M5JH9uKk-^m|@Ylw%ur` zmW9S>{F$6w{+-ORiT=d9k+!=FHbGU-nYd8(b-h9Q0ERI`^BT)hV6c)et@Ya|h)yMc zYRmPIJcwzGq`ZoO|NfW6l@OHx`n`wY|IOe37l+iDK0}fHcxAeq*5=pO!vNt@35YR# zbw-*rFPjYS87^k?$XiDiQ%v?^WT@O}iL2}cy3xBlqt_dUFlv~R#;PxGIn?Zx&`&Si z%+M;LVfe=6iT?J#&AOq7L0=3>`tafO*GLimiv4nlP`}092s}CI=JXdL={&j4`?yK( zl5ExitJhgpZJ#GSw?0}|1xvsiNE$B~Up_Vc4}1Jx>Z^^V317>~jq~0fS2rl8g^ZiO zR>AccQq}aNZVc^>!BvJuC_=~Usu}{BeSMBF`-A{nP?Do8q^K{UtU_o$we<+l8Z+ygR=_50YJle@Z?}>3b~a_TdX2-X2;Zy#=OWLGK0hWz==+U4qU1S`3`*V3WoRvpM~I z9hZ7^j~9J)Sdcn!M3?&J#_khFO*Gb??Di$tZMu%Up4#+ zhCN)u$yyZFKjI8!qp#S>j5P$xuFVb90gH9)*SE-`o))u?(XmOpL?IH3v1{HMn@sy3(6tPn2lH|JJ3u-*~lYCW2yT^-=|_ekxYXEb|yVR zQ{}x8q8D;r{@n`9<2`XIm(^WSft;_X~ICGg;CF>pFcABs_u2HNIqIXv+sLlORHUcT@E zVEj0UCi;-M6xL4eFZm5@AuSZUu9Otr8phGtU(QE*0mKv|$EoQ$8KFR33n1x)s;<*G z{>e08C9J&k_A=r+vu6wY5=|@NUL%0?{%JWQ)jhhiT2$rjzP;&bij?!X^LAh*crkE8m=OW~5s0Z%mV-u1Wh;i06xm7u~hp!|;L zzd|?Q{o)*mgClS~sxVz=6Ut1+@iJm16@o96yB5VGK32cp8~-Xg0(srrUTzE(!*E{@ zl>Zl}DQMZE$OwoOH7gph=d|US>(a73Qq*rPCkmtowuh19yW;{L+n$qiQGg?e=+JhH zBOzyp4`>cs(&+D@Dv#9D0WFyj4@?FC1u_-(82HcE(<#JreL;0?l?Sy3%1p{DB0z`% zMRkx~Y^WGFw%F+=cqj1MBWg^fh-FOHqrb5a;Qc(wxyB5zZGcfEy-iEz`8936T~_MU zSO&gQ@+%sgKXiYM4*{P$ivwu zAr-tF(d#I+o9-yA9nn*U;=~GAJ8!s7;zWM20sF{!m|hI+?nfs12ELm6)T~rxKi|l; zxln8R!?m2JGq7EdP4N~tRX&j+ZQ{eL{&l;9thVOn#Fm%kb9 zE8DulIzKi?mbu>=aFtsEIN}ABtVjTppg~;s4!jd1H-^VNbXe5smU8jOi#|c$0UL*& zf1v;6JvkFx_jH}LiVMCI@Z!|WFP=589}y^6g~V~+0rhQGeP*YC;J z=S=q}BFwLD*Qv5(c&1@C$Hn4$lM0nQa1r0CzrOjC>*(9^j9W-|O4;u&3flf|?bt5T zuk$fc^uPYYzR$dUYd@{Tz0pSlBv?B9N;LftnSfX~sgczgaXV-K4#i*A2o z75jH6>sm`}CImcoNLJlC4k44_qQd(09MU7Z|8b{WsF;(c@*p`0N7;30T(6B+H+KLb zC21T;#>*gn7{6}elYj92!?ouEPdbvxAN>rZv+YMk{|d?Q^j^)Qv4$ju!&gVmDsrOz zKVA1{G|{dyX;0)m%2x@b`Unufv`x1y7>B9&%LN7$a1fEHP@%R%1lpF}+cQUI0jdhk zN3+_4?+>74qFy~WL8}`V)2O7zs%|miC$!su$xw)u8Y%3vkQ;bV1RF^J+tGI7$j#+7#|B7c(>ML7`(h; z4y07v&0w}T;IvG%T<26m_JEM50c=P{C{ZisfpO;~H^^z^^RWZ6@K)R~1o&v8g(f4~ zWT~tEK%g|oxGsL7ztAvM#njIm$b^wv{CqB@mxzk+btpvZAXKr&I8UX}kMh1`x-GR( zPp6-jQAjyP2v(SgfSg0Q|KO36^q~nLx2ctZLgL?_(sI|B1%Bxu|mrF@ye?sw}M=@i*p zsn+DDdD4b{#c=gNqonj6ZF<2D@k4%+rn1E)g&AeK16?o8!mR*z-T;j z@0%Z~tFtJZEmV%xN`DfNrS6U0UkwkLDI-((l8z&B;>7-T03K!aZU*z%wEa~;r@PKG z#V)|mF#xe&V%4dl<}E%~{HecBRb_8cdXh`nbv{+uOpySrssA-x5Gl=aBibX4#mVx~ zS5gczib2=O`ez-B2yM=4|4vT?=A=me5JZph6%RkN4P=^Pum>3Gb3*7*{F53;#PlsV z$l6{YarrXT0B+WJvW~@DR3n=Eb!y#u0Dr}D8$G6h>sDx#;2!ezdpV%;sRv)=xLzc{ z?)?m4qnwVfT;Cu}^zzPZ(Ga6XEc0{XD_I)O14yLHf;cZKdwc~hxr ziGuWDKTAM2NUk@o8$GrMN8h~#DV)D^mEk_ikfr`;K&Ag1{6wzAf-PFTw^RPv0-=a& z_D4Qw>JpX6$l$jG@<5)5lc@Dpk;^3=|ATAW0n;w;01uXM!kQax%*a6`FdQ!; z{P1ZK8jb|zN9OtGw%S&h+(hl(YxqA8K%WO|)r(K3H2$U7DrhMJz^k0?GyWFAfHWLR zHwy&&PKsM0_z(U8syj!m0Yip0*=ZJ8L=Q>>tAD)6vhc?v6P`-6!+@E{`dkRenWt4a zRGRE6yoKlJ(_1T3`mj^pYoI)f zts^)NaKyS^48&v8_HeFs$qnj@S*25{wp9@9s6az7cD@UkqpIbEanvy=M(IoKXHPZ+ zu`(kf5fzHd?#thAhRKO1i*^D&qvV<=y))#ru-LSOaZo>wDsY6(HRoF6Q0*Oh@Nfu8 zT#KfahI;2Ri>91-jWhe0E@d%qU^*1>kF zxnO#X>M^pb@KaCf-#_w<1-iM}1u0I$-~S)B!rUXzH|^$APE+N*6E1LuhaV+}-sMK_ zEU)oSz~ygyQ61yMpb~NzLx3>416d)7))YVkHRw%uP7UsoT6TFlo%CE6@YyKPL2>B| z8Qcfz z{|KxD8ecFbC`Mb(^yG|yfrADuz!r(6JBR|TgLHHpX|Szj4n$=fuz(z)T3cvy=ANhU z$PqC@DZh;Ky{ZvewPCQNFlpA(g9@_QUA^?F&ak;Iw0iE{yLS}n=h;im;uGBj{uxlP z-#4F1!$?C#Wc^qYE|$%MA9P<}!0%QPzkj;<7&conEaBgS6)s3tPa&WpgalrRJ7axF zw?i(LIJnD5{9AcSz^+`mM1OP2^qd_r{;;AKi4^0mRP5 zU|j&tAIT;t$hk@5{2qq^dh@O9fXEhYg~XEyUC=?7#cb#L#iDGQ#YTW!Zq_?z5jjOWi(1fG$WU>fk5xY18 z5siIE#DlsPVGAp8gCrc~qHuv=aFa6AqH22;(dQj$0vr?%k+EhE&e2Mfva8GYv-4ti zTp(!a&A6O|nw@gWz>|?^Sh3P2C&>)_X7?oz8586ZAu9tj!$lpDLF4kQe;4Z~X?Q8@ z1pf-yH};NGsqLN#Fr8FT)$7a@R>IoShovZAGlpOg#K1_~X70@5vn;l)*Vx(UsH%~_ zprfJ~fS5iMt#`~J|I6=H=1u%nQLFybtFVFdS?U^@m>AuSZeL@tgQ26Z?*pBx3A@z; zaQieXvmR2dUT#GJb84~$M0Zt(xj8UH&aecl+rXJ2O$oeKm@e*p+xSjd6@VRbpu--$ zHV@ObzSUc}+H1;hr46pI_ClbmGlr2ZCU+KG8h84x*w3gVTNh~)aU2;Qz)mktKl&D- zg3#16q?3ODxI_+>zsqZz5ZkXG^Fat;Rf4+h8C<36b0HFGGO|GbY?P$zgA!}0WeliL z&V_!oAQ3>Kg3Yb=a2q6WDjHxLZD+o4#Fkg%N^H;zvi}lZup9BlGCJwzxB)$>(03z< zMdN2N3cwjzFv2y%HP5ldgJ8vZSGg;O^qF31smQi|08l2eJ&Atn(Y*fsH_3;Ozeq^B z54Ti&Z+5Fo-7xU%79!-hV<5Wlm_aFZy(=1ma~ve65hw_F+Y&^Y#pWSRdLvo@A>5SQ ztZXF5s2p<#2sr{^>04Hd%0k6_E!BpdW%K+qpm@Uy|9JSe7{bdvIFEvcN z(%*9(!m5?8!77mWR&&oW#<#zp2G4!Y5dVmt`v9%zuqSzQ5d=1Gz{w)YmsK7y4lmL- zIoXWIC785+xaPbIMR}ub^SUhOT^{p5A?jVA;r$0x;jKTv7qQf(bTkM*K9ncita1It z*v`bWqNsJ!Xb&#Og>S{-fZjCyH*=PD)2MBYu zL1&#Pt!7V$N%i91)K>d(7sGYc0=*7-SyLp{YQps z>t#}E_YHsi>8u{QN-`9UppzwBpvcXUdAqBCDL!C5;0^_y#*|dm&Zg;W{+}P;t&VYN zvoaZ{gIi%`to-BiTl?<`lD4jJKNCG%8!WvO9%>l&b=_dZU|jQ-b?4ih(zJD(fVZmk zCycR=K=0$R)}EnY*z&E=j+{gMX{|?^u14 zST_rubY1RpNr$YQB14mIallMU>sTUpq2Qng_2wS6N3i&w<7>Duq0(#fRiz$3CRh3> zGnH>HtWVdmDpx8j!bmhR-jw%_t2>NzC2qaAU+BW!>uS5#-6!QvhdB~1Huta=$KxQFZs6t(vEc;XTVL=DZUcSi4&W>u_e zQihDC7t5V(R=cNoRZPFu>pG2+>Kpab3}u%Ir@1Q$t!eA4J(3O@Dy9t|aNq+<#ZU)B+&N&B@6vhJTcjlc{a_HyFRy*FY^x#W4ZaR9GUO`sEuuXA=c9d<5R%%r>5;tx zxP@k}wDdRDG3s&Q<6qRsJ*kd)C@&D~fpy`}&H(U`BJIG^v4pf*cekh!sOy4meQ_Xt z$L{2y`1pg-=9#D%0qmoZ&POdEca2ouzp-r_i>c@8n_H~hkLY>v1tdX)ON>f z@uFjvnYmUbWYrKvC0W1?mm=xQV*O9*`F`+UDUJ8T4rcxj7Kr5>anzq*zDZE_<-~Mf zk>yljd8evYLJoCbp*eG?xB3e^JcC;NL_TN|dm#?xOOzm~0VTLQLtfU}*fvy~0;c`k zxF~L>p=S;utBTzC#(;fH2U{cnH7$8~K!TavhSmR1yDe6}#-`nY@bv!JjChV$esNC$ z5*}!gdobe7mDMsm;0^1LzP%YiIJE(2u=36vqF@^PcGhMKWh6V!_-^m+7G~YS#dyMt zOHs2)h}yZiL77<##8Ox(p`rW?g@vG%rfEh2qo}Ztvzm^bihD`6x)*)fVcHWH|BB}J zagd|KG(4C?sdXU4aIGI4D_5eR3g%4LI&K_=yYU#3Fy&FhkO7p?{61QgjE7#j_D_Fd zy0%pf`J~1XCV|s|Ve634lUmq=zB3U8Ex36e$qH%*#+j#k1-u>qdly{HvZ~75>WtCZ zVf&E*r3Z@h#!E*ZS&RlD)xzXg1k)UB>f;9|0TSVO@aztmvQ3XpjOS!7bT{JXD=zBpG4e>n;M%M%EW(l?JIPyajL zXV0F||HKd5fboe2NS`?}^Wc#ozOtF)0Mkw&tef8gON`ZrhV`T_IOLYi{0;`Zi~dt@ zhl3`S)XL@KA(&hSOnDs&R6|mLA?SZIIKZI#!mbiwasC~*VFv?apXx3SG=h>NB7T!7 zt1nQvApfxAOb$VDrLX$w{v`rnK8F#AU|-Z)q5%kRnAiEz(gHKpr|4#plPXvVp_Lkg z?8uade}H8_JjR&%4l)ZIUwc`jmrg^1aI{!s=-L2vk(wiTbZS-9bQtD}$AYXawN%dR zEoDZ5pV>)i%SySdXRNnB2Azvv5Lr+(hb|c5%a$(+5|28Fgv}KW3r7o(UdZGu@c(dHS25GWBl? z2`^iMz=QQ75d;~n*UBe4}Uu;aF?tIzega5NlFK__$qL(_${ZX_M zz|RRcEW3Y;k<;iTdeV1_uU7%iEBEG^plT!*bm^CUa$0(WPr5Us2ge+}sM10oZXl}> zP?gYuA~m@k=gW3m=%=cWdC`Dyg*4RO>=wa!N5YJ&MW71trg)v;9fzv!lKkiY(d={K z5!fsbAugZJW(Fuu{i%+W>xGEolmJf{!)AYw*st(Q_QTLWQ498b#_-cWfi2|^o6ffw zxKiN(ObollkXhchHg#8|p1>N?duptiz`2}LT*gy<@X6*C{>W!czDvEUv+l?M3`~wK&FSfkk=;NeUC6R&LNc{ z;-b-Q_gGZQP@lYX!FIOyasaEiumljkaUd^EZUY6)7iNDCx7T2$JiNh4Q0(tt9>Qdc zu2xx!&{97*!DQXj9wXFLA!hT-t3`TOg4Y4Wi4JJ}$c;P@m%7`m$^RTmF7)94gH}p^ zWF&%q;F+)n3&IOpfJ(n^5x$6Dv77q&U2PxPRrGJv>mUZS{mp7*2lNmnChgF^)9~Y~ z6w5O|guHI$3dLEKhL{N2!h;rI%bs7YI>P$J1HB?bHpyy0Rffa?yW}^8B z#5i-rXYOSgNfyw<_u7+kBth1(7X_zLyO7C`T%%;LxI)fv#~<52}l8NAfSAK$l*ZZ9o5V(V5mI?E*)YnQsgF2 z*iU!yy7?@KJq-C1edqy5xg(#G;=O!Gu}Q7J0qN36IxinW-X784AD%1rQH&P-Y z?(D5ua!U}531oX&i!vE}tRC^hW??$^hmqNYa*_r_U@JthZfbzQD5Q^alTzZHTN6as z!GF@?=hx@zz$oyhwg!j^@4Y50pw7Kb1^+^=WO9<0)V({4VO3tJ_(zRvoUa0A18RCu z3s%qWGb+AAF#K+%&F3S{-UPjTI)Eg^-nJ@_UHqmL>i(K^0^MH{I-(>IAC+<>SYq*? z9>^56fKXLOaUNTJLyCu-Kp{#r2K$6dVv7;Ol`rjb2s82z^J>+g;)cd zeYXz{RQ7nNmJh;(oPUcW^Tos_pd!W3SZ<<0azTsK*3TmRlLGZ?4iNd0{HkDOx;`ze zZ{_vT`L`^@ieLpji#x2uk&TI11qRRWU;$Gs#%9@lyWAb`!pW!W03!S(n+OQJBplL( z)&Wh$q_P6L8=JaFska2lOfJq?Uyx@86bQ<{*C0*NUk$xZJw2KipyKZ>i9}RH<#oOh zyb##=Qfzm#yGoJ4Abr#SscE`8k6C{)ZiBM(78fu+j(`YzcOu}a;U!QaY+YUL z5!fyU23EE!`A<5*8-?kFBBO;2o-e-SswK!08<1Fa)D^Bw#^+18uFs#HC>{0Vt_2mQ z4iV1uhj%|Z27#MUh?ig!`j^iW$j!fNsrc^@1>^m@zj=GyPrnh+>C9e`kgdKQMzw2A zz)zJj4dXqt9m#dmy?EGmt_!Y{+GBU#rT#_=@;%j@XW5SHVOpp5S2MLTWc@-;kHwaJ z$8{{ge&>`(%Vy9y-!;?qVPlo}z*yoi#gyKas={qY9yCyzcVjPtUgb0x*T4bKN*cPC z@aUWJ5%u0x(LwEI(#BF9D$7C*RdjC|s11u+3_jk*l)f&wco zKYzpD3|Q_R193w;eHNE&%c08ag@kE5--tla(bHG9fX*7<{{?@n58HxG;{=T; z=2S2d5MN^~zk^3k1T57-xif=;1Qw;lC281HRBo$+GXX7Jsx%EQ>abd6L;vW>x4(lq zi@Xc}iqB8QO7H*=AC%wtqghrDR)NNfAHNLL+v^tbIlXc!^IPU$3OMHU+E%r-494|u zA#0~D`%600zkyKU%ayWc57O~6>>g?RIak1g(xmd2qU?n=%2s>@ODlcqt}KC2CT{kHHhUyxup5rP&7LogvKC8l{kF8wOr(D}lIL-qBf_ zDtP!1Z1bz+{tPzA*Ydj6)ajVNsg6bgOUGG(jamPT!`_KcO8@gm zo*7qU+5~A7%cwpCCY4Kqa_LiL$b)Aa4Vr>eiR)^U-(L8AaN>nfcUPv4IAY)Rq3XF_ zN#{*nkI^Iyu0r>>m$QB$*3f68E>;A9^jFdNqprZ&STIAro+pCnNCS*(r-6>R zLuHShs85g>*jw#Gf{5&2zLBvQU*qGn4S4J13NdJ-3gKYoaVCz-B>-RiwYfP8ks+H_ zW6#ZrQKtZnEP^&1)lSMQg&_QSBN^|T6?0u77-_BEtES-R@*yw$W9~0>5~`jW6S4s0 zv=0nfgJ{P1K5#wxQ_$_%ARz|AEtypiwJ&ArY9sTE4+LkooJ@~~6UOlmEWjqD4Z(w`EM^q^tD9{>R;eX-KmcmhOdWJk=V;&SVy3FT3*N1@aPGSgC&3xZc7Tw zA3e@-9j zUbCi6{*#P%(~&|A*WOu%A9EACL{Tz~a`}~MUeQD<%(&%eeFc|vo;8U4bjU9N8U=Fc zhLmPu1o}n~oDy{P53w)#BBj1sm1h*P^_z?uWvzer*b5=y@v4~m_fk2<%il5vM86U@ zyYFQ<5d?@^!7?i+J7rYAMlQM=iRgP@B>GzJSdIiUEdJuQzwjHs!d0wFp6}3h8=G(Z{2jSop6+di{|qZHoVO1P%=#VU8+&NL zL0aOHR?c} z_DSSM71llWGPhZY$plD1(fG}D&CE6l7WeA8I6N13QkBC@iOQzyJ~~B1=kv9V#^*;(EfbHfWH?A)C@Q?C{LR1v9pB&;60}*AbX2C8_&ur4 zRAG>=u;<|?@)$ypWD&K?>JL7%))d*-D?{#UvOU|aO0})}b#;acRYzC*+z6z?K`l$R zhVr%hlpDg+x%%#IB$t)YG_N0d6JB9{XYrD}WnP8h3Eu`u7VQ|@O;E0#q*#-~a%WO`LQ z|JbrG;59hdbPd+Ac72Y|$!n_~!wr4XcC2q`x-U;+{miB-Rv0HGRV&6_H&%Ymch+0h zuJ)N7E1dq$>pH(+%$d%ud#w1ceZz||NEVy1?to2Fq4&JFe2CLzqd;WnFz`bu^Y*IO z|Kyn4IYf&-{^cL_3mY*WcFL(!Tf~mowjuJk(_hzDit@MH`78NyJ=QFkK_{OAnrlM= z>DjUs&UQ--?t+-8jXvVY_axnarZ?yEOizl;5A3?KL|n395}qyJq1;Y?(v^N<)2yhI zY+5U`SK^X++Xa%uY?#2x#N?!9W7vOS^;yj<-Ko5nXLN_?TBzu%-YbHN@pZl6^ALB2 zvbgP#*_iUa_02xRI(f;W+YIi&@~g8l?XP~kZfco;nI9Lqk#tzrplARDZ(>jsm5VAk zM1^Ug(&EBteWF-V!ibu#&QLzl=`3{hfD=V97_R#!Z!PMlD%^slguFd&vK1RMU7p#Q zAHrpM|9Cri*rv*jKfq<%!?>GQ;_yH6ncPJGtjhO=RT2iaJyrujCLe^3SbX!y(JCLg|?s>Xli%%P?~wvPvb6{}rkCQ+4{%$dwPPGbos64gC{mxD~O zZmTNaGppDG_pi52Tl(ajay_27Bf-1OML&?*u~Cw_UUrM-RYWQJyF@pcWtC&bw-?1;E`NTl%L-Hr9U`PNqBrLo=d zpQ+n(&W|6j&osT%5A4)pd@pKfLRJaZ(vaec&L?S!Yy2GYG#;d!;lEkxbY0z2G8M-E zxOsRIrAkDR3cJjKPHeettVUXi?uG|+Y>L% z%U>HC{Zbm9MAt1oL8Tr@rV350S8NXz>{ycUjbHz7Ud}&fxd15^c`9j#rMZISb5T0; z6YU_!a!(q(2CN zUM%?U-7u_{HmaUe@1GXj+mA7#Z%t`4;0`%iMns2~trnc(xJO5T(amb|PCHQtJ3i zUY~J8uH$ei%mly7NBCVT-1R%m2pcOx{;_+@!DhC zepeQ5rMSCNK2jBda`(<1uR8Tb^FNoVxT_cT)P zkPp)-NW88sX6iz^+FnlwkgDqI;B%@4yDQU;0H$N8@$X_RF}I0NZe$3UkK>hqIBhFy zo*(puqJqQQL+~LB_+}h|B;a1IL|1;GpLz%52z*-N8frm-BNTphLspeQMphGWuUpap z7r@`XVKDxHKJ6ex4b_KzNIL+|q%2ZCa$TcbXU4<0mo10L_g?o-yYmY z%kEGEctPcq^0|=dY$fXpSOZ7ryHbU83{N@nV|`{7 zV$4o%WdXMm=dw1paqH@+sOJI^aRzX3N^#U2BE2z;dLf4}#$Ur<4^DSy=}k%DtY7^G zV=iaA*jUNv*BY$0)wN<6JEf!Hd0?0Jg*+_<&dt=X=4K!RrSv7L-xb1ZcBrEJyD^TB}Mf8Xqc zj9D+QXF|u5RkPT{&u#~x?9fpToCjQips54FJFK?)`g;kd#j|Pq{ab{~&u)!$KWnrg zyBc!ec#t+^;D9Gr+!vw|v-}~?1T}uy6Sr00FdFzVkt_p|QUc1*5 zk)k*Q_mX_Sn*N`sG)@+wne${&fT1_U_V!#BaRw3=+am>KlJdS{glw-J#VQ%0G2Y&;NPXo2Cg}YliB~p>w7dexn-emRrBNYqy zYI^6jIh~){AgFvf=!Yz*;UH_go<+?~OcIV(nscDXkq@H4iQ&->2|w@`v=3L*nj}Rt zLc8)&)T%^3)d{C!p^oP}k;1&+e!D<36;|vKhh>LExfi#F!Vbn?G2_FB z4-a0u+@CaLx&hQo^H&a_*mwJ`pY-%oGp^W@c;spHpf%n>!WcOBL2{*wOX*s`|0jV8 zzn67XUjI+4tN)5o|6deu|3wn}@6!AK|JPxU{D1dwy#MzfJ`JGGIvDF}!R&VmY21i; z^Z^5|FYrks=BU-Z4*bZq;OV3T9a8DXUDQVH$snIV;R#ZiE7I!MfkKxB41^B(dknIu z`_}rxSRLwzU<0umbt@`AQbL;518-#$9Z%_<^|5SF?0qNOlYBvG|2JrPQirTNK=tI+PXsXdAw%tB(2OW~0lIW% zgK@_joRsPPy?e?8w=#x_gM@ftY{?K@xDL#nP2&;B;Br!KK2T)vP^5XG#&5{};Sj?` zq4`M$`Xp;X!>ClB*aEj|+s3FUYLzuWveh3`C9-3KXTb}Yw0ClQT{{oFr^x`@jG@!I zc&w|CD_5n1Hv=651P*X%I1BAG>a>l6BNR-wk&~{r3iJ^#f4L*z2DWK`E_6 zkg!5j)dsg;!jn@l>6nL^R8}n9mPkF(dE5;`Vk@{<@}+`WupAu-ebr`F?VLo`(`?kB z)L8(2kjOzmTa3L$mU^R7NHhz-WTMzB;`4?|2uKg*eN5(n8#$|yE?tM#Cer>&Z6}W- z>qo}`YUjkHKEyCd!t;ZDgk}VKdvCve{5P~}YdGV7*ut8du>w~-|B78DOu+IWXab6B zFBv?&*K|6Hy#%8I24S#}?R8=oAV!pp;squGb`Q33oxpS)QR*RlIAxYdz0zxPSM3Kah=;?CA+2 zWq4aQ3d+dNb8#<*NP6m=0|&yE3qct4(^8Cltco;!e(y+j!%=>1{U#_5me5=@0t5;F zRr|2SHEWmeDL&6i0ss&D#2egMbLV>nlePZKaF;m4FMrGWarP-@p~vmT=g+|NPq}p7 zna2$2v$Q7fBj$Vr=7n=f}#IDjM}^U|i^ zv3bVES90fp8Lh1QNQxf>q)lC@{}6nVLvg=*LV3YKHDyA`1L}< zBnfqBW>oJSD2Wo?Pi2>D*t_dXGpsV)Irce*Z@@b$73_a}E2!3y!a#V6&JHHMV1nX} zsUEFQE^TXk>36%QcqgQ`^l1WZz_)MZ{ee0NcXTa4c3-VwT)!To@8N*MXRr% zYThl<_=iGGkbFIr_Y!Fmquec%-BD)Ud-R0}P&Q~yqF){90;6YXi*XsMATcu68$Lqc z0&S46(3MJusS|BPj%E88DcxP2Qx2JUOrkRhUJ-V@sX(PrKltf=w8%yAOZ3xydWBR!G^60tIXamK6 z&%$gsAJgn-1IJM7>HUs=LjmJCRgC^Qu3`nKNtF$u!o*|&3ZYATA(TB~UXAass8^j$ z-#X9Eh->330GcKi7@Ha_ho*Fqq6DLVT3vTBHE+-P%L6s9(aR?80OJa5NGOe4opvUPt^wJNR_~^xOH6YG%0+GkMF`tNfbp#@t z&|S99D0MAd-Dv$Wk_L+#!)Y&%ZVWMSa^|;6$F8KK- z`W1r+-5rb*l$PF**87sf4r#23!2iVx0%JZJLci8SKb!Yffsik8uBQKJhhWU@M<-?l*)ep zlql?E3|XX0`&<$1E@YO2E*AW`tv0KFu@(G$5SZ0Ui!*Jg{)R|_b*`Wef&LSF7Yi?6E`UKlwdV@^7i#8|Q>d7ga|@lfEx1L^nk3`g%*~^4PC< z$I$GEEU&RCAendbOL#LO@->3N!H+7+$q5X{Z6Z#z$A9=8ld$gVKt|PG05>=ofkB2T z$ov`}cfk_11g1oYXW|V#x~)9FTUaFEjtNbDa|nLZjbwO2@YES~<)#NSX?8vD4vVJg zS>n|u;8hb5wRmLuzHg@@OavHy%K79yBN^dC4=SFnzTEQNqYnnD9 zQ#QEMEk3Ww$mhQw?LAZkYq96@$w!7{nGn$2BAeLbo0q?un_Etsh-wz%%If_2^=_v~ z*PzfL%SSd8$$dk8f1BVDp;!Ch^cVq_l_?&`t-qDw+XjFUe0mFOjX zpHZq5{v8PWSO1Fl_#fK(#~QM^s{$wr(eSn+-jgke4fxl-Smld8Wf1ieTPyQ`B~z(g;|5(_p?Z#1c^tes7;FbS^R72tAv+z7Brk2l=lw5!JtQIzH=J z4|*yw5OQ;>TYp~p>wZ*8WAhOxgC+RcDB=7I&{blfQ(ltl|M14q>jRPY$PJL7{b@yq zY$$?)n-|=??HPAi!?0pi0Cb43|FZ)*<6hDZnX1OD1`55@I#0O8w$44_m(?$|egf&;-hyzSZjF+hjlYL3wF1!V_9OsU$BVpvF zvM(qseU!hpZ80S$*dUqorQk9ewWo=1i}Ic#wS%x=@!f^7u`uYRuZ;+spl zjBLi3>*ORmNP12)N2^Hl(X%7ben0x7D0L-`P+liUq}sBPf%YgJ>b!WC@JDKr zEt~;5YE~SeUo&vgHJj2rs`amO1DEz~0B1 zE}1ktR?x`vI=O6xVKBiSLEtZywlQKCmSzDQCbl`&!ja=RDyDmngmN;}&vkorQ7gHcbr5JhGnkqgr zyzJviXH3qOrdaCZx<$L-(KON?12kYa1tO zUXX4>CG4?9_$XZ*aKt)BB&&Hn`4xV6b63XpD#*gAk8(8UI7kMPd?tbIL4qR@Yf1L* za)@=-e!!QPKO`N4F42wCfElutmvlaTUeY4nliNN2aVWVx_Mj2;MHcfLoN`!EM$sKS z@hU4K%-?1;(kz=rukHcx=vgxp_rU0+Va(@%);}8t`FE<+m4GImt3z{` zY0!EEDxG{-^$X!WbI8(&{3C)@{b>z{yXl-n2O9AKv-KTEei$u^?CYpS%UtYM2T`sB zi7QMyYcj-#6&t-L7vHI+YbK@AyEGyN=jyU21g89BI{abf^7V=;CJ)7OAH`}>Ps-R;K>Z`RYkrGe3{J#wm zqk4xSpfW}4_T`l;oX&LGbBHv}<4NCDC-eZy0-j(;=i=FVQ(N*M##jGSXzQY2%C%36 zAGyaq`EC}wRrou#=nx zT7KGd=XKHQT!o&g0wDqFfMeZAtv+V6!hR3b&tc{T-*cCt-q#oa^oa61CUfJHFx%-PjP=AwZZqH8 zYX-skh7gD6PdgYtX>a+J>FCh(q7K6^JhiH~P@;eg1f*!ls`-lLLm8~RH1W?4HDrF< z!p9hI?bu!Ka|wL4BW}4hwwi~P=<(d#h#w^P;lCSjU4&l!9lX6-o5ENwk%V^XY1?Eo znOMYV&}^);o?=j`-O?-75$1n5Ck*LCCROxsm?_yvP24W$-qP`X5OwQeJo}fV7sDCf zCj+lbUp?MPP5jctb!x?K%tamyk2Q_GMY#8>J@d7fB%06(kCqg(OHg>a7~_;4SWYpF zu=HLfuxw*14(6^#Vw3w?hdys9!)USy)w|-z8lnPnjchALI+~ zgSm+u;kUU*fN`|BVYg{Upe*P#xbMh<+x+@ObeXBcMT{O15ZOX?=lQyjBRW*%TTG3} z=Ni^yKefO3I&yn%9cF!vq&PAEhGY6L?imk%2QkP}#QWZ0Cp~lWpaqFTVfK*LBX-df zd*Lxmcn3l4TXSO9@vvD|)~UOJ73YzUBRFi49vK0;UT`ARKGU7Jh-~&~!$n%>$o2UP zACnUxZJ7`xEAD@k13qwG_|CEU1uZa(V8?D`384R4d@I?sW{p?$cqS=;a_`OViY}Nw zjw9a=GBWFV4@hA7Xiwz!6J*Rv-!cN@*GBlK@Uw~*6bY}vWmvXlI%6_5Mz3My2oDn; z7BE9ui4?}<pwypTg?4n$hYM57MwIPG3gs@Dn);XTSvGNJJVtT;k(FAPvmfdsDfX5IL@y z4AZ<9(lxCvq7qyLzwn5=z(`XZ;>H*Id= zLMiwhZw#Xc@OC6lg9#Px=ig|k16HZsiuwzn=V5D{4%rDWde-`l#Gvgg+C?OfRS-qe z%$hwOk?d(R3^|PFS$b3kGk-FCEla>XG3fco`p;lbu&$rw5NMK>yJt}-tj2@iCSu@Z zse4bM-)k{+&_I3J#u1C5&(lOAMDi{LS5g+z%ErzSiXu&vc@|UWBul>^H z4XEe!rTe)UmB*t!L6@x|+XDjWSmA9hrs@tQbij)(D0gFP*klKy=fUj-6tlzh&A@ zpCQYcn+Baej|Zq{6Hp#YTsS<6*_sj>89$Z@=R2Y6WILq}6Q&!T>5zuH?ts*iC^D8< zNSb0y%R|8@_FRE4i%)%W#EF(ZXe*<-lK~?(yXpUNCy&xaate#H3bE9v3^p5%!LBbQhcS!!=r zfm~<`nz~xr2ZUaWlpt^_q;jo^nurQPqt=rg3290?rA_*t0%Wks-3iMzODEfe z4a^ziwbL?LjZft5rrbH}1IB^Dv4ISJwM{sx_YJ<>#d!kJZ|S%#lt@9cU^)M~DXhXA z(~YvL%JL2-rg%tdL^y5VC3csbwKS{X%UOJ!9^#&N*pxKK&@i%8hC*<7P=R-Yz;hy^ z5$QTrI)^;wELv&|BJa-C{Z%er5!{7m>%`!`!HW#w!8dXZ#~Bsz1)yXLsWTtoBu?7L zor9rq<-1wQzY%Q})$uN>Ew?F}Z@qYqe}u}TZY=P~_l4mfPs|D2c)uRy)>)XLGU0Yr z@i*Q>z_|D2Q^GlSCnXaH16As4^i7?tygCd!x;$__8>Vm-~6{qa^qHk)Kod(N!0#>2^o9O z94`9s6yy6!gQMigM6LQXb6GuJfM;?<&!9@8*ijXUjTPI3c!{JI^5w!wS@N6Rxq}bF zrNij1|fL9l5;yUZYfgMxV%49~{$n>*H16|XD7jiZbgL+P9+==b(bp>3a; zu#rGFpMH7p(iQ8=$Qy(9G-ZRO;v3=x0%jSC9g_Jry?hAtuqCHB2E+IKpYbSYN!wx9 z`R2l9RPE@!>ICGLMY>!sV6lI4zKAn;H_t@8k~4q8ce_l9FhoV*Jp|{1WOh9}!dRfA zGIw|IPv4r;tWqA&Z(FU^dFJwo>w8KEa@W4*2sY6a_rL!?_$bG&Bj##w4ZCX9~fY4c7;x$=#r9DOKXe`&RBpNre z3s4`to2bXyeC;C^yDzqx@~xd`yAua&arA}!WShCeqnMl1M$?TiEOQa1!BsZ-QPBKf z)5w|S*pV3)#lx^)E7dI?*5ucEI)bS#G)@UJ0gJ0)bQBXp;L1%%SEWfq*g;YsThH+i z^7+3M8s=hjq_`0q<_TrBF2_kQ_Wf97q`hb)YkW{FZ3Y#6U2O$&Zi3)M80!~_Ji?y| zcc$$}5%%%bB@U8xWT*j}gdmo0uq>Pc-E0PkAPNJVx%Jt1=i0hkUk*nL#)W{Woe81E zvmxT(f5v5Oz3mBSHvm4!?x%owI*9UkjPe;(17B{@7Ga*OPHds ziRn##-9aA1b4Y*FHcaG#SiVIy+45tKWE{v%SITMngpsBwjlzho@g)qSER4-ARgf_mY&ECC4TeR}jJdrs583-~} z&j)~_tIslCr5|Q9SDG{7tKIV>)Kr9SoLe@Ccr_H2@F-bOE)a7!mFK&sn802K#2Zl` zVyd$4>4;LLbi#zV&@ggNuhB7o|6uPz z)%!&5(-nPaZf+w|G%nya-zY}KmaOj+@tG&e)_ZoFl(G09;Zt-|9RboVK;)Tdn zmZPvC;k|6$$ic2atC`+j3;A}OU(1;*^j6~Li#uUc9`hy#DY}==ox3iuqTn-ko(4D8 zGPiKo0YRIXIXQPEZ9?UenvAm#iydD!mT6daL~axlKXCJPnuF0xcYWgzOxU&~B{jTG zSE2{TGI{SuW7ny^-cR1I`QB;X$A{q0xcLkr`YN(qtRd4;UQhbhpElt;GmNJRZWDh-Pe$3w kFf%Xy@e%(" + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "example_count": 0, + "measured": false, + "metadata": {}, + "metrics": {}, + "mode": "raw", + "notes": [ + "limit_per_category=1", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": null, + "reference_kind": "historical", + "runtime_seconds": null, + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "blocked", + "system_name": "mempalace" +} \ No newline at end of file diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/convomem_new_brainctl_cmd.json b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/convomem_new_brainctl_cmd.json new file mode 100644 index 0000000..7b54da9 --- /dev/null +++ b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/convomem_new_brainctl_cmd.json @@ -0,0 +1,26 @@ +{ + "artifacts": {}, + "benchmark": "convomem", + "caveats": [ + "Blocked while loading ConvoMem evidence data: HTTP Error 404: Not Found" + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "example_count": 0, + "measured": true, + "metadata": {}, + "metrics": {}, + "mode": "cmd", + "notes": [ + "limit_per_category=1", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": null, + "reference_kind": "measured", + "rows": [], + "runtime_seconds": null, + "series_name": "new_brainctl", + "source_path": null, + "status": "blocked", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/convomem_old_brainctl_cmd.json b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/convomem_old_brainctl_cmd.json new file mode 100644 index 0000000..fbd9ef4 --- /dev/null +++ b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/convomem_old_brainctl_cmd.json @@ -0,0 +1,29 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\runs\\convomem_brainctl_cmd.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\coverage_status.png" + }, + "benchmark": "convomem", + "caveats": [ + "Blocked while loading ConvoMem evidence data: " + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "example_count": 0, + "measured": false, + "metadata": {}, + "metrics": {}, + "mode": "cmd", + "notes": [ + "limit_per_category=1", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": null, + "reference_kind": "historical", + "runtime_seconds": null, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "blocked", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/locomo_mempalace_raw_session.json b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/locomo_mempalace_raw_session.json new file mode 100644 index 0000000..47d4098 --- /dev/null +++ b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/locomo_mempalace_raw_session.json @@ -0,0 +1,38 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\locomo_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\runs\\locomo_mempalace_raw_session.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\coverage_status.png" + }, + "benchmark": "locomo", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "measured": false, + "metadata": {}, + "metrics": { + "avg_recall": 0.6028, + "cat_1_recall": 0.5899, + "cat_2_recall": 0.6921, + "cat_3_recall": 0.4604, + "cat_4_recall": 0.5809, + "cat_5_recall": 0.6188, + "perfect_rate": 0.5534, + "top_k": 10, + "zero_rate": 0.3499 + }, + "mode": "raw_session", + "notes": [ + "granularity=session", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": 0.6028, + "reference_kind": "historical", + "runtime_seconds": 2106.411, + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "mempalace" +} \ No newline at end of file diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/locomo_new_brainctl_cmd_session.json b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/locomo_new_brainctl_cmd_session.json new file mode 100644 index 0000000..dbe14fb --- /dev/null +++ b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/locomo_new_brainctl_cmd_session.json @@ -0,0 +1,4450 @@ +{ + "artifacts": {}, + "benchmark": "locomo", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 199, + "measured": true, + "metadata": {}, + "metrics": { + "avg_recall": 0.9413, + "cat_1_recall": 0.8641, + "cat_2_recall": 1.0, + "cat_3_recall": 0.7436, + "cat_4_recall": 0.9571, + "cat_5_recall": 0.9787, + "perfect_rate": 0.9196, + "top_k": 10, + "zero_rate": 0.0402 + }, + "mode": "cmd_session", + "notes": [ + "granularity=session", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": 0.9413, + "reference_kind": "measured", + "rows": [ + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_1" + ], + "question": "When did Caroline go to the LGBTQ support group?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_10", + "session_13", + "session_12", + "session_4", + "session_14", + "session_5", + "session_7", + "session_11", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_1" + ], + "question": "When did Melanie paint a sunrise?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_14", + "session_13", + "session_8", + "session_11", + "session_16", + "session_17", + "session_9", + "session_18", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_1" + ], + "question": "What fields would Caroline be likely to pursue in her educaton?", + "recall": 0.0, + "retrieved_ids": [ + "session_7", + "session_13", + "session_18", + "session_10", + "session_4", + "session_14", + "session_9", + "session_17", + "session_5", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_2" + ], + "question": "What did Caroline research?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_1", + "session_2", + "session_8", + "session_14", + "session_13", + "session_18", + "session_15", + "session_5", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1" + ], + "question": "What is Caroline's identity?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_11", + "session_16", + "session_17", + "session_3", + "session_1", + "session_8", + "session_9", + "session_12", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_2" + ], + "question": "When did Melanie run a charity race?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_7", + "session_8", + "session_14", + "session_13", + "session_18", + "session_15", + "session_1", + "session_5", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_2" + ], + "question": "When is Melanie planning on going camping?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_2", + "session_16", + "session_12", + "session_7", + "session_17", + "session_14", + "session_18", + "session_9", + "session_6" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_2", + "session_3" + ], + "question": "What is Caroline's relationship status?", + "recall": 0.0, + "retrieved_ids": [ + "session_16", + "session_1", + "session_8", + "session_9", + "session_12", + "session_7", + "session_5", + "session_14", + "session_18", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_3" + ], + "question": "When did Caroline give a speech at a school?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_13", + "session_18", + "session_5", + "session_14", + "session_10", + "session_7", + "session_8", + "session_19", + "session_2" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_3" + ], + "question": "When did Caroline meet up with her friends, family, and mentors?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_7", + "session_9", + "session_2", + "session_16", + "session_5", + "session_17", + "session_3", + "session_8", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_3" + ], + "question": "How long has Caroline had her current group of friends for?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_7", + "session_1", + "session_4", + "session_12", + "session_13", + "session_16", + "session_3", + "session_6", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_3", + "session_4" + ], + "question": "Where did Caroline move from 4 years ago?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_8", + "session_19", + "session_4", + "session_16", + "session_15", + "session_7", + "session_17", + "session_9", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_4" + ], + "question": "How long ago was Caroline's 18th birthday?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_11", + "session_16", + "session_15", + "session_3", + "session_17", + "session_8", + "session_9", + "session_7", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_4" + ], + "question": "What career path has Caroline decided to persue?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_7", + "session_14", + "session_1", + "session_3", + "session_16", + "session_15", + "session_8", + "session_9", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_3", + "session_4" + ], + "question": "Would Caroline still want to pursue counseling as a career if she hadn't received support growing up?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_7", + "session_17", + "session_1", + "session_18", + "session_6", + "session_2", + "session_10", + "session_5", + "session_3" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_5", + "session_9" + ], + "question": "What activities does Melanie partake in?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_15", + "session_8", + "session_18", + "session_1", + "session_4", + "session_13", + "session_9", + "session_19", + "session_17" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_5" + ], + "question": "When did Melanie sign up for a pottery class?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_16", + "session_14", + "session_10", + "session_12", + "session_17", + "session_8", + "session_18", + "session_11", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_5" + ], + "question": "When is Caroline going to the transgender conference?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_7", + "session_17", + "session_1", + "session_14", + "session_9", + "session_3", + "session_4", + "session_13", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_4", + "session_6", + "session_8" + ], + "question": "Where has Melanie camped?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_18", + "session_9", + "session_6", + "session_2", + "session_4", + "session_16", + "session_8", + "session_1", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_4", + "session_6" + ], + "question": "What do Melanie's kids like?", + "recall": 0.5, + "retrieved_ids": [ + "session_9", + "session_10", + "session_15", + "session_18", + "session_8", + "session_17", + "session_1", + "session_16", + "session_4", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_6" + ], + "question": "When did Melanie go to the museum?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_13", + "session_1", + "session_14", + "session_5", + "session_11", + "session_18", + "session_8", + "session_7", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_6" + ], + "question": "When did Caroline have a picnic?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_8", + "session_14", + "session_13", + "session_18", + "session_15", + "session_1", + "session_5", + "session_11", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_6" + ], + "question": "Would Caroline likely have Dr. Seuss books on her bookshelf?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_18", + "session_10", + "session_6", + "session_17", + "session_4", + "session_14", + "session_9", + "session_5", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_6", + "session_7" + ], + "question": "What books has Melanie read?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_7", + "session_17", + "session_2", + "session_8", + "session_18", + "session_1", + "session_4", + "session_13", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_5", + "session_7" + ], + "question": "What does Melanie do to destress?", + "recall": 0.5, + "retrieved_ids": [ + "session_7", + "session_8", + "session_18", + "session_1", + "session_4", + "session_13", + "session_9", + "session_19", + "session_17", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_7" + ], + "question": "When did Caroline go to the LGBTQ conference?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_7", + "session_14", + "session_1", + "session_13", + "session_11", + "session_10", + "session_9", + "session_4", + "session_3" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_7" + ], + "question": "When did Melanie read the book \"nothing is impossible\"?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_7", + "session_17", + "session_2", + "session_8", + "session_14", + "session_13", + "session_18", + "session_15", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_7" + ], + "question": "Would Caroline pursue writing as a career option?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_1", + "session_4", + "session_18", + "session_8", + "session_9", + "session_12", + "session_5", + "session_14", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_8" + ], + "question": "When did Caroline go to the adoption meeting?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_2", + "session_17", + "session_19", + "session_13", + "session_10", + "session_5", + "session_15", + "session_7", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_8" + ], + "question": "When did Melanie go to the pottery workshop?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_4", + "session_5", + "session_16", + "session_12", + "session_17", + "session_14", + "session_13", + "session_1", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [], + "question": "Would Melanie be considered a member of the LGBTQ community?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_5", + "session_14", + "session_11", + "session_9", + "session_3", + "session_7", + "session_10", + "session_15", + "session_16" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_4" + ], + "question": "When did Melanie go camping in June?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_18", + "session_9", + "session_6", + "session_2", + "session_4", + "session_16", + "session_8", + "session_13", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_3", + "session_5", + "session_8" + ], + "question": "What LGBTQ+ events has Caroline participated in?", + "recall": 0.75, + "retrieved_ids": [ + "session_9", + "session_10", + "session_3", + "session_2", + "session_11", + "session_7", + "session_17", + "session_5", + "session_14", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_5" + ], + "question": "When did Caroline go to a pride parade during the summer?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_10", + "session_11", + "session_5", + "session_18", + "session_16", + "session_12", + "session_2", + "session_15", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_3", + "session_9" + ], + "question": "What events has Caroline participated in to help children?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_9", + "session_10", + "session_3", + "session_2", + "session_11", + "session_7", + "session_17", + "session_19", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_9" + ], + "question": "When did Melanie go camping in July?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_18", + "session_9", + "session_6", + "session_2", + "session_4", + "session_16", + "session_8", + "session_13", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_9" + ], + "question": "When did Caroline join a mentorship program?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_10", + "session_3", + "session_8", + "session_14", + "session_13", + "session_18", + "session_15", + "session_1", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_8" + ], + "question": "What did Melanie paint recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_17", + "session_11", + "session_12", + "session_18", + "session_10", + "session_14", + "session_1", + "session_8", + "session_16" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_3", + "session_6", + "session_8", + "session_9" + ], + "question": "What activities has Melanie done with her family?", + "recall": 0.4, + "retrieved_ids": [ + "session_10", + "session_5", + "session_15", + "session_7", + "session_1", + "session_6", + "session_13", + "session_16", + "session_17", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_5", + "session_9" + ], + "question": "In what ways is Caroline participating in the LGBTQ community?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_11", + "session_9", + "session_15", + "session_14", + "session_3", + "session_7", + "session_10", + "session_16", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_6" + ], + "question": "How many times has Melanie gone to the beach in 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_3", + "session_14", + "session_6", + "session_4", + "session_7", + "session_17", + "session_8", + "session_18", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_10" + ], + "question": "When did Caroline join a new activist group?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_1", + "session_9", + "session_13", + "session_12", + "session_4", + "session_3", + "session_8", + "session_14", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_10" + ], + "question": "Would Melanie be more interested in going to a national park or a theme park?", + "recall": 0.0, + "retrieved_ids": [ + "session_11", + "session_18", + "session_15", + "session_16", + "session_7", + "session_14", + "session_5", + "session_4", + "session_3", + "session_6" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_11", + "session_9" + ], + "question": "What kind of art does Caroline make?", + "recall": 0.5, + "retrieved_ids": [ + "session_2", + "session_4", + "session_3", + "session_5", + "session_6", + "session_11", + "session_10", + "session_15", + "session_14", + "session_16" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_11" + ], + "question": "When is Melanie's daughter's birthday?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_4", + "session_8", + "session_18", + "session_1", + "session_13", + "session_9", + "session_19", + "session_17", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_11" + ], + "question": "When did Caroline attend a pride parade in August?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_5", + "session_11", + "session_10", + "session_8", + "session_9", + "session_14", + "session_12", + "session_17", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [], + "question": "Would Melanie be considered an ally to the transgender community?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_18", + "session_17", + "session_1", + "session_14", + "session_9", + "session_5", + "session_11", + "session_15", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_12", + "session_3" + ], + "question": "Who supports Caroline when she has a negative experience?", + "recall": 0.5, + "retrieved_ids": [ + "session_4", + "session_10", + "session_12", + "session_19", + "session_9", + "session_5", + "session_7", + "session_15", + "session_11", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_12", + "session_5", + "session_8" + ], + "question": "What types of pottery have Melanie and her kids made?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_10", + "session_15", + "session_7", + "session_5", + "session_16", + "session_12", + "session_17", + "session_8", + "session_2" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_12" + ], + "question": "When did Caroline and Melanie go to a pride fesetival together?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_14", + "session_5", + "session_11", + "session_12", + "session_10", + "session_17", + "session_8", + "session_18", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_12" + ], + "question": "What would Caroline's political leaning likely be?", + "recall": 0.0, + "retrieved_ids": [ + "session_18", + "session_4", + "session_14", + "session_9", + "session_17", + "session_5", + "session_10", + "session_13", + "session_7", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_13", + "session_8" + ], + "question": "What has Melanie painted?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_13", + "session_9", + "session_14", + "session_17", + "session_16", + "session_11", + "session_8", + "session_12", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_13", + "session_7" + ], + "question": "What are Melanie's pets' names?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_7", + "session_8", + "session_18", + "session_1", + "session_4", + "session_9", + "session_19", + "session_17", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_13" + ], + "question": "When did Caroline apply to adoption agencies?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_2", + "session_19", + "session_17", + "session_8", + "session_14", + "session_18", + "session_15", + "session_1", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_13" + ], + "question": "When did Caroline draw a self-portrait?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_14", + "session_16", + "session_11", + "session_17", + "session_7", + "session_8", + "session_19", + "session_2", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_14", + "session_8" + ], + "question": "What subject have Caroline and Melanie both painted?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_9", + "session_19", + "session_13", + "session_12", + "session_16", + "session_15", + "session_3", + "session_1", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_14", + "session_4" + ], + "question": "What symbols are important to Caroline?", + "recall": 0.5, + "retrieved_ids": [ + "session_14", + "session_7", + "session_2", + "session_11", + "session_6", + "session_18", + "session_15", + "session_3", + "session_8", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_14" + ], + "question": "When did Caroline encounter people on a hike and have a negative experience?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_4", + "session_16", + "session_8", + "session_14", + "session_11", + "session_5", + "session_15", + "session_18", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_14" + ], + "question": "When did Melanie make a plate in pottery class?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_5", + "session_16", + "session_12", + "session_17", + "session_8", + "session_15", + "session_7", + "session_10", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_12", + "session_14" + ], + "question": "Would Caroline be considered religious?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_18", + "session_1", + "session_8", + "session_9", + "session_7", + "session_5", + "session_14", + "session_4", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_15", + "session_2" + ], + "question": "What instruments does Melanie play?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_6", + "session_2", + "session_3", + "session_8", + "session_18", + "session_1", + "session_4", + "session_13", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_11", + "session_15" + ], + "question": "What musical artists/bands has Melanie seen?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_11", + "session_17", + "session_14", + "session_8", + "session_18", + "session_1", + "session_4", + "session_13", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_15" + ], + "question": "When did Melanie go to the park?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_16", + "session_13", + "session_1", + "session_14", + "session_5", + "session_11", + "session_18", + "session_8", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_15" + ], + "question": "When is Caroline's youth center putting on a talent show?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_9", + "session_11", + "session_14", + "session_5", + "session_19", + "session_12", + "session_10", + "session_16", + "session_17" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_15" + ], + "question": "Would Melanie likely enjoy the song \"The Four Seasons\" by Vivaldi?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_15", + "session_11", + "session_8", + "session_10", + "session_4", + "session_14", + "session_9", + "session_17", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_11", + "session_16" + ], + "question": "What are some changes Caroline has faced during her transition journey?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_16", + "session_3", + "session_8", + "session_7", + "session_18", + "session_11", + "session_14", + "session_9", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_16" + ], + "question": "What does Melanie do with her family on hikes?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_7", + "session_12", + "session_4", + "session_16", + "session_8", + "session_14", + "session_19", + "session_2", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_16" + ], + "question": "When did Caroline go biking with friends?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_6", + "session_3", + "session_8", + "session_19", + "session_2", + "session_12", + "session_4", + "session_17", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_16" + ], + "question": "How long has Melanie been practicing art?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_3", + "session_15", + "session_17", + "session_1", + "session_6", + "session_19", + "session_4", + "session_8", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_13", + "session_16", + "session_7" + ], + "question": "What personality traits might Melanie say Caroline has?", + "recall": 0.6667, + "retrieved_ids": [ + "session_10", + "session_16", + "session_15", + "session_8", + "session_14", + "session_1", + "session_18", + "session_9", + "session_4", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_15", + "session_17" + ], + "question": "What transgender-specific events has Caroline attended?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_15", + "session_9", + "session_17", + "session_3", + "session_1", + "session_14", + "session_5", + "session_10", + "session_2" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_17", + "session_7" + ], + "question": "What book did Melanie read from Caroline's suggestion?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_7", + "session_17", + "session_2", + "session_8", + "session_14", + "session_13", + "session_18", + "session_15", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_17" + ], + "question": "When did Melanie's friend adopt a child?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_2", + "session_17", + "session_19", + "session_8", + "session_6", + "session_16", + "session_3", + "session_12", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_17" + ], + "question": "When did Melanie get hurt?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_8", + "session_5", + "session_13", + "session_1", + "session_16", + "session_18", + "session_14", + "session_15", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_18" + ], + "question": "When did Melanie's family go on a roadtrip?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_10", + "session_7", + "session_17", + "session_4", + "session_3", + "session_13", + "session_2", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_18" + ], + "question": "How many children does Melanie have?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_10", + "session_3", + "session_18", + "session_1", + "session_4", + "session_13", + "session_9", + "session_19", + "session_17" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_18" + ], + "question": "When did Melanie go on a hike after the roadtrip?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_4", + "session_14", + "session_12", + "session_16", + "session_8", + "session_2", + "session_9", + "session_1", + "session_3" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_18" + ], + "question": "Would Melanie go on another roadtrip soon?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_1", + "session_9", + "session_13", + "session_11", + "session_12", + "session_16", + "session_8", + "session_4", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_19", + "session_7" + ], + "question": "What items has Melanie bought?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_4", + "session_8", + "session_18", + "session_1", + "session_13", + "session_9", + "session_17", + "session_15", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_19" + ], + "question": "When did Caroline pass the adoption interview?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_3", + "session_2", + "session_17", + "session_13", + "session_8", + "session_14", + "session_18", + "session_15", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_19" + ], + "question": "When did Melanie buy the figurines?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_8", + "session_14", + "session_13", + "session_18", + "session_15", + "session_1", + "session_5", + "session_11", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_19" + ], + "question": "Would Caroline want to move back to her home country soon?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_19", + "session_18", + "session_8", + "session_1", + "session_4", + "session_7", + "session_10", + "session_2", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What did the charity race raise awareness for?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_7", + "session_3", + "session_8", + "session_14", + "session_13", + "session_18", + "session_15", + "session_1", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What did Melanie realize after the charity race?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_7", + "session_8", + "session_18", + "session_9", + "session_1", + "session_4", + "session_3", + "session_14", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "How does Melanie prioritize self-care?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_9", + "session_13", + "session_4", + "session_1", + "session_7", + "session_11", + "session_17", + "session_19", + "session_16" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What are Caroline's plans for the summer?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_2", + "session_10", + "session_11", + "session_15", + "session_16", + "session_7", + "session_17", + "session_14", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What type of individuals does the adoption agency Caroline is considering support?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_11", + "session_19", + "session_13", + "session_17", + "session_15", + "session_14", + "session_8", + "session_1", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "Why did Caroline choose the adoption agency?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_2", + "session_13", + "session_17", + "session_16", + "session_8", + "session_14", + "session_18", + "session_15", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What is Caroline excited about in the adoption process?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_2", + "session_19", + "session_13", + "session_8", + "session_5", + "session_16", + "session_1", + "session_9", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What does Melanie think about Caroline's decision to adopt?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_2", + "session_17", + "session_19", + "session_8", + "session_12", + "session_4", + "session_5", + "session_16", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "How long have Mel and her husband been married?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_10", + "session_7", + "session_8", + "session_16", + "session_15", + "session_17", + "session_1", + "session_6", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What does Caroline's necklace symbolize?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_14", + "session_1", + "session_8", + "session_9", + "session_12", + "session_7", + "session_5", + "session_18", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What country is Caroline's grandma from?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_3", + "session_1", + "session_8", + "session_9", + "session_12", + "session_7", + "session_5", + "session_14", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What was grandma's gift to Caroline?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_19", + "session_1", + "session_8", + "session_9", + "session_12", + "session_7", + "session_5", + "session_14", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What is Melanie's hand-painted bowl a reminder of?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_8", + "session_5", + "session_12", + "session_16", + "session_11", + "session_14", + "session_1", + "session_13", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What did Melanie and her family do while camping?", + "recall": 0.0, + "retrieved_ids": [ + "session_10", + "session_7", + "session_16", + "session_8", + "session_17", + "session_13", + "session_18", + "session_9", + "session_6", + "session_2" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What kind of counseling and mental health services is Caroline interested in pursuing?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_7", + "session_11", + "session_5", + "session_6", + "session_1", + "session_2", + "session_8", + "session_3", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What workshop did Caroline attend recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_4", + "session_8", + "session_17", + "session_11", + "session_12", + "session_18", + "session_10", + "session_14", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What was discussed in the LGBTQ+ counseling workshop?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_8", + "session_7", + "session_1", + "session_5", + "session_6", + "session_9", + "session_11", + "session_14", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What motivated Caroline to pursue counseling?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_7", + "session_5", + "session_1", + "session_6", + "session_3", + "session_18", + "session_14", + "session_8", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What kind of place does Caroline want to create for people?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_14", + "session_17", + "session_2", + "session_3", + "session_5", + "session_6", + "session_11", + "session_10", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_5" + ], + "question": "Did Melanie make the black and white bowl in the photo?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_16", + "session_12", + "session_4", + "session_6", + "session_13", + "session_3", + "session_8", + "session_14", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_6" + ], + "question": "What kind of books does Caroline have in her library?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_7", + "session_10", + "session_17", + "session_2", + "session_4", + "session_3", + "session_5", + "session_11", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_6" + ], + "question": "What was Melanie's favorite book from her childhood?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_7", + "session_10", + "session_17", + "session_8", + "session_14", + "session_18", + "session_1", + "session_4", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What book did Caroline recommend to Melanie?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_17", + "session_6", + "session_8", + "session_14", + "session_13", + "session_18", + "session_15", + "session_1", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What did Caroline take away from the book \"Becoming Nicole\"?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_13", + "session_6", + "session_17", + "session_2", + "session_1", + "session_14", + "session_5", + "session_4", + "session_3" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What are the new shoes that Melanie got used for?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_17", + "session_6", + "session_15", + "session_9", + "session_4", + "session_1", + "session_16", + "session_19", + "session_2" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What is Melanie's reason for getting into running?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_2", + "session_16", + "session_18", + "session_4", + "session_10", + "session_15", + "session_14", + "session_6", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What does Melanie say running has been great for?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_10", + "session_2", + "session_6", + "session_17", + "session_15", + "session_9", + "session_8", + "session_3", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What did Mel and her kids make during the pottery workshop?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_16", + "session_10", + "session_4", + "session_7", + "session_18", + "session_5", + "session_12", + "session_17", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What kind of pot did Mel and her kids make with clay?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_10", + "session_5", + "session_7", + "session_2", + "session_4", + "session_3", + "session_6", + "session_11", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What creative project do Mel and her kids do together besides pottery?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_12", + "session_14", + "session_10", + "session_7", + "session_5", + "session_16", + "session_17", + "session_15", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What did Mel and her kids paint in their latest project in July 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_10", + "session_7", + "session_12", + "session_15", + "session_14", + "session_17", + "session_1", + "session_13", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What did Caroline see at the council meeting for adoption?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_2", + "session_17", + "session_19", + "session_13", + "session_10", + "session_5", + "session_15", + "session_7", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What do sunflowers represent according to Caroline?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_11", + "session_14", + "session_1", + "session_9", + "session_12", + "session_7", + "session_5", + "session_18", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "Why are flowers important to Melanie?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_14", + "session_7", + "session_2", + "session_11", + "session_6", + "session_18", + "session_15", + "session_3", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_9" + ], + "question": "What inspired Caroline's painting for the art show?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_16", + "session_9", + "session_17", + "session_11", + "session_8", + "session_12", + "session_13", + "session_4", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "How often does Melanie go to the beach with her kids?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_7", + "session_14", + "session_6", + "session_1", + "session_9", + "session_11", + "session_2", + "session_18", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "What did Melanie and her family see during their camping trip last year?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_18", + "session_8", + "session_7", + "session_16", + "session_12", + "session_17", + "session_4", + "session_9", + "session_6" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "How did Melanie feel while watching the meteor shower?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_8", + "session_17", + "session_13", + "session_16", + "session_14", + "session_1", + "session_5", + "session_15", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_11" + ], + "question": "Whose birthday did Melanie celebrate recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_4", + "session_17", + "session_10", + "session_14", + "session_8", + "session_13", + "session_12", + "session_18", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_11" + ], + "question": "Who performed at the concert at Melanie's daughter's birthday?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_4", + "session_8", + "session_18", + "session_1", + "session_13", + "session_9", + "session_19", + "session_17", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_12" + ], + "question": "Why did Melanie choose to use colors and patterns in her pottery project?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_16", + "session_19", + "session_10", + "session_4", + "session_14", + "session_2", + "session_8", + "session_7", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "What pet does Caroline have?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_13", + "session_1", + "session_8", + "session_9", + "session_12", + "session_5", + "session_14", + "session_18", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "What pets does Melanie have?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_13", + "session_8", + "session_18", + "session_1", + "session_4", + "session_9", + "session_19", + "session_17", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "Where did Oliver hide his bone once?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_10", + "session_6", + "session_7", + "session_11", + "session_8", + "session_14", + "session_18", + "session_15", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "What activity did Caroline used to do with her dad?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_10", + "session_5", + "session_15", + "session_7", + "session_8", + "session_14", + "session_18", + "session_1", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_14" + ], + "question": "What did Caroline make for a local church?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_8", + "session_15", + "session_5", + "session_7", + "session_10", + "session_16", + "session_18", + "session_17", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_14" + ], + "question": "What did Caroline find in her neighborhood during her walk?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_7", + "session_10", + "session_16", + "session_18", + "session_8", + "session_19", + "session_9", + "session_11", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "Which song motivates Caroline to be courageous?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_3", + "session_11", + "session_14", + "session_1", + "session_8", + "session_5", + "session_18", + "session_4", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "Which classical musicians does Melanie enjoy listening to?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_15", + "session_8", + "session_18", + "session_10", + "session_1", + "session_4", + "session_13", + "session_9", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "Who is Melanie a fan of in terms of modern music?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_5", + "session_11", + "session_8", + "session_18", + "session_1", + "session_4", + "session_13", + "session_9", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "How long has Melanie been creating art?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_3", + "session_15", + "session_17", + "session_1", + "session_6", + "session_19", + "session_4", + "session_8", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "What precautionary sign did Melanie see at the caf\u00e9?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_5", + "session_10", + "session_15", + "session_14", + "session_1", + "session_11", + "session_18", + "session_8", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What advice does Caroline give for getting started with adoption?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_13", + "session_2", + "session_19", + "session_8", + "session_15", + "session_12", + "session_18", + "session_4", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What setback did Melanie face in October 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_9", + "session_10", + "session_15", + "session_3", + "session_8", + "session_14", + "session_13", + "session_18", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What does Melanie do to keep herself busy during her pottery break?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_8", + "session_16", + "session_10", + "session_14", + "session_2", + "session_7", + "session_18", + "session_5", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What painting did Melanie show to Caroline on October 13, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_13", + "session_8", + "session_17", + "session_16", + "session_11", + "session_9", + "session_1", + "session_15", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What kind of painting did Caroline share with Melanie on October 13, 2023?", + "recall": 0.0, + "retrieved_ids": [ + "session_2", + "session_4", + "session_3", + "session_5", + "session_6", + "session_11", + "session_10", + "session_15", + "session_14", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What was the poetry reading that Caroline attended about?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_13", + "session_6", + "session_7", + "session_2", + "session_4", + "session_15", + "session_3", + "session_12", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What did the posters at the poetry reading say?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_10", + "session_6", + "session_7", + "session_2", + "session_8", + "session_14", + "session_13", + "session_18", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What does Caroline's drawing symbolize for her?", + "recall": 0.0, + "retrieved_ids": [ + "session_14", + "session_10", + "session_16", + "session_7", + "session_1", + "session_8", + "session_9", + "session_12", + "session_5", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "How do Melanie and Caroline describe their journey through life together?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_8", + "session_4", + "session_17", + "session_14", + "session_19", + "session_7", + "session_15", + "session_10", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What happened to Melanie's son on their road trip?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_10", + "session_12", + "session_17", + "session_7", + "session_6", + "session_14", + "session_2", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "How did Melanie's son handle the accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_16", + "session_8", + "session_14", + "session_13", + "session_15", + "session_1", + "session_5", + "session_11", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "How did Melanie feel about her family after the accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_10", + "session_7", + "session_2", + "session_9", + "session_1", + "session_4", + "session_3", + "session_14", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "How did Melanie's children handle the accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_16", + "session_14", + "session_13", + "session_15", + "session_1", + "session_5", + "session_11", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "How did Melanie feel after the accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_2", + "session_9", + "session_1", + "session_4", + "session_3", + "session_14", + "session_13", + "session_8", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What was Melanie's reaction to her children enjoying the Grand Canyon?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_10", + "session_3", + "session_7", + "session_1", + "session_4", + "session_13", + "session_9", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What do Melanie's family give her?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_7", + "session_3", + "session_19", + "session_2", + "session_18", + "session_6", + "session_4", + "session_8", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "How did Melanie feel about her family supporting her?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_7", + "session_8", + "session_17", + "session_16", + "session_3", + "session_2", + "session_15", + "session_18", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What did Melanie do after the road trip to relax?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_17", + "session_1", + "session_10", + "session_14", + "session_12", + "session_15", + "session_2", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_2" + ], + "question": "What did Caroline realize after her charity race?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_7", + "session_10", + "session_8", + "session_18", + "session_9", + "session_1", + "session_4", + "session_3", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_2" + ], + "question": "What are Melanie's plans for the summer with respect to adoption?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_2", + "session_12", + "session_17", + "session_10", + "session_13", + "session_8", + "session_11", + "session_15", + "session_16" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_2" + ], + "question": "What type of individuals does the adoption agency Melanie is considering support?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_11", + "session_19", + "session_13", + "session_17", + "session_15", + "session_14", + "session_8", + "session_1", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_2" + ], + "question": "Why did Melanie choose the adoption agency?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_2", + "session_13", + "session_17", + "session_16", + "session_8", + "session_14", + "session_18", + "session_15", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_2" + ], + "question": "What is Melanie excited about in her adoption process?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_2", + "session_19", + "session_10", + "session_7", + "session_13", + "session_8", + "session_5", + "session_16", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What does Melanie's necklace symbolize?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_14", + "session_8", + "session_18", + "session_1", + "session_13", + "session_9", + "session_19", + "session_17", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What country is Melanie's grandma from?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_3", + "session_8", + "session_18", + "session_1", + "session_13", + "session_9", + "session_19", + "session_17", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What was grandma's gift to Melanie?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_19", + "session_8", + "session_18", + "session_1", + "session_13", + "session_9", + "session_17", + "session_15", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What was grandpa's gift to Caroline?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_4", + "session_1", + "session_8", + "session_9", + "session_12", + "session_7", + "session_5", + "session_14", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What is Caroline's hand-painted bowl a reminder of?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_8", + "session_5", + "session_12", + "session_16", + "session_11", + "session_14", + "session_1", + "session_9", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What did Caroline and her family do while camping?", + "recall": 0.0, + "retrieved_ids": [ + "session_10", + "session_7", + "session_16", + "session_8", + "session_17", + "session_13", + "session_18", + "session_9", + "session_6", + "session_2" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What kind of counseling and mental health services is Melanie interested in pursuing?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_7", + "session_11", + "session_5", + "session_6", + "session_1", + "session_2", + "session_8", + "session_3", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What kind of counseling workshop did Melanie attend recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_13", + "session_8", + "session_5", + "session_6", + "session_7", + "session_1", + "session_11", + "session_10", + "session_17" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What motivated Melanie to pursue counseling?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_7", + "session_5", + "session_1", + "session_6", + "session_3", + "session_18", + "session_14", + "session_8", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What kind of place does Melanie want to create for people?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_14", + "session_17", + "session_2", + "session_3", + "session_5", + "session_6", + "session_11", + "session_10", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_5" + ], + "question": "Did Caroline make the black and white bowl in the photo?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_16", + "session_12", + "session_4", + "session_6", + "session_13", + "session_3", + "session_8", + "session_14", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_7" + ], + "question": "What are the new shoes that Caroline got used for?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_17", + "session_6", + "session_15", + "session_9", + "session_4", + "session_1", + "session_16", + "session_19", + "session_2" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_7" + ], + "question": "What is Caroline's reason for getting into running?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_2", + "session_16", + "session_18", + "session_4", + "session_10", + "session_15", + "session_14", + "session_5", + "session_6" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_7" + ], + "question": "What does Caroline say running has been great for?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_10", + "session_2", + "session_6", + "session_17", + "session_15", + "session_9", + "session_8", + "session_3", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_8" + ], + "question": "What did Melanie see at the council meeting for adoption?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_2", + "session_17", + "session_19", + "session_13", + "session_10", + "session_5", + "session_15", + "session_7", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_9" + ], + "question": "What inspired Melanie's painting for the art show?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_16", + "session_9", + "session_17", + "session_11", + "session_8", + "session_12", + "session_13", + "session_4", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_9" + ], + "question": "What inspired Caroline's sculpture for the art show?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_16", + "session_9", + "session_17", + "session_11", + "session_12", + "session_8", + "session_13", + "session_5", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_10" + ], + "question": "How often does Caroline go to the beach with her kids?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_7", + "session_14", + "session_6", + "session_1", + "session_9", + "session_11", + "session_2", + "session_18", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_10" + ], + "question": "What did Caroline and her family see during their camping trip last year?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_18", + "session_8", + "session_7", + "session_16", + "session_12", + "session_17", + "session_4", + "session_9", + "session_6" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_10" + ], + "question": "How did Caroline feel while watching the meteor shower?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_8", + "session_17", + "session_13", + "session_16", + "session_14", + "session_1", + "session_5", + "session_15", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_12" + ], + "question": "Why did Caroline choose to use colors and patterns in her pottery project?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_16", + "session_19", + "session_10", + "session_4", + "session_14", + "session_2", + "session_8", + "session_7", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_13" + ], + "question": "Is Oscar Melanie's pet?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_7", + "session_8", + "session_18", + "session_1", + "session_4", + "session_9", + "session_19", + "session_17", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_13" + ], + "question": "Where did Oscar hide his bone once?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_10", + "session_6", + "session_11", + "session_8", + "session_14", + "session_18", + "session_15", + "session_1", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_13" + ], + "question": "What activity did Melanie used to do with her dad?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_10", + "session_5", + "session_15", + "session_7", + "session_8", + "session_14", + "session_18", + "session_1", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_14" + ], + "question": "What did Melanie make for a local church?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_8", + "session_15", + "session_5", + "session_7", + "session_10", + "session_16", + "session_18", + "session_17", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_14" + ], + "question": "What did Melanie find in her neighborhood during her walk?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_7", + "session_10", + "session_16", + "session_18", + "session_8", + "session_19", + "session_9", + "session_11", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_15" + ], + "question": "Which song motivates Melanie to be courageous?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_3", + "session_11", + "session_14", + "session_1", + "session_8", + "session_5", + "session_18", + "session_4", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_15" + ], + "question": "What type of instrument does Caroline play?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_6", + "session_14", + "session_2", + "session_3", + "session_1", + "session_8", + "session_9", + "session_12", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_15" + ], + "question": "Which classical musicians does Caroline enjoy listening to?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_15", + "session_8", + "session_18", + "session_10", + "session_1", + "session_9", + "session_12", + "session_7", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_15" + ], + "question": "Who is Caroline a fan of in terms of modern music?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_5", + "session_11", + "session_1", + "session_8", + "session_9", + "session_12", + "session_7", + "session_14", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_16" + ], + "question": "What precautionary sign did Caroline see at the caf\u00e9?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_5", + "session_10", + "session_15", + "session_14", + "session_1", + "session_11", + "session_18", + "session_8", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_17" + ], + "question": "What setback did Caroline face recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_10", + "session_9", + "session_15", + "session_3", + "session_13", + "session_11", + "session_12", + "session_18", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_17" + ], + "question": "What does Caroline do to keep herself busy during her pottery break?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_8", + "session_16", + "session_10", + "session_14", + "session_2", + "session_7", + "session_18", + "session_5", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_17" + ], + "question": "What was the poetry reading that Melanie attended about?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_13", + "session_6", + "session_7", + "session_2", + "session_4", + "session_15", + "session_3", + "session_11", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "What happened to Caroline's son on their road trip?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_10", + "session_12", + "session_17", + "session_7", + "session_6", + "session_14", + "session_2", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "How did Caroline's son handle the accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_16", + "session_8", + "session_14", + "session_13", + "session_15", + "session_1", + "session_5", + "session_11", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "How did Caroline feel about her family after the accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_10", + "session_7", + "session_2", + "session_9", + "session_1", + "session_4", + "session_3", + "session_14", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "How did Caroline's children handle the accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_16", + "session_14", + "session_13", + "session_15", + "session_1", + "session_5", + "session_11", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "How did Caroline feel after the accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_2", + "session_9", + "session_1", + "session_4", + "session_3", + "session_14", + "session_13", + "session_8", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "What was Caroline's reaction to her children enjoying the Grand Canyon?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_10", + "session_3", + "session_7", + "session_1", + "session_9", + "session_12", + "session_5", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "What did Caroline do after the road trip to relax?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_17", + "session_1", + "session_10", + "session_14", + "session_12", + "session_15", + "session_2", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "What does Caroline love most about camping with her family?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_18", + "session_7", + "session_14", + "session_9", + "session_6", + "session_2", + "session_4", + "session_16", + "session_8" + ], + "sample_id": "conv-26" + } + ], + "runtime_seconds": 37.507, + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/locomo_old_brainctl_cmd_session.json b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/locomo_old_brainctl_cmd_session.json new file mode 100644 index 0000000..9c0c218 --- /dev/null +++ b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/locomo_old_brainctl_cmd_session.json @@ -0,0 +1,38 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\locomo_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\runs\\locomo_brainctl_cmd_session.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\coverage_status.png" + }, + "benchmark": "locomo", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "measured": false, + "metadata": {}, + "metrics": { + "avg_recall": 0.9217, + "cat_1_recall": 0.7614, + "cat_2_recall": 0.92, + "cat_3_recall": 0.6616, + "cat_4_recall": 0.9738, + "cat_5_recall": 0.9821, + "perfect_rate": 0.8817, + "top_k": 10, + "zero_rate": 0.0438 + }, + "mode": "cmd_session", + "notes": [ + "granularity=session", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": 0.9217, + "reference_kind": "historical", + "runtime_seconds": 445.74, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_mempalace_raw_session.json b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_mempalace_raw_session.json new file mode 100644 index 0000000..31716e6 --- /dev/null +++ b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_mempalace_raw_session.json @@ -0,0 +1,33 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\runs\\longmemeval_mempalace_raw_session.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "measured": false, + "metadata": {}, + "metrics": { + "ndcg_at_10": 0.8948, + "ndcg_at_5": 0.893, + "r_at_10": 0.983, + "r_at_5": 0.966 + }, + "mode": "raw_session", + "notes": [ + "top_k=10", + "Runs MemPalace benchmark module raw session retrieval logic directly." + ], + "primary_metric": "r_at_5", + "primary_metric_value": 0.966, + "reference_kind": "historical", + "runtime_seconds": 695.36, + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "mempalace" +} \ No newline at end of file diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_new_brainctl_brain.json b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_new_brainctl_brain.json new file mode 100644 index 0000000..211eb17 --- /dev/null +++ b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_new_brainctl_brain.json @@ -0,0 +1,101 @@ +{ + "artifacts": {}, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 3, + "measured": true, + "metadata": {}, + "metrics": { + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0 + }, + "mode": "brain", + "notes": [ + "top_k=10", + "Legacy 470-question session-level slice." + ], + "primary_metric": "r_at_5", + "primary_metric_value": 1.0, + "reference_kind": "measured", + "rows": [ + { + "answer_session_ids": [ + "answer_280352e9" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e47becba", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_280352e9", + "02bd2b90_3", + "sharegpt_Cr2tc1f_0", + "f6859b48_2", + "ultrachat_214101", + "5dac7cc2_1" + ] + }, + { + "answer_session_ids": [ + "answer_40a90d51" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "118b2229", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_40a90d51", + "db73b7e4_4", + "ultrachat_392094", + "afe04238_1", + "5dce60dd", + "db50c0f6", + "3ea9f765", + "d600c646", + "a0b8b0ad_6", + "sharegpt_i1iuP70_0" + ] + }, + { + "answer_session_ids": [ + "answer_d61669c7" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "51a45a95", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_d61669c7", + "98c198fb", + "217debf7", + "6e0b1800_2", + "5c44d9fe_1", + "07ba9acd_2", + "sharegpt_3vxz2Zr_0", + "sharegpt_hChsWOp_128", + "ultrachat_282235", + "sharegpt_CyJ3dal_0" + ] + } + ], + "runtime_seconds": 3.541, + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_new_brainctl_cmd.json b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_new_brainctl_cmd.json new file mode 100644 index 0000000..9820f8d --- /dev/null +++ b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_new_brainctl_cmd.json @@ -0,0 +1,101 @@ +{ + "artifacts": {}, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 3, + "measured": true, + "metadata": {}, + "metrics": { + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0 + }, + "mode": "cmd", + "notes": [ + "top_k=10", + "Legacy 470-question session-level slice." + ], + "primary_metric": "r_at_5", + "primary_metric_value": 1.0, + "reference_kind": "measured", + "rows": [ + { + "answer_session_ids": [ + "answer_280352e9" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e47becba", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_280352e9", + "02bd2b90_3", + "sharegpt_Cr2tc1f_0", + "f6859b48_2", + "ultrachat_214101", + "5dac7cc2_1" + ] + }, + { + "answer_session_ids": [ + "answer_40a90d51" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "118b2229", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_40a90d51", + "db73b7e4_4", + "ultrachat_392094", + "afe04238_1", + "5dce60dd", + "db50c0f6", + "3ea9f765", + "d600c646", + "a0b8b0ad_6", + "sharegpt_i1iuP70_0" + ] + }, + { + "answer_session_ids": [ + "answer_d61669c7" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "51a45a95", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_d61669c7", + "98c198fb", + "217debf7", + "6e0b1800_2", + "5c44d9fe_1", + "07ba9acd_2", + "sharegpt_3vxz2Zr_0", + "sharegpt_hChsWOp_128", + "ultrachat_282235", + "sharegpt_CyJ3dal_0" + ] + } + ], + "runtime_seconds": 3.384, + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_old_brainctl_brain.json b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_old_brainctl_brain.json new file mode 100644 index 0000000..c1f858e --- /dev/null +++ b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_old_brainctl_brain.json @@ -0,0 +1,32 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\runs\\longmemeval_brainctl_brain.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "measured": false, + "metadata": {}, + "metrics": { + "ndcg_at_10": 0.9253, + "ndcg_at_5": 0.9204, + "r_at_10": 0.9894, + "r_at_5": 0.9681 + }, + "mode": "brain", + "notes": [ + "top_k=10" + ], + "primary_metric": "r_at_5", + "primary_metric_value": 0.9681, + "reference_kind": "historical", + "runtime_seconds": 85.439, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_old_brainctl_cmd.json b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_old_brainctl_cmd.json new file mode 100644 index 0000000..cd62ca4 --- /dev/null +++ b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_old_brainctl_cmd.json @@ -0,0 +1,32 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\runs\\longmemeval_brainctl_cmd.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "measured": false, + "metadata": {}, + "metrics": { + "ndcg_at_10": 0.9247, + "ndcg_at_5": 0.9206, + "r_at_10": 0.9894, + "r_at_5": 0.9702 + }, + "mode": "cmd", + "notes": [ + "top_k=10" + ], + "primary_metric": "r_at_5", + "primary_metric_value": 0.9702, + "reference_kind": "historical", + "runtime_seconds": 130.863, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/membench_mempalace_raw_turn.json b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/membench_mempalace_raw_turn.json new file mode 100644 index 0000000..e4a3869 --- /dev/null +++ b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/membench_mempalace_raw_turn.json @@ -0,0 +1,35 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\membench_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\runs\\membench_mempalace_raw_turn.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\coverage_status.png" + }, + "benchmark": "membench", + "caveats": [ + "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 200, + "measured": false, + "metadata": {}, + "metrics": { + "hit_at_5": 0.885, + "simple_hit_at_5": 0.885, + "top_k": 5 + }, + "mode": "raw_turn", + "notes": [ + "FirstAgent slice only", + "turn-level retrieval", + "topic=all" + ], + "primary_metric": "hit_at_5", + "primary_metric_value": 0.885, + "reference_kind": "historical", + "runtime_seconds": 804.35, + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "partial", + "system_name": "mempalace" +} \ No newline at end of file diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/membench_new_brainctl_cmd_turn.json b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/membench_new_brainctl_cmd_turn.json new file mode 100644 index 0000000..7e88c24 --- /dev/null +++ b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/membench_new_brainctl_cmd_turn.json @@ -0,0 +1,83 @@ +{ + "artifacts": {}, + "benchmark": "membench", + "caveats": [ + "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 3, + "measured": true, + "metadata": {}, + "metrics": { + "hit_at_5": 1.0, + "simple_hit_at_5": 1.0, + "top_k": 5 + }, + "mode": "cmd_turn", + "notes": [ + "FirstAgent slice only", + "turn-level retrieval", + "topic=all" + ], + "primary_metric": "hit_at_5", + "primary_metric_value": 1.0, + "reference_kind": "measured", + "rows": [ + { + "category": "simple", + "hit_at_k": true, + "question": "What is the name of my niece's company?", + "retrieved_ids": [ + "simple_roles_0|sid=119|g=119|s=5|t=18", + "simple_roles_0|sid=101|g=101|s=5|t=0", + "simple_roles_0|sid=135|g=135|s=6|t=12", + "simple_roles_0|sid=11|g=11|s=0|t=11", + "simple_roles_0|sid=25|g=25|s=1|t=5" + ], + "target_ids": [ + "119" + ], + "tid": 0, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the email address of my cousin, who is male?", + "retrieved_ids": [ + "simple_roles_1|sid=59|g=59|s=2|t=17", + "simple_roles_1|sid=101|g=101|s=4|t=18", + "simple_roles_1|sid=143|g=143|s=6|t=18", + "simple_roles_1|sid=33|g=33|s=1|t=12", + "simple_roles_1|sid=42|g=42|s=2|t=0" + ], + "target_ids": [ + "59" + ], + "tid": 1, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the position of my brother?", + "retrieved_ids": [ + "simple_roles_2|sid=84|g=84|s=4|t=0", + "simple_roles_2|sid=85|g=85|s=4|t=1", + "simple_roles_2|sid=28|g=28|s=1|t=7", + "simple_roles_2|sid=123|g=123|s=5|t=18", + "simple_roles_2|sid=41|g=41|s=1|t=20" + ], + "target_ids": [ + "84" + ], + "tid": 2, + "topic": "roles" + } + ], + "runtime_seconds": 1.919, + "series_name": "new_brainctl", + "source_path": null, + "status": "partial", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/membench_old_brainctl_cmd_turn.json b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/membench_old_brainctl_cmd_turn.json new file mode 100644 index 0000000..7af9666 --- /dev/null +++ b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/membench_old_brainctl_cmd_turn.json @@ -0,0 +1,35 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\membench_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\runs\\membench_brainctl_cmd_turn.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\coverage_status.png" + }, + "benchmark": "membench", + "caveats": [ + "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 200, + "measured": false, + "metadata": {}, + "metrics": { + "hit_at_5": 0.93, + "simple_hit_at_5": 0.93, + "top_k": 5 + }, + "mode": "cmd_turn", + "notes": [ + "FirstAgent slice only", + "turn-level retrieval", + "topic=all" + ], + "primary_metric": "hit_at_5", + "primary_metric_value": 0.93, + "reference_kind": "historical", + "runtime_seconds": 140.592, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "partial", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/summary.csv b/benchmarks/results/smoke_legacy_compare_20260423_105447/summary.csv new file mode 100644 index 0000000..5fb33da --- /dev/null +++ b/benchmarks/results/smoke_legacy_compare_20260423_105447/summary.csv @@ -0,0 +1,15 @@ +benchmark,series_name,system_name,mode,reference_kind,status,example_count,primary_metric,primary_metric_value,runtime_seconds,dataset_path,source_path,notes,caveats,avg_recall,cat_1_recall,cat_2_recall,cat_3_recall,cat_4_recall,cat_5_recall,hit_at_5,ndcg_at_10,ndcg_at_5,perfect_rate,r_at_10,r_at_5,simple_hit_at_5,top_k,zero_rate +longmemeval,old_brainctl,brainctl,brain,historical,full_same_machine,470,r_at_5,0.9681,85.439,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,top_k=10,,,,,,,,,0.9253,0.9204,,0.9894,0.9681,,, +longmemeval,old_brainctl,brainctl,cmd,historical,full_same_machine,470,r_at_5,0.9702,130.863,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,top_k=10,,,,,,,,,0.9247,0.9206,,0.9894,0.9702,,, +longmemeval,mempalace,mempalace,raw_session,historical,full_same_machine,470,r_at_5,0.966,695.36,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,top_k=10 | Runs MemPalace benchmark module raw session retrieval logic directly.,,,,,,,,,0.8948,0.893,,0.983,0.966,,, +locomo,old_brainctl,brainctl,cmd_session,historical,full_same_machine,1986,avg_recall,0.9217,445.74,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,granularity=session | top_k=10,,0.9217,0.7614,0.92,0.6616,0.9738,0.9821,,,,0.8817,,,,10,0.0438 +locomo,mempalace,mempalace,raw_session,historical,full_same_machine,1986,avg_recall,0.6028,2106.411,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,granularity=session | top_k=10,,0.6028,0.5899,0.6921,0.4604,0.5809,0.6188,,,,0.5534,,,,10,0.3499 +membench,old_brainctl,brainctl,cmd_turn,historical,partial,200,hit_at_5,0.93,140.592,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,FirstAgent slice only | turn-level retrieval | topic=all,MemBench comparison is partial because ThirdAgent and noise-extended slices are not included.,,,,,,,0.93,,,,,,0.93,5, +membench,mempalace,mempalace,raw_turn,historical,partial,200,hit_at_5,0.885,804.35,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,FirstAgent slice only | turn-level retrieval | topic=all,MemBench comparison is partial because ThirdAgent and noise-extended slices are not included.,,,,,,,0.885,,,,,,0.885,5, +convomem,old_brainctl,brainctl,cmd,historical,blocked,0,avg_recall,,,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,limit_per_category=1 | top_k=10,Blocked while loading ConvoMem evidence data: ,,,,,,,,,,,,,,, +convomem,mempalace,mempalace,raw,historical,blocked,0,avg_recall,,,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,limit_per_category=1 | top_k=10,Blocked while loading ConvoMem evidence data: ,,,,,,,,,,,,,,, +longmemeval,new_brainctl,brainctl,brain,measured,full_same_machine,3,r_at_5,1.0,3.541,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,,top_k=10 | Legacy 470-question session-level slice.,,,,,,,,,1.0,1.0,,1.0,1.0,,, +longmemeval,new_brainctl,brainctl,cmd,measured,full_same_machine,3,r_at_5,1.0,3.384,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,,top_k=10 | Legacy 470-question session-level slice.,,,,,,,,,1.0,1.0,,1.0,1.0,,, +locomo,new_brainctl,brainctl,cmd_session,measured,full_same_machine,199,avg_recall,0.9413,37.507,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,,granularity=session | top_k=10,,0.9413,0.8641,1.0,0.7436,0.9571,0.9787,,,,0.9196,,,,10,0.0402 +membench,new_brainctl,brainctl,cmd_turn,measured,partial,3,hit_at_5,1.0,1.919,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,,FirstAgent slice only | turn-level retrieval | topic=all,MemBench comparison is partial because ThirdAgent and noise-extended slices are not included.,,,,,,,1.0,,,,,,1.0,5, +convomem,new_brainctl,brainctl,cmd,measured,blocked,0,avg_recall,,,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache,,limit_per_category=1 | top_k=10,Blocked while loading ConvoMem evidence data: HTTP Error 404: Not Found,,,,,,,,,,,,,,, diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/summary.json b/benchmarks/results/smoke_legacy_compare_20260423_105447/summary.json new file mode 100644 index 0000000..0a74157 --- /dev/null +++ b/benchmarks/results/smoke_legacy_compare_20260423_105447/summary.json @@ -0,0 +1,509 @@ +{ + "generated_at_utc": "2026-04-23T10:55:38.026703+00:00", + "metadata": { + "argv": [ + "benchmarks\\compare_memory_engines.py", + "--label", + "smoke_legacy_compare", + "--longmemeval-limit", + "3", + "--locomo-limit", + "1", + "--membench-limit", + "3", + "--convomem-limit-per-category", + "1" + ], + "cwd": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl", + "datasets": { + "convomem_cache": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "locomo_data": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "longmemeval_data": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "membench_data": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent" + }, + "generated_at_utc": "2026-04-23T10:55:37.887986+00:00", + "git_commit": "c130fbd47174a0932822db2ef7fe64e18a74d30d", + "historical_summary_mode": "recovered", + "historical_summary_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "machine": "AMD64", + "platform": "Windows-11-10.0.26200-SP0", + "processor": "Intel64 Family 6 Model 189 Stepping 1, GenuineIntel", + "python_version": "3.14.4 (tags/v3.14.4:23116f9, Apr 7 2026, 14:10:54) [MSC v.1944 64 bit (AMD64)]" + }, + "notes": [ + "Historical old-BrainCTL and MemPalace series come from the recovered 2026-04-18 comparison bundle.", + "New BrainCTL series are rerun in the current checked-out repo using the legacy benchmark definitions.", + "MemBench remains intentionally partial because the legacy comparison only covered the FirstAgent slice.", + "ConvoMem remains a coverage/status benchmark here; it has no dedicated comparison chart in the legacy chart pack." + ], + "runs": [ + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\runs\\longmemeval_old_brainctl_brain.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "measured": false, + "metadata": {}, + "metrics": { + "ndcg_at_10": 0.9253, + "ndcg_at_5": 0.9204, + "r_at_10": 0.9894, + "r_at_5": 0.9681 + }, + "mode": "brain", + "notes": [ + "top_k=10" + ], + "primary_metric": "r_at_5", + "primary_metric_value": 0.9681, + "reference_kind": "historical", + "runtime_seconds": 85.439, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\runs\\longmemeval_old_brainctl_cmd.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "measured": false, + "metadata": {}, + "metrics": { + "ndcg_at_10": 0.9247, + "ndcg_at_5": 0.9206, + "r_at_10": 0.9894, + "r_at_5": 0.9702 + }, + "mode": "cmd", + "notes": [ + "top_k=10" + ], + "primary_metric": "r_at_5", + "primary_metric_value": 0.9702, + "reference_kind": "historical", + "runtime_seconds": 130.863, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\runs\\longmemeval_mempalace_raw_session.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "measured": false, + "metadata": {}, + "metrics": { + "ndcg_at_10": 0.8948, + "ndcg_at_5": 0.893, + "r_at_10": 0.983, + "r_at_5": 0.966 + }, + "mode": "raw_session", + "notes": [ + "top_k=10", + "Runs MemPalace benchmark module raw session retrieval logic directly." + ], + "primary_metric": "r_at_5", + "primary_metric_value": 0.966, + "reference_kind": "historical", + "runtime_seconds": 695.36, + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "mempalace" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\locomo_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\runs\\locomo_old_brainctl_cmd_session.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\coverage_status.png" + }, + "benchmark": "locomo", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "measured": false, + "metadata": {}, + "metrics": { + "avg_recall": 0.9217, + "cat_1_recall": 0.7614, + "cat_2_recall": 0.92, + "cat_3_recall": 0.6616, + "cat_4_recall": 0.9738, + "cat_5_recall": 0.9821, + "perfect_rate": 0.8817, + "top_k": 10, + "zero_rate": 0.0438 + }, + "mode": "cmd_session", + "notes": [ + "granularity=session", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": 0.9217, + "reference_kind": "historical", + "runtime_seconds": 445.74, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\locomo_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\runs\\locomo_mempalace_raw_session.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\coverage_status.png" + }, + "benchmark": "locomo", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "measured": false, + "metadata": {}, + "metrics": { + "avg_recall": 0.6028, + "cat_1_recall": 0.5899, + "cat_2_recall": 0.6921, + "cat_3_recall": 0.4604, + "cat_4_recall": 0.5809, + "cat_5_recall": 0.6188, + "perfect_rate": 0.5534, + "top_k": 10, + "zero_rate": 0.3499 + }, + "mode": "raw_session", + "notes": [ + "granularity=session", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": 0.6028, + "reference_kind": "historical", + "runtime_seconds": 2106.411, + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "mempalace" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\membench_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\runs\\membench_old_brainctl_cmd_turn.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\coverage_status.png" + }, + "benchmark": "membench", + "caveats": [ + "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 200, + "measured": false, + "metadata": {}, + "metrics": { + "hit_at_5": 0.93, + "simple_hit_at_5": 0.93, + "top_k": 5 + }, + "mode": "cmd_turn", + "notes": [ + "FirstAgent slice only", + "turn-level retrieval", + "topic=all" + ], + "primary_metric": "hit_at_5", + "primary_metric_value": 0.93, + "reference_kind": "historical", + "runtime_seconds": 140.592, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "partial", + "system_name": "brainctl" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\membench_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\runs\\membench_mempalace_raw_turn.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\coverage_status.png" + }, + "benchmark": "membench", + "caveats": [ + "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 200, + "measured": false, + "metadata": {}, + "metrics": { + "hit_at_5": 0.885, + "simple_hit_at_5": 0.885, + "top_k": 5 + }, + "mode": "raw_turn", + "notes": [ + "FirstAgent slice only", + "turn-level retrieval", + "topic=all" + ], + "primary_metric": "hit_at_5", + "primary_metric_value": 0.885, + "reference_kind": "historical", + "runtime_seconds": 804.35, + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "partial", + "system_name": "mempalace" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\runs\\convomem_old_brainctl_cmd.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\coverage_status.png" + }, + "benchmark": "convomem", + "caveats": [ + "Blocked while loading ConvoMem evidence data: " + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "example_count": 0, + "measured": false, + "metadata": {}, + "metrics": {}, + "mode": "cmd", + "notes": [ + "limit_per_category=1", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": null, + "reference_kind": "historical", + "runtime_seconds": null, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "blocked", + "system_name": "brainctl" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\runs\\convomem_mempalace_raw.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\coverage_status.png" + }, + "benchmark": "convomem", + "caveats": [ + "Blocked while loading ConvoMem evidence data: " + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "example_count": 0, + "measured": false, + "metadata": {}, + "metrics": {}, + "mode": "raw", + "notes": [ + "limit_per_category=1", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": null, + "reference_kind": "historical", + "runtime_seconds": null, + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "blocked", + "system_name": "mempalace" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\runs\\longmemeval_new_brainctl_brain.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 3, + "measured": true, + "metadata": {}, + "metrics": { + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0 + }, + "mode": "brain", + "notes": [ + "top_k=10", + "Legacy 470-question session-level slice." + ], + "primary_metric": "r_at_5", + "primary_metric_value": 1.0, + "reference_kind": "measured", + "runtime_seconds": 3.541, + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\runs\\longmemeval_new_brainctl_cmd.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 3, + "measured": true, + "metadata": {}, + "metrics": { + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0 + }, + "mode": "cmd", + "notes": [ + "top_k=10", + "Legacy 470-question session-level slice." + ], + "primary_metric": "r_at_5", + "primary_metric_value": 1.0, + "reference_kind": "measured", + "runtime_seconds": 3.384, + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\locomo_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\runs\\locomo_new_brainctl_cmd_session.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\coverage_status.png" + }, + "benchmark": "locomo", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 199, + "measured": true, + "metadata": {}, + "metrics": { + "avg_recall": 0.9413, + "cat_1_recall": 0.8641, + "cat_2_recall": 1.0, + "cat_3_recall": 0.7436, + "cat_4_recall": 0.9571, + "cat_5_recall": 0.9787, + "perfect_rate": 0.9196, + "top_k": 10, + "zero_rate": 0.0402 + }, + "mode": "cmd_session", + "notes": [ + "granularity=session", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": 0.9413, + "reference_kind": "measured", + "runtime_seconds": 37.507, + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\membench_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\runs\\membench_new_brainctl_cmd_turn.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\coverage_status.png" + }, + "benchmark": "membench", + "caveats": [ + "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 3, + "measured": true, + "metadata": {}, + "metrics": { + "hit_at_5": 1.0, + "simple_hit_at_5": 1.0, + "top_k": 5 + }, + "mode": "cmd_turn", + "notes": [ + "FirstAgent slice only", + "turn-level retrieval", + "topic=all" + ], + "primary_metric": "hit_at_5", + "primary_metric_value": 1.0, + "reference_kind": "measured", + "runtime_seconds": 1.919, + "series_name": "new_brainctl", + "source_path": null, + "status": "partial", + "system_name": "brainctl" + }, + { + "artifacts": { + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\runs\\convomem_new_brainctl_cmd.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\coverage_status.png" + }, + "benchmark": "convomem", + "caveats": [ + "Blocked while loading ConvoMem evidence data: HTTP Error 404: Not Found" + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "example_count": 0, + "measured": true, + "metadata": {}, + "metrics": {}, + "mode": "cmd", + "notes": [ + "limit_per_category=1", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": null, + "reference_kind": "measured", + "runtime_seconds": null, + "series_name": "new_brainctl", + "source_path": null, + "status": "blocked", + "system_name": "brainctl" + } + ] +} \ No newline at end of file diff --git a/benchmarks/results/smoke_postpush_20260423_123452/README.md b/benchmarks/results/smoke_postpush_20260423_123452/README.md new file mode 100644 index 0000000..7657f0d --- /dev/null +++ b/benchmarks/results/smoke_postpush_20260423_123452/README.md @@ -0,0 +1,34 @@ +# Legacy BrainCTL vs MemPalace comparison bundle + +- Current repo commit: `b9a75f6b975c5330e3d35a2627a5e132b2e06e93` +- Historical reference source: `C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json` +- Historical source mode: `recovered summary bundle` +- Command: `benchmarks\compare_memory_engines.py --label smoke_postpush --longmemeval-limit 1 --locomo-limit 1 --membench-limit 1 --convomem-limit-per-category 1` + +## Datasets + +- LongMemEval: `C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json` +- LoCoMo: `C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json` +- MemBench FirstAgent: `C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent` +- ConvoMem cache: `C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache` + +## What is measured now + +- New BrainCTL reruns: LongMemEval `brain` and `cmd`, LoCoMo `cmd_session`, MemBench FirstAgent `cmd_turn`, and ConvoMem `cmd` coverage/status. +- Old BrainCTL and MemPalace are frozen historical reference series loaded from the recovered 2026-04-18 bundle. + +## Blocked or partial runs + +- membench old_brainctl cmd_turn: MemBench comparison is partial because ThirdAgent and noise-extended slices are not included. +- membench mempalace raw_turn: MemBench comparison is partial because ThirdAgent and noise-extended slices are not included. +- convomem old_brainctl cmd: Blocked while loading ConvoMem evidence data: +- convomem mempalace raw: Blocked while loading ConvoMem evidence data: +- membench new_brainctl cmd_turn: MemBench comparison is partial because ThirdAgent and noise-extended slices are not included. +- convomem new_brainctl cmd: Blocked while loading ConvoMem evidence data: HTTP Error 404: Not Found + +## Output files + +- `summary.json` and `summary.csv`: all series in one table. +- `comparison_table.json` and `comparison_table.csv`: long-form metric rows. +- `runs/*.json`: per-run payloads. +- `charts/*.png`: regenerated charts with old BrainCTL, new BrainCTL, and MemPalace together. diff --git a/benchmarks/results/smoke_postpush_20260423_123452/charts/aggregate_primary_metrics.png b/benchmarks/results/smoke_postpush_20260423_123452/charts/aggregate_primary_metrics.png new file mode 100644 index 0000000000000000000000000000000000000000..91ee6a4b5b60cfd85a4fc065daf6c30f817c8c73 GIT binary patch literal 72024 zcmdqJby!txyDy4~poofzq?D3McdImtbc!^S2`Jqt3ML5BNQWRynn{C*N(+K8r9rxT z5|cRh5Wn?(Yp=D=S$m)T$3ACV*ZYb%V~l4!&wc;u887ar+$JNXBqbsuB9oVsRU;xg zl0Zasi0;@?_)X`hudMKgu;Wc_$9s0>jxNTJ%!rhY9qn!G9Br%~&^enua9<> zol}sV&eGA*-a(j)3;l1e;Iw;W!BwN@^c_Bh#9mIvfr#j04D#Orc_+V6q60)k^0HFu zuJ332kA#Mf5%!i|&1a{UUp^cWa)yLNs_gW&BP2)8-BCX*`&LcCT`F{bUjNjo(DNyZ zZ|D2=O6M2$q6TN@oAK8Rnh8$d-|NIxOhiQ)>@@S?s2S+!=p^Aw{~XUB+(_Q?{reZ9 zQ<8`N{;2OyYWep^U&Vj<^%Ek>zrP+jbGYpv?~9=GMaBd)yI)v(Q`J3ng0d#zLJcLuA*N7HY=uxP!dTU*h&hO}F{2XIG-d?)>p zp{!^>c<|sTey%s)XV3eR(4&;Itm<-U$}!{f#`Mw4`0Hn3(2j8-1Gd#OViP6pXShxKU zG}c{UVV*_nvz=Fg??IWz3qJfU_fb6~#dC9cxt3g_euuCgJD>CJ`;?&58DXc@$-7g( z@{&jEgJpi#1=)33Y1n4#mo~R5$MBP_ZDH*O2W(>KW&G`*5u{j#JG!nNI(p(dj`ePm z?08S*%6E}`yTK|+0=hf@D?6^ync5*v)NOPor<%Xj$nqqW;C-5$`DC1aYjO@*Z2B^c zvPw=w+`X2_8w|gmM&P{?=UBd~U*Ob#8z<(WFi_#vQaZ~*bz+9`DK6W9XJm10E$nB; zy_j%roqW?9IipgGjWl~_8 zVNvNhxT9p0^*u$X+HLP26B&vToa2M;C#V%`{mGPbHrpmWd|>p)Z=|Tk4`8dSg4P~x zFV)e~&*v=cz>QOE2w`H94`Cd~Z_VU1wMy*G7KVsV&MXc3?D6HS^_4orpdzo`W47x2 z81`Q9;gvRT#c=lE;sN(2XUy8#b@S%PQ-gzh+bcn@DftvWYUS7t#5^4N#35T<=1qK>Vgz!8su+-&&}-Tf&37Nh z^#I8k#$(uON?yyin?i;at_K8c`|RBf7W&ICycK--pkVGpyf@`3nk`SAiM}c?4@sC~ zRN`?POC!WKzjAA4c}(k1eZXOLi>wd8nsRZ?zQAR8cy+3yZmPL zBG53X+-2dUTokui?%bNwh1Dg!rYF%n7J+kH{$tz5sjb!XnL(=SF3z)0jvVh9;0h-u zr#Z~17~Y%(r{eN_ViWETx^=iEMxd7O7GeEogpv1@<7-&WCj_Aubd11EV4{P;qc^M# zq!u|U>Vaqfvfeb_5`zvBnN%KRnd!nf2>Vy>Zn4qb4y2+Flxj6@-Cgg10&u7?f@{HX z5NeQ!XwJ^e5=?;;auJ-dwL5E0?u$cGE8?3&e&p}tCMwql99-k{ifx~5iNBy=QSpU| zTf0-cQvSX1tO1*jCdNQ3y4d2sT*kVDUt>ui?!@6I)@CGzpv^PR!v#Ee+ko6Jc- z5%X0`RfxJS=OgZcxA|0r!YM8O9vLw?p3IwmQlZ$cD@*4SjK@LexgN78=KWf=sj%*1 zuB)uCudO+`QfEnWShc4dfQe30A1~qaslsjLAg4i@sqM-{t3avh%{?Ud9B(7D{0CM>oO!@W(c+wp>f%&>;z=J&-l$wo_5-USOo}#--L_Vz4t;xliNb#N*Nv6OzP8o&!=HVjOdfHZ{&|fq z>3}1Cwi}I8H(J;#!{O8-I1Ow}GVdflAz_}=nOWO>xGJns7UZ-I_ddYmHGbh5o6^q- z<+^lSqEWT?fjpBs{+UM;-_cK&67UOyb_2LFWvIl5pf;Ih#~R9by%%#|%{4F~D~+u* zf>viMtxI-mt<0#(Q)s9$jBVU2ea11SIf_T5;=H@~jNdWpSKg=cT^0;i1XsmA&}Mrs zH?WMmNVo0G4${6j!$f*f)HT;FerjV)ETQ#H5v)1Xli1$fwe-5S-r4Sg7J^<~PLA@- zp+JEF*IN962>&+sxm8i(lT^oC33J8hx`$^AXHaF1Q|$+20#1_sDtVYZ=Q0U%@8EeZ zLz_5}nOZ)w5o&|v++EI-*L>r+&0fx}hgYSo?Hnf}>I*>ntE)264wUsCtlJEx*rsXD z??fd=JMYRDhi6Q_U3vj0sj23Tvh{@u3}tx8d|*N@-0!t~epW5vEG1cZmZXwjLc%PY z`^sp8$#EH(kCa?@lb&{28=BclO-$+R@-Ivbdsy`raOMq zA)*Q=(zT-;5^1mY@#*DvnD|cStc-EeeK`HRYBvl0USl=j%dS{z=UgDo#Aw4ibg$>$ z&U#JP1vbs*tW5pVM+xMlvrzcHpAS4*`5nJ-K`dNzn>dv=RJ(VQ_=HqvmQIKBHo6xp z8^5!_LZ-pzNv=?rOsg`X}BKLz;1+kf#Ty8I;(C z(Yx>)JLtLc1}T73TYzpNJ?4}Nu(pI|H_Ony{Cc#L^^;wgn5lPE>YX1olv@o`#muS+ z^W~>6cdFsIW*EKr zO}Fihr8J9N9#|ZKix-Omu=rr-(N|nm*S$(QkyVOegBxEeq-znykB% z0=txlhS9!|+XjJZl)URH zjDN{kR(71v?2QhKLxK1m`y;WgcUVvkvNCnTPScU`J3Bjm%8nLZxykPi9gQ8GgDRPS zzvkddfl9mOk@u$)X_d5F-!Lpx5whWCoX#<-rkSOjCdeQ1v}i5g^}ecGX!)Ez+aI?@ z_xlDd$dEk1QBe zw1ykeo~C@6XDMOVunlFhW0*fcURJFlNO^AIvRAbLs;k+vK4qkfr+?oVUjaXQRM;Tv`6(^et#jM5RDi268@P7 zfTDLw7{l5%bggjaLt^((jo%pb^M0Y&H4~DZotfhn45Hr%_W-{vUsyhX2P{>+DFH zjyFdw1M2JTYKh{hW5*G3bJw6;zombV6GIV>K0ew$4aJ1{#GB@o>;o9hhJe);;?$t- zA1moI)LtX>S<>!vA|l0^115pqTw*@Eed@`rkvC*MOB<$YrfIzCc_~%<0ZL&2<){w5im6a0&Fgw+DK-_p z(H}WAsEw@0cGccHrcL1-%)USD+uUb$Pp3b@COxko>AWMKd7&sD8bK;{Y>RITnszVt zMOLnUP_CgKVdlKrG1?Gltn#Ke$N5svePPY2v65v{c`w*JPH}QjU=s*(d9Xul?KdrE1{kEvju4hMqHsP~%yfKoKeQI9c;U__cetM8muUt=68LY~sk*C=WK%_z>{(BOR;dGgob# z2_9S`RakasNR4&CDQfKQ_dSoMb;Y07FP_QnoxihK(3iNH4F9wVKaE;^6(Skbuv9ueo+WA}U} z7u%{nQvRNHD_SVP>vraS38dD8KEV_>k0~DDUO#?;llNWt&8MAv0x2lku&fuve2c~L z_218FtVyw;3nzaYoE}t zI%Vti>%PQ~58n5aXyZ!!uylD(izFU7nizG@&h8{Ot*!gBS^C(L#TK=M__qP>3@5wH zUoxr>r|kJh;7GcZnRJiPXx(Xar)=AgT>Sa+iuLnf`DTiGPMI3RA3ObF)>5eAGPSrI zax|g=&65{Vc12fkPNuw>ZckNQ=AJz4Lz#8WO9gt7(T86DpQ-}vVnI$x^nJk)k1T=9BFni)Wg(2WH`-`^LooiKI1gbbyne8h#U;fHYyFFIMe2L}h4T9ZYr&gFN`n*!ZCS$s**K4FutMq_m+Bhex6!PjQS7d-*G zgr6Y)tY1MCfam6gbN<-S+^Xs9&U)bkiT+P}D0*->-0JhLfy}!t(`%XN!&#rWSea=( zFBy8w@lEv>=WMzG%~Y%Mbc+I71MTFNZGV~BlLkN=4bTBo;hV89)}GMFpa-k^3u%Ak z8Xc0UEKREP!(WdFm_QZ6PKFY9ChV zu^w%0i4I#F4xo*0YqKsGo{ZM!y_y3u1BSF`x;)GHzIuH)M+^x?QIbwPOYEr?q|tI! zZAe3BF-19@UKO4!>iB3<8NP)fqHYBet)xfc^u1SBla8v|b7 z4~5~68kvm~@jje{^#uWwwl)voQRX1Tj4=wglMoGhtsATq7%J|(y`7UOt_qF)#?DXR zSg31v5_}cw5<3?tr2&h`Sb8a!WiC?_5pAo^?)i`dT(Xp|wII7wM5*J)h7m2gz+;*PXPauRES0Nq{sLN*B9C;ULkTEkonSzYP4puVx)*B_u99mgz+FMV5 zS~Ia)UUcPH0v-JE=gqifW=n zmnYMvPx9f&UuQY9L#rH1I*6=w%60{fx!z1K6(_Zll7}M-X-ZK_M?M^t{4U*?p7lhN z(%qVTa#H%xPw$*Pg;Kz zlPSiWL%SFsNnaYN6WEDQ4hg*3D>CU;*#E@v5XQZ0LX8az=zqj<5Eg0-x#?={<()Q0 zudUT_ujG;D5T;i;X$L+UTFf!blJ=0wvfTT5ANy#mar}nmFAVLdrBkJU!S^05S>i3q zA2@~9II$Lj3~h{m4d@`jz%zfWS~{;Dc+575(uNr_Q1tb+e_N&|#{Jp^EWpQKk0D3? z36Mza71&JqPH9k1K0CWrLl|w1A6#wi!e7shwm%8~`h)l!BE%lMfLfNCOH|r8;n5TQ zHM-(P+3HcK{JyBuh^R}~97rYfAf_!72!+1TVQv6`4)G!`wxHONYQf8hl%$Wri`yz) zmp-dwty?YThSI#eiw1cUfV^5T-+wRSlD%1sH$92|EDf)A4VdIHK~Aktnx(XZgq(wt zeRs7()%?i~*J6+;Vj5OwH@CJLJr*h_ur~l2eJW^sf=J88A)u*HaB><|>09HnTQq9- zc2sXRM}`PFm`?$f$kH!Wd+_z;*G_ng?`a0fY8euE6LLtNO-3fppW}aBd4BOLTZlbL zElKvp=v2pt5Pq~SCrnD@@V)wQZCV4EWnf>mkCFZ}B4Nt5#Z7=y^E7ZX{=7n~ekJ0q z&>6|whcw4lo*lo?Cx(c#$dU70DT=#vn`#xQoPUo0Jt}yffrGZ|OJ83ix83d1QsPrL zZanz%7Lf-%_P^yR^?_A3x!Um<0lP-1i=&|EK5dme+qNiV{P`&|((=Jqju`LD@sbx$ zNmf4qfr}p6Jnt}%Z~D&tiQOFnRu7a0CUd2-i~Caj)2BpR=Nv}9k?C!`XV>8dz2wE0 zR~Le;mFYr;S4QfB#$Y}?BH+<0G#%@@(?2sxv-i)1p3c|%h@M?KFaiZ2?0sU4Lt&={ zZ1qi!AYss~P)(WO zZcG&|s)KSUU}DpY#gqpe{BsUNYtY(dFH?R)m1GTz?w^*Zk&O#Ek>u*|!~_c>ezWW3 zmv~&=|F`Ey;q^~1)$r{9^|Rl3szqHyA%-3(Ki^l%s0|-;nVpy@c;;g2%4Nz0VK|^{G z6chwERQ1t#^ED9EBkF=^MZ7!-__=rd=pH$w&pHHin<-q=ElghpmW=QNk$`NivgYYb z>fqg;{mXL|O852G>M5`Y`k8)L##4J?Q5T7tpDWq zZ@k8z4>v?`#aC>O$BgG$(`D@f8bwrpk~FxlY)p#bp)mKO>0B3wYPenI^}CBu5h)5` ztokNm_CvmoK#J8;~*pfMV21`NTPs6PMQMAI7 z2Z4egLuy^=3?PoX34{{Hs;a7d*QE#dm<-dDF7@Imm)Aj!Ru@C~8l`|O)ub!Pzj+!0 zsK^|tvZHMo_dZ}X9FWosMcU!VlS78L+5C#0$@rhRq*Q`>*2)U6PMg-;*%%301`cp1 zNWiS|Y)8Aaj3!}G$11WP-Y@q?i|xr=tp!3xI>bl7#9@|nLGHrIT8l7n4tV(CrA$hZ zcfJMDav|~~?pj4*i2Kl|$5{9X!E6qpQ2zn^(GwT%&+|=^Q;KIEJVGkByS-T}hIBVL zT#vFQ3_E4*eg*tjRFsnYp(Y#qHdj~8(r7Z5ipx^Y=EuM{-Ahz=%c={;bjhd+X$zRx zIamw(pW|PMyUf4O@IHC$*!8BOwR*tr&3tXB)0(<-bLq4N=dKum?0aXd1UuVyN}sKM zzW^>y)BF14@}%#eq{|a{B|Duct9ppj=#|YcwlBTsAkS%YVrO3 z!;*pS<=p&lzh_H&T7^XC2dlY2-r_aR8_>$sR0j0|HS-KJQzYh}sngjW!DSTQTWnXp z?YaM{uA9eTL=j_h9tz=zeMk+|n<)HcsP;GBO+QP69z4Da6AZqjwUtY41WZly=XFIi zs@hh+2-}+u9p@RKWYu)FM)R@tg|(g@9vNYPPY8q7QV-RswlD~vrYXF6W&hhmX<@yn zAV_j;4yr;5_&<+7D`EI|VN$iU_Ln)m1J)YgE*dMz1uFmVg#|8?Z?8!TRalO_ziup@ z@-6f6hF$f}dCTtY&6SqJPmw7!av}({fT*hP#kT#V=WRiTR21Lac5u`AGmQvqGZr+e zcVOxUC*Kb0*^OmZrlgQYJ;Q@AYS<&Ve?v~)P?DZZo%R&Jo%GI3CR(;T4{Sb*gME~I z*7a~c%9lxvjEqV>*0XK;N`9!{gp=++AL!2vD}&prozbd0KNYDj%&Kwh&q&X4Q(Uy| z#VWh6PSP{n4t%}nx)upV;N5A4TXInR$Kri%HmO;v4#!K&w#l-ib zZtaKLfK9)Hafz8LQkH|EY+2FD(!O!csH%cdDY7lAW9IJ~YL!01TcHLmc=XfbL(2$v zlGMz6&BxAZSW!Zf3iHDjwSMY6EHJ*#@09u|`*PEUGZdDnwSuE{!HEY}!3I<=S^*3^ z224&ZQThn@G*KkM4AL@7Ute8N?9MkEgE1S|1}~c5col4zQL|}agd|mbrZ^PVeps4E zD#^bEaC0zBy2Cx*zrVa_&$!3(;xvOlNzw- z15it|R)_A_#Gmn=VL<-?nd~js0gL_D=Sa_Ayrllm)WZLpctXxuzzGg3jBqhLCc+Nm zum6Dsh-CjYvGB#Y#rVKeoIly+KN#|c$t_Nr{gW{yIo#$OP#?wfp*&#!H$<}6bnQRQ zfUYjD5YB!L43Q9MH;NeDB2#dRp`9D%o4x>Ka#Jl$$toN@P|BC8)V!~_E*o+z6%w= ztP7)4aI4H^fzxMaEdu<~P8+Q~%OBGP((T5>9-t||1|yHYT;q@VX%?F~W0jgEI1Yh;3j?{;nE zRbXL~ww2Q|C>+XHZ7%WKL_!I@4W^j+!2wXgkA74G?=Ooa27uL7NGOmw&%*j{oa-(A zio*%CEdv%%fc0QD*HiQ-j<7ILc?eS+2-D7LrNHaRYB+~pb1Mb2()GusW+3q{kH^?4 zG)LZ09xGhaeLS-$zTXB1!bNT!J=eBqO*!5Z(5m2PjE+d82jTu5^*jzl;eg?SMlcX_ z7j;tQf~CE;mu|1D0L1w$rirdJrWI$S_E?+l90TT%Z`F}zPYe1ma@aF5(?8Yfpz96> zbE64xh4SDS=r%|72Epn)6_oincfQTm-*8Jt!tMeQoaQlff1OL*WiX%*6i`G0A-%mVr z0|x%tsRD8$k%SYMxDOwX2Mem;3a`rkN7D&k0U&uurz>?d2)iMLKC~Y4sln=MPjDGIJ^tc*UzWs!l!nS|ArQV` z0T3GuBi;mXCKaHX`LblJ()Fae_n=grv#4? zq#_P?xDA*_1fqI8RYUj*OqiIr)8Ih8`S}o+W5iWt-5CPVK@kZof!W-iVQ8%?7jqW$$R_@lw9N;vA}+#=c}*iGtPmswO2$-YW(3I>R`ql`P+Gwf#WuivF{^oU zB;SkROrdlrhg3pB!fkQrF-VYK`(^e&N0N7l?_3-7?hsY+O<8ZylZ4z?(+i<4s@3&Y z`&4^ENahTiCc94FgJ(cVZo-mcL+FV*Mz>R4AEt*;<2Q2GgJf|1HaJj|f!h6>_X>H`6qld%lX>F9e^OhWQRSs=F_bzQ}Obq_iXs<3njunCjNh zi$ZJgQ2Mmlp(+tIu@dVt5S(w;)CjY~m=!nZO+XijW`T2M(UY5HSg8w6Ejv_uKCzlL zKXOcH&}$&RkqO#|`XKS72whL|@4-#Ckv9U(H;Bn783716l>~WT zSBHkzF~9%}S1rd-j3*pOG(r%~pv#zx7~mm*6dljUj{DTJ`?vJ9j_`>kB)7x;M{fqGRu3h}Mz{GeA0$&Qi8yC@ zF4hp&bWJgO3qMYqNA01BybN5X+P^-nG4kF_?UoQNY%SBgGgAaNeHw7w@0l64R~NYW zHGYF2nTM`+oKgd@6_nu~cf-V=7YVy8A);ZXZP#@pKgS*O$9T=+kEMTCVQIQ=AQAdT zLSg33sUM}7*M&A2%n*=KrlD)F{f9Yz;Ls{eedeEqA1-LW zUJjKzm@-H6aBo2CM{r@&ShxNoal|HIEDN8#R`~_jX5$e1ITO||xBZZ$-bNNLRk3?E zY7%Yt$@jpxq2F!-e5myT0O0@+s{M7ictTzBwy*yi&C5g zD057EAixMx@Z%JJA9qzF{W^ZzKFfK&#lq|`F?2lS%}7h#R?=n%Dto(BzD^&rKDc3 z|0ocd@quVMuB#SJwT<0^m@(m~2d?;gKwy-%zWaVDe-XqlxSlDi$1_aOj#bh!C{U=SKqjzmq4BtK!>?Gw>q2G0$=99by*>Qwbs_-uYuCPq2ZR-f|L%Gy zV?Fr$xwm~jME6#X-&%lRAu|BEq9Yk)hP@^wx&Wz~E06X=xW{>O^Y2VsJ}r4$5{ZGD zqOAdQo-ED_kAULJ-Ab;X0HtQ!8$yXESuW`pS_WHHCx*=nXBb+tYGzr^BQ8e34A}fv z_y)i<^LN#sS$Z1gU)fWwow-udWobW4+p|4NNX zGzT0HTOPxM!5I^8(Hci>vjZ!sWz@Dy)_#%w4J99UXEBII{{`u;K*TBM3%k3!+z{oI zoN;{eiG+e#*1Cf7`5t`Y~@gOwK0z! z{xeFxgddE8hb4Aa-<5`O8VGBjo3V)ZGMD}YW7_~u5*wfO&+ARJqOSK3%44nreqJ7< zP*tB;5jqpgez~2oG4LFhq3BBAlmox_wrhv~XRsm*?{z~SEaN5UFl8K6WN+|_alBR` zg*M%42xkE#8!r8fG^&B57~RVZUg_IlhTFGXE8LqQ>DKW4Gy@~F@MY)^^&n`sltETd z5mx-EUIz52MVj|mM2>6usD8tq-(VrL4QFq%vd{yNvHh<>}#cINmPAA}uZbuP=fJC3hG<|Dw5%~e6yt-yS3s;j zZPk?(g`h?otqe6L0N1Z2k|NuSfjZC#HJ5Y+d++Fp7aJY3wAlwQ)$($LEth* z5^2~ufy90tuUO=Mt>^y{Pb!XqX4nqjo&we_Xz>ZIknarv9j|u_az0U#Lg_H&rM9=X zQ_Pyev1V}Xd9N?t_^i&3a;*eK72$e1A8H-}*fj+nkVfhb4heP#&f*HF1p>X3yJ8;r zLQHS2vER>ZgWhLa$c=%+UhJSH*Vx<+Wdw&Fv&Z^;7$V`?!$yQJkGoJYZya^WPu+mz0V4<%fl0Sd z?0-##=Z$`Ql}~epss-)mFH6-Fd4}v0oYP9uUBX}yr_=^ z-d9TY-%g||3w<9_z&Z{&6uZxE|DpNC(l)ne-Z9EQm5-(qZtty6$&^~yKN>w7klYA-`AZw@ppCWzk#XEhU&^f63Ul^ z9n;UUYqzOi;nFX;gBV-V_YLN)|7xDzGj0*=I#+~6w95Lvkya3XS+-MCJ zccy-4cwCjwYs-=68x)GNe;A^MK}3|2g^9)${!NtM|N2WS;4eu=ZfCqqYZIl|R5x z5s>^4EeSo*wx$rF<5k4aZghG7;<+Hdv8VvqpZ0hl@y%_rTGK1Rz1dMK#tal8b!K2Ye4znh$(B5KOaB7#D%S>*cY`PGS4-L&w zQ^Zr1rraM+dNhIO5OQ2pjauxGD_H1DMZx~A&|%7%$A^ehpgx8GQOeNHODe5G_zsln zsF?&DUpmr`0NybJlz59scpy5D0>XN| zclXDng8gvYqOk6ZUWh?c1OlJxK3oAs_{I##p-hmP1|gxV9^&BWCmn4N^1g}Gl+P20 z-2%7`J;QJFiw?}bz@M(S!DX$jQ+ivrcDXwoqg^KF?lea?=(X}}BRqr7ZF!6wJ7`m# z!9CQwka{x)U(;pQ_+BS%uk4OzljhsV8+!|{PJu7{_JEf}@Yj`z`pIFL(M+k@&wO3T z#wp0EHW$5uy07Cr*+#zvyS_j$jA31Z2K)<)ds_|7IBnVuz*9R!oFwZ+ptWMkbZWPu zC3hS~oLBzNuKmkhv?(0&#r?20`;Y7yKw`3}`|8!7Bg#?SH$lKbN&iRdbP`l&Vq+x$ioGpI!lCnmj*GASEM!Bm-3}g;4&6fg5<`#6C^LK6solVzcE8lSvhjIae-*F!8il@I6W4&6RJd*okD8)Xx@y{;+(4z;1QR?y{e2NI7olhhWoWwHGW^6>92j-`c*MJB*@*1b(8dLNJ>LUZ%=S2myTq%;SO#pFP5aI{cp1z zcvE=HkOllpED3R}5LCqkbnT-GkirG^7i`?LbxcU6e?QyPI7LWlJ(WEo=4x+*wb8Zs z{9B%9T&`4rRq^9bE1Thh>}uz^k0sF>9A2BtW=N=_sr4^$h=8(c_~Ad0%3#Qd@cb97 zQdqyrwT8;Us7$F|?4w_q$3!5AiTY<)-Ltp4BSqds+A{Xj;9uGq+O97w>Il1~{L0UoGnJD5YeyjJUO>y_W$BJ?I%nw?0dG+jFZiuM9UlItr zZ`e0;Z`)nkc9NLt7&)|Q_U&xLd!`uuGDj)=!j8c-za^uG3!&-j@rkdUpb>HX3wdM$ zwHws0zvYoaQZkTbEsjJ zXDOy%Mw=Ji^HW^}We-452h`*jB!xv=ND?KOGV^F#a-Ax;y_a#A5Q@5>22^TO{THi( zRD%G{)57Mx>mp4h4Lqq|P^n|98SnC~cC6i&H}Xzw;5|0IM+M!YkiEpQC+?fufBLb}^bi1PTps=v;!y29*<Ti-W_{!ONdLqNG_jY?9S9ixX?Eg8ycJHD?U46oTo*}${ zMtXlxiEM9|f7GJ*lc)UioB!U%cGq0k93B}X&|TqH;MNT0$fJK4AE)I1Wt03LN+|gy zl+!>rkVcFZbe7RG`yf!(0bVBh@@OlNG>7%;`zLei2m*uFhH4HWlq;0W$PfuLAyXS9 zV+Ekwrhx7rV>`XS^h9VAD3A=q07&QZVXR?bpveM4zuJpM7of}C*R20|U*=#MqymLm zhRs=m)B*p5PqtW}zsa-H-uha5l~)E&Zs-rVCl{V=43l~}`km1(l#?A3Q^jzPd3)48 z-X}*tyvArff-=mj1&`pXqjp+hQ5`G|kmh@W-UASXSRbM6I_>&?P()sWc!lT`;0iGS zPdrw*188j&Hj}=C9e^=2SwZ)5iVW*O2YQxCybApZaY)R4#55a2ueGejLGHA6u6k>l z%|Wyhs$#U~#&7@<7v(XpTmW>~SLVb94G3xC}j47EA)j8;1RS6bRo94$T!<;8Zq;b~GVD$^%sTo5JXxT<@IG zO_C*n{iYe0f@omx8}%|6G7X3z*4kyVuiHufr1QnAHTm@3Om{a35eh$$Im0W1d>(! zk4(J7Udyj%F`B2h^=XT;Zeg&U|Mq=W9}dcj`R+;{?fW~%b^OSt)lKWR>sCGam@lt4 ztGRCVRk+ENxwxrf?fMxh>Het9Wr1J<_k;12g$>uMw^Y-K)v06nUTZ z1LjT{lq~ZT$c|y!zkUzhG~z>BfKbDns{|N=#2=Dy=Ony6GeESf^(xxx68F^9O#j%~ z2A)ISvsMK*FTSf9ls!Ei3J?fdi2n2w{@wjGz}jboyd9UhkEOUq3nW_SWY@iX?hr?e zxY?7wGB+9BLMyHoy=sf&@fxsyXN(X$((GN3rW)?+OQd(o=KI}5X$T}(9Y2j>F4pc9 zDNBQ0?QdaEErU{!axDRdBnC_C-ZDoUAJ^edQ+Tj97%l|K7lLq+uWNr5qrML>B!7Y( zkNZK>qR=aA$@P2D{%n$UCJCFpQy~QD`o?#m@z=Ohvtg#4Qh+v}F}(RQ?E1Qd=lYv0 zogyWOpk4Tl*W=G343^CNbSW8N@jBWNEf&yMvK#+iXeUS~4b9H0jeGfk%#q8eit4wJ zLIfvzp6@;G{3YEQs?%L-kDLt(qxG#WzfHjPkggT=E8KylEU>Oyxu0R$*A^TQ)gvB0 zz?XCdHc!zJ*$oCOvtJjJYb zU2pT=?I;~7=p@(cbS=+jlsi=+{L66p>@^z+RPRebFHLY0lvOP&cR-#W=fD4nh$lE1 zD|lDeslv8jn1Ua*wh5H6MY%V@(q|I%zEA3E<mV9^6~N6oGoZI1-|nbi~DF%nE_@*1E|q|WOQ&7JM_wdzZ;hs zLOSD7i6KE|5WG+<2$9Iq7Zf7tN{v30=)JS1RqC@>)%)#k-tRr;!#}a-0yj1{Z#K9& z=8MOA=BiY>v{b_udQ-jiTVs35ktNofA#(Az^`bpt>(KZFh>lN7(6hhtlqaFytzAdw z{7E_b9qm0oCy!q1WBwb_Up#>jOwi}U5WW1xG*E9w+@6n}$t(Op!y@ajrnKrVp-gV+p=(v?x%p_t0i;ZHQ_<(mjitS{V64kn;^5l_^z#9lZ)sKM314 zDmkvhHl=8ixhow98Q2|{tDr#3!%oUnpxE4Q%cj@V)8Ah5!{3?zksJ^;%O%>Ct;-pY z-@enHwi3L3>AMUgzz=I079mbVbq&`FN z2?Sn_G|X+-3}(Y2wLcC+-f;I6bRTAR7VE}}z^==C&$2qJK|xRBy9L2;qej7Kel#mQ zYJ`u*AGLPQF_@sQ(J$dWuMcxAnWFR%uS%i-(79+(^V8 zt!Ghr{}!%C9!ao3!Y*ySz`tcHk{yBikU)=|TTLUtlTI`ZeRiBEE+Tt~3rCN=f(S!Q zf(JzV@{MbcuJplkh2)SN3zF6YdGqOv`+oiSRP^BRFP|W4ng-Gpz%8eKJNKL!h8)QF z1gIbK5I#3-+!%6a;oP=+B<9itDxRx|s%Jc3w2q!=O@-zjs^(zDG-5>b%Pd}P@?RW| zXZe3b%{BLS%~y)s9T3<|{Qu)<63qc#UrU)#_$}>i|VF+yo2dh%lYm2w9t_+G7G!58TJjaJf48VhK zLzpH7p8bG`coDB@?ODj;R>qoG5CZUG|5GK?D_DJzr3tYMZQ#fBkdqEamlPky*9 zXb7OzRV*S+O|gt?9@iwaEOS4YECv~I&VJ3e%e$#epTGsX-*th9n2EoBae*stv^nbD z*7^b$AaUod4J23xUPmy(4JNL@^^Mmi{@-Y)0l2~;?F!dC2!r(;_S3`K1fOA2h#W(5 z2qf{KpmG{{EG z2bDHxljt|J+O&`BNHht7Pxqa`G0jKH?pGQhG1s-aW*jgzPQU?D;NdKfcDLQ!;5!i~ zFW5bL)rl8UY1hdJ}iob(6z8tWNW{=wvCxqRWQ`{uZ|BuRR zbMXaAew&LmZ2Rdo--xfMgOf1lV?4;pU@Ls8N~|sn1Pko>WM^SF6m}k_ zg7GkArgwb@_;n6*NUk}Y8aC@!5Uah8Ct=XqqG4IYXE5?a7n>gWV6#r_%@Q4=;&@0^H_serdQpFYD zE(Oh0iU^_PcN8B%_7E79L>}CpF#t5zr3#rq_)614LoKcq{RRX}$iM~UTX_gs z&Rv0c@Z)SP?cyKtn14j0zJI@Prk~~eo>O7&i=DlQBXW`n zYh4|2g2tW!gp6G;_sGRk*#@=tk$0+0J~h&aZmK5|f_ru0ZRXxCTb{{LaclwV1I00a zXLQzF*4pV(PUG&nAEiklH1}vf#7)2recbb=L~&Z%>MO-lya=hQMv8Ltpq59)y4Z?K z{)eHp)4!4r2eToE_#Z!KCNd6sP~8e6RgczA+1_PpdiQCPY#BJ=|Y+Rmwc39W8^x;Zi#(;7( zdlaMGd-h-a$H!X=G=yh_32T}fm4?R&{}*-d0Tk7?wTmK(3W$k~Zc&Vg3W5<(vKSFC zAVH!5k=W#%F<=%I1r-URf`kTz?k0m`Km;X&TLA zj&3@vwdR~-d}-vI^-x=4pYkEqF*TTOaH;|{p?DXsynNPa-d$2CK(i-h8O8-q43rPH zUO((giLLNv)V>aVU#vgj);Y3ZUiMmA{qZ+3P3;+zlQ)*k?(!%Y3+~wD>@jPg;mo(H z@f01!iSsiJhU@2#9T`vw>8YO|(lgRv^*yh(z1`Su{x5c-+#Upw<&=7h{M*b-s86M?(h`r=6D+$vmBAScg6kum>vrP}$v&&K% z|5&#q_B28lzgFCU=Y8N7?Y~Vb+xe7@zI-+knnfv#+nanlP)b#&#IpG-QZJp~PAXdD z`N$7^T2Rk(KhTKCIWtlI!*+S$G}CXz?zKSS>EqG>kQT#Dyu;^dq`NGf>g4CE%l>N? zz(c=`1EW)Y(`KTUMA9KfQ2|_Xl`-$3-4TwU=FX877mVq3X=o@a5&egf?2)`zCb6y( zOCpMZzhLz$5!)~F$RRu(Y<_ii(JNZR!~dWrTObT#;NR3_2Kv&UXiCk3x&u#0BQ$I` zDVNw;GEsjhbByDigA8@y#Q!f|;H&(rYr@370S;9O8}yee62OA7r~p;%?-)9Yd|Jxu=ALU(sIsum7rz;CQ zZQlZJlCC)qMD9o3@ozkXB^B?${?%!@jZ}&{tym%!-IG~#uy`_Y z+rZ`gepZ#vxD15aA=tvRQKUTk!(iB?%0w>NOEKq6~2*r=xE zky;bI*mk5;sgmYNmvI?8Ioo|S!(HC?NxF*EUDQ$1Uy?Zc9*gQF7p89zx9EiC*!0m^ zD`1|)6IL>N@5MIQh#pQu7BWX~vc*&~MJ#crBdXMN*AsuR=FeD-B52$LAS8HP&RvyQ z8UC~d{jVi^E7WnbqzoXRLB~zaFXhr1Ef*j@TWo+q2Cu=y?R@b1?wNk?`B6t{WixeR z4ro9Xq}ZX#gtn}x=o)v(aivgcf8)MOwvpZG1qJ~Jc9`GT@UX%x1;U8*zlvB^FJoLr z>rIZ1W3dfYFZ_1#?avRqf*o#^pe53;Pf5^Z{qg6;$gwX!u-=xI*KFIK9?o&awKdo& zf0j+vGp%Y-QW4=fJ$K;$M%H)(D<=kb-T%-sbqTd8*CCtNqZJK`TDCEB385$b&Cm9>Hdr z>s0SyJZ;cpW83oN(!#0r$JPzu>oo7lruXmGJMTF+;O+Y(xGP#TV`uku)5@1|0NV~b zj#@eWAxIj&|2Lu($bhH->^~v8lZYpY(10`xu%T(d)3Xj#!+aVA^cTDi2%N}6PErdL zaRE31a7ZXUN1c~|oVe=W>NtGu$A&^6CW&SZgyoeI8%X2+IPcGo{!xx}<~$RXr1gL7 zN^^YZm1L$@Fjk!^`b1i~(6K7|-7@pgJFaSkoda~f4r-Jgd60b=T8`KO9u!2RvggFG z@h0tr=V%E(SqzrzT0P1^3?cdHT3B5>ldY!lziRA^*}p^M_IT?s^mWC6@lTH;oiO`S zIb@+V|fw^9c z_)&ApUS}ZA&~~W1(ONd5)U^2UxUZ^Fb-=BjdU}l7)3bbuNH5Wye@&S9O<*=x!fBxT zw-@pyd1cJoEKCt{$=M?UBmy!DiHfu2U~olP?tY*b{Tr7f!G`?E@XnwG>PS4l<{U#> z4ImvVL$Paqfu0h?e$D77)WeyCb@F4~7XjfBG>VcZH!)4SA{LT?Pvhedg7 z@^6+jE#8ObWHn+Q9{{sWLDP&aYUn`na^beTwd3iA0y4Lo<-3g3I|*{!=$8}c0SL5KtGk|&xj_loV_GW8qj3&@u(3mQUI z^#BslG|CE+$1s>R)^p=szo)0?BB{8tktQ_V^D82Im8@WfkY0!AL{5fh???I~sp*$l z_!nO02p+>1;J44Ky#c|;5j5|p8*jW0{qy%`6Lgb51z6M8@#*@*UJvq@{PEl@KWV+t zZ70s>2FOO%qb#?aS)YT)Xb8L7iVZ)b=zgrlk_3WN~S5L^! zmm2$ZBwY^bFqjLC{HCQAbwssDp4ER-Ew1^aT4X=1Ke2t9*k_AtB^F|m3a&2i_xHB7 ziaD`*{i4GIqsQ-Uu=1>=8wTZv_TI>!qy#+K?NH26?SJfWuBq?Ti+xcQ-}9rq&gFgV zUo3rO#j@Rw3HFmii5uN7Obv7AThGD94RwbQ9T-dghk9BP$=JyA)1 zg4D-92;m!%-{I@&ss9N*v{Yc@d1Ev;Wa_w0UoN_<;Q>wbVqGajaKDc^n7x(mZ@hb7 zMC*uVoZe6SdP1TA=vDP{K+UPqbGSv4K$hUQF8~x+z3H=C6Xl{W=oSB0-XIcFEMZlDeY*kr0x&%gK&Zp=3pL zhSQNU*{GUEMVYp&EO!{INiI0@-5;9cLi|^!;77edm>GgvpPCef7lSz4je?T3BGusM zcBW3X{NU6*widE?qukx=i4SivrBhl0H%P>lh zp4yThtmRox-z&v174|IgbWA0X+;ffQRNIM~Wt(&}!$Hg|0Rl8G7m&6+nK?v@B!-7s zAAV!~<4`UdfuT{0R^OE3IqsUAgD{gBn>1q|B0)%%=q{|%fBEkIK~ftaYnVUyTMi+n zUqu_dx?D1@G)Al7=65*lwj%(il4j~(c_c6c(X~H}dGf?xF|wu+|9)~jNgy5P*Wh3v zX-OF!G4YxAAi>rl(KS(MlEPTH&lb@x;$bBEm@`n6xCBWgJv2=%@Z1sm>IqX_x|zl> z;%HP~f-kpAq-qYDJtV2>iPX5!wr76fg5Dw9j4sSuwPR0v`Q)irasLyj8EC3K9`oNg4QcmkqAdaO z41Vb^;Tk0Mu9^ANP^@~&Uptc<7s+eotS zd;wu%7DC$6*FNnD1iPz<+=M|s7rWd$fO3LPzd<=x{3N<7mAv^pZ>y7>R^0#7V#|w54za65me*9szqb_b-*1$8#$My`iZDcNg#an6s+S zVDI_S_y!Tytr);lpJ8-~TTFYg?-1;hXF45<)5G`Q@J*%^GFI{nnX^$TyJ*qc8@86V zpQ7kLeQJlgCtr%PZBZqg)XtHoe(S2VDj#`fxIOIF9^Gc1b>3r)e`gMH@m4Sds}AH; zzJAs;GIg0&kJu=)jf3b6if+YLp$xljuw~>7`ams*j8DH#*_$EE6yU1J#GA&S1ML)B zwzZn;A^rP7bo|6w48w#&WvW9Ar=y5w?B7pq40btb_AK4_m&Nc0?#Y`eReU?`}Eyr3sMn~sjC$O^<98<9MLZ0LoW8HAN?c@tzgG47% zYc3UQ;G_*E>cHL{R=)7eabzg)Hkf@%KmUbachdNVZ;{lV2GaGh>l0*Tx$N42%}MA%o zJp5>2JFZJ2H7WWj^>h0tTK&l+*UOtvC2TEBfROp~;|F;vd48HKDP-zIl{|j{i*R0W zb&?S-x=%SoQPa6VwEfqhyA0iY^402ZBap|>8Y$lgKb^{4Dqj`&9B0$!sruJ@xvn*- zYD>%cb&f}!QG>+pR8?}i62yn=r5*tVaAptQ)e9^kLNeuu&`Z9Q5&p7@nAycTwUaHm zyuBmk1_c8<)$2eZMTIcpJsH_7bfk}#tSDzJD*KUfr1<|^M;f_`-Q(M{qd%f4p%X~7HLdb}BI~B9N5xUR-K20eapv!|_*QsnXk-OqLyjE@kumi& zZnL{D@8y}8!Pp*!RMdPJIQu%wH3{9glZm{dj-FprszYb#By|NX01+xL>D7tFCXCWuo<0;r z`P0ly+|qRoMmw!|fFbclm>r+FhI$_R`D;_9LIy>bSPt@orM2V`w4OKtijI++lm5V9 z`3oQSLm_e=X3+`+0#aGsIu8g?9uS}+xwTFa_nTPRy>f2aGT+BLHr1=M=~2J7PBNqX zCbcD=a|UM5^xELYH6bx;;Iy^tuUVX<2Jj-X#W!u5WWA}kC}_c-4V%0Yhjo9Zq!`j- zyiv|1MLjOfeNXz|9m}5jeo0kvV|YT$c)s`*Y5_Nop=oazThRkuc9Sv~A4(Qzb#hj@3QI*x)3BBO}5+qWHy6g7U* z?;VF|_h|?#aNU&!p9$Cfb8X6t(_j145)UtXwB06_b&>cP@uQHQ!7ha^u5*{@LX~)f zjB@~-Nt{O%;&TzceGJI#dZ0W0sIU*iA}3%#CzeLfk)IoQg)4wfV>$`3w{2$<<`io| zlDd;tdlQuuCrR884fPm+2MFYxAb?TsiX91*yaSM^r=QDxtom8wrC>$egb1`{_ny{! zZqgDvb>H(8&yQtOszvXEIG}i(uVY6b4o%hdiALXEohgnyy6QLX6G4}EIff%za*r(< z?u*nCGc!m9cx9^75Q%`sQ@xzXI*I*9Anx^qj3do`N6Q?d%A`D+yiF};XwQ5k?vX(2 z>U%o$Jko1gIwEtWdZr1*A*?KnvA2!T!Kr=nlb_fKJKzKS@3OSFXUut)LSv%viB6tP zr(R#Ts7~MG$f^8F;}SKKf^RbzCIQw%BQBd zHQC%IcRCK?exNAv2o~TdB=FC#WJEJajhua(<$IHKQH-wg_b9~@3tH!;*?OZ>+^QLjwpDT+t~A7>E6Rpyz`aOhf`nbjmT03 z<}6Eoxr%5FWr98CB6uULp9D^#3;Kks`{nlt4QXG_Z^8i=58qksQxfy6tO&mV$7|m` zF+@}Yrc=k3`5SK;Y`GW!B}C87sh3}^W1}2S$EHt&zCRO}?{wFMzKRUZPzsi!Zw7WR zu=i46;R3}K0Gn^76x`3XDSN#GP@6yUuk%#&UdH!Au}tQ7sSH_Y9Gr%U>()l3DQ3Yd zdl*sS!IsaM_REcDtXar+{_B%vO+Cb8$aE(1ioub$hdaK#%{m?C@nw2TvT6)qlY+vz z2j075NoazXQRXzLk;l}ZqiQ)4M8{wz#-X(s$^s8?OaT9SN93HlPrqsw(`DqZn5GlD zmtem`Fb5@jcks=v&kq*^UVEgl!1@86KS<{_ROlWk=@IR<78aK+ylW9CQ%1YUxqz~~ zgl?`yf_NY8J0bG2KbiQaQXQCwVgJ=1I5~CE*p4;_u%!~uD-v>DkS&Eoh&`tLFOM`h zwtf}B0eOvVNKg54kv_H~o{k#mPrg(Ff(3RwI=#NOK9TgI88V4&X5QyP9qL(wh)G zWC5R3O=V4X_Twj8f8!beU<+y#rJYXRrq(anWz*=D@j_tuWkuwV>GyHwPnuOobi`=8 zAl21BhfL0h!|GP#DS7<4l>`>TTT{Dj-}HA^u%8!3#K|hk7f=`mInE&qJ#WYP{NL^H zxvx(u_4iIc25kFU*{Z_#5GJEW_Rpd4CT1(=n(5bSC?kbDfecsdw*3T3Zt7WKE&6j9 zUG=P4qEMgepo|Bs$vy~AdmeUhGCjj3%J3iW(#$r3ei_`s^+O2W3AW@#h{Pr&+m3-A z-r`^}{bt@KD-=$_M2G~8zuO&CZ&^;m_H1~Q#R?nXhL6L95&VnlA&uhc-(f5GjynyK zGQluseu_=AZwbY|7TP7^=K1$mCR1r_Cnr2eVx)|7H_A^aR1U)mo$moi`(G}fAkM_L z(kOowqy>^KA4UCLa2wUa!2nx@u&QEUCsKltQDl{Sw+|kq>5p>t_6&RUDk?*IS)Lx1 zNN6Id5kBQ}X3aW;lYy8=BToI%@BMB~s4yrB;`~h8tjkc*j5iPk#w4Bx#CR=2IPQ>V z8dmxmahd7YSFrDxK`HXqzW9TL(qirAQ*hsj)GXV0^%LR1O+qf)Bkysy7P#coF&F2a z%NF?jwPnh%PQUyO#EjYbbn2DTZEN@yW_pVtP4wfXJP;1eic@#WS((j)SuRTAo%!9-H@rmO`#`@|`Er&VoJ$Pk)F62811j#YA)lbMVq9wc7@5QW6}EtvQ>;i?mB z3wm(<7EnI0=m>kt9)&~wfIvuqNiyWm!lxHG_lTb?N4peBLKsf)|?d>21o$*Mmem|9ip+wPbwBWzR%K z`3AMrj#{UBae4ETaX$3%>0K$_g!?u;+K13svWp_CEfAX*5#U+m(G*<)h+F=>CjA3; z&wTSP;BiT!qtj6+0c2!C-`b>#6g%S+Ut(tTqq9*-wjj2zzK+Thc)7D9D|4>%<+;C9 z`#%8Y#ejEx_}Pl*A~+i5Wr*HN{wRg~#1ng{6}s_(#jolRzT?iT7`l91^O6yWc2YQ= z=$AqFW{S%c)@-^yMWdFS#zD+TzD^)9|MWRuz|4gu^vTZ?K$r5hTIgqs46t<(Z}@Rn zOG^J!$_&~dO>=*H#Q>e}g67c!rBJP^@WpESn(dG9y@y8e0m1$b=iGi7S#TEuy8i$H zCJYE-IXZ@QYxlPmy;DDqnOuq}wT*NwFMnc|kDubcDIo3mu)-0ymW(73DlIQxdDtxs z_OJg1_>WSj+xWuIV=BuUBFRs*7HjooaeU0fBF``S7DOxfu-}vS>b9N^K{fbG0kWyt z);h9LOT_d{vKe7B`k?f^aM>LpyH9;JT`kACys(f|;%kq}kE`QsTip3-6p=4KoV(I{ zZ*zX8J=U1HpQn%(O4qE&PBazaJhtr;}qzJl_Q?fwvKI7M1 zeg!Mcsbu7HY0`tQ#8s59gzHEm&%Wskq)-*K7s!ttgF@YSWV08N<$hcY1XB1Ylhye2LCfutZMtg`ko>wJn)W-kY~6GUZb*Ts z=q}oRAK%QXUX8uR6r-@Gp_+?yX==Tw+qdqqQ7??|Wmb%%k+-OFJ;}Rz0<;AJUL5BV zrS8u)%QdU!oQ#nZ_bv}98xi-)!#3Q4|HALy>vs-MRC^+M7e_-8R2cw;{O1}bQ^RUa z(bn>b6;--ABF~7I4^08|5FvYG;^_i8sEG5o2eWP4&XQD**sLwErC==UJDMTeWI4n0 zt-LD@Cc7SEbI}}NRq{2Zbk(F7X6iRwXU4Bu5{1(%h|6JW`*W5$v5UB$XC1y(9 z4e}f}7;UW@yjd zbtqp%z>;#(3}SZD{X}JS3j-D-i;eI8k;jo#UB$&T>J{HygcUJ$VjlN9#Fu4Nr7GeA zNc+cKl*9e!NJ%@Y>v>ImPmq zr&syjTCsckF%A2DKP!g7svCMTgnSYsHNJtnkr?-+e^G=`Mp5<$Zj^~RbJ0%sk1SFaU&A}un^{TCNw z>_55O&w6-4Y%Wtc0vnOXwH)Qdp(fas4^76R|ALLHlN?&(-4f`T`4JZ zW581rr!Y?`|Ju#8FE?f<^KZxJE}N_jKSN^#4Sp{_$|M z*yuC55T~x1ZSlkCx~3y*v-O8pERxEN)C9aK4~%82*S8n{0&nsaZm9<9u(w0f{8X(O zW%*(jhXqGIJS*G%Uvcm}L9Z;DH9io;x$>9UMpXCdC<@7B0aFs9PSaO`3=_Pxi4-6^&A5TH*YzbP3SJQNMhgAwB&Q_Doo4d1)0mS!-s`oD!hap;ke zZzQCoTFPS_MNMl@g2|1=83TYCc61qg$aM(pwterOY@~YjCBDe!c(Xit`V*CGfI};Z*Xy!@#FE4+xs?TN$7UmYnF<+Nf&md znwu5n3ZZh`gwlAsKj)H7YVXy{m;>mX@1pjFK-Xn0G4KLT<&cW-{QKz)n&we}m*Rys zu#j6Yk_>dwy>Zdxj>Nu_k8tdAx<{0GeYwrxOP?T~)O2@uj$C3~S7802Mn#U96sruC zvpM$eA-i6m%{`dmn)hbL_;b*TLV^`mqHWK;5r`K-y~QXYgypit@#J9rgE66wh2NLx zTT+a(GEq&CAj;vMye(gnHem8;!CSa9zPKHo*@oS^y3TH7WuZsk-kGdgN`JK`E{)yQ zR@(zfML91<0jcDr*Vl+T#L}8L6-Nd76J!oA9Jczp@AAQwwz3QRi-jB~2FsSae0Hlp z;rzMl+|jqpbz3J6%&7ZqWv|cuZE1gd!eG@H)qQy90-OF7`uS{2W(=@qqkl2c5jNpo z!1J2E-Z)n(a|B`6+mwLz->>) z1vQHFGuV!VaU@|3rnvWnX_NXj0TD**skFlra*is8GYk$06+9GWNjqLE+*Z#l)D}7t zY$~*73{|ZWKzfpYL$b?NAd)f=3O=MhdX9+4-IHkqJjMWysLlxpWGpI*!XLB~F1O<3 zMH7Lpyp>Z?hTcinS)xy$maTldEatMWuh$vb5HpNqd9?3qh~_@W_EIIZCg+8Mxo?on z0387Vol)<9YI?UGv5LR`(sT3E{>^n5y>^ueRMOP4@u_U+Ua2l9tDeNUjbyE@T}bJX zKRr53ou}PG)#=Z+fAYvK(D$dzgP5{=w^=LX&r>Jn7dVtk2)f!G$kGmzG`253ka2lG zDpX3@*G0Rvj*YA^dq|B}qEMSS{qIZ+jzv4OA2qeH-q>GvIstF!Z{A8mVo zqHZ)K`-CTOVuMvaza+W3j}R5^f1}3L3^kx?I(*bwN{ft7Y}RIpu22m#$A2Se4VxAB={4=5fBNIPk0R_QlrzUcQTO(u8G%=U zvy{fJqiYIrlZZ-=1+r*UoXXD{5@na%_6+xFlx4M)oidv&61fnU{8m}k7+A`ej3Zn_ zhz$4^V6M?OXO;VD&rXbLB9mD$m_#xUvjbjnHkTe;<$XM&j8_PQh|Qdx1b+? zO__34v9Z;C5}iStovHJn26Iq$<0Mqx-HMJqmTY3E)w6Hf}Zwsla)1p-zlK717o8l&3;p-7Kjc3^DlT)X|#!BTAL zW@zDoroOwB1_U|)?O4!OXB7Hk^6?m%*Njg16JA5LKVf#G4qt{%^8uwhtM??|CL7Bz z7`zu~4{gz!lI#qO!1{XWD*)ZQs6js(`Ar+b*ktrAoSvv(tS8tD;ttLcax@kIGZh+` zFX{)gfU`toTo!?|>sw8BI2w%~OYRZI6cxrZ6MKcz(3or;^a{=T`eN!HDA1Z2N^|X6 zKCQu)5{UZeGh&^eLkD3CxdqM`zo^5V=%tANNB`YFppFUvDzb=jva%AIU4x}Ya9Sb; z#HV_I!-PZyo&%mp@?3xatRAs;W1}2(+c62LS}~xl9oItZcHS>n8i&tb6abFX;KJ!= zf~`o@yAFxiQI`O;C*EAKTkY(}o0~{M3KG0-RtednVOIQvuBAp+Jza67jPq)uL;z+Q zl-W$W;eQFNV6ccGcVrvE2%zs+58jvo|0;>CUJGH^eO#YPhJJ=ConWLZ$Uu$AzjJ1WVaoK?L$U+!H-9Ng*K| z5()Cwj`Vou^U@l2txX!yq`4|_XA75?G%0*DtB`-=Aiejkgu8Bcr4RK@%jPQ{`0)m2uhrjRZp&-;R7r_Z(7#J{u2_Pden6Wa4V? zzNve(TIUAqwtxHki|Bpt;E>3JwN*9d7M@6D3^5^R50U(z9yPgaLgqsN9_jphyTLZ( z@oYWQ3sH~Cv44qO_=Gld9-|)h`D#e$im~BUZ|AvA+*D*%s0$@tKdoF{L4>6r_h1wm zI(N`rKjxGa1vtD-V>0ztXZ)UIBQ!v2X^xbo2Gh5>fsZE)vRca^^JJ;zE-f z?9|5xTD~B4C1%Sr&pWSy%qx+YgDdp@{e`*P`e*uu5&zgmcTG}P%pj6g!cJha4;ERT zx*Bu$PIX2k>l{fQ$7CYpCceliY@bAY*1@86(?DbTbB-&%Zc;^IvmLRUEI2U~n>jh* zhVUd>Ud|P*P)7POKzj?8$X$7Vd}A&Zue@?v2H4Fts}O}D zSrNIYAewKN%p}hi0f&DctRwtjR%IPFwR0FbC6=Lvicd1*873$HGkS@=;IFC09xjds zhGgKPG*d>IL?kjo9WxDt{yk{%N~REW#FoPT9`LK8a269FbD+)v?jlMeVk`yy^4`(Y z!|LBt7DXA!d$3f92%6COczm}f0n<5**8T!zIV6w5FB%s^^g@PrekNNaL6R2_V-cY7 zTX^!_X)W_AS`;EllrqN$bnQIHHNAbCbjGob9VDZ{@^Ys4taP;*wDA@*{(Ui|d2M6> zO^gc{-3p_Yq2-#QfWOypQnD484xNXHkOH*Jnog&t>{|={rsc}cYjG7&F*xR!{@K~$ zUD>GA>3>EDsRRhsKd`EE|3Ag;2Ej{K1d#j^6cR1qbg)GVT;Zkv6ud+x0_cAHbHZIc zj|1)t@bUoY<)r2XQya7^e#%aNdV;-j-v*uXsJ#`4^e3dS{0=uYDL>psWFio4+n$R z*DAWxc+1l8^+yZ;8O(X}K}mz8tm@k`@brbRKp&i%D|EsDO02R78er z{aiHO_w$5lZ7Tg=0PpADnr4#}9^lrG3<`zOw-8>+1@?+_s8@bO;3?mLcHJrGpC{Nh z@2{BF`8)k(J0vY9_%cAal#D_|Id=XZ=}!uMEdfhbd=)%&M*-BerSNmkP@nE7y@-c| z@)dg`-?}X@4p*6ocuNXuPpF2<)!uxK<@sm!!yA#DXP8iU^)ODvZH>0@1th47RHikV4k!Li zW$F@3@GHMjytB~Ua;xn`wnC~TGN!fSH&{(zDD1+AuspxRnN8@YWq_-ZKy?z61Y5eR zd5JIpI$=eG*{#DS=nuN-GqzeISV7bv`{b2lUg08!&4OpX-wDY*NS|7Z3Onb8Ih=ik z)S!#q%Pb}Tlk&9c(vv+i+fXnEb!*RThe>M|*L{+(34rE4sQ|GHkPxbfVuo2+bPwp9 z$Hy>6;m3JhYG*Bphf{XY)gLT6xN5%=o2=pISkFJ~S{~paL z8xhV#R87yAH7LBWcds<{`K%gNkX0rpY#`9j>5&KpoIFlp0sI&za_0#KT|D~lpk|$A zlp^6%)T***J*rUoFn$~6MW}b?^UH})*8&N1PM)TR(U zEh})9RR%gd z>=I`@o0GLfZs?IiFX5d(rwa&RlU+PjvTw;^9z#~aQ{F*SqdsgCstb&G!unU}cu-lp zP_LW~Fl7_KFhk)J0kO9OjVJ1RL;|<$h65zAmvceQ{sq)~<7w&8zpGIx?K;DeJw@Ir~?r&X0JTXnBy0 z=ja)FQ{jIVkyc~?md*3S%j7`{--7CvhzfW+upqZ7Hm6&6#dpi8( zV8k74ceMMgPw5+`^8LZ7Sc;h{Tj_Q>9pTNE+V+5R>W$4Yv2MnnTFBQsP;!{ z`k1+0K%bvxxKZJuX;&-p27pT-o6H)`d2=+T1*I1K4&@UCOL(5q*+iu2Nmgo_a1OFk;O(*2T5<<3{hog@fri`&=0A*4EjTv#x@zh)d6l%VVok2SiYkX?l;&L@ki|wMjVkzsQzqkcI*Q%Jmol?AuxSyU#yXj2S zWEI4Y(k3PEZSi`G3|1nkIpbHieTeBzAu)D&sDgRaZnP`g6VQH-RZv!qU3F2 z+I6F7+>KxnPz}2ifHxV1XZ(Pp+{)F-Jcf6?J8xw5w74UVZ(!NC@S&JKXxMFXqI?b& zRX|&6ZbbSJ!SaQUB;gw13-1MnNm2HgH0r0f1M~$CF3!%^p~q09)X{>xfD?7mN<-#A z@VUOBsE8_;=2Q*arlD;vRQqL*c2O8KX5o6p9E(v;<7JXA5rCyN31kG;)gPt9zYnn> zwIj)b1vv2Kc?AFeKwBDb!hvmRVuTqVv3RG9hrQ3s#X1E|Xi25@BQu_ekCo~=n{4GT zb|Li6-PgI3-mF%YM~R2NRP|@m>SJA$-_dG=%FLAl2Sf$pO0bXX9Sn9ZEoyld04w?9 zpdILne=J#2IW$PXRoczed|yu)tEzNMW|zEA%?M5DilhT8pG6I6@5 za;GOt{=7%@(LZg_Sk^%2EWM9Clg&d~jtL^toAw=_d)Gr*TR35ZvJpIA`?!T4NT~m{ zUrDEfZ!H%_S>0G)x$Z@W@~6p>n(?mTFWSNWAD0=|#WNq9|Hzl=&EH|St3W2S)y>O_ zEf-^97k(_ASv#og_g6b-{Gc8{qvv(gEvw#K(Ot9WuV{Z>&<&J7E7 zZsJ@jNCtp^E?`?2e4!QRVei|ie;1tDN@bVI4E1?#jc$dg%2c21Dk5;ipdA`6)?@Ry z*XX&lQ5DY2i(d5$OO^#!4uYMED^DDiHA2%&2V0Rx=BJ=ery(DwOmq}#qiXIg>8DZS zCg4cbK~vO$zjMm_a2`8Ioan7MsLZ}+SHO;4(V5DO^XktWm`KezFn^U9cjjaBtg(a= zvBNfZjlKkY5lb86iZOV0ySDa)+s)RoRO6(K@gGda>ADr8My*GbAN8yiao>O3ye9Sq zbdzskOB{(LnrH-~Zykj!hyvkRr|~VzjS&A~5Mv2i=Vs3VO#VIx2}11X7#V}741M2s z>;RZpYKoGV4mIJvpEIZq>X?9Ad)pE?PiOBKhz|-Xg8+=S6T!$&2T02vOIocobQ#ZB zw)M2P>~M=eMDmh@y&FQ^&*F;2kO8iiX)KAs(fr5a%&d|0sR#Pb`x*B3*r8tmJk%la zR88M!R0F=OwJ%W}KsZ72b=`4Y_0_576=I`UffT^i$JtG+@VO=684&@9S4P#Xz}IV- zkYzI1ivX%sdkoeZ14^VVw@NR9w8s!(Y&K`2SBfA$_kO?X!>w;G*Af)qOl)}!C9O5Y zO>bk!$t>1*+2frrS&D7x!#%oX?ext^xa(}~ulF*pj~|sdz?R|u^K9B+h&1bs&v-zV z!k!qo$hgZZ;J zd|an?qqBadh0V=3sk)Jlj@=D2n3MV3H**Y79X~Ug_^}4HVVuevmOdvXJTN zW#u%9?YMRzs{Q0!D}*G4$J(LAiCbJ%;s!ua_!kO{{n`WQ-xx>(n3W-^vANzRN1$`? znF9tBNp?^4Js#0Vs&dukNys~H{&JTs#~?j7P{X*9G@oNSeC<`UHL{uKrnk5W=}AG= zR!lLj+zcdI=~uD02l6AuUgi65@ZP*MkIE*FIg2G&5wsL&Zk>j z{bx*=`d?MEPrT>^tg3gKe}&wQ@}~idul0{wxz*BzL!S+5YdLiBXha`O=?YG&e+BmS zj1w)bS|=$(WTBig-jmWT@~*b`V@9v$>EdG?Z#%`KHE%7FQ$`E~9J8%~eykn97BWy# zi0kq{YBwdhW@a4xIum~voUew#x5ksrBM%0%M}DKvY|pR zwtOjf=daRd+NDBGO*gh2DYY}?K3-;)Jtyw9tO3pBxD&N5=dHV!p+^752(T3aCoAqx zL>BZZ7x2)ucgWM%9h{$&Ha3=WYAmt#7Jtr&bEk|8^;lyb3!#bC<)T5g9%hD5PwVr2 z;IaqaE6`=G{_@yceUT?xH|A+dJ^&(qer$5Nhbr71IlU z9=l2Zr?NXv{dOE*F`|o+Vy-$6coHDn&3?~S=W+&;;}@6j^LSf2Q=4n4*b1gpbE>>t zQf*zfrD;tmV=TSW)!xkNdt7D>e;L(md{pk7@)FUjug7k)Cf%1%t*lOe;A*+r-&69D z$MTW<0L!jIOV8$VM$c!FxEBY}{MMTI=82`{)8n1tIMknPI}N0ZfetWPF0Va}npp%q z&WWnLxXN?guGP2kMIcIWs`6rR4%eExwbajUo_dPrCax@O7l>(yhTbD?0~3T-C;Mie z96wdPw~MSZIg(xuN%~!RY%uxZjFnDcXNE;x;Hht)zvo$$UAM9rb2F@Yp&1<1(Vz4& zmFs-{0$ocp>l-hQu1b{Jb+X)|l0B)65suz@0E3j1Lkd$#?saDm*wfT&E6SSosXL5) zSb8e?m3CPx7$V1|^gYMD+Q|8#p>{<_ZPIe;1Fk)OZ&qREh2yeDWplIwlV27vp~m{! zT)rlycf(HPM}$7FzKe=Ci3cGhxB7o2cOJdE_QUnGJXw9$Qupaqi@9m>M7LLJA&iS&_i$hDPf=9BustUL#|3{i(g*L`ip%DlvMgT zPDKfCyp$-bTP`k;dj-#yVC5Cvj1hTW_Grp}3wWbOL(^{`X)@_~dmyQ<({tk2l`eKa zjhNwrOV@VD2RVL?e-|CES+Rgd9Q)H`&8=p0A0Q1f2*oj#A@%T9-Pd$CEL#)g1-5ayuIq<$hxBF zzgJ3FoV>|7Sm)e)YhS$G=l2U(Y4mOHYCzkeY`#&$N*|tZQfmzns};2i*er3r)Y@(I zONC*d)qqEMr0T@3?q1NNrXUrpWy3B;6^|gY&o0TqHuuEkUSFQczPt00{wm2Xz=kWW15>X4{x z_nom=D=+~8K*{BQ#b|C$+&gE1&TA3nlfC6mvjYl(H-z4DHh<#KRZ|z@HNxq}yro5L zd+>o(tZY|_jnSev;ugcBdPy21-@(z`|3~L;Y@c+bN2P2{MubnN#|mT+bG&=pa6EoD zSl&10AQsVpFn{QDwT=IDA!->nKUV{wb~Q>qmuO?h=*dU;0pJ#3fSIh*+B4AIOL$GV zSCl`4Qq3n|^`B>GR^in{(a9S9)Mz^-vbIIv(HIZHM6C+`XTbSdjiiZ<4wfUQva~8+ z6!Fy$vw>rx=ody^o@qLDX3+;9+AK@!@<-i+!pC13?-(W^6im)e8& z>6LKm)Ei90tgL6Gn3RIy8wc{C($*>Z*}L$tjga$-QI`QgGilLB>C3;R*hMZ^i*GpW zf7f*Tj?3z=6dpzvY?L#*){Pv|lxhV3yoS5F_t$#fAjgRvDF$jHkLA<$r&^DBm3L(L zE}1V>Gklag;JKsyv6b9&mW^~CX;)Ray3@69om}fBg6BmDo*G zk2!)3cWb=7wBzlZRX2YnyMBruq^@*pkBUSHb++N()BiQw>PE2y!B@_|&WeD;k8Mn> zfUAR)jkt(y>6X)`5Vi*P`C;PM0<*s-9zR4i`qOJI4H4uF!}e+k1ZEf07R`NvNPK!v zy2T%cW%aUIg(%$^V0)_23N-5-@H1ss6#e5LNbVNQ6}*Icga(XW7f~>==nu-_NEE^7 z0XGZurUQ7GD>00+15I8ML`CaA2>sz?#NM-!SQwvbyyNw>_vb&|S<|_AGj1PXu&6s$ z>!!b81N*$Nf+xHE1q=S@5d?eLMlh7%$o0T~{LP@!+&|xmbY0WGx?1p`*-LBou<;1; ziiBV5nGJ;Qp2$_HoGN2`TKU{|n zr+Tv=+U{Mi1dunh_kIq;s=l*xGHFI<$D>f~JfVK7$~8sJ26VXRokgvqx^M?T2>;ci6EbPLtSb;M zl)DagqS~Qwr~9s4TH7kmY^a|m2o-Oz@h@WGxToQ~N~y*p2D56@BtNI3Bj#nY z8bUD#+`nybr(@qCW?fY9xI@#WvRM6sxQ(taJQD5iI+#@yf4seHS!?fkPl+#`7M?d) z-!jWAU#iuD$T$Pzl7k<0veC_L?5+XBx>0fIUy^2}pDqueH)kNQP8)nH1&_wHx^f_P zC-Gfb!2TJa;gCI1RwEVa@?)7fnv~cA7cAIVWF*989E7U91ILd;HJJ+L_Upq^(n#Cp z{mjHn_}W^{lW}jPhNjknci}URIcuwsYHNYB$`M8HoK7a3&pdx`eM7ik@Q=3>+~Ky+ z6OU(aM__vO<-9sbe`6s6DMwK0C?HEwT)?=(($0I8K*YHJJ%=Vy$x$J8dc zc4Urrjvi%ibE@}zXRuHe*BR zN(&VA(jN_>cYWm~_t3&gBCAX+wau%QT3WCDpq}d@0UAh&&P%7_h;ra79ZBzZxt2Ww z%(x0$aBWuv$V8v-@~|blp}NyA>qTQ_jx)(GK~Ce|X!{Rrm^mJv(bffzc0|T7rqmK| zhB4%1ZPgo}na?g2^uXUmCK6kqtv{|~wsE88ohOo32Fo#j;(oVwXV#>r*8U9B5}_cJ zEe)Ql)_VQ=FsXBTvkvVOmd<(7+%x0SzFw{}W32(?_8J<5?F$VyXtI`D-CiuGd{LLb z`j(`mef%OVKKl=!H4J$|*E@x{#|caAat{Ei@(ml;X83z0&SYQOXBAU73Iv32ZYW{V z;?^Iuc+ePqa;e6lg9}2xN5{k^EM9-smpQfb9O743SQj4kV4uBNqzw{7(lq(W-pbCw zl{{LuRHv~Ue3}K)+pj0Kz7FW6$XuA&2KVOdfZn&!RW4&VC(#4i0?K{Fl+Qr$@F@O9 zfAluwV(Uj_#VxD0e;qacGE|zWov^Vy$12(@PrHIT_`-2n`*CW!vB%1A&A&-;F^&=x zYnL(0Dn{C}tW>HGB9sKTmr&LK45`rTTX(wf$M zTUw8K33oBHB$ZD_0s^91&F^M??W&hTlZA#$E?u&PN9v6S4)DY@O(u_i)3dD-L*k{n zIuD94l;stUk>c^{Li3j+I;>qG9s}A=q3msTrdex-xtClYpJ1P?$ub_?@>(t=UZI9} zW#{O)Bvx$(RWEaAT?O$phO@W5cc0+OSPwaMoZ>Vd*0?2{-GqMVSb;X3`z6MY=)3>= z5LfngouABE!WWa+Qdl)6$|dz`wca-&S!XW ze3^KDtLmOf&ztuX)kpJNr=Uo0?xXK#6+SLhX-rgXMcJi`ZJ>6Xn9sPMiLa@_tOLs} znL|xkI$j8i)2p7PsumfNe8|&XWIo5o_>2La@Q|3T+LK~`iHXE&JY>y0Pah*$TOR#o zTG-#^5M$%R?TAri6Y&hY)&YuUQ}n%Uo%QtIP_CvLmmVQA4|oM$51;aA%s#6zYz*A@ zbMwL5D{NTX6u)RGi?qpEH?V0GPCb-My{~SPC(fTV1X3Pd?T)}KJJPe%QZ8TTVk|ZX zT0ekOx_agUnbUB33@u3FZ`_v=`rZGi+_hr@Mv(~-L$x4QMTdbPHhc)q@i=tf3oSs&2DY*xlRVr6F`nw z!ts1{xFS+}3XWTu{rbamG>l!p%BI8n$n>5hBoPibb2L1n+co=}G%g#TeR)AgqoAEB zjaqurB508(KCdD6!Zg;m*YAjfh_raGZ)O_?-#jMnzLg`XLf1n!nB$;H76i3Dh$~Aw|D9XZ|;|n@hjH zMqOKlTtc(f8#?z7VI-mju=;H)CmXMe7fVOBW#zhWZg$zhWm#6HxFU?oP#i(5^^0ot)~uk~KTs;-BgtLS_2s^2Ke2AHRXc zq~Kude11{bw%_V7-y5#y;}Cr9DP!(_sAv>C_jtd(b${Z5B#0;*b*zVsCaI224vaae zPPA>$e5;Kv$G>7#GrlFoXr<BI+hXGoULFW(8PgFFT06r+G4!WgwaJC-O-eK1 zW3>Q?WJB4DHun5el&^+vKOPx(yv|vf2;?kv;i;^P7@kxKz_$gc`)#5Z3(ykrC&x2V zC)-?|nCDJ_n5;3h%IMeplr)9$uP)xdkE0JeP+Tk%=(rzFSDh5jB}O!oF@e z@BSwLpjfhvhzV-Wjajn!alYD`HUXy^#DX_n7da~0B8(7Ari=N za$-DPcOcEIbxYPm(;$i1B)H)!gA70FUb(G5SteCoU-H2>t3RaH`W+7g11BU+BM;2m zc#{Mmv)tROlj6BE9l|k;VsB@QpKG#+1<9wF2|No)=sKV`YK>zjLZLUd ztf>8rz$%T3=vDNW;}^to_uIN|nf#brvSi8H@xj&6p5<-q9!nnUsZr(cRdq!U>g?K8 zUH`K8d)w|Q2UzI7~Xm00$7E8FCT`h)5n&%8hVB~~vm6tp4) zpM7ug*y!m~E)$niN3_?jRZZ&r~i=`CwK4L$EF{lvdk$1+pc8p>Bj7BqZi|lRjQ9k*%EJ={RP)RHV6cDz$FxulNtuH@VV@HG~Z^omqXF^q7%YsT(7v)blt%{gxxc&b$ zIoEi%&FG^f)t<+L9U5fu3Td#87}=zYa3w07{R|WhF?uNpvi+qej)&MGyp0S3r_`nI z&SfmWtLk6-bGixBySi4$I{;ut73A&bAe>A9w_JU5*$zFo`5OC)@FYOaW8`_=zJ@1s zQ%8(~z4I^UC*Q6cl!R6HMPePZs zT^eZ*aFG?qrFBXXP(ru!<8mweu4&%#?wTeYYc?WEO8XfR)PY?5S11&dl{WHi#C*cM z%ejvqLX=RxO)eHf_O!xOxOv1*mJ2=r17w|4cF-MqCCdb5Rr{z{4mhj{Xs}d`xlPq| znFZLrY54$?1*uxqjs>!V-(U?4eR)drtFjpVqpEIIMQ;|Z-mBVNBc{T3(1l}5 z+L-+4cS4u?E?8wLSz~>XyBXYr@lqKP`vJ_)Yq&6mSi#>Mj4?WpQ^^J9kfxFjW74>K z>H+lb-iJHWi}%rmKvQoh_Kq`DEsi+UX04R>9K%Cr*juc;r4q~p6$Q1==t}qQJ#Kh1 zShzy}4}0Go)b!f@8Bhd4u^>f}VntEFLKCH^*hN4I9fBymDosiN3s?{=C?G8qrGzFW z^sb1C5I}0^B2q$A2mu0Q&m-J--`~#e?9S|OXJ>Zy{&(+iA^Dc`obzetHVY8k*+y(R z`*iIa;{1U8{}S!9*KWfXu^m*$+ zn2>{-W53GPSa#{1K_1cp`nIYY2;{xFCJONV)A8pFpYcoRHyJU3P}CgP@q6D1 ztWIXBHk|(l4fj8KrVbuh!+_u+P~47r+yF0cC3y1rD5j?!l5-@2n$*=(~qVzKHP0a zhnd%5c#9M+)9|=SP=m2DcyVy~<@9wrck#ia{*+f>bzBC_OZV~6&+xY{4vM>sNRQ70 z$eO8CYBL!0yjXJif7uD4w>yI-)D~dYYy(y6OWWfhC>>9SO$Cd(B1q6vrk~+}zXeE+ zt_b<{@n=QdnfMfV>~M#L3G{(cA6nt^FH&&^I8#2a^L7B5@YOimK-DD>+?Oo(8&3Sw zr9$uwAh~NZru<7E%ea_K#@4z2>;uP{)Hhwxm8yoEsn>vZGal zOU=}kD%Y?{W!H}+<==42N53A(a6Xx>e;b8V>gU;y5nrOL6sXDT=Uji*Mzq4oWd;~? z4x~&{WLgb*+WcXWj~>k4OU)h*sXd>KHV8NMD7abL;YA9`^E2aCDcW8@=__(3Ey@lYzdGH#5vNOMqi|DrhY3*ns$aO484eAn`lE+@R_GCqGn3>zqvt#83 z>Ji^|V?>ja7OQ}S9+k9+e=e_N5pie|F8eJdZBREc7N*Gk`0j)#A6-*v2baX6kBPA2 zdVF(4;PIKNBE?$;aKhFVUQXJpv+}6U#{O&h?8eRi+6wFYTy$k49R~AY_5To>TmDoS zjAa&*AVtD|0UM{h!XzyKgzD1$+%3( zBe3Q#MI-henmkd7)P;`D77o5~l)K-CY&$S54=-RExJ7e;JCtulzAGy13o^P@ z#Tx<&^BRz91NjNYGutLPy^c7guSiz`)Q2u0><2-nrCs9Tg5&_JY9wc^(@A#j5w8!p z7rZXbiXl04eJsV%?T{iN6GpGk*Ab?PAjG6iA5y)?1XHzMcxm^A$F*crjdJG_12gT( zFb7b2E#^DM)>GTon_X&a4js(BX!gWlkUPx|KIh}5R9Ykm5cA0z1RR*VBtSEcU?s@H za4#kstw4N0QjRLsK(+56wHn(XIuRb%D%~aIfMXZFQw~GH^{q#4Iufp_KxfL)Xy(B;B?G}mz08FA~;2n(&WIW zc@Wq4vNl4M(T6cF6?k$84n79J+v_e+#Jj?nS5+VZ6Kx)xbQCD=UGo83_M84aaCOpV zKLH6H;$q#?xTJ8a60kn2g4=_+RXB2$~WH0-MZ znjSjyjok-$kHeAMpmx(RLt^)8-vj+K& zu_;PQfJ4KE`)&}-{}cw!wd`Wp95xw$wU7H^>H?E~;Yp&2ODU&JcjCJ?bG zO|~I{4fw#`r+8@h_JBWSA9)dGZ&u#XEWonp?h`#_J<$A&wrS8LGca9-O3tW=4p#ih@Hp z8e;08XRs9dI}N6}d^#Iz?>Ku9`gx~T16k(HwDDq)Is_0ii<~D0?4|T_MC<5xCh!~9 zR9Y~L&&{uYZ&)yu@2k(^`hG-)`Vlb0Z!QCH2kg_I+<`YNL?f@u)!^x5drt;NpXJuH z^)L96M<(B#U4{~G+sPL1IPITLvt&#&e)nZ3MLoW^2;D}J_IB%(707t%KBuLsG`9P! z4w5=ViDSie)(f@vkI0QK!p)ZjLyen3N#Kom?d>B+VNa-a>&{cnB^Digv=iFCL`RZG zn!;gWDXrPtZYA_q#apG-&|H4XSLY?u1Z8p&h5-C-s%C$JrZ5xwZQJh7_FStE9>*O` z%A)3(Exct*f7BgDK}oG^I+T4iU=uc2MBY)f{@EOmbQ@C1Wj@%W86eO0`wZu|+r5?y zmuh-@ioQXa9QKL>B?o0Jg(QuwxEjjPWS|Y5jR1ZBZax}bn?opS{!GBBo9ZmT!6n-@ z2C!))&4uJO&BvE%7tkXtK@f9T(iLV{7&K)ZlYNtY^HylvPbRaqBb(uKvd~M~J5}*` z?_nTj&PR4k^w^wQ2QWJJNOh~vu{(Hg*7NwK5#Fec09>kXu%*>2MZnf%IFtWY{oWHh ze9x+K**8BAtB$+ImIFa@McI-B*`&eSr*Kk-JAwr&B=(mY6n%;wn)0>z-cU3Vtet&D zu_i*Q?43HHf@Zj6{YkYIcP!TKAmotRg4fAyMNEFJsOl;$y*g07$UZaS);jm6+IAOc zP;a3!7H7r^Y7R3Jr7#$ zg>%tfe=dOcE06{othzzEw$JWZF*u*@RVP;&P7I!KpVBLd z5M&cBiuN~0?#ozza=C}KFPZ!@f z2pXxZbx=TxYHmDJG}ke5d2Mw2%bxVjXlE2)uUBkfzLBwN{8?-pJTHec8qv2gEK9&9 z>9%K?*Vt1_jqEz|>@r~%q+H4f%E=jSWD6-PakmjXKCi~;DwPo2@}D)|xd z&UIn-Ftt59alMd&bVUaA%=8Wzuh3#OR~W1eVCd<(`2Z}+d5DBCa_i!?dF=!Yh*1YO z-hYH@SbXP}4sRNtnDXLuFjq|V)2%!l`S8;Ys98khlCa0n4nZ(jp)*D8X3}$m>ohn{ z#-N|b1103bIl@X8U+2$h*{j?H@(%_8ziGfGrSEhD$zA$Aox4^L=QUa|80}OsG8gxF z$v;TIuFF%=76`E{DRZ^F`|L9yRSC5Hk(g~%zWs7ZoR}KHCat5}43btd*`|I`MqbyX z-6n05b@`;D4Pq_T5^n%O?cjiStrl|27iEAJBpYgL3v|~5<2n<+@-6RG0`^d328O<$ z;??|+9t5ySE)3XhYL)^Z4!0i9MgXgkC|WTbap?NJG$=4YrwlIpGoFS0cV1HNB6CM)PR8bF@|F?m+HqOfD= zgU`{54{1KX{B%W9u8|^B#;<&{h$HVxZ{bHTGN_A8^+O*N!Xaq~aY*AUb@=T$w*L{` zy!@m=PwSzw>Wjt_D;t?2xS~?#kP%-=gu!4CGySh0h4y!_3&02o#Dlk+h5jC1KEHO~ z__vPQ1PpzHMsRZemH)!QF!k?0Rd<|?3|De6c)7lw|7BTm_tD#>4~4J|`$j5s1tDnN zD`B1ojoC=1n};TxihFDlfB}cVTZ35YlDS_IGk^)4(;EaV8j*{LN)mX6hg)#BH4;>@ zud0CpMjhXKqz#7FW^>NsLvG<5oi)lg)>E}IZ31}Tr)0pU$@HSH`m&|%{-v|2QGkhl za^fECaX5|hsIyPj>H4}w%CdD!?L@S)*H8c?*Ea~%P!+vqmx}D8T8)g_(#0cdBul2R z7J&*h4XBR0L#c^H zWbjvhKOEu%=E=@^3d9EhKaz>sQAl$9;cWr+3z4SnQ0X^Mw=dt+-m=p`L582I1B$$3 zy zSP!lPmBLYs#__mLNLlpmwMe^E;RM)^1b%+nqRZT`DlNPzFHq1sXY|-@{dZrR590mT z&4AEyY_K*0gmnCAr>i|BTtW`s9bCw{RnmpGaOzluCwsclP#ZR@UWyVYr7gO?Uhn=} zFG6POs385fSjc{-U*%4qVzvra5_SNLEt@KC=+K@$m=j#Rp@=*L{ku~|)`sUOAPE%7 z*=X;(A5E^yIhZSmHU-U>E&SJ+`qLZs47tUT)yYgCu{sNpwi&n=UwDhid_Wk$l#;2g zH1`qM5T5=5{VQcPV9+)It9BaDqvLd>jfhb80ej4N#c{02DHZm|$g{TgC%hI!41%cS|G9ieik9Pm{Mu1F!jb@AzJir_wbIcFZ z&|-YYW**>Z{j%nteAwOvQQshqWzi7_7x8PLgjWbCfT}&p?)`*brzpk=V8FwyLly*l z3$TVh+?5;n{>L)D{YunytNn7`5w=%j7BvAHZjvhcIG7IFJP z>3$yN93X#mYZYjtOWjP{;7}#}8qblrP`fX2Y_fKLs#^wsCmy`ilA0c_pDyXHChEnl z9jE$upE7mvk$5NwgBxVU<*#tyWv%gXpFcE`E_*jL|<(|4(x>kw%XYDl@@Bx#R?~aj*q_wc|jwoR)cAQ zl@j|%drj1>_MFJh>$y`aJls4+__Dx>vp1_Tav(Vrw_7)t?EeZrJ*GugdADLw;U%x7 z*?u=S{O_taqXnVHB&7{A?{$m?eoBrA85R6lpZ*o;Ia~q4Wsi1)N>)fm@q-;mkh}Lr z@dz=c(<1gcPU6t!O7@QcZ2?FG1c}s0R zam0+~qiwSmtLQdHX-4Jn-etcOFmB}__8v+Wyw{xJyq`vH!<$g}ya2H5yCH&{8hAEm z(+-C_B_jAnebTTAu5PYV+dKQ4cAD-l4_Rs)=P+F<+I;Jgh41ltEO}aKnjN=)74Pnb z7DPwXvB-GQVANk6mLl>WwX2f5)MaDIs$!(IDV?53-@{o9`u#yl&}C2!(wbY$K0ews z-UvAm^b>g3*jM!%MV61rL3Z%faDha1vH*0J2)GV(E3UebyH*Z8BA}Z-X&`prh5rM5{iSZ^|{{+aCGB-UXoLkn=><-4^xG zqJdI(u;d$93=9C&5HN@TB`I(DW$FKdgMy*cOwP9OHm?blNF?hf0LR+Rl4KZ8~z=dm#8YMe=5SQkIutEj1d2jr;aG!sh9=v{VUN1k14bd;25d+(3u65oG+cgd}Vw*2`; zts3sFdKCX;4&f(hdz}ql7l3QZ4OTu?uK&oU!l*D*5sdi&=$t8?5UxiSEcmG<~#!^o~%ovz4Qlfpm! z*-a^@PDsC?Wjkkh#zAk=0AM~KDY7=mv(2rIkkt>04Al*2vFmbzDE8|mgWWIM5*mYmu{B+{3;z5h6&-iWE0mMrIEX8> zJ?1&oAZ9z4wFe=KGyQJVgJ#56+V53ZpnylDsI=^U8Z90yQ}6>+eRj|ar29Q)^`za` z=HjM#*McD#HZA1L6yz5U?2}%sIDv2Yb>`UKH!TcSBmNTHvv4C#%J;0J}k_A6E z+rZ{CD5YJBuD`x*^&BI$rb4HW3~@Ce*CWB|%9-&l!NEd^v$sSV;REUUmEe%J0QF2J zkk|#Sj|1oT+88vQbBA}MCFbNRh(CDUR4@y^^?>Rd~L-mZ?^fkF9JPiBc*$8-KQ?e7Y18$@}~ zB(5cVlby=8YRS1SWKeR%9JBDJH_>kTb88pX5HJmTBp^%_9VhbwyR8b&mT^k`u%siO z+#eSpnFSxr03cJNeM~sb{*Fb!7eAr7OBGMvJAN3}j_K0%x;?0AB|}Oa@>wL~R&!*K7X9i3Ox;^PS7hg%C2F1xl*? zCxdV+KedH~8hd>4OM?k_Z6_VVgeWyqWwy_hnm){ox7)?{5G>`ilE;M+`OX!Jw3TOr z1kB0ngg4)aDsmlnoVeaNU7%h3vH9-c`fc^(Zo#8pk!aSu*1+7VF=73QL9kVf`t8_d zXd-m^x=VF#t&wO?j@31I#&GwPZunKrAIvtZ;Tu>2L_;4EXbnmE_u!s>XBTFG-)R)k zY*#Fqa{;2Mga-Q;2gxGwN#nCho>&_y zJ>lyx2qYkduWB=Carv9fx^cjr;p96g(68k8F|L%%85@7C%u@5w=zdX=4)?K+X*Y0;m{J_-DT9KL?Oxu8Ynd-e@s{+P#fywmnbY)7N4}JFO8OehEggLwr zkDVBco@$R^t7QJgU$gU4>=|TrOfnf)636pnkiZ1)x0ZqLYiNd7hNyeYJvZ;l-&5sg)7r*^7k0E$@Sx)xty_{U+@}WeriYK_ zN7vwL!-nP$h~)%UK|_<~!(5V7)nNo6@CXg*DLiQzu-!XdYN{Z&YLZY!2}5hfy3Mef z+<&iy@6@gzU~Ab<2N<#gbelF15Ese|acgS3k!7WImuzg)**+i}@?hUe8`6FN-1m^H z9l5ljh~^;kh&4X{eX?B6DJ*3AUZ77q$;;pA^)33S%~oPl&FvO-T)n#-e>73u>AGXY zj8NK*R)*<`b7KcK_XB|8+Di0oVLpD4O4hjt&Z-U)_569>9AX_Q{hYA-N0P#Fd!=&c zQPKqtFqe4C#kRxpJS63J{jMfg(6_{yru8Oy^3aZ^GVr%%rF%>P)SU^s5DL~<+zaif zMbiO@IZVN@zhTbv)E$i~M}UswAEfu0j{JOP+L;(EbyO+phdmCLe7kpP6_e>8i(3k_ zQ@yfX*&ZtC#U>l-$ET~+e-44Kp%sSz00BC1QXOjRl8L_{14C@c#lCdz*ZYOh<&V14 z2z1BDdI-9eLh%8Jj&BF-#4s#@_ysV&cJLaHL7i5goJT??e%D2q7iXZ!rl5dRIDJVe zPqioX*}iW$F{yICho`azWWHA5gUEzv=?>su1>Z?LOtg0YzYEewJhG}d3~S)rz|StC zQ!<*2twh!Er`d>R%Eo!y(@Qdkk^0yafBE_h_|3tE=PyM_8kx*>MC9dW#`olilp5HO?DyFWh-*1F zcO59zvxZ|X8}uWEJjWhybwLMX#NiwMMi4y!+qKWgZAEL+!DP4YFLkXy0JOx7*4)#C z*t7gwq>MYUXKY?o{#Duz!TGhCbsT;gDVRnmzfI7C6!GOKu zMlyi@dz*}PByG-`?S%$hHu!hAGL(cU=JeH|+D}4^I{nNZQuV6wmm2{oeZNF5-blUS zLJVt$?dPc6tNe;36uBh?ps@^*1ftx?Dnw(jM1IZ#6OYb;)aD&*sPzQTYQ-0}=v5>!aERKZ!N^Y=|4N2$Zb1h<&_QX20oG#^SUw z7kgjh)Y4Fm993}rH4o?F`ckZTFATBXGK=rTAq@4}FW z44e;Y$~Mfr9>gW35V~x1SJ&^hjJvR=|A)v_&0aK}r$P6Uesd9}+^ud{xlJ3t?A+-q z3DF-#-FXhU$l!JV5(8^eqXFn8*|s;*?h8STm&<6|Z{;@Ae{S#O8C6UgA0Kkq=RqY9 z>Hqy7^uRpD-SBodL72$+-aXf5p)(s~Iu|Y1bWHClzW=EhnDv?3nTOGLr+M*A^f&FVO@AAhIHB|lf)D%<)5s=P-M`6W0P*<|;2|j;qY`kMx;?H%(iJ$T z*A08F$b5Vqw;X24Stq2>#8wL{sS3HCf^)|Tg%Z$Uv84xX#0(CX3-ullVPOrnWyd^- zCbI$>*gmobw&W20iSJQSeGE=h;Uo{`tHs)1M6dLLQP?0X-Jq z=>z0g0fXmuLl%cJ#Bw|G!)17i#sU(q0l!2q0YwW^*P#GW&=#C*-38&osYuu62{YOl zJlsK9SFnwVm)}6Pa1CF7CQ#e*li}7qn=gwUg-uE(P`k2W2&pRSI__rcmL`{Qd7*x> zbFe~q+`?z;%%3UEkQBXS=SXVf4gIq=IX;PO9#v0hhn-M?1Y7%NJbH4xl%>1VPcoWijhOf!3cOk!BKG zIX;**km07xxb!&X;*v#KZ^JHJp65itcWx0STdYk}E+_ASB45QeIR1x~*c(9!bmhRy6C378mz|x_@cWocuSGry_~b#!U_Z(mLqzF!bP7nWjbPEDYNe-2dD} z6qu;*YZNu%tG~kuGKyH~)Io%OSq~1esjPHj^e#p8Q-be7r$$T+lr3YT^e zdf79;J`tF?$>CWxMXq~$eum}JsAh=l<@YM(kA3rHS!H;q2h7#~5&%oCLqH5>~d|U-a6sD1V=OLf_eGz25KiKzdZsQz0svMnkXe>Jbd`pGNJ3ZP^P%A z5o{Yo*z1AMdl&KkJ(z4=8_x8}i0;5GWwTd@ zM;?|9f35+UVM_#ul}=dW!^JCAXpa3g|Kg8PGD10<)aXO3_hb zS3f0SAjqwl{+9y?IiMq{v(+%)u}=l-d^NX6V^WCVJp6;OQL|D)dYNNTxwAu&)ie|K zf3zm9VDy%)8x+9Nh!7C^EC|1TEOdH>r8?@_?L&I!Jti$)g?3W!)JMk2?& zL-BZDit z_34?im&Ga(pWO`^E1K-xd_ZEH45u*Ky2@xA!%dEvgRYuL?3Y0xuz8cTm>zjTpqm){9!u8vEo1VhQit{ELs2xR# zwNZ*k-(Sw?#vAcYSNB&_YGn>k@J0%{0Uh>PAZFBsc9uu_i|F!~b(DlFDGw>I6~=e^ z*oq)I#tu+0#A3I;ya8cM$bImUQE+BrZhLD#@qzXkXH&3|kxJ%&9oL>O{4+B)@>Y0H0wTAxGyz$&NIt^5H28&_N3xku?;O-NqDg!)r3K82)SXj*(c zw*Wd8x0aO{Akn$_|8&K%7}~PO(WwObxtI1_&kfm_J)BF*%TKj$7)MO-3#3my!@oFn zKoJ`Is}9`H56&?S=F1Y8E12LTK^EsDan|1uirR$|9noO}85688c^3(dOo;~OzqB7e zLPrih>t_uHJ;reW15z3x@wf;Exms|ss%yC3WiTeb^Km22$sw2U(ETXQNHA+DZSWial%@{ayPQsAEWT*s!BqDK09hajruEP zdgHD5xVx!36+2&eDB4t>^iW!nr*OlUPAor&75A>?FWe!JvyG>PCUv&9+G| z_=NNs&r4SN0Zjfzut6%ijA$YI6EZmf)^#KTL@ub8MJbAq?3e}P39C7PZX+5umU+qbI8b2Q%li+YI@5=n4sqQMvsZBBTM2%%|RC zXdoFzl2v*A`WB`wD13xak^x%5_?KdD)OLE5|XEY!A12y_$ zc6?HKG`@1G=(O&2oO}0vCs*oLT?1B#qeeF_`iQQ}Zk3SG3^b42v`T znNAweA3q|uAmP|}Apo!zMg(Fw7(F%*xDRIi1Ff7-mwfV|Pw2lHy`K!MSaErB>VV^{ zE>I%%%);)%t}%#okL+(*nPAKP4!~Dxgkj%pvD1m%4t5e&dW!fwBpA8m!Sxb;N0@@c zKNYf-k$X>oS_BJlP-})Ucfj%oYO66Y~)_g||mj9q<{a>l*r@{zwA(>Fi(`^WP!Z`BFifOrpx$3U{3DUdXIXt{Z^92aKej+~yYa`utP z27Q;nBDw>9pXek1BS#~*Vm1bCQPP1#M-;^b`5L(ZiOVF*t;K9S=^WVv309t4;pVMIzT7Yb> z*tq_8@UZB^tI|nEBxY-&y4ZQ%vj<+alztI0VDSZM{POQnaAKmW3DBrH5FYI8({Noc zo_^y}FWm9m+~>Ad-xOBI$PB@`z<{1}Ux)G4_$_wap>TMw+aiiiq{423HU!5Y(W#Ng z@H==o?WnK?<+vs%Mwms~_mL~S=BS2uxpvrvz`8|*`~ZRV4KXVy6BPw$NL1^2VN{a1 zhw#0rBZs3lHoF+&W#HahwHkY-p(+X$ZZMq%!_my4sm@5~#dv@|Y<#}mfWQ(PKCcQeoIPv3EqlK^j$J&ZSb@b2ltz%hox z7@jUcn$h3KaVHuQ2N&@e%u9>j?S`4#JLfOpqe$dAcrYc9qZf2fDY&vyWb;)Vr`th> zG^s->4Yom{WZ%3AR!lF5kA~&HptGY$ou}`~**;4Vc-fWPV@}?;1?am4bX!F*pNzq~ z;P#pf-k2=tijn?)?8Q0ZKK@nZS0iJsw)@^h=+`*cHL&ZfVjqt#m9{#P2pNrSphzF$ z{zHmH>j#kiiUII>S7-w8aJpbmdpQ{eAAJ|CMr-HfP5vg{H z`uqhAUG$(k?y3MY$e58}5a9f}Ko#9s-GRX@2yWAxYSzIO+>3h9Iv2Pz;~^WJ^UpcQ4Y7=xhZ_U)B@Cg$+FeL1WP_F<2%5$a}d_56BPh zw$}HTHtO+M!t3|RkJY-01*2k{gS+tAkcFZQh^gdz<_i>MhS49B)?({|dde>e0Vthp z^?NS9a;!m_2X_Z8>+YfEsHWi(7TIcbzcGsvg~iRFN@+<(>*F<0X34!gx9d`Wcq>NbCpcjpPflLXE>%h1 z&<-ZNR6umnEA?=%77l(>c;9-H9|D2Gk$hP4M9b>?BQMbQ%?>#CSaDCC>nBcAO_pxa zI-A~jwnKK}t7;5}quXJ2@fCOO;*xmSjEsT?8TP0i-?t%sCsKnUOayz(#%Bsxz@i!T z(pvwszfdq1>FJ-Q`$u6>n*GjG%GV|1oiUlFk#7^5^A-?qUZqm*yG;J!lehPd)-12lhZE;4(R*ekA7-xG!oQDJ*`()=y^q|9>o;8c+!=eCoj!1peVj8#y_q;HKAdq1Yr9bxZv4)k7G}qZ0~AO3;_Sx~?Y zbaj>bwj`cdZ)QJTin!%_Au!`(SKbnL&inBjJKD)_k+xyxfMJAgIDwMuASa`MCvBVX zdW`SP{)%d8YsTB)=#h%lhA4&McBuM!uCPF;48XBGyk)_LpPI8^$m&1Kmg?i)E9HHa0{Oh z$j4C|ywgGV?+Ux}3NNgcajO8Rwu@?sODxwfi6)V!v|hV{oQneQ&epI*UrhzfMm^rT ziX8Q;0*2uE&6B!Xm3<}1dJZ}e1LzYv&u_0)tcS9hr_5zv7K?x6A;-W!&WH}G>ltLI zKnt(p~m0MsN8X?%8JIX)Sz0tZCd*ox&ZRLus@%-=}z$QrN?D8|aj$ z!uz1MDhIxD2R&_9sVaMu!t>UOBJQ7O7>`ks+<5WSmDXo^V@DTmsx7Ieu-^2mQV=Nn z`SMaHUa7q0E)!1hE^nkdwBKwvMp!81)jjaGrQV9UeJnA@p>Zvit)4T51@lFH@~3yh zfyn#&F)fjD;fEX{lF!~b>54tzFzqP7$gMcOWt(^#T9RmJmIdni`Esu{g+d}ePa(fl z#!V42Tk`^!yc-I&isMxb^01Tr^oN+8JQp~9r@FOI)2WxfA}|+ZIrg8k%ryXPjX)&D z44hF#qU}D9FN8`)q4=Wl+~~v^!NM5}ri`;*whXoArT81H86RHIxjch-)Q>~q5r0X7 zf6Nyi^%I)}WYd()ttVm+ovcYIc}VEmig|uw^~AX!QwRpV{%IsI**X)>tHRi?+wk!h z!eBMKfUfpyb4uv2i4UY;jw^|5nHAL&)92mB=p$2lYQqkHkeC9MRN zeb+Iv^JbZR4}G3fRy}mc645YPgZc7`(5XHajn&lJA6L+AO(Y@rS-k%R_ig~KWMkRa;9Vnu7| zqjL=MgUc;kqH{Za*Gv)``MH<_J9Bp!%>RgExbPWT4D@(SskMH%ihD3!bvk?cg{oUri$lSZnM*G?{X{nfg-NYEkazow-VpQ`1 z{pM0QnoSGK|*Acn7cYR0c`Hnr(ot!?{|OajZ)!oGuDoRQmdd+>?tp)#Db8 zAxXlv+?VC<1t%6v-)Wy#nFiofZgi2Edml9aMvF2H2eN(@XeOm@)Q>;rQbvqky!C=| zbgo`UF-&H&^qmLEGZrW~GY{+7pRDc-O<32hSNd+V_;<=I;vVbAYP3fC>2@G~yWL7r ze$qpjt9H4Z_}q|3aK69=qq-ODWz79T(u}WWSi519cDm+FU$(gp?3c@AWJ$wtjj)0e z7;Dmf+CV4LTe=TXjcVb2J1IjB*lp|f2bJ?QBm?p*&^t!#r<#QuiTZxdpRcRhdx>!P9H zxqCKIZ<|T}aw!V>)P~Z}dA}!fgsfX*8&o~EV>TK)A9Mz}R#Cm~G`)v>AG#1OweCcB z7UyqoHKoF1w9g!|PKKQ5^{p3AS4n&OByMaMIXV{{Ztb_}GPY6wikQ~@K|YP^L6$v9 zJZaw3K{FBhWt20}$1Nrf(sAL^tx{4;0+$)}y)D}@J6n}~gfP9^XHE2Qx1IMI1+$Fw z>ha7CkLFQsTn?AsKk28+KE3S}3ipvqj=?=FZ5v80y=SE+pLCIBJ@qO!^pq9$yiHQJ zzdzaup_zN~WLAg}1YDZLRrGRNMcvX-i2}_nJ-_sum2S8QYnrt!ZTtjuw<*#AL64&_ zy_FQP;msW7rsJ?>3vw9S}QxR_|eSiixh6G%4e| z_ZdbxD8UP=VPoe2_a_-G{p-^x;$|0S5?k+mKBv-5Eh{BBHkues^YZK27IiJ(@p_&} zz!mBbj0mRkS&+KcJ|IWA-5U0V^=^_^XiqNdL5Jp_6GHKpTce&fmHq^29CwcEAE>K` zUr_9lXu*58Y6XZvh4n^zLERdU%ojG7YA_VB5>n8UV6_8gl*2FUl*GxDU4d!Xh{4^J z*e^x$yy}RDv4*YX9saF&4#|hrcNXODk3uI}xA)vl+hF06Ylwg4;+Ps;tKirYDoQhM z(U33-@UWPdIv6Kzv0i2d_ffhp=9De=2-Vk+C5QEd ze%R(754jO^W1Zgv>LDY(`V0>2ljKvjuy-l_o&S}b`6pJS=ZzZX>)mELVh`zF&z?VN z;{#;7fZX-;o}QszqkHre;>{Mz?SZA5k7CSZob^i*7fZkE{ z$+9ND?bRKVmfndybHY0+`d61GCJi6{|HYH>B-npx6L#mdro19jtfgK91%yrC07`)DYfRxw6ETdoKx(i;YpSpWeo_yPUYH_ zBg_oP_;N)s1vBp7!FpNk4yZI6JM9&~Mbf$IU{0#3znjaQihpKSdoqS6QlTIjQwYQT zxHqk!X|)eg{`;ReQN6K9t%d~r6ML$ta{i)UkA?MCq0cF&)Oh&jXM{YAZJix6Fh6XA zyN~Q-$5aL=yFO7m7pQ|40jp3|6M`zX{Ft}49J2_K!Ub>4O{V#n+!pbG zE4J3FgF>*L8+iXDRqBZ;$!tEyk`r{-`Hz-Ff>wPzMq%D{I84neWqsL1*741D~5i1%>LVHNmHhg?6*E z7kL<3{HBG?2eYX^`1z2HVMe3O|A`~W96ac;C$zM9nClhyAG`s#$Yq<^@KWDoeG>qM z?_wy*=~9OCI}h+MlQM$>G$TP$7=EBT0*_@mmi7-DMA->&iyQK6dP80!PWAQhZXfKoQRI$4}@ zZ$D-de?+mX#d|8+bNm+Le2?%5jBXbAp3+bY!|Q7>qQh+bl^=E9$Gh^!34aamT>#JI zc-RHHjmTT^n`*Wx7tNy%Vk7o-mBE;nXG|)ue0wzRv^Dw;xH4mE4T zx<}XGKf@}=L8n^ZhRJ9CbGu+%3(m3bs#QV(lVh|5UR-fG?4 zS2j6Yu6NR9dX)`!6yA8I?7M}X*rzlfp1z9u_;CG^eaYcFLY;11NpkpPE%D%;k3^CtFaZ5Mi`ivXObgi-i zAhfhottqijC)`MnXV-rDB?N^!(awC@fHQ`r?nT-znZ%kO)((BG85%0oMjM+jJx5Y# zOLA-TXdUM({2u+u(ESY>R%~4rIMy?~mW2ZTt^VzO&W!tWE;fX>i%H@Q(t2AYV&K$U7acF+mXsEIwqxk# zJE9*+N~4H#EYWLIV06ZNo#3p1Utct7nX8}uZ0hICkU7;f;h@zPfAJ>UM+(!5Fa`Lo z9*A_h4y!xP7DIAG`O>8=S5gwQIlY_wKa?3CvP+!TJbG$N`5eFys{53;TuzgpHv8zq z(#Zb`^sz-3&9GZjc3wUzgp89yZPaz8#JGp+1^EP6e{D#>cJVF-s2yUmU9y+}Xr{k3 zY&gdF!#ff;Hg~ue_u*}GzusM6@vrxWFGYy2%aPdraaFlrm-a5D@OYJYW9RxbZS@hq zB zkL9G^ zjGLEXI2OPVhq8YS;BYiLl!_0Yxh4Knb5Hi{VwA`&V!u$-p=PS3Pal1&^`4|n2^6zttSTOt{y=%kkWWVW@ zgt`2=#@?f#nJj(->zOQ);J;rTR1QLsHaquf(%ukDwp^sj+@lK?ri~mGYD%$XJnc{SvHNC=4bPwp`H1CeAHIjI<{&zGfZIE)lyD&Z;sy z#aI&<)}VBrXAV|0wW{47XNK4?-}1brB<}Oo4mhQbtya)(EM6_7;LMRa0eYhiTH*(! zXU5jtx7rAsO)5N6yUz*s6~R;Iy+phtsrx>LE3`R7uUcHmt3|;@QzbPxI(&;!HMTVDtYVDI zPAvw1*~FjfOBK~>;j`KdY2GDEeE5NL&CQ%8v=c@M(8n6t7z?vez`x}(c5I6V`!26F z8?S)Wh2xjl?`x#c*;wPa3y~j1v=}yQwVaQeb*_~hqxoHI{!R>Q34M!lA+XQqR?$HZ zS#;lJy{)iU37`qISxnk7%O z*IQqYeIZSuuV$vTuD#+{5EPX_~Ee@961Oft?Kxe28nVrEo_bT+f}`%yn%?wy_ha ztL={%lG!mM!>JlOO7SmyCaPq2@M1)ZS3gT=8mfgT$NYN&LUhY5iINn;9R+Po&y%%s zQ%E#duEP);vyr~7C3aj!q#RDg?eZ+!7WCu1zfl~9Kc<|G^w?$c*kO}m3DbKvBkP@ac({fJbq&T)_nue^EeFRjZj(wh zje0uGK0xff+8hF!!rP+&!R8DSe3TsHuJz^kAEMq`zh@@6ZhprH&UYf7qla6$u4Tl# zuayozTj9%!@vE(H;@&6S;(eL7&iYv_{#lQ4kB>fY8)uBn^=rU*u#UZ3xS(0m-G1l@ zw8#&aN~`7ae8*c=hbhV>K#kZF;##&UTp&KzEqMvu*7(_hA^CGTga!^lzwZidFXy@Pe z8QF@(H;Xp=!&>^q+cd*cHySJ~U8q(b=}Qgq2S1;y*kw{iBR>482IDMAs%c(p4}P-u zq@4QQndH(lJ2AWEo(its8Oe-$Cu~|;_Re3l23#y%^_JKpDRGzk?p30PhIvJel~)kI zqv?@^byLv|KnE%b4*1iMpBmU+-RIiMVlKdm6NCRh%taMK4dUbZZKlz8vSjv1~3;KYa`=>E|i!8ZhH`i$d`e?GX#+Gw=K5;@$>!#TkjAue&e0khw50^irL0 z*>viOeMgbESM*FIuk}`IJYAtB-{W9INP+E3)2!#eRy4f@XKpp`sLoi9$;y4Q!XE7X zgk{vNWtL&FgB2Y)R4veeR7(xE6B$O8qNxlKH=WyBu@u^b^KDDMcfRhgX^%2zJm1~- z^ZOPz4X2>$&EdNzRkDk0{Nfo5IV?@bspYQf<>1V`To0%-oZGNRRJ*spFHYsI>1{9b zTN3`nxz)3k*&~VTAWrK|s7CnY1s5>(M5c>4Gxwx6tdGsm$s18_LA{Z0yXZfbv-gcy z@b)pckpo10+Ck5Nu?@}IfNZXrxc2E7_MIi`kJSr1#8)Y;^ESLWldvoLnMC66fj{53 z=Yv<`C5V*&k{%XE4}UBq&>n44cqOhV(sF~wJjF~|P z8!oEqDD)g*9i@p4!nR?k9!IDW6+d_JUV={P7$hQG46kEYoL3velJo#{MY(sg_I8jV zUoO!1=>UxlQ{xRa>8IQy_*$A-3zTHbkf#82J-wWprsqnOGsx?;T+x_$K6g&HUTot<35>!xkc^T-|T`T>f6bu!Lrk9&OYV?{i+3cI7+K}NVD zY*lhqct^+7>##B~rd*Qn6WsvE3uyGm!R zRS>iKq*TTjbXV_mzFC1iTMa&afYP$fMkeYGDG~j#6TOOmSoDywkudtJa#9kKy3mbf z;1#Z#^a?Ri`);IOjB+i2sMhpa3B=x-6?KzK`0RB?F`_vFZqI@yDgr}fb~q^}WnMgQ zTgYjSQyC{m&H>oT3nfH;BkC5tV25cDb=ceB2x2#O9Y{oj_M#ez=y0gN)m({EyTI2i zflo@Yod0rkm8MaWx{e0jQuNA_=+Ij`x# zyGRqn{=Ly6P8Pzca*Dmc$!2~jsgJez151r6B7GfdEeM6lJC$<-rR6zO5m-O;nT6n` zi^k~l!JhgiCiw~{_av%jp->wG_W+HrdFE6m5+VeeO8YiF>RXw3~t z6d(Vh#CVbOBRQHzrf^r8!Y^7Rv#m<)iSHRQIY)&KJI>r7@^JcZpo9=1uU(uE`6@!y zmRdKrFL_F;6izGvtnBm*91Hy(TWWO53oHVq8#meFXF2liOX`$o3&ObY95GMHO`ZK( zv)U$&UVd?$Lep6&LW^waz=#J{BU7TmHL`0 zbBIJRY!Jxx0q3lWK85dU*vpp-(rOHQSN_7hG^9P23GTi3pQ9A`gB4%z-wGu>`ubgM zw_6S^q#%b%+Ro?(YfNfXB$y9;z>vA)f#sxZ&mBEU`UlR&borblVTeF9Dchi90wGTC z*veX;vdbh~aw4bLv?3NZiV+8)Z}Og13ULK>J&JQ>vPY_pLWOD(Yzp^GA-v{dC% zFsKddLyuGIjxr;X%KJ zYY%t-w(fo}{BTy%2@q^-dAt`tz;TyuG@hM2=Bvd^mbb%yYOU}=mYHv~;QYc$h6$IG z^;i>I@x(o0)abUr@)E23Cj)DNrO?4$p}Ez@FF}-5Yx%+yEa4ms@`A=QxXE`T8CrWFZ@^MVTkPB7=O!Xa` ztxg!0;KH4@p6zQ?I#XwTVhX=7Z`x7z8HAgJnqSJC-Rhh|XmHK`*|lo>+dXjDzcR(f z#Q{Pvj|2CS*?3jzqj|DGDRM5kr6<)}_TyN;8c9Y9TBsbbYB9_Op$Ezh0Q+OmW9S#4 z*^8pVQ+?#V3uiU7i1*fe2mpuYKJ@(cZ#;2a+Tv00g=ZHeBpz4}i{4Aq%ZL0PSqJ-) zOX9;^b()6EuSe>Z;usQvTEgpW2K`gd5i%9Z>Pg=dP0H z&W#@yFkhHr*u!DC<&5u58`d9v^kz!6>sxYI{J{socT*;C=AekHXEmd+2W}ep9N^Rq zn2)I0l^Ei|0=n);e@?KvQ_V+}BH}EE;5bJj{`u)4ShA~=6?_5Nj7aG|f=+N;hkTjr zjs5dA;aUgY4W-rHlWW!~lKrxJUSgZ~=*S36MN*;LL6nDh98L(O;;)E!WWE)i=Scwi z9q>^JBBx2}JlCM8xubY8&lN?oCc#m=9 z;xnqCCKbkq-L9A!ViVz(U{gXTXb(@!1ia4b&}L4%dIs=%ykb8SJt@7MMxlXQ`i^0V ziK7>o2YX866Mqa z(JuK6lS`T+taF&v%J#kG-qcxuPf9@%q&Pm4SFV!5>{=S_!uvf)66dj*Mm})9)_|{) z%bXm2?yu0a`xSKhzxgV9`qaVl1a#+|Ck|PUv?U=Phf_cU)*)d-U9RFU@XJG8a%6fU zUK^SW(FdSF_0wr6^r4{Hii1iP4~t!Qif3MQ{tZGu*5btQEQWJC-sY%N{$2GoecYO; z$`RaO@FWidX660QP9RiMDdB=Lt+ex=zF*r%c0=BSiNaxuE&c-f+4 z-u?$YJ?~H$64%zFs6<_RUe_E%pP^f=6W(Uk7=N>M{s~;;XLw4>3d~OST{xVXU6$7j z*x-}Ao$f@Rd|Rb8Ak~VV9f#}fg9Og^0-Ye$r_1qBA+>SS!>}0RRPbvdV$ZGSFnF&* zw>AKrsTy}8;S~a@oPV!kCu->j1(_J+*B{;xa)~Zn6+Y(%#3}Ya8wK;CqVhnz%14A% zDd04_sSJUFwwfjMW`Wdcd>(Wq)k;cma${;+e2-b_wbgDeol}Kt+afBem4?gG%OE1@ zf|x6xj)ysp!;q5~&NH$2`Z2veyvi_Y9<)gzz|LGu8zE&eqlaKP?xHzDCy*#NjTJ0bXLCGDzXho<~N>qVXHpz1$Q z_zCh2cik$-OVAH(QuZ;T>IWh==8BYm24j*q7{!#JB~Bb- zct~WYx{Tw?_olaKB9U@^Lfm4`>9bdW#u+NTs%17%O8!s+n3rQVy?nW0lmY+=9Cz+^ zEhd=KfMSXsLf&xykQOu$*!@gMeruL6VY` zoC`rf36eo_hMucD|G96xd%NH3?l;D}J^G$8&M2wcwQKLS*P3&FVXkL4m1L<7Fde{P zFjR8auie66_QF5+uv6@V7x&1PO?VNvm%eSUYGY#WbjQ{hqj<;uzNL-5rI``Cqp_`> znT@poFP5KI^c=gXz5RVVaXvn)e~#d_vAxIlNyniX&a(ghbuBv#h9L(1Pa@~=_!S28 zN>1+DWi{v5)BSs2v8ZocoMk?7-8*6b5sD9ANCQ4L%$^J&U#;nPcyRl+8k*@vEL*UAxV9O-}wb5fwcqc28;5UeiglZzfB*Y<0jfOg{A`{4U1( zmbt+k9r_ohmW_lRy}pXX97eD53K$CXdWsW61~1-6XfdSldg9kgJ#Mb^L;(LWj}?EA0n0bRC)LanT#gQxWy%A7?%^ zw786hY38nG;m_#P!)^P(?P@=}Q{|Id?y;G6I^O;KarfV!%pUGNcHO3IS))voGQs-t zvgP}xaJH#PT<|UMUzg>%Drq}LHS4CdMj=1FJ^m{Yk^N?BDa(BZjv)=2N z#ad>Yl$;N}Tn=*sl8O@Z)n>neqriD z+cnzMT)mIK>OQUw1;ls+F$G7gY%R8=!|BgAiBG-BR83b}cL~f*yz2XFwIouIGj6bl z;r!j}CBHvYb}(7Id13PULG4zw?D7N*T3lVcJKT75=?u*L7}u5QsKMl-e!&if&~QD+ zZ}z=r=|US?+dm#3%8D;&;QjdK+p_;^ma~CKzkP>H#Ix1kY`bf++Lrk>3-^S&Pv|$S zZI!~W43BvDE&FYX5BYH{*adub8IlaC(=mE>s=ZB?E!K1Gr>`IvHtk+%9Ly1?9qU?) z(r#^w)XlkyiZ%!H`kBJPIDvr^n=)9|l~fCT)YZ z9j3Z-7GP|JT{n%QZtxbTw7>N|_T#O0p7hl& znp%GI*>{lP=K7SuZggJ!-IsD)h2L#&7+Y2^zQzTMMSDmUOsS#SVjRHp!F0tD7Msq_ zv@pK&){W6AO)cXv6T(b>9c%UgLC>jQsC`-F`~9n_c4N&=xm6G9j<0?k^zF0$X!UAt z{h&bO^NXsAE^}q8Tmnsks-cA!S9)+U9u-x}7q?$kcs1qJvF-f!t%s}BWn|vn9NguC zuw=QiGX|?mrY|pT2cBYPT6EU8(c~G?`D)r2_loGvfH}{q(HK`AW(yG}@#Ric+4V?^ zyt0+sW!)J5a)`YNbo;zvB6&F!!)eOfjD=yIx_eAl&cQr!DW0p?Enk2o{wv|`fXi5p zk2fsFuIq(-vll-nP+X-J8+l$IV^gj*(N}0QT0qdxwtV|(Uzag&$#{}V?wC;kk4Ev% z#uPVPpG$d&(|}95?L=yX23pbebeg)@&oln9%P@n)e7(4Hi;PAelCx&GZ_X5k@s+P7 z*+MALxcbRBTr2I{@%599z3`mv*cYn~T^w27?Hk>y-Wy0 z>+SsdNa&bxxQ=zP4BP2PkB&R`oFEUQNvqvJ(I_m!z_Zx4Q$=?4`FZP#ZzeH9uA4)} zwVQ|M)FtBG=E_6{WlJ@jzCWltb2V_Td^7i5^^qHDRrM70wK947Wn3K!0*XS{0)MZh zM(AgGTC>8+tcQz^WXn!;_6Q)@H3<>kNWkx}JHo(ujzWCTw4dxeYar5a?tS1pLgD>(bCs zC-4mr1{bBStrg9dO#N!vR#lb^+w;LW7v+<3%h}?w=2am`7n=L1m;~E#+Y1eRVa30q zx%p`GZn*nMt^IiH&Ng3?lV8iUUVJpZzxGPD^Ui!!;1tJF;N}^r!Tj=hmH_VuA4#R?gC_1y6dTDnamACisX zm#ULKQwqU&?67FJfPM%}aJ+4cfF2H%R|cRXX^WWwR+47bR2VXoAU zT0WXG%o?(Xc|!aBy5bM6XR;m}6-%yuw@aC_$kxs+-kt`lTM50fa;yp-Xm z`+4;$&G^w(+7p#MYMs*e4?k4Gh21V_5h4xkZB5EFwzwKdal*gyEEfaEcNQI*?{tZ3 zWGaegYo67(V=X7H#Qd!C4Y7}bbx~Z>NJ7rS|Jh-_L|L%y+IorC_LM=8lnI#>jdksL zm~N{S1nDowKI!Sw|5U|=u$^oTG%j(R5x>PR!Z}x!QN1ekc_i&~ZLj>hv~nGboC1wd zb-SN|0?ipdUW)2WGKWQBt&WaDI>%#OscD>#h3sxj<%P6a$%HsKgba5_G6e9|%2N{b-DW6oKfC<%bBxa6eR>OR;_Jo_od>NZY7*}|5WH_m&$djt3$eP)@=#CWI6f5qe5~M^99(sXub&-aN zne$(w-~V_-`D=dP@UjP_kZ3*O+OGl2OnTT4MuC^?(Mm`VL!n(U^`~=J(f#oRz zUp$|b(^Dv?cu_!bk@&FKd?Bx`?hsbkAP~0-calcV>TLMB>=0GWz4A3nNmxG|dTfKf zZ%1C<5MJ;1*D8;p9SnF2(cu?XGG=3Xu6QhNeM^X-DV?Y~@BC93GH;DD`^rNw9Q(#?&N#lr-4lEBq1W83(wjtarzVA^ z<}O*h`+{6@h&q`-5JX3p1v3TPw)qzEH#iuTvGKJV)hQjN26MP_BCofH;$NyT)&Mh&2KR8d3 zPz6g@N}G1&sL_>}lh^w_H)j@c!F+sWOSe!ySj5#neCB*8fRJyRJhW_W-5x)mL!srh zvpyz4>87Ahipi1d@a-ex(u!m|2mu8^BKjPep(WBcZ)AS z-WO!ASIS01S~DfWCB0Wi_0HJ1>uSG~ogt&ZVc9QkQ#pmdxY`DjDKz~VqwB6=X{Vt4 zt22q<{`~qdi3XX}Q-^M?)0!Gk-YOR#c)Map8^4((o=IorKIA+j7tpQmo+J@ASG6ELQcz4Fd^S5~&ovys2*U~+*wey`hfzN|Pz#15!7 z!ghzYgPuFp&}lY)a-|8}WmS3V-Y$1qNb8|Q$zc4vRhYWQ*Z4M5o8(7AS)%k>cVFn9 z{n#$gJ<3=yY)FE^t8qbj5awt7uuXlz#%oi@6{>Q(>$G%IDzO5VJ#PogLRs$I!bNAr zS!6hpOJ&_XP;okvDW;CKZ<8T*lu1fSqGg$2aM4$e#^>7ec_<&#q&zl8*ts9Iu0e7h zU09&{C?`Uf7A#K|SF=f%SoK$^!b{J*AxY+#RqnXq@$Vm2Him?sK06&JYE?R4yS}vx z(~PEs!oX{LS*$bI|1IDK^OViwF)gsTS7C^fbj+W4wT+_sskz_XZFw>b70#1xo)Arh zH{$A?K77c&Zx>zSsq0l2W4rS0-q;Cp{~y9!^n6gRe76!KT^wg+i(k&Co120kjjYh5 zIVU4BS~%!=!7F}fks%LfZ^tLNx*c`cx@00vcY-N(~zf}0Jj=zh+r>+J8~A3qyPL1r{vOmQ#SNZ?95 zLpt{Ji}8AfIq9vSDFS;*rHrwL-(-cN@V6B0<#o5g_xW0clReo~g-oh$9=JPqRQl=r zxdV7d3i`c<1(=3=h1A@C;l7u(ZI>DVKsB?x_S8yk(1&4;zji2KGLIobpd^)k&P7!{ zvtY30;U03~&D)A)aRsaM0KWoP?1CLwXlaUMoCxFL+v{WTW0kzMN2lA8WvPdUb{QU@ z|2&&dGc4p&6avNnIjj7dW5xCS)b`b59C2fF+qwj)h_*J}FARE{hoIEhV)VlbBvY^ezmv7PT^h0tKRwoZLGdwVlnna%*XN% zF$+&Nf@Oa*UAB7?N8>GBVBu%@HnQ`sr9nf{U{VuPKjDD_o%*{Vhhe^JJ!{*mgCX79 z(t8~hWN2xBxX;(nuW#N~@H^W7gd`!A268EP*GH#08SY0(3vVK;*-d0~A{aQ{WL#nC zJCGP|{pI8EAS3L5P zL)WX4gnXo9oJm)0Qt(vCzTm7QW|jz5c3B6uDdrYiKgcO)&{cUxcUcjq?unJsCOb9U zflz3jS-|geoaTPkZ*jCn*Q7goE%IVp;vIX^=^`KtC6GH2elvcj`Xdwr?6vCQSH znARf{5FDTS=bFq!hB5Bi#sCLD|4NGE9Myr7IVyV@c+^Xqu`bui zjOLkS4e35zm#CxQGm@yJU@>aM&z~ht2e>NeYwk4Y;nZvRn5vfRhd=gP(iI+S=JyS( zjy-pZZ{QkPnZrC(x{gKAg+?Derfr%iL2`xWdW~f!Dx%8u$W9~V-)rBYn5`#=`$b4= zgfhD8(xx9raE7C-b=Ia0meZtkU|t_p=`J2g+2(3{YNy!Vuhc%FqIp%jdGyX$PjEvK ze5m<2o1z$Z#E-(uXvm*|ncw6|klf|y7#gwKHg1#e!TY7*p%k&S)=Vuo3bd%*iwXM5 z8GYokg@gv#JsR&?0|{&QouN&-nX8Sj-CHk{k8X-BHVV{@yU5U)+G_@nF~nyURTK?U z>`SKd_iuBaUTc)5AMnXj@yq=@A6@5mc<{wqHkC|3%QXe_qr9w;Mzb+Zj z;#Jz*=2N@qzsh5>mp4oGg468l!HXEv2V9s-0S|y2lVxl)-L)$E=@SOIU&z&-unhHIjJ3p#nMKg zrP@aBC5r*m=4$-mn~BBBwm%^`@x)ahwGTZ;0VX!|tD8W*1=}8P28c{!EmCz*Kn=>L zLY0Bp+>f(a3P3R=Z-JpTGmW#W(6W`vCnR?W%_Ssa_N1o~SQp+1gTwJ>q`5{(aRJSsf zDb7q?xc{TF{>Y7UZ6n+8rL8c;GHiI^MS{wL=OryBh2AWG;Rq2dwgftcD@1bTcIe>p z&|=$n19wgY^suh6B!*X15N&zQeluY5rOU6V(}$#Xq4rNf086~c12=q2nywSd#5nSd zkOx&pX@rcI{#+5#$F%-(Vhfs#m=_AZtwt{G)SfZ@%sejA^ki0n(e&GlC2fGu3k(K| zK{MS<*Pg1YS>%*VX00#B0+Y0O?U)zvD|SNKYWtPWs3;x_)0t z?~eikM7EeZ#_EHXgJq%7yC7$!4E*bJf&k~ov-i0)$kWIOYAB zH@yW(H|ztm*G58>f3@hirw@IkYY{6+lqn=+91Z+%vfLMEH0ZI~w0b-&cD|f`?jHU! z_NOBLtV(7f%$gAUK4)H)u0p8pM^~~kg}DYIGI|5#N7i(!XN(SZn8qXx@#+s7olpFw zUzZ^q#8o}p|1souvDq0ZT38CY);2?)!vf6e58J3~pZukw^6gcqvXoe%dYgya&!_aw z>V$c5Sh-WoGnWFl>Giru5>A=ZpEWGg1csVH#ojphInN_J1uqGvHvGUP;JWW5y_^Rm z+Bin9W0~)oM~!K%N1A~lX~6!A(MM=+U+Xfk?|e4Y+eq!Ew7Bxets^6kf90tH}Zy6;%jAJegX5iCA= zCQlUe`{SSA=yM(p>S#^8x_Gb*RHXr-@EgDmX(kgKfL45zcL;qHsBFcH0j}w|oe(?v z8^70t1ZhG4ri~O7^HzpPZJ4?Owo={Q+3o-mGbUWu_A4Iozfh$X4P>KlV1{*LC^303 zfnV^(Jg8;tFpkg*@4k%-DtSYTJ`UuY;R=&KRX}p@^#hr>QKy0~jT`Tq#VwcwO>epa zan^yYV+oCewFN8%Z7Sej_MFm%`sC6)c(5Wa{Xm*Ui91e5B5oo$9?3Nr<3&KqzjV!i zR6ZP@!S*U(dR{qiFk8O8T(Z5=*KviC=M}I81%q^$m5Gki1vZ0HK=E3!Um<*Ufb3Swn!|l`4PXsWOWh~_)K!PXgGKY;#tuB_1{0M zI)LSiMzq(~?tt4|>^(hnsQ0_^$X;(qqJk|;7l2PqN1{Ya>t=`Y#ni$9S8nAE8sS0r zMSB381UI0-F&M+w=CmX~(2%V3I}I+i-{4CHaK|SJpfB@Q<_P=Sdo6Kh5SWz_!&9b7 zd|smT?92bSOR#>$Ki)ozLX#7i`mfplnILD}5>@-ZA+0gh&%l33A@ zpVXE)ei9De3ZAW~*j~{o*f^9BSF%;%c&0biH2&{1=waS{k z2?6r$ssD6-+d1kttkbm9v!>OlHpbDky$m3xB^~^}S!b`*gS0V=Ce6Eij|! zJUz?@L$mYB#Wm{`rPfEx;4+jUm%9l4&Qar<_-@yth}6c(@5Oe}fV3Bi0teyluCFrE zB|R_z;=q`?m$-N*QbN;EHGpKe33yhPwjM(d;AX>@3M;mGoVY{AnA72HcX437Iv^f1 zmBH%sh=3;6`8-l4)bp?&P3CV*Uo=;f^})LAz~;GjXXi~$LASQp zVJQf?9Y}>eU1FAFkJ2S9gGcFW0aoi65wF$3JgwUjh;*h{K1zs ze}JZ1+F?}gbnGi&sNz`;cl&$uW$h-B^3_J4>ppc&dVnKD;^m$iAKs7C@3yp%cH9BX ztuEDG-1d-1#Y9?+tY&nZ?>t7vk1{6Pefg+&cjQ6X5n^6>yTVHU(e2Y!*@l`UPf}aMr zfty;~xVGF6Df5+QUWV%%s_#h+)+#d$pSs~*^{95Q*pmzZ+mogD%0har0On=z;hm@B zo*po~D4Co^E4Wxb@37vpWTQN14+}!7vz4ZfRRIgH@KKUEKp&cLj<&s`5=*UDaa*p} zQjKVKo@O|o<1BLl?ou{`E0F&;@XO6%K{Qv6{WKWw$;ahl^b8;FwWTwA;{VTm`_GS^ z3(t~Udg{rPlMuC{x%>Tjb3wsWFJO9$rQwum?YOBDOF#UzX3axU z&)|ZvzJw#(a_+EW(AnCzNJnX(qw80X(axjDPhz@n-~F*0VU?;+6jJ2WERk26lDK%A zWkr5u1Rtacu9;$KDaah_dquI2Ii^j7bf7g|IUZY5;anc%U=fdi#$F@)XTjKQ|C_}b zcLQ@99`q4)m_2(*o`7L_3Z5v9q5m#z|T9_(DX7{I=+EJvDz((fq!~&Ooc* zSptH`;zE;R@8RJMp|m?fcM_cg({lN^YW<&Gxnj>j-PZ4_Mw?zu#!#V&o6>mWRB95m zS0^aP$?q*+7As7uZWEvZ9M09&F~@v!iE-7#Exf4hwow~emnkq(U0ojzxhTA$FD-4N za!OokFxwRszEMwzBhtTup@E0f(eJC2zkGh=I7e+o zK$H$n_aj3H@1wUFL#{s_k}r@kV1Mc0jG}%8PHxod=VhD^sIyeEuiGw*)+xkl%^k)u z)=n|s8K~G<&XYJH^9I*;${9j#)9@+bW%1xltX{=>#C9ujA<9_l~K%W81D<2sG@v<$;5yOoH-0V-y*3JY5%P zk+3Vca2IQ$pW`D?-hJV%804bGbpTm8t}cV+9*br(cj=#IXnb7hwK&}aXR@0sT~u;~ zWU6Y>{(Xvp)}*?~p9-N=v1^_XPCB^rN?&S+BIPN**7D16R#rYDBjKmSrk}~3fRvT~ zB5TaZv2Wb^LSoG|CY7hh)IF%A3k<5BXGIF0G)d&)+@h%Rp=y}A$jjNspoiVpk4l_9 zsTtSZ0g=jRb9PU?kY$xakj`ArpMW)K&!G!VpN4D^G{DIzd)9;w2+bhbXdV%LU+3KI zjpZ9WO|DZp5qtWefIkt!_N_ zl^NsyUuh6Ww0}chx^TEjxbGa%)r68{m={fDDTFDpK!5-Ds!C$1C0k(E3B8vP#D$0T zc_9biE}scriV2{tTLk)9ZNHoda-?E4NOD^Mo*_;NQWgI50q&9cVHB#j+{?j@pj*c9 zaB45T?(@^aW@k?p1-4u+9cw!-+Pw*Ne-nt9&&g_xMyEuL7PVtNKL{H$MIi=_=GL4 z9`|E(@4JOzcu!h)5TDF#l1zs=Uqa>$M^uv02rRT0Mbh+0EE;jSeh?9!fbMrXiC=7bXilxhCSwHP3d zrplOFvnEIn!m(D@>)3(;GsL`v{^Rf&%Lk=_o1aP zRlCmJ&9~z}zn)}qe~YcqZ2T#`{&jLT_xswmUr%))%Mu}5r_kCAFoE15$)#3N_t69g zRp9I3IvaO3=i-|s78_Z{a{wAo0Ewdj$k)adYCOAy!cY5{1K(~{>~7E5Luvg>z7!TE zKe-RWJV>tXK=|Qt)$KIwWAa1ICk&tXA&(Fc*-sI&e~uP39*ER5#F0|5i1J(E+hmAV z$N^r`G8HjcB8%dUP)2s~FG5OEf<>me6EIc;5G3%i=0R-{5_Y&oJQhOLo!sX-4qfWm z5wWh5nZ?o85OG^R83Jg|CRjnb6SIr~T1}=X9u9O6WVFq@H_gxy?-4SUL9rk;F;UaQ%B4j{*-2Ud|xA*sQ zwV!-Ypf`joF~uRAK=Zlc@SSg{zC{L8VlkDg14vWm!0u-l$k1Skg)i<>T8qH)WG(or6n*yZD;Ex z?tXc04@TE9IhiE+2f$mn=pmPx34yZr2yt1p)-HkFl?Mu94!s@ZA?QG)eRblzL7#)f4z~BVui&Gnh-K4jRA=@rAxXlF_^7W z{Ky5UJXpXREACLA4je@vLw|1ZGBVh%QCTh!Zk=GQf*Al+ zuB4_I>8|YG6T@h_6!4u;3zTh*0%MH!uu_{k!K-Qy6DF0>Br>sN8f8UQ!_}`gP4;izFt4l-2Dh%P<6mT<(8+ZcdO9eEY{8}d`~2}7rEe% z8^yhR&4Z8yTs%%*LuOm1*aPuNzS?I6KK{w5NE7l=fl_$O8i$d?ny zuQoOYp#c$@if`+zt-vuFp)5_(YTKK#|HUcBJ^!&8U4W7wCMKSycIWX2IpU)EU*o?BjU+cB0*j|_!awl`3!2rz^ zIsr8kTTb0YQW?kxP`Nqc-qO4JI1 zhS-+?^I{b-ND_vW zj!gquXwrk4RYFv9;R{ET)GX+I(T^)Ea*H{6;F(W=lqcU`;=r6(G7laiWu!5nz|;}H zvy?vRfEMHicxA&K`yB|n9t>UnHN)26;4lN)-VEtK>R#YI{X!@M7QzgP0)wEGGnCf1 z+Ge1F$y=B+YuCK0;yz-8_;U;@JSSib7)L#>8KUZJbP*9Kfc|L}1y#f$K-irjWNYT> zs%RDg#Owf#`W5%o$=BHk*PjTU{i3)Ilicn;!_Ps05HUz@gkVR(5F+*~(tQWsVm`4& ziE_L++^L$Wr)n8|gS4PHTATpPrdsyKxjDx2og6j4?5fb??n{41WmM>y@XhsqTs z-m?4r>ljxl?APmolnit)TYHK5#G6KqE9}_1za>1C32p zM5x#)__i4k<(bGsgxHXb^~`WS13pr~X}b49&xp!^Y-6d;cTH7)^HQU4hXodVA{p)~ zNQ-=@*gi=S$mj}`buSc6%t3Dz0)Nt{#X9!d1EvjJ@t~i&UAm77PNST)Yg!QacQuEY zN)tZ5+AkJL3JgbZYs)izA_&WK=v_oPcE9Xx<1qECh&*I8UV?)S;vN9G$KZioF+flW zrFw2s3uJpJe#d9Rh_qEY?EpGpHOvWY-epRCT@p zhd{1MC~YLbQhui^xB4d(weqK5-~W8ZfYvngZ!Xd#LeEhS2>{wqLD`Pekq{vFM~T~y zKSzX}gy+DxC_?>~l~BXpkG3gaw@o!5^Mj()lDa*Kc{WI7AX406Qmsr;ZB}}(L0}90 zWdr;D`T+MA-(j`BAm%77+B}Q%AAQ-`-b@EG{$B?40%Uc=@er)m z#hRh?HOnpjRsZnLlb1-Gm~CgiJPzVhtPL%prsok7o5bM}6CrH&evX8|w}K|(h*yxM z5Kk=lJJBnsZAy22hns%=IG3Uz$zqmPSps{dU?4*Qnsp|iHf&feKzaf)i2AyzJS9rT z{5B9`m?s>eg!=O7Cony)tBNTO+5k0Q%GyRJ!pnC222X(+^%zPS)DSZ5t7&A_@-Cph zAyCm`KR{>~gcV#G2W+2Hu0$YjbA+y~-0T%yZq`i%^l1aP)`mmYi>s$A>yA6=mcWCT z-E?>uo~DXQ1c~W?vIDI(3nZ0X*YaDLozP6hB{Xx>YEw;_cn0kP>f_=mD>J(`tB$P+ zmzVXACj>Ds?e->`MkINd?)3=Eob>4&7HzIf^fYxXZ6G0d$8Xj{8vTGPQmOor@+B^YeflAW|?E@t2 zaPk~|wO;$FZXTk~c}z#s5X3cyDUBQj>pwsK>NDzEg6Z}Ofeq+!N#N$^zl_9MUr&{F z!pqMU>?jK)2pXkd*h#uD08Db@frAQ%vEMH#thL?pJx+`}H^uX52lA8k^t}y_8N#pr zj|a};J;@KCKVIK^>R&;tB$_yojhG?Kuyo8pOgt2!_P|_?{U%D}^>3Q?dZXH;7ZQIv zXrYm)LO><}_%Bcs^mB>nw^i-HibR4cvYW!ISJLU`fIc)VHCHA10rW{Y^1tALR!V`A zDEnW(r5CdwLw=Cq`3{&$)2;hUQX$g6|NBGQYjvG6EMS4j)XdXI1&AVq1BWtAa&%r! zKTu|+epl|9NDAPUN2x^{45?nL+t9*R_H_zc5VCcRd%#gMxGf6Grj*`ko3T?*#PgNO z_+8Os?JZ=hh_$~lxVH$>DY3&3=DY{%F7bAIU$$#vfCMPdbebEO08FObp(xTXTHI1p zp`uQ%1eTrQk$iATxD8TgIIkxtP1BMD#?9+(gD_mlu$YId3!XrhNv{G#p|~Tomj(>V znILl`kREl=}BCQ1e^t2w- z^BlS#fH6%$g={7y(_{y>{*O*J3njpNae+UH2P)cCr#!H2RVWj__POj&{1af$+_D(- z+;VKUvLpEcB~Q$*5B2eADWLuk&z24u>;%#!iHQQimom+K>5bVh98+ZnT1^xI?{>Oxrp5jzF+TX>{@@^2vK_wEGPE~J2y zvZ?FOrbNT+z==^f>jBq;yfZC)U!WA$n_ETm2U8ohVrBem5Y0I%Rte{nDb#C-2Pl~i zF4Po|%*AxRSgLJ#86OZkRHrzAa z1g1KUEicI6sQ_2rv>78`%o5yiI@%ooxEN#7oD=BqS1n|e8018mTnAW~O0s9)NJSdz z8_fad#{{VRuP<33LLD)K$P^*X0ApH1VweMVm{Qh5#K&r-6cD?Yf%r)(op&PaJAf(&WuWkHPJ&qy z5uC{lHBU|e?K@9qa1kAN<}oNMKyVvt14`&XhN;tg&6LQz(Px*q4`2Y%{6^hdX^4zD z4%oyDJx@^Wj3qVDP8S&Y9A1(i&`S{yRcJVJNFy@J^$mI?82o;RD6$doq${9S*}scF z223;1+GfD^E<2l|qfAvNk6@6^L=tLdL}DyEI}_g*fcFtnD?V^PR8d$C979%;8Ylo@ zdY_aikiG*gKMmHEKtJe{2o=dgTyu^FYD;x zpS?-;v_loyJ@St95;~u$G0Bh72m}2PUvy92!s&2NU>Vc-x4%QZ(Qz1Bv=apUC;x_5 z&l~kFg{-_I`nJG$CrWwW&Y$rScbNa5W9rXS{zisFG^hxh5>0}@R*3L`TZ`6x8=WWN zoc74MN}?MLmRSdKpa5GPw*P}Z`t+tSkK>W&2%Vvx&Iz!3WUXeQ4=!bQ8SSYIK7Jt$ z85I#h^>=7BEzq{aFhI}nD^DoxT=R%w7_ZGEjB$qswhq)}48bSdHIKNW@Y?A401;un zPy%ps>^G`PIdr!e24Qon_)?xg^VMmoSY#CQBbH4W|kCUiLo zy6~i|Awwzd^9D{`9*N@Gu%J8MHSk{eM4&>))Th9hLJ6tz&zROIk^Y$_?FTA-5YX=g zJE*%<$u1(>m-SDfZ=HEsqJe?!FH+5bnrccMymf;oY`4S-HlF|lj;Qi3yDoHG!qLfN zm{~B=nt@8D1Xixgam!A?)-`)Q*n!E}LZ~B}MO*7hE0I{Bxr@ZL2{`@+f@m9{X)O+- zHLnCKmSvv|-4CeH)u2}sPVIdw;|2!tR;g&bK_%m#0jCd2HCNfC%D( z&{n~$Aq6H(L;EaBy^6f=znlyhF(y9O5^Hp|H#r>usDM!fd2j1SjiPQJLvNc(ll(y9 zTQ)}aRQN=9P9K@pY4rY?WSF_xZ{EqrO{^O1ZfJ%!yat}#B?{WetsdnvqeEWF)wxPQ zOB`!-E`*n*5U(u`;;H~BBL`hPZsoJ!)|LY$#F%x{=A|5`oG***bx93k9+yvqCxdh- zYp}85p(iN~>{VFA*mLL_WU4J#Rw6^R{=Y8K@HX(q9WV~hiQo)Yx5P5ofeoY(^%|zB zF9xEdQnHW=F}Sso?x{RXGDsuK@&`J2zf}11&|isS(53(#Ks806*v}d9Ozx6V6jBoZ zxg|Z)O0WxG^mT)1s~6f(?En$i617;J{%>ZcJJMqjF^U*w!Q#d@@0+7hXUq~X!^*7= zDCm2i&xDi!lMC4yAtl^Si#+}(N}eM50fcfmz_;bAb>~5njANf7WJ84zjl; z>+*(2fqQ`Si#TyPFm|27=z_A?fy@Y}_n6fq+X3PSOV6Pv(dtiH35kmjxh|25aeWI~ zltX=hblM$q*F8q4IghyUe}*0>$+~pNYwcE0-qOszm6?L38$>kGCOEM>{ygmGI5U4?eIpT8iZdWlU0^|Ru8~eP$NwRGL zt@GdI9y}PE^Bc>9sY?HkD6|Zmpc1#z&q*VU-SZT4{}VKgI_!P|8js9=UTk z5k1cYMVK5#0Px3VE1DE{i0K92j^XDdLHl2bgIq8<{~KLws|h;Y={V<;i2a{{{Z6h! zNHx$#s{u_qXq}e``3_sFaIT3H3lhfvCT`l&|M=IUYU1U?5aWu*QH)FiJqbHH`OVLm z-~QaW8Ff{}ug0iG$X!D3d+)#c2a})Dh~TZ%)N|-!;ITPEOhKkk-(y1L8;`Iw{Ox-n zdyhH9CMAJ{y9ro_a(l?B6@XjxL`dPE`6k^=x&@Mb3ecS59#DKC(K5<#iWs?!J5HV^ zfw-en0y?~{&=24dU_PJ`@z%(e-sgP>nR<~5gZzzCWnAb2Tg?l4F#PuLbRY<%4`dU^ zFGnMQ3WGevad0t$U23Xb|0|u?fN1pohKUObq~{YQch-g{TD*4bq}UP% zkwz#EopGo+%pS7nuL@}77t?e8`4rh&mi1v*DJb~w^)EL?&1{YXGS=6v@Z)Qwuz ziw>+@>j3IY1&mPA3lp$xROX1^_kJLciM+J|aeXlW3w3#LpC*ymntTHdCg9a`C_xsN z0GeSk;u?y=KhyG#SpelP6>Ua^Hiw7u+zczecn|`GaP27p@&goVj6+2$2UHNAWSOpaAvJDjT-|)Nr@!c( z86a82!0y1F2kF3>j{LR(J5k;$7jaCTJJLuiuO5;VM1B~ttpMu^w330CM%P=LEli7H zg@O*P#^!#{r(Wn?Je`%%k>{w-eH&=lMxq3757R@anoth!vBkHKzI&aTSCqe4Dpfu=iN3A z-N^rE_q_YBi~ob!{y(=_;s3%1_oj)=8)YML=w@jL4{O9R-TR+9^Ko^Zuvdfq=l!QU z-H4T+Vd+F87k~yL#!0w#zas(_BT#rm_tW^ z;aBwYE^KZXHH{#*x3IziA^`n&Rb)XF7E_lBUM}cEgx%Z15O)9_X)*P8EAbs&KzB89 z>A1Du1ilXa@i+G+;N1Sr!#P1|UxXzaq7Q~Wat{z)Vz9p?7pCsY|BJ9XyiG(x-X(vm z*8kp-V`XyWIiROgGe8w~?E+>Tc6UXP5E5Uu>pB!u(P%fpk~bW1+n7KLD^3imemelZ z42}nu7rft~2D1xk9wYJNi}0~7pnTQTTbcjr0yI8&34QiYM*pv1^KbGK=bw7{KmX>U z@BsfSOZ=zb6}DFw10FV!G{jY)zBklsK$KR}U)#!~F3vJdV$_3nT{@y}YlhfIC|vA` zdJEb*l&BM+Zb@X}gbn(@+fAf9hSfmuuz{Um?7Ow@rGiO9f@mfNdP!Mi(A^Hzt{smk zc_R_EY(Mfsw%bkU@5)$!c{CHA;246f>$?XtC;#=aOr63c8We1%3@@W2|}=G-o5~6`1!dtFEl1F5DPI zN{QA?GY>(k9fl$l_%eQdwq!*zZ(77FF2o!=VHkyvH_w9Oa1+pyRi6&K-6j*BA2XXg z%bByv#UGC6!|&^j?7qQ#H9%y2X<2_y?aP&?;heT9Ah4L0O35H~0D76-#bI*_&Im^E zLQA3cU|AP0uuZZ0T96NGs^m}FQJNZfcUKETh5hhBx~{RgVkEQ*6LCh{$oU$Fns+D; z-`oeizOuz` z_to}Ff%VP|_#qfYRAz&GmU}Bxt#(H556F~`$@&vuKgj&eHT>TRxcrwvCK5$xK0Z&4 zOWy;-&_d=7KSD;6I0%M?SfT?B=pDB%5<^mAdZn!fgG)5pdVm{)HuNGIC<-jby2xd8 z@XSySIEX`j&EDOUjIKlGaR0ph*FCkq6uW4T1fl^N(W~IHs->g~e92=w^YF9s7<^VrYPv#7R)s1|;JzH!f?D#9-n{wX(!{Uo>=?_Q3+{?i6;L>@=66Op1S}eEye$N3ni$iLR zEBJI}il`IBYId4YOi`YLhGF*&Bqaei#{(pvQ6{*Mhz?FbTB=g57~%z-2Lj7n8{FK) z9Y~y7m0`DNJZzDp8RAJ2mUwpnn6U*F0p(=oeLl`jEu6#kw^EQ z;3KUNWoIO(9l~c0>_;psu@eJ!0BA=BH)WFOso0x-A5uX*0B(7~Y?`)aN5!(EdgTU7 zYbk*3Tgc=J4{Wq%FO1B&8lMH|rijmlW;Q%*+M0qK;gYZ#))$bPcpNP9Vwh6k;%r9; zfxeGJDn+p5O3$QMAdv7|;<5pnj z=U=>KsQI=RX4c|9d#r%IyCM!J?%0fr8sE`bL##ye2F#^cqB99tps`VhRFVVOJ`?>x1h6LY ztOMbl-2gl9cx5Y{Q&Tu_gq+>K&d(|#L)a6xPg!)N!8;tEDkE*xxV9R8=FS`tqs#9o z=8dk3W^Eu@lGwRoyE0BK0M`2v*wWY}s`P3(XcQ?(OT+S9X|z>;U8|CVoco$oG9ocB znE6UAMP1+F|;Mtn>r5pVicEEQUqEhc7;S3ijz_Z8Oz_*qbc6 zXIPUnmI5y0A>{o+eAC!|Kj>zsu{rojP9cP8VLFtu zc1oeFS(`bem^76M*^0!>Fm0r?h>{Sd82eIYMj@rLWXUo}*+RC6@Ad5c`6IqRx!um` z+&X4nujli+9@pb>zZCj0LnzI(-rFia7RxJ{4fXR6`+ZzNClSxFA0KX&=k>7H9}&C& zZ@uqV@d(for?zp{_rmC^Eo`GBUMIJ}Ol*aqUePYj$nXL#n(1kDLHCiV8N*BoMQgU2 zU_kJ0J26gTNLV#d_nycQ!X~))$;mfxY=UtSUJ9ak#KMG$)}5TQ>GgOqizQGQkt5i= z=uf=joZ60DM|Wjm>Hgu6VDLCtmxyUG<~`*|Hs?D@0d8s?@$IsA;7FN197K|7LQ-07 zb^CHWs~@F)D>qV4SJ^u*!K3l5PiPJEsIssXQSBPxg=(k%hu=Hztv2aM_9Lo&rk$?j z%v#h=geep(!Ap0ilRo?rfslskt3+0)i(AB`*<*Wy0t!<21% zNayN~ZG0+pA}5hduah(GBHs#wSNaf=+yz~T=9f;qwNccXAH9c5QL4z_OPwaic%GmE~y&E@Gxdtdj7%xg~2o+KXEly{%iAL zzQGi17VBMn^fg?B)2Lk?>p<;@Iqsyjf`wK`Vmc}AKx)YZobbBJF#xzjw(1K4iD~*B z$!NyW!xEQ?K}h>O`z5*;qt0si^6Rq;_{bkrxV^LotKcVf6Lqp_Es&S$%{N-6GQp<- z@AOKhAMMobt^8H#*ZKkG^yaChytA(S?kDME)LG3kNpm+1Ow43KZ3IAO23@lV0ja}h zFCdlr{y5kG^wzt}TNfb`&yVJD@o2BB$kF}K@ZMAe6y_>Z&r8J zcRq3)yVZ6&*RVCuVkXexBYV?c95IkCwbA%XbCpH81n-KK-31i>jTW<)QIIjRbdwyj zcfdx%WTjNzCIwM~MT))~%b zCWi};N?^GzksQk5xaHYd*dh96_Wk9$5 zxy@v{;>N{;MINCUi-CoFOs}Jbt1M@{%{e|Cd=yTVFg959a#bDcFrClEn6jnNl7BB& zfZ6;Tyf}ZEE2aJ(3YMqlqoqy#m;y@jAK?onKj>9Yq+Z27)Zz9~m8<42(d-KBmVzy> z*b)h?IX@9p_){$L7n|WW`pf#?%8cO;zbW-J!y&K=ZG2RpC-O7h--9lK8byV*fnaSysUntdW?u*cqqq$W2ZNLt`ehTc(PMJLThwZ31$vasjBjRI z{OtC;Qpf_FO_e4ta|F$rS<-WgV4RO(-apc%(UIck-0y867$qUgzxC+Yz{YhtOQ$$h z&0u=Jp*b4&;NRx$s$yc`THlIHwhf&dn~RKQ-B5Vb{f={7_>*9I`|K9(&eI2&;V+>S z8u|o-@UX{Z8(tOFEFP~cY)qRK?NEdFJahP?{N@1=ir{im6Qtv|P!HVg*K%T(;xejW4|z5FDKuV@Q%H~W2ygYp@?`?a`*FM>={$- z=nGI4b&cuyGHI&(>AUfLF;Zu*ITxpQAC*x zqHq-+Dp)$?Z(!oS(MB0G;*3QCFi8D)nnI|zzp#d@&)KD^{X78^w4J+ zA;x<}I2l1&qF2G7m2ek2v+TK|zF7GoI|DJp&$6bUXZ@%L`t3>1mh#e;a%6Ayj(hEy z+IsoBw~BdNEw&ZhmMPpKleP_pG)vl|@`kz4`7j(3L9eS{t$EK56@62KKx`%Ypmylj z=$Dsg!9(JR=27+VExVnBe6SBREBRq?Z?rg1LZi`i8`b4#d4XC_8+^WQ-+?bu6$T!P zXSz_LFZ_nh7Mn~dlk;%2+j zcSO2xJwivV^nlS$8@O4@>+_`z47mdrmZ6S%->HN&fdi{#qT^V68=?ASa@qrf=09-=|^-JX7c(Ad)rzm7I| z*2qC$pcc;U&;ZMa(!vYyBht@AGJ7Q3^nv7{?v5f--x|=jb!t&^(WGeXD`++j3-F<5 zuP6gSd%=*o^{B2gwNn%bdASd^-%7w-Gn{}NWs|8Gb6*9#>W+~g`U9crd3Fyz>Lf@P zNQ$k6X+%wF=+&*7)L`e9n^!&<6a8!X0#DwL1bQ%TE74%Pl(54<1cyVt0er*Mm{Em7Ma+aV}<-x!s1XG($DYEoYbGSHc#+Nn-r*(q;G_@?Jm-H8lGaZkEo&qR@L6Z)d(*MpN9! zK?wp12t(|H=6*|%VXrq{Kw8k(GpB>SYUbs7N;4_Pkl%6Xj|mtXrY_abFB?hUM(M3x zjgdfQ*;x4ypSFSTc-V?RhqYfJF0Qx0uNrYoyO!B3`$sO-aE)_X`7j&(z66 zSLW^@=n0&WA4@^WB2~l&Klw+$>fx342tRD@&OyZASlmWdW+t3LH+T451nh-uX3jtZ zFU_Y}3AR%h6dxil06pQ;aJW^J+;Oh}VM}k9_P;^Tbnkxf9xC-2l2?$4QLiR0on}^O zL{5dFb9k<&vY(D%RH5r+t@JC;B^S1RMmr`^{geHIoT-+z0@-3`4eiw3_IXl?s;5IE z`y|;xM>{kvv8igK!7=^SqR0E~4%*=;}(8fLIpGZJj_( z3HZeROoaUbr}f9-P^+NBxu2jXy=OE`Tg=*B|25yMdh+KQz6^f{;{kHPB$$9|MTECP z8g~J(Vih;?^*gSka>}b1ZkNRK%7~YhCvq3iT+?XD#D&wwILmJ#l52QG)r?$_1_N7l56=K1 zd)dCZ_SvIhOL50Ai2+L$mpK0T;e9hGjM%&=hUBQK9+0?Lp{AWy2fzRcr(^3hgkrY(eov^LH{}+@o$OzW z`w(RFTY}fhR$0W2TXxyTd?WAS3ew5v2?{k>EmJNlj{KXTK*dh0ajTKnOx`2kB> z=_~UlV&LJ=`^V5b_d@Q`=U8=J?ivA&TtY=vEb&~cwuT^L`c30Q(Pkv#ET{1{&dG^h zDaV`QoQ#_+HfwB@CoJ7dNhtQqAiQRz*a49cyLb-Hp!Whpb(qCIltX)36n&;{-ks@Q zco6z6-v?xqjs!x*J(oIY6n6J$jBjt%rvqe*lM+?lAR(>GYe3glBZ^FJgTKpC2oiO6XOSmy)SLa>CeS?8O4V zIawF)OFyPb`1oVz#&muzvhlB)eG!B-M2(KzVkkwuP_B4$mc7T;rbCMO2l-J$jMLf! zskjwnrm3hrgEqOG?_s`^m{a79F(TJP00`DM^sprHJN2ML7a-mD(GRAu; ze&4VWyu20aoo8k{V;^}*EHe&_hSfDvaQ+u;`X#oH9tK~nde@1#gzp4_;J-}w_T&0` zjM>dXbqv-fN*66*D5Y|{#-I2oeKWRS^hTjD34SX#shbz1`MjAgD?z9q+N(PLG}tDs ziaK#(G^Q@blvyd>=NYy_+c$awt`Zb+cnq3=Zs2=%BANMIJo#(Ud&Rct9&5yNa&aeFMTHfqX0Zs3`Bc@xLMfblV4*&L+5#J=xH<7qilJ5&bG2ycyG z6KaioAJdeR`s-e{Z%J$bp~p2*xl9eCOZh--?Jn7qSYa*V(l{V*nvKXadydI&|IU%f zZZAkwX18Bo0W0KD=ys<9UXvf2M2OOq;mOBhR|rHS-=V|_Ux(MyGFz{7e9*sK=x0iJ zYGRnC{vS=TyTf}c6w}|y-N=(^USg);pmmwu&m;;MF0a51WtANp&(bwaCt6qHwd@Ox zVZuQCD#B2o1?}wYPK<6IBVoKfg>fnkZy<4s!Ln9yZHyJRDY-voSGZ+ejyJ)#9#Sbf z$mAa)uW`8=?x3A$3)q6O66pg>f|82K(41%ot1Z#8Q|4*W{P5(T-18G_JYb2zcJ7+I z&mcV#S)Eb0ukaB0jH>qKQyj6dfR)cIOY)u$SXN@vQ9L{;F0@|Br2d;nW8oJEEFWFI zNSH6x*v0kNj!;n0J^}jYju7~Ui=F$Y{LItW1cJrrZBiJypzoTw*xTEvL0yQbMrp-n z`75&I0(c(&f-{wM;{<5-mM)F6@+6ype0R72TUZ&#-NZ@u|6_$3>Wkfz?X;LoVeUTa zR>QZnXJsW>xKNGxQYq=&q=2K2;RAICn%y|6hpqDo8Agf6>aI`Ye}_Y~+~GFk{>c2R zHIucU>;(BIYh_HYO=d`+GHUqdDBXBVvBJKOR? zRUmDds`w>cF-~ALbE|!4hDz0lWzkPK$nND_w5W1SX!IT<{`9olbn!G={|cCos!C6nNEdtUPv9+Lle?saOfd zJz|aNWXEP4DezM-+F+oRK%npSyH2oM!eR}DI`r8!ZNQRf*nUD*V5p7yWvt&2kyr=e z0>}}&p}5_LJBaJ_ArcHaQ&t6{zT;?<9CcqlQKJd$8Q=I=C}98NPb>8mg;+pJ^NHun z<}I$E8OCG2d?L3#MOWST$L`^@Xl>+ldk}L1`SgqLKsTs^QYAAKTvUX61YAWs|M_lP zyBG<-=Cn;eZZD0yQX+X3d6s_Jo1Jiv;DSFKwwnzW{tYIe+^J9*u-ub ze%xPST$=XRtS$j(>eQ@7STXU%L-C%86a!s$KuWWxMWRsvg zH|+5HAX3}|k%#P?LY#rioVm*RraD#~0&d=pUle7oNG|0xI6`$o|Gp_tM6kWK!3p9OFpjr)y_YD(+(EG|F17He7V*vDkbV$MTHJ zID(tCaOjgsiYEW34Fc*P%bm6kYYiRf({jZ$>fMeR5h(di?Aa#UO-`3TL$1COyrku& zpI9TX)w)?xql{l}%+g_q?XnZWr}942w9AK`=H{wbb~}HzY`cS}DviBD6wWcBqhS*n zW>05_Ddv_kWQBdG`8ylrMVG^REK`|g8U6UO-`7KsK@yUy3bQo)R%i;0C9HJ8T%Q@O1X4N&5k&%#wK?y-L$7NZG zCP_1n8cW-Ys!LX8q>sgawa10oJN)Eavm(0&T-?zr#5NFWp4KiCKm%I!Uzt===5SEU znOvgx>*=u!%lLH-T3Z;(v07e%T6&CxZz8`5zVEc`^&>u0m0zWVY?tQw%b5D%3`plM z#xu{UWzkk+%mV(i4ha=aQ8xa=`6H7)awgz>9g&;1c%?r=y{i-6m~(CYT%k%QuGm)R z(SSPc$pE{$`FotXon;(0_grJ)29GEjMYV(BCzXDmYPh+q%Az>g?tDBgVTwR`cZ~VU zU0#&SN5@WYFB?Qa=TiB@G2zTf1&^CVVii7NCcBapu)l8fmC)NC(HsP*@|~(Zo~LcZ8NPf{1B5!vv3A^%Ex8(0T4qSw;D_r{gXFMDUEB zAcz}$n@J=U_yrPn6La^HL~XE>;fbtArP}=|&y2)oIRkqnmdD^=-<3j}#Ao#DYpBuB z=u^cB#>ZjCnD_O<5{l+5r0ug2aT=Yoke4B1xR+x5l&P$z?M~0Sed9gtp`zahTmr3% zxF4_aSrWyxPa^6WKYl{;to{;VbS*+gwWt)>{p#ZXVT*YqC}Z3}j@d=YhN`_R9l7M+ zv;=cvPl)CzciW}Y4lbohNBgu1T_rnY!z8;!yGndcPQI1dtkAOK(fskPhV?vaUADs% z$Gekh?;c_wml9z)`a-iaALE5G+iikW7aBmj}tU2@a^LAow%sT7y(>wq^{JQ+=!&G zY`V_jF&@9v>8L+k?Ikr6nkQeghY=+zV=c+1N@C1vY;FhZf?5D=O;5+E6mek;bykD@ z*xOP+Rs_SVAqFtme3$Hu8!m_{iYGPLha7bn`kd#V>)a@<LEE?R7$*3QmvPk-JLarrsLCw=Fox9TAqKuRl23N zxgjm9#NS22-C>Qji(G$1e;Fb533f{BaydNXvzu8Ugd4xM zZnn=O?9t%w6=Ch@NtHw;&U(iflEe^4x6(dS$+|$TfcE7vLEXsk7H@y(sEBp1Yy%fB zN&cmCpgrT_0UXgI-~%_7a;vKDaOh;TjQqQ1NBy$zT<-ZLW%;6_Zb#~xDy>I>B{&87 z@yXha-56r`DNE<_d4YOOZ7=GqY6smfx*ua6C$Q!bg=lD)n=wLS_9@BNSzhr}xWh0U z06nA8!v{{fS^dke{Za{ak&uFJl-c`;HF8LSuEqVsuR)(5mo?@T^~|(4#f>Fp-G~?0 z7c=0lG|%Xo?)hLRugi5gCNae!a^X%r>aNAQ3R4D=Rd@YusfIca^6v*>S5E0H zWPGbZCeHOjTtn+$Dsk0*>(?;vi<{HfF4efEV8PQc7IKrQlqHyIa;NEfB^&aWp%8|;a z9?;8Ww1HTEEk6}>2yIX=DODyB3jR+s^LK(D&E$DFC>&kDM(`d4=tBM|sMHIS>X-^B z7_EX8q7ZMOSRP?ZBXHoVcwP_h%sqYD$i`4`BIq1FwA1l4s;uv8c-6r8B4eWpztq>^ zrPjZMwDM($8aW`qQpL*(v|1Q>BI;3!K0g(HFCE0W`8u>3tWD1!$Qpl_Xp~Vi(52+H zL2=!c*ad;)(%pW#o3cFKLvN-t5d)C3rX)zjIO*%NUXFtT6NX7QJQJYR+CUNeypm>i z99Ej{`j19A4!vLQ5|og*A4$wM zE`hkitnf+lF>Pg4sxo8#RosEwS!uKkw<7s$q=tHGMELdB<>SqqmN1;AbM8F=>$TD$ zQngd@|H#a3ND7^dBsqGI@i>e=i|>V<);2XmNsA1q&iD~i-*@W&E&o+L!zOs; zRnVU}p>c=8XPKTJl6@@{@ zQ5&2kGLgI$mXG>PGko-{XT~ou{i(gQo(0*0h$^{LMV`&GpV2NJm?r=rLnJBGUQlAu zb=^fnne=zmPPG0+-QClKixxHJGqMNl9M&lVBXnU3gbwu;Wns;YZu)8Nf>E$5%G(%S zJ>2OQSpUG)lh6s+CDbWLCi? z*-^N^p8@Muy_AqQ8}*;V=oP{VBQY`Ao=MQ+i(pIq#m%wnwuH*=ki}SNVbLI<#c4@4 z>5_zQ>*<~tzY|I@kgC^64Y;x1iOX^%84ZXi!g8^8D}g#f_r{SBKDrw$AJd+vO`HgO zpD~=~XTA(zK_}!Jx_-HK_`7tQ7w8`9-S0tAa0ai;xoq=QVK5A789}s(um$AheH|*@ zvqIP(1zoA{;~6nnW_+}2&*j%04ZNuROEM!Oh3hoj9RUGGO+@EBHueEU z`GtGDfctT_lL9izQ)CHSzwBk~azQj#@ns+1R@(qucH=EvO}MCwGPKI2MjB$YiJ-^H})Z$qE*pK=S?;Kp}AJC#WhU8q2D!nZU>x%mVyVHxCviT{Ugs$Njw@x3RAe06hB%nm;)(ye*Tw?xE`}i#?dE|Qs*t?%I5X-VMv?=G_c*a^-{cb*2NtqCfT9>!k_cU ztk4`k;Y^7AW*3`Adr{Mm1vy=C#NWxpJj{_?SaGyLk>@CS%Rz4Z~*dQM{SboP;&Ie^0-NNlnQMm{ojZ zX=;aWWZBomNOj8EUqJ*RQYMVI$a_L1*_78{gl_R`Iq=J&1iU{VVQ7-1GxZm+my~dIx@Fp#a1mKKucV)P9$KoWaQl!R*Cp2Xig)6Q z91ZP*;9?if1cveUo=69NOMU9Rd{X}|0ja~V5$hvBmG0?8qn)XRNDx^m2KRB7p5+xU zhb{zRM_YgnK^gH~r1Eb1U89aPG!>f5L`=x38D@aPGJ?L2!Iy9nCNPI-TUjor*QWm{ zb{ch-A?^S5n(9NW)}H-zs@OjLIepDQ);O)x;tFo>OREhMM?%s5@ICx8GLbk>&!eGp z;~swlvhC8i%)_-CF4n>|cw4hGj{M?OT}j*^?~Q|v!pBi$5NgY&YG7i_d$dGqzNmS$X ztfXpbR3Ct1lB}va3F~RBPNQG35%fMm^+S zUx=k2;=>Hu|5ETnd-^^Crst^QOYJ6v9y|%7>L%Wrx><9Fo&b2?OJ8S-x8UGcBnBM@ zcEQcQHl3crt(gDdOpbQ zyr=rG{B9wPuIL`ONT+P#s-Vf-G(M^*y*pl;H!KqbzLde)T=AATdmn|xqqFf*J?5Bp zE=Pzgy2I^OW*cP$=BU@VMEI#Kp#=idLeuDIF`FwXIkaTQ6`Di->j@+N>V9jS(A|X7 z9MnXhjRo1aul6s)E~WXfFECj#edQ&L@g7C%qyB;{3N@;=j7^~l$imnwICM>ES_Akbrimnn7Bx+UwRk3At9ia{_ zQhr~oqp^C6^oYA?q+~bUUtb>-(Jdj0wAK6@j|RB$ZFOeqPl0K&^mn*kM=t)2osFz? z_iuC3HRW#^{(;R+()+w@2*KcN8SJe4W9{4rO^=Zx7&uA~p0N=_b>aYJv&L@v2PWR7 zfR|`Iq{nF%I%)cUpOh7a{#433he|$sw?m7eoC}6s^#w?o*YlQS`s=h3OX{pb6kZ)W z^J>-fXl2oyoM|j)G_1%pnKLt4D}pK9GwHUB%y92L<&S%tLl0QJ$aD|5V38%qTVJ=Y zmY`XTPl%4^lpM2{kb2b*U6*5bUL1+uzE4a7y23XEdUbp9Jbc864zlOtKkURHhaWP$ z30Tmgl`TMIHG)95u5CAavdSP)wV;?Fo5M5*!hECZYYRd_JMDA68y;pc`)Z{*R zUMz>@`n;Bg*+SDUv;!p@Ew% z9K3H=@+%iZif{}bW|^M!LR_Lgamqhh&FAVUX+YVoEGIST^^q4pjq|SLwjbMn+?uJf zrTafsV~*Lss~f)mc{cIs4sqOO`#I4Ei=i#w4d)*0n%nlN^!@a5M8vO${=&37Uv5X& zIyC;5>U%8oX=dh+k(1E}u_@OI99ss5c5M^(56kq-J9O6kL8;62k?R^SOV6HrVE#6s zUt#?q;KH2luPJ>H^0RlJvZ&STT9a-5{yQ)VNfX`LVwd{rSrg6F-e$Gjm8JYg}ey3Zo zuWx3V^pUX(5bV9n3|g9>yz75&L)5e0w?T(JjURc-hIN&YGrZKc>9tvI2;moXlQ;js zU(p<;e+<(uW@4oiIsM?qlYN~mQqJ5nr!S&EGyWty)|mdj+FMs|IRQkn>{XgiM|Vxu zZwWtkd3$+3$Rj~F9(*xoD4UMU>U(xQta@6wTl3hi@BEmd{H_6CA+NmEACt?&7N&lO z7#Wocn%P zezYy$tBr3c+cBq1PT%wM%)bL6tNA{8%7vU3)1IF2J$EJ|nZ6x1IUIJJc)Yc=(s%LP zKxxNM@8tuQXSHWqb6t;)m1RswTwi?bFgaMDn!&eCS-SVcQ~qmfrnb2oNBezhy-{Da zBzbUcBP{p)vB^a92mCF$UNJMKO_x{A&jDueP>n52ZNT&hf?2OfK{K%9qJv2f=@R}$@+%FTA1*Lx~R`9-J4Rb{h|T5pN{C4wBa+MzKcE)S51~c_oNq z=D&oMSJ6X8AUIeQq+NSz`pxS3p?9hNhL#->b)|SAjFnHOV{C_JVdu*ewn1wibBsFf z^-=zO;IC|y^em^4ihd`ao`9jHzoN;|-AxUa9U~SwLN+D03nqc)v-4a|CB>|wQKzZq z&m^gG1M!BBN=QWNr&R`e>H^!^Ob!}DjpdTmknz0gi<#I@9SO|GhG$NPzfEKH&k1AZ zjbZ!ulCPfE+VLa*O7Y}(emZpRKt)XGS~d~Ecsg4Nm-mA&JxUw7`URm_gWC9H~#zi41-&82@aeP0Czg*q`4;*=zVP zL&kh6z|5PTv?6Z+3f1N)KI=Ogeew!CXT{jL z+~y@(;ey7}L6sCzH7kH06Bda!Sfyw^ufSkTeU-Zg^o$AEz1&A%6ZV6X;!D(I)^{{{Gk{tr4K@PC;~zW+gN|Nr{WKdft;sMR^J U*Qm4bFT?C#%fl9VX5N?oAKPLa+yDRo literal 0 HcmV?d00001 diff --git a/benchmarks/results/smoke_postpush_20260423_123452/charts/locomo_comparison.png b/benchmarks/results/smoke_postpush_20260423_123452/charts/locomo_comparison.png new file mode 100644 index 0000000000000000000000000000000000000000..7f5f7c91e81ca4dce1506f440423e88b57b953ca GIT binary patch literal 43742 zcmc$`by$^ayDvIWRCFqeiXb2eC@M%zK|%y2rKP(?3F&TB1ZfLIQex1lbW05-C5MvI zC7pxZ_rv_wxA$55T6?c^{yFoy<~2az9dA6(egEnn{tx9OPMx4Sfx%!-Nj?x )rV zmxnKt9EBJ6h_-Eb;kOr4w^y=yZ0}@XYmAXKu(y6{W&hOF@Uo+^t(~cr{Vp^~^sOJ!A6)o^%IjJL3vHvSm;3L|sp&^`2e zorCBydOaV3IfGtL$Y4m&>k%dlF}!#mI)^y~ulJrE#}L8mr)pwvc)>i>KXeaX{{Q^* zPlJ5p@0Ao2#e+L`R@(INeur+opC2fjT!_(e8YsvX`W{p5to%Q(&ikUGhK5Gz?s{LRFSTXFM1)aDSEj}nuI)cxQeyAAC7WlLFa=+{ z^*EuAQqbL*f1!NEcGM^2(w*dwPh$C6?r!v(hN-AZro_6MQE}*3D-Stw-S%fX)joBl z-)*77G^?PeF~@44)M2J4OKVX4PvWD+-(RPv5<}$zSqs(8etkC9b(+m8%3j)J-de0Z z-6^>J=iLR3M{jm|Z(H;xUHzc__a&P;GSeTQ?wguzPN&2z6EgEVq|!wEND2I{cNWSC zrLkUngy|@&%CvQdM2S#2H5~_)nf62p!Ia_wEe|R?~U- z<;wP%?NIZs47Hrq&CyW##Vys{$db{JJ01M9neTE`-Z9BCbMHi`pL?BCxy5|R@uz_) z9J>`qhj-5|nUHE?D|^}tw@|6dbC=1P_FAm7;T~1+N~ql{qdu3(m`?MYirZh)>pC3J z`*yw}W9%?Zi3tzaG)wDBSIOA%thZ6~sKR)srJlrmisd7Wy<;TYfZIe!*AcXz`th0e zuIreCS*gQZZ^0k9`V1SoHh~qHU$8&~zglFLOhhVq4B5!#$wiNPZqKQW-d^uD&ne&Q zv#Qd$JsEAMS|-<5XwkH{J+$BB973VzZleL0W7aPbc!9NXzG7qW^9>ug?coKM1Fp~U zMjvz>FD^L1owfM&chk*VU-4OATKSD@eNSaLR$(2oE7p5I2lEUTH&{eF^k}&3;^@4} zpN4BZPIHWLnfQ2(uFYrNu7w-_9u|}4xgAZy?#58YK*Q%IwmV-}g8C}A=e0(;**sdJ z?K>om@66?Q66aXaNXX)Mem#j}HSbRECJto`SDP%EjAbz&s@x^kb8F_rb!}FtYEF)S z&|}kbU#d6nZ-B+6JL-VJFvUSdh!{Rpy@uDT65x1sZw$U*--h3`rF8s8@AsbDtm``a zEG*|=dS1KSgR@4vuoFrIMDL6S-D(_!CsrN|w>kOG8wuL)!`j5sN^V0~jwfG_P~1H2 z$uJgZQNGf8$C%n8UxBdB7#+Aqp=lET8Sd*{zknCVPP5H+rpUWY70#q*^iVd36@HmY z2n-*vYr9J*cyaVpV-EYv0NJS9tZw6BYD}{prTcr^O>I1dPT#M7c<-^Zgfp(%+cN)E z{p|3{5{|p~Nt_3Bg=JT&g0W7i{r4R6{$iV7@qVDB>B{CbYk^)2q(D z&KJLXGq?H>VbA+8>9>V7_;>B~nT(w9wVCb=_n64Syy@hq@Jk-6?Z&6?u0H$)3r%Ys z-X`2*cMZ>{+-DSecbWSYUeBRhrMt3nmAkXVZgSb0ef<*d4F1mZ3jZVCMz|2xD5pV} zmCsOe!gV~>x{VJXJ^5`x#a*XjvF0zr_T_8^l{k-e4FXzJU845%2!unaYR!D4;WHvIis+)XXBG;HHg&S{~Y+}>Ge4T40MF&rXC zyf+4%^jspWe;W9yY;XO=R{Y65Mw-1MzvHvbXhRrl$=F+$UzuN8HP_oE=(?S0pyDYNN4o@iD^Y-l$1SmD`3T)u6?GD7wRu;agg1uv`rPxG@>6_;uCp$v7|F zo@|}6@>bg>R<+w{;z5*eUPN~YGAUbVtnrH*8&3aZlu2*Bf9#TSrJRqWib^I`*;Pn##k` zW^iTNDauS&viS0m{egy&+-pGtr8~^GSoNAO!-w<*%hKs&sit|@uFLq1xxS)?6{xOu zjd(Yx|BXyy^_ahX@p1u*1#10j6wYD7SP~PX3fbGg9+i({J#xyI8@cD7G_e-l)MHP_ zWGHb`y6q7r&HLN5?OIeU#GbsWW~s?j6q_9LN#lN5LSFUQki=3}&*|J>n9*I(Cyt$4M|8wXXyw>M8V^LwN?B zNBey~@wY#{9LeIdG3dsA>f+x??!Vs23w~U95^0*8UrTyN2JfMk54CafC~3Dp6ZzHI z>@v%O*=DG6B{t%ld)o^vtZu_EDV9CBMt?qfCu=5oK}9oK4J#t4fMk-~|G|-`?&vzPJKapax^?EG6Cm%3r2qj7g({;u^o|dE8mOTiyN* z*RXRRD#g(X@Y;8agqp>)RlpU$!4i_!wR#j2QA+HMsXl4+GuMFc z-U;cOd3D72V7Vgf@A=uBs_d@gx8IkJ$hCQ>osUs7KRb+1%H**f`?kbi5nhuuwXFGW z>bK^JxP&{iN!N97Y&xYf9kU~@?@PCP*7^6gW^)KzYALtz*PeGo)|3U{OJ^K1O_u5@ z;(q6aw_;0YouRr-WS3Fs*ekfMwD8zn*H*&+ju-LkDGohCq0J$iy{Rb##ddjAP1ohB z({AwGTw$MOntYrfx1$-G2dix3Hg|ju_IQOJb^#}}w`s)izZm6A>BGD{;d1Z zd06h!8T1hyqj@bJN*e=B>flRhBHmhUc^M$%BIxZ#$b{I1BgVs zKVj3+DYE&t%E|ra=3c*B@qV2zZ#1*GHHr3#NO1`Fr6!+nwNoc@ibn!yOjw}~PeYf= zzFo(n5-xXI%d*6{aukul-$jI=jpkfLNm~gmtK7avcwA)AAOkullZsw=U&W?WNmP zP^R-4*1eL-Nz3YR$eh!*y=rq;YGto`oxc?z&iFGUoixphYTa;K)aM+6%&Kys8}z7u zrLYr|v$}N0u^DGS=rTEOR5c#1wY+^+l?j@4Dw^H_7I!%^Zi-&J~S zhipXlGV9>1s&)&@=^t2#zAEiflpe=z^xMe?&(^!UV~yB#HcD52zc@XS^KoVeAoVl= zg{Dxzk=;^FQhL|8TL=`Em4pdpd;-0kX>4t%&|dKlK}A&q>;BVndbX4wZiOqs{n@*% z0%DAe+o5|Hec#WMv2GLFah_W@VytV9cW2w7{;b9_2f!#SeC^M#F9P2D0JLqxv}My< zaksH>*0%wvcBLt$EZb#fpR;M+cU=GVWD*AjPkW}fVD`$K5!4Cx1>th&_>+8dOl$mm z8&&(VAM`w33Uc`7i$^=yt9BGq@OcRfmD|}nt}5gM9@1gq&~V+b#@P6L0I})?@k>hb zcf+y)BXDJjJ-icd(5!GuYv#EC0M;-ex&@oGcdOiZ?RSGvla_S1p+_YH8sHTQk*h_h zii`I!Mqi2qL*#u?%7?5y^KvvG?~*`Md(~jMJWk@J+*;D6XEIYgS$)pK3d+l3qc0+w zZXQZDd3EM1v@lKzE#j+i@0|IkKE3F;p!JkfW#evz=;TlbM=J?$V!$l)g(i9(Vht#} zM%n3sD;B-^ror_6K6TbTP*mCU+I~v-yh0t^ljfO`{kF&NXUCSg`4_7Xvs5?&JZD{? zrTMDBmGWwq4$qlA3EUu>t#Wgu+RE*dX1U({BjcestT_!CBrOcpFBGFD}ltTn?4(OBGT0@_e%_Tls;{`^ddP+FImNK2hEN7n86vIVn# zT&KK{&TT|0tf_&FO#sD;R?0j#ss0WMaizP~xxe!)bl{AQWh1%LmUy-DO%xm#DmKDp z{Q*^0WEwYxpDU*=1CHXHL(OIK@IMeZfbM@I@DWYfdAx!}p^B#CTDRH_^y%wtXdv$by&K`E|q!an>N16@B~AS!O2vTpbfr>oxsL#smVt$Gb!S4 z4)v_v2LZxE`NUwHRgsQThPA1?suS+lk14fL6# zXg`KxtdBlS-$9E_hYpEK7$t?t)|k%x$E5ercmJ=om&l$qB*I`iSg7sK=EI9-k z0hUQNa%_9+!GUyhgu-A59!H0f1fkR<#1RIe6uVELyx%=V;tY_G2|pFEFrk9*I8 zsd?BuprG>Y98R|QAqYKexeqM}DxPe*FAE2f&!=in;3&(BITn3|X+XJ5Cq6#OR_m60 z=(A`AjLLL#aoBX~M?Aret#SkQ^y`_!M3>U_@^lOA3MhO*!i6~QOC6a&L%)RHy{y4^=rwc3n z!G7$GL^5bG>F#uS0(Ev+7xHUQJnIwwx$9icsLyL!RjhTu7m*pgs3LQLl0x&eVzon> z&)N+dlH!-gEoKf`TkCUCfJVw_XxIw66!{)(LwWCrb({Yv2o(B8>EH)#Tgg%&_O_pj z+r_D|z?ua>N3$bbXkw_#oQ}wbJDC5|Fc1rJ2TPnevF$K`c3BV>oJfxXCT`E~w{60L zn8JF@=1Gi$Dipo44wjtqx4`ELCHZ@REnmB2`;8tgc~jsG&D+o%C^}K|$q~`Urz6?S z9u$xyptIPj-8n1MPFWsVMu`v^#H0B^1M7S%&GxajisT-kzzzGhyBlM775h%f|9|*OW=;lIwtGMO&7Pha)2OUdRaqqlxMzPyyxlH*rF?Q|jHIv%O)1 z^^y;rD75Hiq=^9NScC$Fi|S`rVEYbvx@1?qljFZp<*B$jKX6CwOhbWrN0OxHJm7SB zmaKfmQb0qtVPlOfIog$;KA94i9F()S%9a|aJE21Bn7YwZ=U+7B)K5>7h%v9?Ot&WORZ?U{+i81Jn)or{wU(XX7S&;5f;^)h1@S z(h}Y-u6D|^pWYYtdB=H$S9D6f9RZaEMw2Y6y_Z+R)frd;%Di)@ITdr_pWDyUqE`f@ zC;YU3*7fHFlr@@EXQ@{X^*nY{5h;Be(Z|L-%A{6anLL?Dl|AF`ezo8t19feTQEW9V zU4FHFXba<$MpG-$J>Tct*ADCrFvHDEjIz9RMiL#i@jK<;H1v;@AO5`tXb#=dO6g(n=gZ|lgwJw|M%L$kk|8V+S^Zm zUOD2;?QlF~;%dC1n6XcYK`Y;6^C|TR*ONrZSh)ULv0GP#RWm$y4Xw~E*E8n>ut6)rh$QO2a=cV3C7{4e;MdoOOoM)|`&35kIlHo!&Yt;?`0X;~zF z2}U+2n=J6z1I|r}ql-zZd$Le$A{ZT}JFcScvNqkBqMGAha$JNb(lp}GZg=Zhss_Xv@s?7aY+v1Rbrc86b2^-$yZZty$Iyq{d7{`=84=iw1l0YU&laaFu8#Xy^0BDkj?Nb0~b zmG>54ao+XwR*2sO&?r|l8r^Pk2zB(N;Lr*?-?9^@V=G#ef63YJ1!FAWf~A338`B;w z>yTo>4%hmnq{Mmhs~yP({4+k>Ro4tG;h+-VnJ+ftAA2p*QMJFjkdKJfQHFMdjUOVX zWl^DeL}&HX4nVOq5?eGMy%FmGP7$jMB3}6)*g^75H@`GrvB;5p$26VvPEN}2lB0Sr zlD2`x#$M32{@VYZvGfqmMF2|DbVg34RA(;~(%4pkm1g^zyd|$eS(J1_R$sy z*R~jEXxSf}$U+Pnqn>`2^xO~Ear~9vYnFixXVsc8wizG@G5@omtzMF=J^(92j{mp! zVcW|WR2<2AKnmyfdyaHUf^}!_PsctBy8=C8KeFfnD1&^U995CbCyNq$I}JFfWv`=h z-?6<=6^H2N0ew7UWh;8!1?nLd$l=vflpL3>%mFFzoh(Q7X4Ssm1~>kjQ@FMGF|Y?K z%ZFzTh7=?&g}JUyDVXlU~no$)SQ<*HJ%K z%CCT$eFr%@CES}xl7OaI(*JyWrA)dm-MPL&kR*yN9$?A4qF-Z+})Dh7(I+G`6$mZ_$8>oNq{UE zg>*qz%h0(N8%n?!rJ~M&QNp2hi1hKULJO@A`QgOg?+kudKO4=6ETI$(K4rktAXEimVY2hm}ckrj^j_>+h006p!s#PJXS`D!`!Tj0}W ztW%;3wge2pB@`YG!3h< z$wjt&!KU1%wSa7r%7bxa+v?kEdcwAqFc4Ut`U++y@dI}8T5$`3IB{x=_b;kUmP;C$ zGsb=}AIbrtHf3YTt4aWK(*@vT2RN2*kq1Y^h6;fE@xnpZSuC&-fdhdMBob+_{q3Pn zfUz-y9vcHRY+~M~#YHmshtIj055Sxc?|GDv7BVh7D+N zYI+`FZhUofs5&D9vLGjTV*&v*knh{>>z_9iyL>TNf3z3p5TS4!b4|qb=K{1|Gwn&Z zC2bq<*rl@yFAkZU%OFL@-nFF%fF?}7F+B7?k@_Rww9}pCZJLW9871mCd{Y_#W;*~t zysgc6OH##<$~^Z)S@vSsW1azfCDJ*m105Bl^ki84cl^}Jh{b0 zPz@f&-F05Soks7r`1M#P5TiE}(RQsg_F>8E%>MLg)K7E2I_LDkwq}0mMK7F0vY602 z_l+_1CO}GJY1la0gAzWjE^;g3in45Eq!iDL>YCbv&VQPIw0VR7Qu3h5VQIww4 zI1SF7zsSVJi(P*if53too}-9S|BRRl$X68|4+*J%f-FE&RdFeHvN%+|Wzs7VbmaWi z&k84KMrz~vv6JRhZKvTvyXc^6cr8O5!AeMXlZU?hr7Guhc3)C_zWW=o3*R?5_jSeE z1h=17=2*&Atl96C5Yq{^UtihMuTP{hZ0nd^02JnjLVNwhe_A zzQnT_Ip4=Ek>i}p!@%jNtz6z$H&8Aqnw!DNe)0?)8W(A{;khvp<1}a%$NutFH!FMa zm7vDbndDvTXgciDqD3~Q9t;-(QfXd!H1>hXkRkyup2q@Te*q^>lpNn1A0Qp5xrnK&q~iAfB8dpL68nGqMMf5T<^(wn%w zL>8NqUxFPNKLu@1h0_oj5C?Q>j=HYv%F04}cL#W|OkwjK;lc%=-ZM~}PKOB%<$(1* zC{!n!#;Ge*c_t&^5kJ#x&^(7iEA=S;j^m$*rUyuzjDCvSv__9%k441`+20P1Cjl=w z4eo<3t}(Y985>7KRukqs)9|{_q@T=g6B(i|y0u=Q?VCN6Ux)8BDZ6xH)F zX(TH;6K~Y~fpUXWfH%_OJ&1wIqzzfh7EWNe8;3c8gbEAc#8yH*Ta3 ze;;Yw4~J4Jk0YUxuGheNbX^fHO}(uB_@lAyRl_m1b4n*pGxBE2#R+;Wdz`$*rYZ5q zA0UsuJDDi;g#Q4u3Ri$ zVV;Sp?hL%MU0wac4%U~lQ%dH>g_mOBGuY0jYbiWBr(~35puAacf=fSonpS)82UqEK zc(nWS_Yw->=1b0?yNB?2={W<}LU={X)P5Xbp3q<^kYP5fLPZ-o1)WV0hC+)5Qhj%T z|KhZ&nEYsLACRsmL@tGES*GRx0VLO!c&`=#$*vz?;kE5# zX;rC%|2FJv|2+cqrm-(HzVfc?>Nf+E`WrtOWCB}DZ2}u;!1okI8kC*W5TJJU(Ci?# zuSkFLlx-fzL-`c=`)}dyh`%k6HWj9(`#t!+Vd^}u1Tdll|$Aa1===tXxaz{xF zr7t}*GO?w>)!hWV=CW5aWnJDWcW2z3yl-d5t3@;M8%sb1py`;d9|x=(v!`Z!LB(3( zDn+nR;@JgY?u`{hhR*Y-Ft+4$BDV|l_-K*5pI*(TYBRTT;r!%nFwO|Lf#b&OXJbRn zZ*M|6wFgd73WCZOMFWnbr$U1RzEM_EtN3%0YjRD#nn;P^caTDa0zsygGb3+FPB7L5 zVio|NH$T&dl#g9175(FA3F3^aWW5MAx$qaL^Z4rHxI>3=dGpCpmYTCGfoo;<({D3Q z80~Y~7lUG2;w9pfj-_uoC7by9l~5Xfgu2S`4%(`z!IZWHKqTWhvaV*(k=bavz5suD zv&|L3_fOJ}{w&gHL8Se781V`Hqvv&O+aM{Rnd1eb)mSnmm%yQW@B{CQKYo9G(FwUB z_1U#VK0i#Z!9Rc5>!0Ex=DWx$b#=wwJZv_fGQ_};)zvJ)8R%MDYtGSr%^)^0+TMDWw2_v zTcC}YK09*Z0SIGy2&^(^K{{)XXW_|5hBEyne&m#=pEEbuHRj819yG@Z<)ZR5CE~t9ly~Em4yj z@zNjIrU9kM0E1Q`2pYP)nv}wqj=1Ulqe^@4qtu#Gj&8;B=&9R3FZ*e+SF9?6O&Vzo}00MB1fTJsHNyxtrGDSUo`ywMFyObGKt_pw;ZMgiu=A*(wd{KF(9vcC;o10W!YvU{YoP-9y1fp3gifcNv! z6xdnTIyV$@T}^01D~lt5DzLB;xagXu+7mkgZ^fbCE7uh$0z-TPN2V&SufC*eV%+71 znp!1b=jtRR-!vlX=N^dby2>^Y#R>Um*Kx^c35S9)$5a3e(+M>Y9O& z5x9e?xS7k~q?lRDAiuc0au7iSkCB;G3tYCQ92_ii-MPXlU|FB9#kqfe`sZg}SS%f< zq3ADU#OjqH{dS|Yqc9FGP^xVKqI=`p0y|{ApN6r z3n-4Jw|^BZ#O7ICLE3j{7;$)iqe$8zr%uKV&SRHfyAG7U$#^KgIxrnyDhSzKlOJ;6 zR~~r>8Mg*hMjO$Z~azVfe;f2NRB}EqbZ76LQT}7)zoT-TwG+V^N7Dd7s_?D zbUV?fPmeg{J%!!W)>Z+BFdh5$#uQZW##!8%UzD2tz+bKp9JYb7mot+6&i~F@V_qN>(}uQpup${jC@Ja}REwpd_yJ z-&px|KQ>&G`SkISfW}lGZI|&d7L}*Isb~6KXR~m9y2^JFAf^NbvV^ixQ{|+aj`i0g zD-Q=zz(D{L^ogye1n!Yn`C?8I7hczt(x%>^9Q5`sXAyU!h&f0shUt3kuC0PWGeI~u z1g63&6#B8@TId3iV@?^#!&yu2#NH~Cqb&X$wuU3+hQ*f63X}sL`#BMRkH;4a?XD_q z!QxWfUY|4f$g~Mgj|#Oqz{)ne(f|XZ{Z8WgeGr_^+Gsz6VkH>NW?W?CQ_+>6ML>Ds zLKJqRFOi`?#B`-5H(>i!$q_haCl{FQ`OumtFdXY--F}>EL(n8*;sP<&QWN`sVH+s8 z?+St2WcC3T>khZ&tqJxZ!m1%@_3?}U=*;hIcNy(Z0!Mpd1}zJ{_dmhyy=&>;&5%D# z5R6n6#r-;-#-GVsQ%-_0qO=CMdiYJMXCQqQ*;au`I|F+leYO%6MouQpEGM1UH}Ve< z2ELuZ;bP=v2=0??J%-R%xqz>j#7anH;a%=*p#qdo0j26BHoltdCzLkhcdNZ5D~8$S zBpVDj13B`}FV=$xtWG^(p7?Ga;7{C(f~kNzd`C%X)Gh9TaIH77Ju|T!^_q-Pe9WMQ ztf$_i%yF(ki?-G-r9iPjpEwxg5|tyoM20HxHs0UQ*zLbOM(g~k<40HTtE1FppTxdx z{4@#;Yui^lYoO0XgaUQ8p9+y`3!6%|XWJg=o7(mcR2gMK`QfBjxfZZ9UDPeR7ujLC zvA)wL@PLA36hiY<+_^VJK2EbD8KQ&)!E6f=hJRfeNLno`zr~*XW-a`gP@a&TK(b&7 ze7(OeF@dBtVbr4e4aI$JP%2=ElXnK0xsHKs=RQ=RwhHJ$29U+Y+Q1 zxg`4n>6ga@*lK8ffX^Z zMw*Kr(I$QlCXeO(3^l!SH)$o=j9Af7Ds86vuG$HTC!mT65Etjqj~%GgAnAT!D_^YX zcOQdi(RRtV78PBT;gamth5T}d(p(!}wKbE$ zm1O~8plOJD%An3RuSr91R(|Y1+%r-P;Iyq}wi?d&uJ2@Tez*_2e(x*CyFO>I9qrqhBkdx9@k@CL)tpT)Lz=DR%CS(SN_*-VhBWZvARMl& zxeqm>b=b_Okc=4g!O}${{lw4HoY$jnC`JZ-NiOXFab{maPkC+l7?{BUjL(BppZ}vv z`}HL3=>DF3@_6+?n2tDrasP9`0zv>Ybq%<|=|((b%ZJoiMtHKAWOAa*0Qs6W^8cI_&_CJ^+Jw9LH_PH-U-wgv0}l97!S@$^|6_>b9D=33yN7N zLNS=#dia>~(7k_tK(=#ypDCEW8hy9F0>WHUAbTY*GNBLbQUK3#Nb%?e`qBw_S=BQJ zfyv2uH;7|(B8%lVC~#b0{bJF40{9@ePz-r|c5hN>UlK{!edZvdYJ%z1sby86GRo}* zySx(=zBtfcKi~l)j4J^xJTGAq_YXJA%)8AQIG?$N511=5&{@~ zSGmgyO?6gY1JyJlMUK1RQYItXk7VC8l?+uEA?U$sNX70%>lb2)I;OiwM@8`_lvhfF zEq!3ILzY>;7q)#TXw5>Bc9=@gl1+hZwr&SnK*oc!%1d5gNwi=V=V<7bH}LH6vE&%X z-Wx3t?jb~sx{nHgdmi@q(Se16QIr~TUy(iV!+~u?Xu=AwO($8a1ms0C1$@uV6gp>) zeHJa?4;o7GAAr%ki4J$58c2r<{xmcZxnp$h*0i5X0YW7q*ZMA8INy~ZW5hZQh0@P_ESBtqBlvn*pxF_2M%72Q8b+a_=BrLRf%KdVf*}vG{1EUwR)Daq z>r}ko^4xnRIZ1#Wwg;l=YZ21#_btkWpgNel{@afCfCN!TkkkQH;Bp2b6hcPOaXzsz z_c4?+`!BLQ&rWA6{{7q@9D3{}r~bU1666=^z;H=PBb#t05|;h+>1#4m?aKh)cU0}I z<^T-t-PZ__8^p~Y2Jpk?kcQHHCHy_HWEUD zFk1*;wt^(Fxq=3T$_ui5z_$l#^qyEMMYr3n7A__1ERBH~apqOBT(M0GygM8^HGiLk zN(l5$%@{43S-{zGjg5`VZf9{i%P`@S3@GU4G4G7b&r+l&88eW9t7JMz1&E16OZXIl zeV~`!9As@Pe&wH%2({4chuSbm)-qP*OX88x%hX%OjUf3I=t42tO_hGIV83fgLYh>vy{CnNs~Cd);kFfcq#bji6BOMt6)fm|M%a zg8>ZDfx|ffL_D$ZJGsNW6KIx42keV+B7IXjvd=0f_k|&EV&|Y^5qiUy3b&RTTzuQ8 zA?W+R1Wq*EG7b1u8UgtNqTjHcKFGm>rg01xC4c;M7Uqf?!=i4PVN4FfPM^L8x@-xb zTmmCs_JKouHw+F$`u$tX1hY^FJ82~N^_?DD`8cw1qoy<$ye=#}!JBHgd#ZrXOXL_(+{DrUCCK?-;SP%h>`N`jfFUo^6v_zp4=^`iqX^G zPQ4ez+?1YB^Vx7y?K1$_$?epMXprd-9DR1*FTN3@mWs^~2I&^mJ3*GAMH?(E+}&`N z^R(Wp37YP-X@VpfxxaZvPA`YrRlO6VT>`7AU|+li?Z|P&wqa zf7kD2p69FKY7oZlw?MbCGx-IDN?=I;q<;wg*E~xxh)X)6Auv*Ea23|vWjOuxxrS2b z7KZ|ebY$_SKouJ|$dtoF3T`hL)5uRd++r5-hEv5bDG}={eRg*ZEZIn)w)F8(Sff0} zH*%pi&giFYbG#Fr%cKL#QHO#w!JzDts{nxHEE~u~G7{6-BTehHpV_l_!3715ilHe-ac=c3M6Npte9t zNSESCzf<5;xlsY1iqS4ov;FRbcY(fot={YrID+!z|>KZ5-gSI8t z63N|UcML`+f=641C{T{v9)(f&%k-)|r{O~BtOkG!ec;hbD-KO%+kpy%>$6lA@!XnD z8O_prCql}1<(Qj7i!&sjl{MF)8mAy@TnP2ok5rMqoViHHaOZz8Lbb0Mc$K6Zl_Kx$O%9)uEd&2_2 z>}UKP$XuNI_8|Ombr6xbDCN4verTbdk$Nf6ByT&+_w)G~=Iuvxvhh!euML4takrl6 zHwt|nbl`@13rH*SecA>P8VjwlC?6VRFpt1(W(K28OXaKLhu@gAa>9Hg%An^1q)9n^ zT7cs;J)8e04&ic=MI<@UkQ73`&o7r8OAnlZ-bF9;e_yGcoy8ToC1xceiuk%#>XHa-o_Vo^w5#C%sTewVJb37wcx`Fh(sfMmKL=uwW*A|Z z?3gMP$*zExfaEKC`_&Dw%F@=ha5Q!@mI3!`8UGnOH7o zPCcFe8sv(qYTMLzK{V)KFc-34Y-ong1p=`eX#TVVT2ho|fms;Jq9Rzl3Q?*a=Tkxe zsq(=uOZtzI)`N&9fJqmjZhx1R5p_e_3S{$J0G>82{PkEGQ4!G3AG?Cp+KAt6^V)mo zy8h=Q8kG%2a7Pa1p`82B(CNx;=%agFFzR9t^dcDz>mnK*SBuimf}rl0g-(JDFO5=n zl}vWcoDya|nHn>&!VcgyxDI^Z_3kW1z98klu2=hW0~$&+lC5Tma9nWFp3{)W2ZYt~p~I!1n2&ieh;d^Z9N8iw^aoFR zGY-ZKZ=(?*zU!Kxeb|{GVwwbaf)%-pNSA9d=!7-$u?Gy&t|ZBK_TXJi{GJ0UN43BY zd3K&-%E&JuZAI%iA1;0JwPi|-qZCRO1mpK0%OwXz%5@76(8@YworYIIBA&z{Nvjlo z+m3e=Zd5XIz|p)kpLXEuMOXwGHf3b_Xg=gXhXSUmL3(ABmyyy9S{hDkzj!o6nnK(9 z{$Lw|7<%WP4WRLtP%H#--mR{E0nR%y+YVw~K8%Vc2jA|!PFD-3AEv5d9EKvB%G-!c zvirS8_IYJEq#PH9^ap%wKL~N!f??!Y7JEu*RY_ zC_&?|*_js@x_S)*SsOoy6=*8|(18a~GGYbbiFQGq!9Fn89G=BZulZMuArDYWjFhwr z9txV5o;*>_0riMxzP=Jrt=G6+8uR)cNNT%>fEF%8m_Kp^9QV+F_^xePa_dOzB+ z*K~YX@6{P*-`C-XDu75Q3GDpE>Ei%b(G(izI+G@aUqV^|jO$B1iMA0(sPi_;(e3aS zYQ&IqGy8fC!^O63kag61O2d)N>QZ|$=GB=4W~vl7)m(Ly@nYT-LHATXtp>lf4}xYuM_?W^z5t497AsryS2I&0_xy8 z3xokqriAD_Kxn5?X*Wj_MdY#K=|p^3~n z{G8``C+6IM)2*E8PPL5snO7I>M5ZrR%pGr(<$P16p2k&0tX_}ii84h){|gvA@xYGf zRZ+g(fju!9{{tKU1x1HkF7w?O*)()hkmZBq^fF-+1)`Vd!dltW2@t-Bp&6Nj>~KM^ z=TkRfyeAZuRy;HFu~8Z%YGWDxDX=?jXn=aK;MHq3QF3h`ZZClD4J8ehjL$_vSYB>U_F z8;EC2NL3zhsZz6X!ydh9vul)H-3v8^ec7Wqo|h94{<{N+^eH}-SJ_@MT!oz6`&@c) zM*2v!Tfx9k5=~oDD({usrSlCtE}SwzD}r49Vss9gxXwAL%eC@qyx@{Jh^-sRD0}1= z!MRfib3P>(B~KQF%+;Tni1=V76+Gt%tMB(FGZQ=a#>ACu zKl^mT4e|c;j6(+n*u#F4)xUv>tU(OlZ&IV6VoG9dA(k?9hTI#a#j^_4((QvRzj108 z-&XNfaJbmV4_s~K;jFLRfllI`c9VIcN7rpmDqhWl?2<VS|bs7~@RjLcxXJxq!)&H#81J0V<4X?8i!)iD0QiR!|IYq)}#U}&2dyq6$JF&kG zY=R3WB_(^EOx-x`d4+LrQ~w0Uci8B~C{p~MdakWAMcE=C2$!<#1tugj&cw_Xrt6h^ z)g)=QQKzIj^slq{k)>6%?2U4dEh<)MGPf5pw03@dBz@r!h@95MUZ^!y19#2+X?wdf ziSDUIB0>>6XH z8$1)+BZJTv!)akSzn_b-rq(&zYk%(@FnTgZ8G&J+a~+__Bc76R)% zo4qDy&dQ36Xg1hLj6br!8)D5qaV^N0{6L;U>LUFRighB3q5o0|oERTTFh{OofzcZX zvvw3R%h+MR^JEkb$G)*}|Fhr1a&PyCyZ_Rp*C$KxZ|Fb7UVPiOwpIJYb$7(>QmQ#X zaXB=T;7Mc92@?j&qos$c4FiwTkyVOeb<%zvbJ+Lbsj#K{hbR8;fJ^ostVPTTOXyDW zz4y4Z{}HbK_uo5MEts{&{{@)o;{6m!4&8%42V|ymz(Z5j5CU6;f{4<@;M$%SevyIs z10pONoyAb4hgo;J%1j4VnjaWuCm4FyT(lU)gB~E-izc%>A?y4W83A45+MQR^{owmMbzIGuThH51;p6y{CFa-uhRC*^~(~BsANN!7}*lmn5 z&;&s`h&)O)Ayx2jE_inGimD3j$P%^lM`I-}ck(Kr8efBH!do!_5xOfU05C~|9^aMpkoWQkY3n-od;-8 z2R?(N(|x7o1CnIjTdoB_XFd$eb!WJ2+xaA@g;1o2NN^v#}O}d29TIGH{I=&xbA-1>?>b1iN z@Y?Wgn*q$!xa1oWyOJr36Dj+6FD)SP6b&^^$L;St!QxS19z_tPq3df!B0&P>O%GS% zb=_F?-XZz)F1P|Nt46Qx-O6r5Bu+vs8^TuvcSh~NPwdaX0DDn z48X2TeJ7DPc8co##xDGL;$F_}ghdD(7D~bKpA4#=e}rbu`4(KDzIUR+feA4+H;eKz z?)P7Ukpsgto5{ge9q407yhskBF{ulhv>P5ebix(ZAdHTmNi7{3hax{f^OhCnt)R2( zvwlV$kms0ph3gu#USW@>`8b9{xH6sSH`uXZB8344`lcGw1@A^7EA$qRL0}MnZHimC5!F1Yau!B6S7g zSOpC=TPWNUiwi-wS|320bjLa{{s}TyrbeHb(hG>#)|#R(;LS(uN}xwJeEJOx{P4wM?nkya)R7yg>M8D#WogFT=mAo>1DqLA|?t1r(_+o>p|Q|rB7l2t4C zQ^@ot8ZS;91dFXYh@N@WjCssIK1-#x>dbxJRf_9tw5h4*N5tMdlvE}sg;_YYU-hYV zaEDgqWaNil zPfZ8d31T8%^=@)NSjbg{@nQ%ybj|dm&+S1IaM~Id?;F;`b%oc|r{2qyL$jVJ5C8}3 zIP?1#cvL`t-prS;_5Ay&lMi5m`N+dfoSJVUQ^xN>I)VV0)oS49N zsDSc>PLxrAH>dBE_>rNpS0zNXY-N8G=a$hp@(|8OocxgrI6IkzU#y$m)R`Gr5{_~b z^D3@!#J6%{KX-MMo;xSvWklod-rnHzZt2vep=i{z_QZp|aLy~eC$AlZ>ii=1vzd_H zN%}n@-bB_g!dy(j^aA7Q#6yb^{r*<_it$fiV*$xN`XCs5;H066xeVWhmD&0(Lvo0$ z){RaA=tb+-wV-Bd13x>el{7h~_>VumqE3z^^-kkfx;1j+_eC}YB5h*xB40QhS z`J7dHCtrjdrM{Y{dy+b2C8)73_4*&^RoOdzXw1dI*y8=6(#R9#3{n1vakg?>@7~+0~u?BUEPzin$clNoK`{~=_S&~m#&hAs1djj|xyPSJPN_#Yl z_blWd&vr$M1`L9Zw8VVs$)ihJYHatDExFA*+O;0apP$~2_N||E-5tJF;kks=9f;^m z%x95d>OH!LKLxQrPn+Q%6kpSA-lq`JNZh(P@m&kt4m<1f_ze&YB!OW)qGixpTO%tL zyG-%xzCg%pcHgrSo@a_kzA#V*;_J2D=nqxTgd`Rshp^v&1NyZrBI1=g5TqbnAkt}e zpuk;{x)i{8Gc%h%JK?%L^CdOqmHPf?Z$j(J{qsU$ripP#PLMDrz;BBN)@H>iDRF0& z^6UA;#UAw=4pv6cairgXLv~2%mBp9NYG?}P&7NzLo(;f6$Raw4^^Ae3ztl-9%QoQ`7fRm1#-*jbW{*Qt+bTb$UCPcO7#ZRolyg6 zAgdaZ*rbqUN6KYrk$^aHPm;0?h3qZEnLHSAO^fr$I4~$=b9A_U;GtXwLg>; z-*vhLx2KNxtKR?LSHjfh7TAN6I7k<@@Gxa2o1g!x_{t!=JIfAeC%!oN+lhn@_DBzu zfF+brp3v50=f{gqG2b(_^&h#EigNh<2MmFEAGTiek0nCs?o z&L;-tYFC5z)eI;$x-TioZb#9O6^0j{6M37MZZDkdu%Z2g$3X%@y5P_hT0QzFilf2@08u&TA^fLDmqDIQoF06HVG*Rd&tGpq0 zM0bqg4YsaQJcQ=sixTt+d6NvFxoKiwucFTy`zZ7DjxNP1F+&rFa<)Sq5Lr%A6$wc! zk>t9__dnWu^JuE~|6hDl8l;d6A%rL@oXm4YY$0Pwrf4#hOv%g`P_Yn&%NKzUF+WUTX%ie_qV?1pL0&_={>z(&*x)$0`fScH+Wse z7l=+2^$nb@kw|;hx7ea8Lg?R8B3ZujN67D*5=vh={NXv!!rHE0tfbqh@922KV)H|| zJr-B(ghDlcD6NLl)!EG@FK%x6z+YN_%q6Oo`?^#%$A~feWNcEnR^`4#irCh<<*S`y_8zqGby66Av+`fv0sx=eBS<$U;dM>!@<9-mi)SPqP}NB0kchF z@h6;wht!re<$M}B-?X^?twqr}3)C!t48nper9a4d{*8Xp8RN7%X6|D<9d7s+Hg4M` z|I<&ynumV@5ygyea#US#;8w!XiLNr2kL)a0%F%g$#~f@A&bD^9&O{Mq?L}*IHRML& zg5AZ=)@-Y+U3-?|6J`0uG4NKQ*dZxn2wi>zmNsazm{QCx2bMtTDAjGZ-KGb1#@_h1 z315AoYXlPPh4sOAfD|`!9j@9&)hA!r0sPE- zB~JM1kFNYv8enf$Z3Kl>r%sTf~puEg@qw4Ilpdzh2lX{OJD;GxJ{{DfYjAJRh8b z|NFac>p?24GXiOncK;06@p=Hykl$D6R9Z1o2oql9!IMN~ULY?@r{j-vn%MEbpg7vN z(oz*uu#YE30)p8N2$v)Xafn~3r0&a?(0rwXX6P?WnMAR%*6{JEZl7vSSjvQUJo`QBPQ9=&)@HG(W)$lfaGuz00wL^x^Yegu)2S&)ql zk+2g(J$3`d+!^rOK|S$7_Vro$Doyv>YH~FGpbKzjDtFj3(e(U-YuhMa#^Li*i2eYB z**?EFptDIsv%d`EX@|lZ=lO2bQ!?)9g42ZZzgSE%4!!amG!*FNvHVsZFL^`kwg7`$;FSa@%M;|LV#Hf;Lx+rnDzf`}kH6 zm}KYY<>jqRVO9|jh^!TF$R4SNbj36f1@p;l49%$o(Nv~x6f2wq|KR+?MuZo(Z9?5q z91@Rf-L#HgM=OY_qu3ZIr98SS!tP7jMuV;~nVR7Px81XoBv}wyho0P6%N?vf)jvT| zQA__N#k*`{&Pz*ayYjO-bGOx!?deC&KWrQSrH9+s+Uy8HeG>V(lxbyyb>aX$4J}`q z){kj@2A%OYi_~KjnHz4x_gqTH*IOi*c@}eR1xuDCP%ot zOAIGNg_BH1!8Qc{B(Py>145EVUxT$n4tmYwC7xoz8#Ro3VB=3TExIH@{B{OKrr2zO z>^vR=kX(^a0Mc4M#24fK5U|YnQi30-dRbD(4Q|}CC?Ty2_>RRx$TuF42zz{i>mH3x zCaP0K>vi^7sF9jpe*Fwx*39K2ROXEupU2IGpR_}S;0|#^M@Wa2*UKp^|gh;v=w8s;GuvmXIQq-;gL{+bWm{NIMn% z^#uOYxhe{?o&E)UH=LpVumt^el_)pH+b|JuZoJKHAwVT7_b7h3|GS*$yDPE(<&?c@ zWj@Y>pU-&%4tNL?;X!p>`c(=4OmdSO8v&uelKu&xIYvjQlIus4Z~_&ao`lG zHR!1j>NN{Gc~M6dNrxg#L%?7$DmMYCiCWO```NWTuDL#1tJe!%QlV}n3x3Omb2_8X zXrD@M>;VB@GvowvjcTBA=AXDb z)x!_`@zn&nCxkpZOriqfcage!4nAzVO1&= zoDPebTz4?T&S=!~a&I`s+5;ED{8R|_2OKJbbBxyKc98PT!s?X<3>qe>z3~2moDu3A z-sC|>j)(c5ub?Nu&N^NMkq{ZM=Xbo)?R;WBBagD2f4ogK07!(YM7CwlI?;r>=q3cv zR$^SpDH>hvyf%R6$j;jMiF%T7LD%|uDen_~u>s=jlmnf^j67hT51){*4<@L$6g|1- zjhSIUP#xL@^hY+x`D{VOp^JO9#c$xK(jb*UqItCw3F}HL6cP`TQSEAows^R; zjB}6e#$TnX#6UQ@-WzQgXpV!+Y)j2FBJ7YQtU#bEH4q=eOgTG4h4=~~S>I35JPLDk z2w{)mS7C({;bV6JWjZ>3?sJm?M_k>Uec zM`p%oA%3T-e*0+xZkk0se{a!sEi=h%-^GF7kZ_o`5*?bg5yIumCPTPL93-Jxh@fr= zPdURX%D03|E%FZGfyUe5$?|`k{haU=gaKrL=9(H+?T~ZDL^_cI48NkTEzk+aVa2%( zq!TNcp@Z~q2MfUVy!`%AV1rfFzY#C-^c3|z};IQlPNk!Vba za~=J3U=EV}Py@dTvarPxsABU^?)n?gkH04mAx^DF2JWaK63CLwJmIEkMb<8NT~GiX zl(@%8-Un30zVGmZJ-uLd1zGIZYsdPdFrfhM4$l&aP1IMeqw`0yj9L$4y8GI9xaaO5 z@`WY|#f?2!7bU^fL4jBMKi+^)u&YpGa$t2l=y&k%a!ike+EVTZJkGclH=MNs66POH z1&x*D$}sf9=|ebRg6h9TJv+qIxBewuYWo~+B~r7LuiCErE!H`>2{}2lbz8U#PI|@86hu zgYMBZAkf`&{_KG86aInLC4v`9)UO;&jAY=$-$#i-juSG5#I9@Te8H6;wVP(Asn7SE zI#lY|1)}(Khy*+5>h%WH=SV`BRvMI<10Im=hbw9B8YbFfII~W%4en6>*(rzD`(tTW zj;ekX21EuT*q*oxc9=OcQ*`If&FBV?<_UEYY;n8OoL|wtJd@)U8dT< zrFlWGCy&AfDOPpTr_w#0!j1P`sbxxVQC7Luz0#?R0m|-~QpuxpAkyS{!4by zn`b-q+n!17`>tv_?AXmaj zuC4iMrxk4_i1t;w{h%(EdSYmL($xJ3HcS(#&#yxKRPcrnaGE{$KK+4J7WLI{cjDQq z{AmmA8kQxiSiL>)HtO8G06`7!=iz?Hox66tAiD|FB(1RJF&q@ev`=)i&haK!-+=KX*%3E0oQKS4;9>j=jLzFnXr4d&+Voz|3-+j9LG#B@7xF~>MT_A^9{Ac%81=y{3otrYXlWW{ddQt z5P`be(*BQ4lzZeOP$E?@A>h$R$6}y;zHDMk_d_+s_0RHPt(1L^bZ&3HY>PG!^ z13LjF`EC%qQVjeeKBXACiY=|}>~Kp=Q6e-93W<=}gIUDT=ICNdVf{CZ%tBd;a@{w? z9Q|!mZ0%IHfe?y{qC)S5U@R3)#fLUEN?BeB)&nwzNkTAM!tgZ&6rj+G<0VO-G&vi@ zO}h)Pywd__#%n5 zs9cc(3>2_0D+Ao}AUHP6squxo`v5>m0uMou*-lw-vt^)YYZsVk+h!l)hgw+!Hgp|O z(t%~j^2&JQDF*xLZVa|soC2dnTrjX^Etl2lda*P{5CY**tp(nM(!Bow0WoKyE z3JNBBQ_MFrzX&?Ef}&q{Z=S750j$O&8GohTsecB*Izwn@EF(=9#FEr<@guNDXfq5P zrd9k88>?Ya0m#y%+-W1n69LjJ_BH>vZFT4%hGg1{ghl3aLAbr%;palT%%eQ5HN!`8 z?`CuPu?@Spo6eM2_$_D`SpGp$1NNV2s6!rP z@}b$c1rdDq=ute5S^d^82t54|CtAlX4n{jXHBmYPVZ;HyuB$<jiC+DvTVa?l$cM@c@leniu2Z#US`4;e>MOZ^NZ zsL!fUYdVp7XaewU<27$1x?jLnFHT|4_VvZ(Gv#>*4B)H;fDgDGy%6#TbTr?2!M(%y z)Mg6)`Yb@_S#Cc?t^*{)vS_{qB=ux{iUmZYwxS#nv67Pz7Kmc;Q=W}LKu|j&I5nrq ziy(WEK%+DnBe6PysNx>uA)q==Kr5e+F+|-?vr{(ur$8YqH~bnMI94){F-nEvFaQr{ z?V%~Y_-hk<*c}Xo?<$*z85em@KR}Sf>ZaUs-$^cLpzX_zSrh+msISj> z+d{1y+B>WU|7^^`e+CfzHzNq^%71c^@}FBG{pWo9r=lm=!1*uP{HvRCV{=#S93dlt zUKJm=z2a0fRPV&Um@3zXj-C2@=fC|W4IjSf|5erazn&6KBf{sGj@|9hnzX|zilZv2 zqZa_^%K%^SiBG?~035=ZnP3ylL|CV&l6FfPS_DyFY`FTFTIV|h4>S1@V#Z+=a--%c z-~c;RJIw;TNZq~XhFb(8AR0XSg*&tiNL23YZiZ%sDv1ZIN*8AdQNcaX_)7mPwVHc) zJ$28xnsScW6tmRjb;=aWs-)2 zAaHopKr>?fsANRyii)Iy&@ zBcxn+yeiB3>pP%()BOl#(??N}{tWHjb(L5+bt4U2Ley}J180ENsP21UKO2VTMa!;e zy`a|rZ3gg4$Oai)hJ$=r>-L!TX-FP9l@0%$3d&$iD5DATtUc|y!Yd$ikjw~@{J<^X+!L2;S$%gvBE$3?%|D6u~ z)0obFow;XGv0zr9;+@ecroLoGfz#`38ETt7Nn^z<3CTEqyVW+C>=^%6lm**x^iIku zj^UeG1%u~l7?!&g9VXQm<;Bm#`(+OO9x6?4JKM)Ua}TP-)*_5*Z{H9Wd24wV#7wBvj;+ab^{0Hs=~EBLdIW zN*Xd$X$O*psSWCgLFzO3M_vibJD8?p`)6GweEgTFu>YMg=szrt)CKpEJg^`d7%_n3 zY1%+s_5j>HLhLy2NGOnA^^t`qMl~Z`k2NM5+E&cKtZmMV=+lR*>-|*|F;vJ96eewO ze;BV}AXhvT_KDnUz~!7-EP=YDqZe`GJs3zLVYLAwxklPCK?eiV5jc~C?1kweb`z@r zs{Ix~*#o#VMF8Kg8^z*Z$I|f)s<`Y~N5X~g9}8yh@fgk#4sGi$R87Q0>$FF4kxl%Hf4Ev8% zv|jXFx#f#94biu<3RRe!&!tMWO@sh>mI{s+U7D|gp6@g|c22)eK) zu!;fpIQ6XvQ0u|uAq4&$8QX1WRUb9sb76}r670q>2eUpI@BbOGTM{37aI6qMcZod0 zsz*<8{=&J4cU9PFEZlZeL$7hT3!~OvL8|ip#DvwRovJ@Vn*Lc zg8^zd|Ar@2yxNDRI;&b&PDmMFzx(%F*mYI9;f}1rp;!YAU@Dp7-d4-V2jrEVZxdYT zQsy?v>Wf!DG{}SAX~pbA_Yy=49Ky)F#qOzwejwYEaDd7RB>?qLyAZ_2yzk^iYF<8K zx1#Hi(C>*m5rEIY&bAp{qTo`pL;fOlU~I_)B23iO~3m>LK03025&Nw1no^G z$OV*;W|b3Ukoe#Z5how2=#u~p!76K-mj`$Cl|bT(9`j5%2)PwKmPKf2G72trF;~^} zvAGSMPSgMWA9^B@mZR!OP*?(D+6l4W36*R-6b_bdRm%R z`)E3|T*vCz8)O&(bSPZuh8xHVu=&L4A$O3C)+dXVNsG{+5Ja(EX6-YJOGy)$hp7`Q zFmO3}RKIs?y9%wURoLL1DBX3|8Zlp-Yl$V;+8!tDWT|6xQMVy)kcGxnH zKX4wh;XBTst8LAd4g={UZH%h7hiO8Q2bU^wrh6g+nFR0(tPynY#bEDv)J zMqn^9DxggKqPKCn2Tgx`)k4i*gMndwBwp9P_{Lv9UJup3&6?g|^>r=J;)jvqlm~A^ zALHcIJs!VHQOVD3mJ+l2L@x_mnOeHk#7^Zp69GY#I|FY`v_*l1IYm{7cVnQ?-h|yB5Y#FZ16&mgwQqEOGz^i`Iy&g#IEbhIYU5t1}vq51qhh^lzVLb0G0?`QZb4| zl(xSZYK>Ei(r|O#_Zz;sih%%&a$p0_Fv~Wb01;7>-^0-QsI2yIm3pj`fv4DFNj< z3tyJu!yDSh)PlM&T#v%Rl04455SwAQ*QZ=VoFl+urFRg(n!MSKkC|NDNP8bL?FvXu zG3=Oh1bWD$G=u|YenII1&UUZkDLl*s+G2wx8oqZgIg+?4)+2NxR~=4ii9b@-YPrd# zwP&FQ$B)P{0FkXU^U#lK$$iRRwZ*#WHfRm=S{V<5b0V0r`gh;P_~?ZILEr0(u=37W z3LRG|BIqeFH`Rz;HbxR2rJv15ogwOWcx9p~QDI1=eBoB4C7Zx-?%uV%EN$}DVxjtE zeFJ*h{zJZi9yL)WZa%q#2HW&bjpvz{xSzpD&NRS1a%**`2-+({GhxQprcU#^T)^cP z6N>7R2U@a~);?};8>@c*@Ksdva+r#}MkTx7D+(Z(BCBSW+rOcGvK1Yderzh>s%ZtV z>Y<-dRMXq3kfem|eONqhef&7=X&?9Bs!Dnr#F^1*Igj?aFHi>Pe*!JhGq6f$je$*b zpBJ3SdK}&L37b2AgJl8PrDu z4Y+x{$pbw3Jk&mJi!)gJec2ttZGdsfD{{Qa^TzXXAi;YEx+u{KUnp@!a8IqrE$tc9 zrm+b4o^O00BX$PuhbUQicD5?mW0@etuWTvm+Z7C2k-Hd5d>&X?%SD%|L+&8HXh8F^ zR`Pr=l)n2$fs+4%UCgyV*;qJ3rx~N4=KCwI`3KtyE z(TP3nM%0<-<=ap%)M4yIXC^ zU~@2w$uIrZ*^TLP8P>}vv>Cw>W(OU>>uym{#O)uN`}Kg5$nq0NMqK&AZTwyT9=ox( zgy8-`=Q60wM5G9zDP@=`i8YbS+#I@}LT2J#d;tic2or2d($IITC}WR7Wyi)2-S<~J z{~q8zmvl{|95rKi?cIg2_8Teq?I5tQn}12&E;cx@1;x8-B?=-7)^>>ohoTmCEy=33m5e)1g<6%C^(iaa?#`UJy>ma{nj$NBj%bwgQQDCV( z!eh=O&=W4V#y=ZgtCimT3kC$FzFBtkzDCk}vM?a-OT4A>@vb6dV|rXSSur@xxq76uI4gzc4OjY&#=WgxW-N$$g2y2%l1!-8en&*dqSr`-tY$9}odl=f(2Xi!W zHO}_Alu344YWFiG*QHp`L?o08`wB%p8G``7=C&z{KL%h^4lDcCiqQp4ZM>II-V0A` zlg?p&!YzJ(EKnSU*DkRb&Z(`t!!g3eubKCG^`&$Wo$8NCE=ftNDV4#6WY-lvCYHH1 zxOL>R&H#YxVysaUd8onGT zuiL=z-p}J!2~kT$QEk2_fj39!or-^GxX1kCXaV`>8pY;DlQn{q5{q6n7r3W9cJ;X^ z`QCd2wQ3upErF%5ILct&Fc}^JZLv3#M{xREy-HhZord-%-eyaXRC;t(LP@E7DvTtS z24WzyB*8UGvAgiclW`sWBug%}<5!a|ncMCZ+1AUnEsH7P^y9tjZCmSU-nPEm$^DUu zNsnQtpyAVKRVS^8r%X~VCj4RhooFTR6{*rmsJFK_`T5LK=DEAQWZ4fsa#8|_VDyQX z+`RUC=j8b>&Az`cye$Q-P&IoU=|rv4{R!;G0z}hEC*HhY zRZ3EV7anZ{WPLmPR-?z0_qmyStb?j8b#!zjHx1IJ?wI&?kS;o>F_U%v8ceE7_%U->&)GT(_$(=?gZ!vZdg>(@Z7of_87y1Lzwf(JppR^Ho84v6`cjCDsfa>w{Z8~GL0oaNw^ZTbd&CF^N1-nPTew^McrqUD)6 z_N&7r%j*)cDrL>R&SlNbwgNY|dYVq@F;(hYNA8SK{HZA}!MpF;P{o6V7*1tTLOMNZ zqjOF|!>Mtnyj&n|TKdVjPy0>=@}2{+OxUg(*CKy<<%686DvWF{MnShf*wCE*YAHh8 zLostTgZra}5>+Fp;dlDnv?8dZZmA0>grNlB%lwt}j^jZ z252<0zua3|Z&!k}tDsN$X_~#p9MRq$l4io2HTGn zFG^ewRR2CpZnJsG$W3RHqQQ6@wm8`<{rtOQSA@9*gq$|9MenC4u9vp|v>zv=n)!bR z_k$Dn?gM%r-sI)DG&{BZf+@#3=rk+SMSc}$GicB6iOaNkS$*;D=!}--Nw#1HOa8@p z&o|G8QzzOU%DMDy66@$&d3fTdlb>|-=doUeI+gxXK_BVi0xgx3X%S1#9qtZap9eJC zJ}@fLSrJ|&)iULC%H|)9cAWlMS1j@(bOOcCpFoYVz}@q_&fbkY)4cwX(b-n32=#Eb znq*?RQ(AZpUx$wsDM~8}0#|BV0u+}NT4ro}-vvz#?K8}%c4xcIJ<(UlEnua8-%}=C zBpM1oWd)dxA<&y}b>&ue%>E0DtFYs+6g;qeG`R7Orh!x{ujuN%rTGb7zl@jLjduys z+-pD6t!AawRLFcZeUsmyOTdL+9=7X3f_SoMck3tg$;a@Cfn?X@YHN7_odBsulP#`l zh;gF%)j?u%b&dD5gU>g`+m-h4s0q%<607vP+oQxNE7flF1Rk&#e4gCMCC@NqBhI4q zGJ9C4+_1y^R0H4VZzpmXcxd887#e-aqol_e7;%%n9BTQ!3g88QoVigpgHCIeJWjeA zg$l(S-PVb{^O>_102&smrk&t5_7a>Prcpn!cj@NUS^peW&P_2!f`{L)FH}6JX2AG% z<(zm_0I^m0W^uI{n?sSUl|V3kB#5Wft#5zYYu~X`sWLyNvF&qykB1p-hZ>zt$ayj{# zXAHC^M}2OZlTk8331Z*Wj~$(c$rE6AsFu8@OS#JXWSO`oF&SEaC0LD^vyF>&?5UXX zj>v9<{FlxH^{+Pd9Ib$W(c)iKv{661A#Y*3B{^#+@u;WTR80`g>wt822^9K3xc^fJ zNDRz$n<9&Z5E$BL^sj+52?bU<1DhJ>iaK_^a1?F713=tGhgg+C0J$^T(VJu zJayHrH5R43ZG7ZqHLsgD0{AdsZg%cWyn2qHu|lw$xKI4aE!Ahf3c~K8%zS2sot9>gA|n)~wkMy-uizL^=H@!y5!f@_$-6cg!m;5<5PgS;cKpwT zdxExet@=+}?Mh%mxq7;k(*s#;+S_M!s;_SM2X?|@h*HGh{+RQjdN&$TxJUHuruu3M zHPgxoYwD8Gg+>`y)M!xeS=69IIv~5%{KW-SI^DZAApTaLO6}1*zf^#@C@A?(# z>#dU+J5py+@9J2&XJ)%Uyy zuhppc!E5`!l%z#2ukSdosD6ZHgO0pDwdNb|Z%Zk;@+Gh(UW_EV&v7MXZ1_kZlR9&tzQ ztdt{u$g+jcu3Gi6hilx`{naPS@e6Ragpmi1!{x5zRugZ7otjeh2$4JNu>4gv^?G*M zdx9Ar4g-gW5;n&qt%mlpmCavzP}z5TghAxh&bVrVuy%s@?sCHe&y)*eM}A=(?U|=m zhXr?yh7A#Y-@SG+kG5+domjf)k{9>03j`Nz9p8bVw~z+<=M~$KLfP|0rNUSm9>I-r zK}WfGp44>ih-v6udbqE==bUa>=Rn#})4bqaBOh&o78gfoNzr>P-VfU5+HLN~S={Fm zw|xSXX5pTig=g1_yU6;}ms-Ck`uYQtervE2(2%G#!)b4VTkzi|>;2-k*~ETJ z!Nuv-gz1~j`vq;^szzm@#-%CM-2M>5jcQ(>On6Ha+08ugkT@Ayy@=2e;113()IN}P zvYyq^IXWpjxSTlQEo}eg+4-Wuln7n23x#~bFRYO_lA#-5wb~2BWI3oFwhA8Dj?=q8 zm!$GX9p*%c@p*yY>ZsF>{W@RG<3&`XZyMcuntC13hw97er0c{5Cx(c+a@56NgKF{| zLd$^s7pON^++ArnwO7laI3ubk28W*t&(36O?$U}FDk?xEGL`EvHU zirMQM{Kc}$Nr1Q3edxVMAub&hZ#iL_(PA@UUZG5?!_E5$3yj3()RvK@8>TzlBV1)v+T|;O(5J= zw(;hhp=s2$@(qbp=WqOBTi;Ye7wbPu*iEB3z+-atuio(RP`B()Rx*F-aHS3z`iN+> zj!u^AtPpdKg|mxA)`dyB481zIYC!xNTdUSLgI7BbKsx;zc&;XfQ|r~b0Ynm8QrQDz zxWi8}YM;-PT9-^`A65vG<>S|YJ*^$R`jY{A)+v@+o7rwg zo{1Tc4*)I1uK+AwpNZJInBm^)^_Ce=8F`fzUxRhUuK2trDKvC$F8`A(bfEpqVA8d(fGZ`Sv}${JZ+6u zokTK#S1gokPL`|x2B8!Sa=r42uXO2pV0?ywy-g6*5JkUxp9gMT1s3@NI)SQvpEN+X zwl)@Dv2vb!73TQae|dX+K-&4{vXL;_FYcuMbfMS%9cv^;M;sjkKgB_BAVqS^n!$6i z1t&jXqG!!F3y76_hpT;7cJ{sx$tpk-d*F6X!eJ<*b7GQ!VZQ@%pwOauH?gA+xS#zD zT+;jd#HJK>M`D&ljS8+3PRUi<6!In^a4q$dOP*QDGbFJWR&Q)(>2t9rx26Zd9;0yO zfuya$RB`mm#X3|%jV_`P`7==B{e_sdn*!pipimZJw&eZ_n2j#t7jo~=#oD3w0DK#M z1`MLV5HsL=0lcHMubm+p9My1WB$>Ve@p};TxGz7=xdS!Jue;q55Y47kD?gasu$w09O+QJo*=EE<j~ggS`^YqoSZIK zfTyt51`Gjx#yqDu+q&UrzJsGT;{fYg@3-!?Ybj75PCW#jg9bQDNst#NtERZ<#y&Mie_vt@Kz$Gp*F z+Q1Qp4HB2(hw86t3~cD|Hww}FL2|N;Y{gi}mWpg>O~1&LPeqtElg1TEp!Z^p<<$N= ztyy$k^6(xh;5HZJbSyHR1&kf9`fZ35-}#qRP{ps((Z_y##=S;vegmSG`^rz2N8c78 zaWf9O1b$*PN}t?`hBgG~YJP8zHmbWh+aPn`L^Rd3Zv_T1FuHkw0nz4^_g;S4{kZ3q+|#Lt~u)yZDc7#NcXJw z6BpiJEBo%Xbxl|vp!O~AWqybq7?E8IhvmnzhBOYMk}pvBF!7FY@9OzGbq70_nH47g zIlVpNm|aZ%cJ~BZoYzEj+6n9Prh=`Gj(ydkQ!|hZxg%V0X|iyiOEt4b_4xh|c;h?3 z#&oMoXLFr`P}ZTMb=-`(@DQx6WvZ;&a&ujfc;C{C11X`p`_VUJwKPj^>5L+^x#WQD zx`s5hmLPvGO~B#J#f&X3)$<_cyP3utdgP<2H9S{NMxRs9M(1pnIAp#qMpmF?o}{uS zVST{B>&6Y6VrXw{f^YDgd}6keAg?n z2ddsD#fJlRe73ur4Gm4iROkxp%K{g;pv@;&=i>Z5zWMfD@w;hmXKD3JBig+Ly4z(O z;ePu9m1t__#@D{_X3oU#VoE~EPqBgZ9+J-Z+vyH8=2{W$T=e^#-qXft9Zp-{bDzzL zJg2pMGhrnr%~+7yb^#1QdcqQn~=`^9jv?H zSve!vf1P{vp~+;6tS>-=VVylvp{s%f59OTAr*F6JO^V>lSzFkD*2x`6KsJ>_S~lL~ zb}Q*<-`~%Z)mc2FLyrbC)al9%+%=sz1!t?Xnc)(@ga&Os2u=$_k8ZyLwvI1jU!Dsb z6J}_8BqJKzZM3lZ%e^A}mF|c1>4|p36A>hr$3n*5FnAX{!Z6j(yxRH65RBE_{$MG+ z!@K8DpvB+bu~&ZHwa7dq9@faMOW)#tJK~|MwPy6E5UA#F&Av@M@z}}CKYSgViP4U+C|Y2H7MQRjahwr?^1nt;dFoD&I? zb3h`KbZy8IJ?*nfQr;VztAf@U@5TUk?dZFL9Zwhz+hweQX-r*FSb{XTt%F;gXDybg*9tQgKF!2!+e#BLOp4;Qlm+iIJplYsBpk@M zc3i8Kk}~B5X}$D7OotDSAt!nk>c+jdb{-P4_6;kP^l=1z8G}QgZ#qZy;aiv6GIV;u zscD*S>wUy#0k*~`&I$!pUiXx|t}$$_6J_t-L!6s|PW9M#U$~<7KYVLgPpN&splXoq z`r|u5Rqr5l^Q1|kwfdJKp$muhibRnnKLdX7W#K^f;pen^A~vvdeF5%<%hKq|)8p@% z37iwHQT@=`85>QtRE4HiKvZU!^C}#!czIT9YFMC!fOnyNg4WuXss4FxjaLve|4i9u zNo~d|M<%Pie-6#4^j zpe#IT%XwmdX6fzDo;hC_*v#&AJUxdchcP{dK1#c8 zFtED`nt9W_u>@!$a6=Yb`vrM?BHp8Cx`TDl5!%MM3BXhZ2yeWDm?UR^Z$8Bn@c z5krT|otZG*p%YkDr1OfDPisULT3#QKE;V25Ar-&nY;{h0}MYy zR%5E^GVB?-a|%hxCXFtbX~mq%e!$e|I@Cj{B5we%)mJJ9wF(=AR-DW|B1rbTX_72b z`j*VuC)GKyXp;162Kp9OG?l+P0qDHI6_z&2bbQ4Ql(k&~Aj>S17cww{U{6aMvyZdr zKW=c;hVGG3EjAJnUu~~pCwzJn#Al`$=fOrxbj$x zPifuZz~9bFD#HrzA_dE6ZDmhPWZmd81R;r|QCM0=l~Jyq@sSvJ&)a%#NBm-74)Ou$ z+0rRV(K!Y=_XXArq;yWGEXhi=2lDvM{#ZA4Hz{}4)f4n%;XX3}X1MZxJQPN1IUc5* zm(93iK3)DCgi|udkO#CFGIp^L~{Wm=OJh8sn&30h} zo6Aa}?yOB{^2H8s*2=RigXby3b<`hO^KHufyWXjGe2p zzvcN&lv6ziPL4vM*jCJ@Yc3T&O*=d_!dZLVp!gB%5jgJx*PiM1kHT+^4F2VPXGhu| z+NrqRJPeNfGz&0Gt9E=qD6_3E;KCFLDcKU-@2qv@v<#7PkF219Xf0hd&qg zZ&>2eY|SLW;ckfKvf8b&r^nHE=E3lcS~BPb?};2sX7UX9SP`|JFH{cixI#V$*qz4s zo^2Z75A@aBo7mS-qcL(=%Bio~M70SxPPUBfX1?%-l^XgOp~dJlJH0C@ag<#?XTZzH zOMHb{csY;B-5OF|co@SW^%J2a?PC{N(7hf-Uk>~7F>fq;?UfwEIzZw9z6TWB(NX`Rqwxo>nDC4UELk=x2;4Jh*i7X2($B@xX8-9-pyl5kSkT;ZCi z!_Z8ckeW+~g$p{orz0Rpph^ibE6o+BmH7Tw`+2hIciWPJ2ps~F41<6B2km9&lJvY- zsR4aps;xMQ-e8YrN%#h~+hc!CuZ+nXn7_A<9Z_M+vh~+A-(p3~k<7MT9@Q~Tcb&Q^vct4q9v52ckDTpq_VYJdCkOl(fs_$3FFzt*ar#KQ6xW{WUdhYxpWpio*J(Kl3ynmNHF!b% z0Kc3jt4y4jC6D1JGR^y11}^z($DCq0wv+es`lYU0e# zNUVh{w#N5`+mz0dmj`7k?lmSUDR}h+Y~-g&up}64KGJ+UZI)LcAP#o;aXEtz1Go~#w{gRMU!E&hzZ0s^>&WwU9ksvZ0 zPQSn5kiKmra{;9D)LP$uwN_l(Dx`xgM9l=~X9BtA_J> z`VM(C9>{L>SZ|WEj;K%CDSyv#037#xmHy}z`?-$a5kM=6 z{5ux;Mw^_!hpSomifh2kI_viv$Jsu=CmP=*YH9uzlo-TcSFzHrNi?X6lkT@|t{Ng- z_PRL_C7kIJ5FDSgdSdg8*eLcN_g{ywKPNT+pglfEWtD%!;w=0c6qWt&4;qWWl?b7* z?y*(6Vn#`2w;|5pk|#^WeRXqR|%ILS&jg6k2Y%(#OsmL;Th9JeGzRg5))4 z7aLBoK6`Id6)e5Q$adyp8NjtQMOhm_2FqeNbqs2*EMvxxN4Nm7)H^pcpmW0L4tsG8ilnMDYD3i@MX7X`XSt`Ivu?B;9b zyqZ#3tsz3`DWoATIWXDFy;z2Ih;^gqU26^v*j}H5%ji5#cKrlqEzLe0IpqbOCMsm> zC+xX=XC+l0IPqisdd0Gzkakbm*BMODBax` zA{i&FV4+F9AJ}gOGMKXt?PH+Y{{nhhR;6|E2hBN2KElw@lqEwx()Ji%+~qXIKe2w> zLyS(-I#2gVkZ32)QA;B<-}*pIrFb4kHsCC(1D55A(SBfxR(DF+xcLqE5iX)UO#6JY zg8P;4$E?Mhee>Zre&hB{D;iYq5 zGPFDWC-3GX+x?ze=EA|`+}W&3jBbBq2I*c?KCoMV0i*SH6##vfzW}$=cT2hIy#OUgLf8A2F3h{I2R#ZR z@D3myn(8LvFWu-x{iQ*6WvCY5f)M*p;O}&R{r*+}kI_74*f^<~z$*<_fnX&@b3UZo z!gdZZH3qtS@@{rc1kFBiM);`gHNfiOn5nbB(NOLH4CHB`&2dh+I;$1=Bfg55JH*5U z2grw7c}&L`ulc6EhJRt-Opiy-ho?B$-vJuUFVTCBOV5701E)p#B{wZUe7Rr-!WI%s6Yc(cj@NN1sS)+M`gE(a3*9Qt0O)C{z`*l-OMr z=a}C^B<^x+yNwI-3|GFqmOI13V_}nifqOOOR#t}6HEF!Wri4XLinL$YN2Ta~w=O*% z^B!phZ9@jN9sfD?Ije?BuGi5q6>>qC*|rKR@vfb$b8o`gLNF{j-{yBPgQ%(*yMXyIn|o^uo$m-laF)?-Dy{o=;{{ z|At&CQT(JwMR0KNcx$x!;S-d!ckkZK5y0bknB<}u7-hmjLRggFL`O%%hq&kZFA|r9 z(fVu5r+drL7?*+ix91soc{L7`Qe8cC?A&Q}b@lDIi?=MVlAOBy?#W=0)xfgrsZ*zP zdjwn-zy7R!&d9*P&>tT_FY*2>bG-XUf5jEmoM2%u&zYg};)90|H>|s}Yvzk(sA5vu z!dX+D7e)y&J~oW#Z322@v4xXSzsh}dboAqz)o?{=jF1O^zw2 zpE-TnpzvK!<%2-x3;ZsNNjsI!YePd(7X;MxNnA z^&GuI=L%N!KZ$y&aIV&JRVhhQc*2-Q2yC&#qk* zQT%7Hl$%XGTZf{>*3RzOz^BK-LZ<8UzaBa|I#R4*^eSC@m!{g-yH{X_Jk)nzc!@Q_ z(wj7gJjAHyykB)k&vewAUx?*LcWlxKx%0AFbZ1hmNd+^>_5Mhd$kN%!&4s>wY4SDT zS7^^_u)NSwx7FX8FUG327G;(;o!93Ru#4DRh5r4;^6^4sMSe8GJE1w4%PvckC(E4| zKGx@Z5oVY%J$F(#^sA`UK8pE1!atn3h*?|8FzQH^r&)y;(QqGH{WE~~U{%dHx7xy4 z{en?GZqV>ThBZw8hs8D{q}SB5W9mJ++WLO?6(oAZ!u;^*8yL`LqetVq)bNC|9+s_b zWc@t+{{6dJ+Z-l4z3jHLo127o_{-zxnZCMFg0&?>R;QGt1CasDgl}FyVL{xo+}+vc z4^XI8YKs$Wy{itZ=}Vk|i{Xm@l+%bu50y@2Gq1~{ac^q8r#Z*2*Y0L~J^v@`q5cZ_ zmN53#Mw&a0vb%S)aRJ*0zEMX+pg*);PbiZQ3} z3fc@;xb)5rmA9g|=f9k4ok$a@Kcmc>g6qoGy+fTpRAPHiSY=v0MfO7FP<=h)@AZYT zA90*$PnEB!xF_RZ19QvJ_$Yj?anqT~YS+2y9Rh~*dbWP`xE`lbuKh(yG%`+ZjrEhNzm0H!pFx)=)p^&1v1odrCZMycQCw1?)TU`_|vkwsa`u9 z5+wCFr!Orn*P8R_#ot}8E-H(|cV~4rrZ1%ubCahTBUrM!_4yL``_rC;F+C|xCodyxog92^1Ght@v18tpB^c` z%bro!_b?e=?K6vi`DdV*^LKxt1;GOoFM+nKbX{4V6BDiQAwmV5MhtntiwahhPLXBg z#wl%KVd1xp{uj6#D%z5yf-`p(GB4kIbs|sW72%7nl{%e{G47OA=K|P=y!x>(BwRlZ zd9zCLp>Zl99x5LU`x|R(@9rwvNP(eS5t^&s^$f8B^Lx|hXvY1IjI0J%^{Xm|C0@GZ z@ay06A@$1_ym|8`o5XvnNmtXBFgX&guxFQ)`BLTM-{)6?fpd*M7B4C)%GQl;4q1?YRYxGZ~N_@tpjHv=|qH%Jv>)uP{Ky8gzrz){LYNyC0#Zx{rceX{B z4d=MVK09<=%Q7~_b5_p8;ZHx$Z#jMD&h7hxDh4id!-5!-N?cUNlLuKgU1^G$xbV}n z4^b!vm4n{!f4ZDwD7mawmD`FkcYV_D81HOtWE~bv5sp(;FkzWiP{B}l()4C%n>w?G zDBxA6KWe3Fer#5qXkK2u#Jf-ZUf#q-0Qn+C|DrSF`f{%*p5J$JxP^9Hh8Fp|{ zwovTtpAEv+Qd^>xZ(5mP)U#MV`$-J}8uQIaZnMm8<=C^znAD?ZnZ$Z;IsVqD)I(oW z%_5~&&MJ`AeV{6nSMsPkGZJgD%1`ux|8~mh9B{>X{A!iwJMkG$`Y$SDuA@m+-rljO zjY)o@rdev2>_S6+Sj*^GgAidl@#?_m>hItVGsAwA*cw&(nDJe z-9$$VvwWObQHkak; zK=1i ze3Ge}T7rX4`ba8-Lo=I7;1-f(!Z}?-^Y;&EdH)NI&t|hN<{SjaS;^igZ6<) zvGZqMj$Ga;nb#v1d6Bw7lRNHh+ zvMV?;HBQxQ;>u}hF#;}1q}2RL@x>NB9L=^FsyZ+|&}m6psV6C~i@=2`&(D+hrp42( zC}f=oHEWA&icWe)7aVc;L!ejH$mE=uPPVOc-p0^T{hr${v7bqdZ4z za>`9xFY2bQjd(3O|S;Ul3b=-h1w|=2=fS%LAd)X-`BCsb$1yHs*)%%lg9XqN)dBW%t`6xg|`$ zMI@9bVY0E%%^kY^sx7kE*i8s2I?A2quFf&u(B(v-hAFg=n4LN(qG9lk?P%S#w%05S z3cB+Z6&1tNeQr~+PKGy02cs`Da};$&JcSU$zoY&p9T#+;mSei@kumAQIxat&)CQAH zCoVoD9MwzJ^4B!qK#zYVt5gxz?-ph3bhccmy=kqtURI|9$%N6$9kLxtZN(vHRSO}D z9Ri;7ExK?8o!u3?=u5Wuu`B#tG=ye&y)4r2IXge~3(Z4YbF};}@+{jm1u@=;N~%$l zJu8}K9Rp#vS6064khkh_HlDk$${wWHF0!F_Rb_=`a*ic;ji0=6!HI*TtF15{t3Pg? z_#<5*;hA1WS!5ZXwqX?IJ@h4rE;DFpIRbM`1vtywi_F7fi2aHkUdqSys)}v!JY`y~7u~FU- zcbp@XmNkTlGGYbMgyk7RDULXcsV0|arQ9eTlZ_K9cbHMKkBEk(=%9V9;ZnJwIT}OU zGtceJ{Nmrd6RszR6AW=jn}q*mb{fz1gv;Es5U!N9#KgzP^DQflHPtuLv_aSxM!~K% zJ=K{a*Gcp)g8%+yq~&a1Yk zXf^p!@hM4%T&0tXzEvnCsM>NByC{clQ_U*wk}dh-b1GbTcS~7+K)cv_29Igbwt%#2 z_LWtZwtc>FQxY2AJ*j4s5~gAv+i&WNdtiR-ao52!IJAnl8{}&q72r8Mx2H`#J*mwr zL|3>IwY)s$Yf1RODj7AWuVsJ!DCo92J)2~Hcxax>q?zG4$*Jx(NTAa1eLs2W*7TT4 zOCzKMGu|t(D9VoGAT(w!O8KmTy)+v9V&aPyWV}zCWtVmSD2wZbP}s!k0XmarlS(k4M`kjTh|4Bl}Aa$N61ss zM5F{W6{lL&RHjq4&^#91vOfH*B{yG&hgIPEGs27##r@v2-yhIg)4%^xnw3`dVYaz$ zZmG>hl zb>Fay_TNoSTUh978$632;uiHYr0J;tn)#%J)?BlaFLF(8~RFj={Lii-Ec0lF1E$ZRsbinvX`rH%M!JU5>(gUl_=qulXN2RpSY zUxs#WSrx9*YkQ@~Zt~lOrS;IGcT*q4F3$x-mGa4iu@T%cVX>^|REG4k{FEb3X9s^2 z6({Gy;6ot1_sGT;D)q=7%FSpgEIXC%#k}8Ffw~cdHz!djS~A74y!7h|Cr7=Orxmr; zNpkskJX3Hu-E<{J>K#B^UPvHbGaxOk>6XfQ@%w&RBq$W>Cl&#!M4o*#J9bkwP2sM) zhliD}F8qeJb`S45`aUXMO)fj(a#)%a>(=)6%0Xx%3Z>_VSZao=8;cX2+~{oGa<%s~ z!W_44hC{$i%=DpE=X*XqR^EVQZ{fYZCz4cDRCV9KKX`TG!lKrM^^H$S5bor;wa`em6U)|z zH;G5$h1}fhh~-IE%hH-$;o#ulvY(W7K`_f}HOi?EW~^)Xf9uW85J}-}(*X~6zz8Qp zMyyy=z4ZN#Qnn7?wmT#_HkF=#uRKa*Ko!GhFMe6jH7P!vO|xTEC}kxBbAA77gKG!9 zZJ+%5e5lgF-ac3*T}gA0S-Z%RY!%^8;IO1?dPlU*bclompA2>j>!Fk)n;s4b-woDI zIB-RPvd&z1H8)s#7LVk9aM1p8g-)|*QzfG&7OC+*ocYHH&q~?f9+haNd+2A@PrWnM zCE>IzzdYeHo;H2|Z^}#1%!smWA@}umoY1Mu0_`nb>B^01-@LuOA8C`MKo%Cpb6*LK z-#DqQUfaH|E2dOi$>3?rc~|N4Hhc=-{rh)b?7H=Ksy_X&eyw-DSzC5E)*+pLk6eueWwPpQz}JD_ST=uqzt0Eh)X=Dnwo*Ur|} z0Z!dA2E>WBUE(occ2)noJe*G+^v<~sfFm-`J|P)gk>@tSH#RnQYq?WC-|>&u5&;~G zRO5T<3CKb_lBCXh2<$(^;F^@_nE=3)q#(_2-G!%A=sk$OCFi-j?X*OIxKtWqjFlyb z9-APD115n@u~}Kyt?}Ij`bxL8Fu=xSlBGi#lv8B$jT%Wy9sfklG6?{fLR5VcCL1f9ER9dSVe88k(cWkH5v4+iD+-6!MjMw{>z&{{7H*?A;(Z8vggrJ zw0T!h2fa01Z?ai@;(*@P$AQZzRM8%ef%MUTX%8X8ygyLA-H8u1&te&$%y_kS zt4gZulbEz*WB!gP5r=8w6Z@C_#{t%Vp;!9vNB;-$f)oczr{qb(`ue(L|EI_GSgb1W zi&r483j(~np)Awq6ixH|uP@2QO+i8YXmhDJfl@fVkL%95c1voojo}oI|@v*2k7LbD-QQCsVul0R^WX8-Q$~qA!m%0a`GD zWI>oVfLhhsiF;llr;k_Ji6lX_qI7;Eo&Zrc{O8F z?YW%eEeMEwhD0ldq=Ny&=J%r z8llYajAXs{RD5i+iA;+}s%;$1u^GwXKpThLahetDO%bgFlxX_sU4Te-qY9BM431F_ zDOqAWBp6s;T+A=?Hz%U>9GOnbt_D=D(`dKyZ-BAHKx&VxFk#pujnbYcfU!55Ma{VQP3`KRUL$KKSpX(n)`LNA4;B>i+znZ3HcZF?$qzd`d6wDkBygP0R^2T@emHYF~yC;-^xCYRjnF-*| zvK}nj{;VOYWYeo~{xqRm<=(e7sP*=4=Qxe{#o|-@Bjzg)vqkm17M4dHuu;#DKQKAE z*ip(!l5~X@J@1Pu=7ZgS50wKQ0qpUQ7An3oJ>C`jKfNBtK!##9FVZuqdrf9H-A=dU zie=4(XprCnxZ^kWwGbqJsdC>)f|*Cwytuf?Me+s^tB|+nImLU0oMv?v*4BF3vCK(ZITMJy7MEv~ni;x~$6urISPr~tGc2Gt^0y;T$1;$kSXo2fKEB?QaF1j=m&tzeTbWFr>)(|D@zI3? zfI1+D^ z(c_hWzD07|EsWkz*U0BvyMR_eBv{o>_IB@wk2_PyL5iX4qnIe(1oEdbofPu~jcQL1prGE8 zvU3BEfC#~Csx~J5u$TTeLWSr<)I7a>oWPISwcmZ*E+A6607SMs;MZKP z%hQzMed)@~Lmw3mdBY2ZgN7}lxYBt+6qvTP{*}GN7xGe*<09lW)B?^5#We+!pwV&N zss{YEBJc)DC}iuAUxe?!bc8(mM;j3Inaqs7zR=VrfK41Msz*FG*+th=8Dzp_!5!(B z|4W-spoD=m6Fe#j4V{4KQ?UpW=3-96!=0;DMT+fs0x~6n3q~#Lqa`OONQ^-%B6{zh z$qEj8gVi1)l1b}&aN}NChzb!oUg3!|8=67K6*(&!&29F81>{qDISo_FXU#!Jp$hO7 zZI*U1g}(ksk82iSMR*g?(=dPzg<_2bC@P&&yZdt((I$15ovl)9qBKO0KoW7Bf-L~L zMFzs|B;aNc^kHZSgjL8P8ASAf%WKEIDfC+SI~rj<`Hu-Pf^KTUU6y4K*_6JcyfIp+ zKlS2?Jq=|`B;=Bm}w(tGMV z(ue|QoDM3;(iwyVTpvoGLtb(Xkfg#Fh3svWXnu;wKOa#V?^u;G9^J@U-MdCK;|xW zQ|!OBRAei04X|4{3Ok#7AWB?b1R!5+?FRan97y!uwJ3w))PQvNbFt;`!O{YUx#6L~ zLOy;XBBCEfq$);~7wi2`1W7U6Ca5BCL?2a)=`6jpbN6eeyJSv02 zJH>Wc<;c|B9E}N0p>Au*CKGYZtCN>-*R8=kjsE$=4oMecFsUI5c9KWvf3OBtFN_e= z4ZRi?5D;*6BK^WU)mUKdIpaME>9SEbrH0++1o;5WqY`r0m`lV_byB?i`_Kg9n%&Ln zT{nwh?+8Gi?~-_oc!I0DxEOixby`o@NDVxm!I;|)+u37dfRFS|Am!rB$K z&#cS%zJgTHZ_y7AAOE}FerluoMFEIjp=W|g_JJk`w78cyHm2f76#ndAf_g4X{CCi6 zYS>Zq<(}JR2M->co-*(7%hW9BFO%NCPUNH65`{OAhav78i!<#xvXNYbUo$i5%v})> zU}s_eW{C{uw@#w&pm7ZG?u)(%vfQ?c$S1cI+OWUN&CCA@c_odBeuC{l&c{)ya?wnrq@?)|zZ^yod1QDi z*a6fWotjE52K^}hpDdf|u2It_0AdkI@ zFxrm7Aa_0g_f@YPLlqb|ebgorm-rJ5YP4b{=KXJK9$8YL)F7E?TfSphcLcL}f<{PZ zP8nnb#ag%jUUHSna^*9){AY{xjPi^Va2 z;bzbQF&ra!0SWQ?D&2~^a`g3ebaWi1Q!j&L zx#Pp#1AT=S;h(Gqy2Die7Gj*?i7rIlMCKzhDIoQx10{4f9??uEUU>Gmvl-@5?P7;K zL3rHNR%|uE1U@{_Y1l=gn3Unf-}{Uh;sv*6(mpa{0?|0D&JmBGL11#nj&EtLUj z5(9{BCef_Z1qB7c5WU@pnujY0-t+J$*ak~gx&4lyRV?VE^fv!|%Eo5y2p%924j@Lt z5L4-827nzaux1cBeNtp6LcxIf#_}@B(5*VI-pXp^ZoC=Pi}2m=nt5JYd0jpeJ;nK% zVR`2G(E{{+6iVbXDGE#Sb>3+6<*QfC2%-WG;O+GppD0Q(xnnSD^ZWua1ybKtTo|HT z?u;6kZMw8)8$V|7l>=I?E=?=E#T?Jdz%L)~(=;*QKk=kSbqYrrt2?=qzty*5V zVfdmeLsbS!Ino+;l?!&nFJEg1j!X>;V|w_}M(XG46WJF4qSOp3=Y3t7ScyAssoWPQ z5LVY9WY2LUqdI4(08M@-JomcG&Tn7dmjdUe&iN<13k7~?YUH>3@6puXXC3!6EEjI zj29@SRLwVjM?{IYH=Uy;egFN6os1H{aPXj}4w8yVegNw?*&HeZ?v>G~;awo2O8`1C z*=D4QY!w!I@HrMGF{qg^LgtdUn9whui%at8O;z^YGt%oMlrIUfc2Dfc!vjvcFh4J_ z)@Z4t1*wI=Eo!22k*CC5pn1m(fOhL64a_Rr1k+8`vz}=;b{&fKQ8qD^&l4G(BvZ5t zQzYYLMdP@t7-4i;y*mHEB%pj(In>fFe|sx&f#$UQ&gMkEd;{mjf9eBJ-X-))AWGVo zeIOI|e?lPYnjmplVatWF$`ewfK0lxt3G%gU|57M+U^r(Lg$)u4Dsy2F^DwK(9e z+sL^sRDf0m$%o%*M1Y9$RjCPW;UC0od4jIx5WJKDX;C)vr~nPWi^530!!Ua}t|%Lz zEN|6=-tdJ8710gFzZs`rY$1uIKt>H(M z)-a%q$F94nbLa;N2?~ei*N+4f^A$*A?HG**(-IaLZg$}|?|3f1cK_9*ni0azbjZF` zCjA*-vYy`?F1xQHM^BU(dY&xQRm6AUUwXkiMzkZUQdIu#SKIVX0(YtXT9vblap>Wy z&PuDpN${szA`lGTQ>YvX?A8q_{hN>iW%ne-cXV8!jg zF!%6}L8VWK?O4N%NFKxv2$hb|>Yiy<5RVmWrGgRK!x*&+&5;B_D)7q7VMkof*~OJ` z&mazu!%Yj;94rcHuMZAYGdKEgvnVtB2lG^OhL5Oa27}kC&&X!p<>$C}cqRPih#^MSYZZe2LD64Jy zAeyaoX*{%{Ra?rpV|a1;$34wr*W{9tk~cPgTx|ssfmoZ~Lpg#<+S{2sMuV&PxA246 zk?(d_k0X4?@akGJD*eaSSLagHY@|)VKImqVn z$sJlm&hk`^)a-Z|_g(RdyzW|Z;vzve6UXm1d2^MA~#rI9Fq5ZGwO@*#E_GKN!RX5cN;Sl#{qi#BkVSK z;-jRb6o_Z}6|Yoit~xVVx@3VZzWe+z*=OMiP^Sk#Le>K7)!~|m3}jb^H@1}XjaS+q zYOT-9+d4W1)_xIx`~3OyOx?mO-Dq}=wAahHff1@57MI0J=93q1ceXkG>7{tM>uV%| zOCC9Y;NuG~>1=@w2KCvByln6RlKqc}fCUJsvl>8)a7N!wV*w-gXLKOVQB?{brZqc! zJP6d4(l9giG~+H>$oo=tM1BHRfJrvu^b{duEGrAN19=-O^cFyJeKMip&y^?rJ;1Jh z-~hw;f)Q}$o%mp8r;t0vHci`ZYYcwT+VAi<{xJf! zvOWQ=TAzFuETA97d=RM`t?%&)Pf+U{FXfi#9?~$6K88AQ;6NffCoSzy6Bj-^jX`TG z583KP;TSpFB*VsEpka~WqTrAUr#gJ-(Nnnx!h~FDlU)LBts>LUoPtEfI6EdB|gl{Hlb`MBg}_W zuOi_7PFS0+aY}%aoF`a)gOfzGV93w(e6mI_EPiEG&lw33dM+VxB$z{oG^FZzK7!7LzyU!+!dlGHIdNL}O|$3YnPvta*BawGApgQ{>}0fYt~E2DACy1@0s;!>+`E zAb;B9TL(*QZL@P&3m|afg4&#>*gM5B!d`l-vqP9-zsrJJNGTI$@%NuU69sx6CcQt2 z7Vqa|bJ?uQHcf`=X6u~{_(HNOwcXG${uuSWk+Gua#$?XvjPtDT$A!u8XjGq)A0FH^>$OX z1DKJIe z+FiXV+suwKY8J8}yOB{q-VW48MLB z?6Z(1;gomQ5)Q!>M~?rLqIFO8AVM|x7AGMZ1gKLUWD&sd^_Q{29epnn(ZUBD2LD7$ z22DU)j)R(M^w+Q2qp-vf`4n6r830=3$9sw)N@87a#3I$cb9tNm$wo#hwo|)f>KZ3{ zz@5pB^s=90OcgSEp{%M5fS$V!A%UF`BHu+SR)9!ngmh{YDu$5+1&N`)2GZ|UP3M1E zT3EOd?gf<}j|WTgjK2olh3u6Mu;_3=BP2C6DAmcqnCb7e=lZ65`=gn~`qSWEf(TVY zL9K#sGl#Z()^8Fd0m3q0kxg&U0*e0(6%`xMgCWwPtiF|%mBU3)b1?%L4V9p)%CO56 zb)`N;&l8YVcI4;_Q(t`b>Qw`QHO;OigoPe(vmhK6(*mS4r`h+IKp_vp>-cz@{YE$t z0dno#=WkEZ#J~GkDcSw;QrQn>&oIw>4J|%2o>xJ;9t`-*eLf>Z#Aru5)bNRY6uJ}w~F1G)c@97;Dk21%A``3C}o(k^lL=~TfP}9=ZL*(gB#gVb z*q&m#z_AslDR9Xus=Jfu!$qeyB~&kjXX1ha)M+8c-r7be;T|kLDpy`eYPe~ zHAhbn^E7eK7aHk8zV1!7*YX-RpKzc0g23&O6~I@+ip|=rSJ=R~sX?I<15m=bW%3FN z3Yj_2!o$PsSInQ0)%TBIftE;}4EBZ(+J?;5Y>eq@p?D#w$}jz)GPDU~!*=*?c9+F* zUyf@W!CnyzG7S=%@W6-TDs3iAD*p)ZRvfGkzhKlTCkhpd#{V_Gk{r& zS|l*lX+0MTS>O@hCOPy>7A(K%Zk6j1*4OKBmP4iX5}c5!Ccf}0gV2s|6Ruj=-_pUf z)08tLt9(_0mwwD^>;5W@5C`JwAT(^iSxz(sOCYwlH_yoROu#(ARxH3*kNt+^w~l*H z=~a`dJiw$>*~hF@C2PCzHAwM6j1XtzgE%bRFyt1%X|Uz-Ya`+mh_0Jn96gJ$^-2k% zhlVaHv=ZEa*5ANka$u>@>A;fob{0D>0!@qXqaIXeIyJ{xTYJiQqd&`z z*tf1=m%5kc6G%+QqLvv!1&oVYjvhNgY(c5W2FPLmz8r-&+$g|I-9MsDbjq=HnVyM5 zuY&TA5LCY}B4E6Pr71;`n&WtOUEs-!HzkGP>iIKBNygss|Ne#+K{}|CC%;4&78U|w zX1)O}ERqQ$gaH)Ld;H0Vx_@?PJ3$fZ)Nd>4?>F-&njsyP3tx3qxY%z1ikIXt)1g#g zDhCxQK=e(7!`=ntE?euiF)dX3Jwv`KoAju+eJo&0qXwqo#ym3i>+fp z&!O=>PVgEiHiKgCI5opw71UFUux0d|f2{WlSpAd@bC>Ak{;qe(x_bnq`jZdfJ;6IFNC{IfVWfuAd?wnsNL+xk*QHk_p{lAdREo5`YJOOq{n}N55BGd z98OMrF^Hj?R{_D6fu+xI^QMab*!fE!66wg=K{U9atZKc4;f5xksZtzE{Tjtyp8Vv; zZj}L8K*>izVkp?H;9;d~I7cj$(|;Ki0v+tTTpL*^<-G5Mpw;If{JO264ZmlgfD}B4 zTm3Q^?K~uTM+G^eoc{+sQ-nsY!BaM^Pm=z-u=zsiZ3ShxZs2GM!GD?Ry69pkhB=#0 zTBfXGB3X2Mb#aXI?KpUdJPsQ`t(7op?6&syonzNAoH}N$Q7huOj8_JVmAwC&m}}vi z5{lCE4qzi*-f{dL0Q`zn!@CQNn)$|y0OsR8E(;u?*xj%nFL>}L$@Qg@5`+A$?KJJI zig(m}V$U*9oHepKA;L2prTs8ckXMXYDU;Kx-s#Sq09*hY9k{H zi4x<-tFx7oEpMhJ48)?av#nR6WiuFa3l=1f%P2j@m#RFLw3m6-j zB{pRjv^a787RZPZy!;>O5uqRyGQ;dIrOM3Xc+PDG55kEcdNU3dm6F#mmF3^lY%8ZcxXf&! z&K}Nqtw#zn*efAcQH^zUk&fQgIQ!9~M+0Tf?}kNoGRaiAyD)B&WF}i0I8}HK*`B!L#|}aH?WODeO~Be`i-|I>Z_+}YxNkOV`58~in+(P8R*6eq(0Z0 z1@BRrww!s?D_&IuLoOjkU(|IWcPQqxTbtDt49Q}L8o(N>06lGB!M$Nhmx6`_lgEGZk+HNZFn9&Qb+H}D_C8d z?k8dGZzQB5BKq?mF=05_EF;G#Yuo2I>4+Ci+%8ccv{3!k&R34scaUSom1HjA6;JSJ@+VGM7l8P_^zhZO3nQ0Y8b?F zV$N9*aYzDqk3KHMxMh1InYa#6)%zN$pKHK`O>>&1%0_+VTkO4QCki9%fEX_t$$CFe6VV_u(QQxS7@ zAJ*4<9b2*Pv{_y{5`5S%x+%xrE9}oekyL`=?ji3nasMuv>Z${E#Sgw5F=6eqpX`+B zB+L)c&si*P)qA1yi`wuC6B$eMze~Sn`8-^_e)C(vTJVc^ZA?c=;^p?_mIzZ09J4-X z{1MPJJY&F@C&enoBk_{GxFYv)2|(~~4>tp0)H=|m@f2*Gi=Pu1)ze-@fA~Lyw_fTB zImhzRQO;VWJR=D$!|pV9Xr_(#ho47gL%%X;^Q>&$V1Bg{8R(<|l%h`b0dS1}5J|CL z`!(J~J@NgIw%zWqpw!qT#aJ0ZWr>F*;Mc^rSP9PEjj1S?q1H>gUz^T0F4GYGD~`ep z!qoaVl~hz4APaE~IwiwRM8iA3_}m66Rg849XDDR=d`)dkQ0$6ehn62z{kamJRbcn; z5f`Jug4zzVZ1pVv^y%ip?t)QO{pH~*v~rn8$6eCdktL|k41QV1gmQ2y7Wi)89Kz|A z_TtrMOq!?!qfyGYAlAc`urC?PkoDK{b6<&!<5uZPoaJiY;)R8qSHY3vD7;yek^%s? zlh7UpiSKf8ZP;bu?d9NMiIZ$rTIYk+Uc#@Pxn;9fuh=^}7P)@&uBvocSI-n<*?2sB zrO#tMBxZfHET+^>TNtD*Tg~}dQep3W+hE1~0b8LkGg4ix_nz{GC~P%gte5kbL09K) zNQN?DQFZ-JHe0yA_(b5u#EC2~{URZzh5s1CkzSd0RxM3g2sV=TJ6mbha(HOPkznb? zqMnzZ)E4kVfUG%D;?%@s>pIPs??YcY2g!MPuU;M22cU?7T8ar}*_)fN3@EAmYQo{9 zY*kFyWfCTE#CBlg1;gaVHUIp>t%G$R$V4KUg)F7MAlK3-_55H^{qjBfqFT!~bD}h| zfcN+juAUyG_dWV{+vqCzu?jU0F9@sD!!8X-^>>*G@fg`QtE4laOE{m)sr@V1GufiA zfN*nfx7g#$zsk~0z_cDMb%YT8h71?chmVJ&SJ&3Ukv$U-9t1XA>K-|bQF|JF+s4zR z7=gGG&9#cl5NsRMVO2RD^hmAIvOUl9?8k?Hs?Yo@9>fkH_Pbm;x%M02 zwR$pgaLEZwn#XN^e7x^Ud>YwvzbCsQEtWH@thj&S>k7 z>&E2hy}!hx(w(J8X0lnzojN|)*E#8xDieP<=nj8#R`l)h;h}PytT=cBTd6a_-Y|VcnvWP9#LqdU@Zb%bwa&n3k@+gC_SKM{=Hw)Ai2Y{FYcT~*rcP|}m%M%3n ze0?eG3kEoTGo&JnK(Ce+H$auqN)c>EB?H@V7^%iVcI*!CapIDINYyk}b+R>%rLQ8t z1&blFTdxsHA1R5X>MhqMK9DWxj#0#9uY24OY4Ex&TjsQ*Fx(ody;!NPAbRg|3&`ci z2O~SV48Jinh2F_2J!F8z@tFL{An*8K%E+SFtR1tnt>o70WE=nZ0^pf4LFNdwi)xW3 zXTM6)vnkUzgr_eI;sfFJ3j*&?Opom9Q+{LeC6qZ-r+CmYt0H`i10_@t6LQiJv=_qdx<5_|Y>Q z!xOCHy}QxBjFRt1le!j}V$GlYcvgyN!wd}1uiiR*v)1CqNn`5M1|9~pbt%KUR~&O{ zqbEW}7JDU~xlQ%?BGMit%j{Yhs(=oD&#SFwcYMPXI;S)-Wk>2VL{_fM2?g67uZhTS z(-M1!v#g=kR;T*`m3pNDWp28})(4v^gtMEzv`@XNh-WZ%za_=HZiy7?)p5=;HhlNk zhuWYc|DUM4q5Y9tP|%4;r|YPQXyF5n{}uXF07V-uFJ|nM4WE++{i-}R8dMolp`8(S z^HI;e$+mc#;kL#`3IFZ3H@OJG#;5?*C|oZDzap0ix&<@7MER)PU#h7zp5Bs+fnv-GH8Yo zEedH>;DX1SYV1 zuFbTu?X9hE0}z!n0O91Gv2Gs1+$IGyoDu(ZpePF!CDks;5Z~ zdcz|E(ArKj1??XnvP;e2&Wv#zvYbOQz{{uF`&R)W!B1c@>hziFV@Fvb?k`%`f`W%B zX;>_EL`1aka|qp9u5;*T|IXvT1S%DZ9!%4nOe^)XFRw!33|Th`sGm%b>X@*{R)TQ( z{+0kJThKn__w<7}&3nC(nG3#n9WJ5fLGd8A{82%{uV23s!q0PRWj`-mfv=o-h((Yq za^y8VD7M0L0)6~14~)N4$xxBvoNA3Wvn`LZLd0@;5_ zr`h|w*XZaHki83?1(lJ`HX@lFuGD6F5JPG5ECAq3yJYzNXbl|)3A|a2Mwf=iTT{Q# ze)jU_{#Gx^_ewm;Ix~hrLRJ@HCMv(*L2C_mr1%=-C+uoe z)2DTn#NSh0m@~_`|CXYxFdlF`oiYchFiw3AWB^x2U`mxDp^dG`K0K7{fDWvR_ei4& z>@#CG^vo75Q}@kIMzTDt6+pM)P+QVK;DsM(i0}OX&i+e_5B~=dx;GujTXlc`{yjcg z|29bB(_=OLMPnIGAYDRUm^axhp{c8vH@|hO!l2k3A)-m)4mdPMh(Us&%t^NgmXO4q2O~R`<;hDv zSxK^i0zp03o?oVscbTWv*qUk!Y~wEs#*s@)1q$>nCt4RSaygElK3e7_+@JqNdF)U6 zO2*c;a89ZCxOnNl(K)H7F`CKdrDwQ+x&nH{d}W(j*j21*9?z=UOcDI7bTC<^HbyB& z{Qd<#;|~W9>)fb*)t{Y)#;tq?Y|qFNs6=sBl5AszQ16q)LxBN1{ z*jqN(CN)DnhXX`$2px+v$N}Xx|3TOkEaTiYHCa*`AcPs_5@|*`rxW2 z06N4b=z&6gxr#Lg5$&G4svt6lDO#ESG3Iv=w#J05RYCzpjP8MQM;TDLC&R98$Zn}p zza}qFlx*0WHbHokgx-9wSma{b0>YCrv5FS80OO{e%?G-EbGLK7Xs8?HC#+k1SzY0Jm^x%$)r#Y57#ag5^gPir!AzX9SkGC&g;Bg2@lKjO$-{`?v zi065yFbol|*ze!VOOxyaI9mH5TxifOUH(d#HAK}bh{zoZ>y7kmjxE5xl?HnM2Xk*7Q04lqi%vvZ zK|x9-6i`A!B&1PLN~A%Ugi3>SH)0}4he!)ZOqxZv3P>v*!XyRh?l|LPt$pvgd;j)6 z``ml}xbq)aESZxzzxR9JF`n_nHuxKT2Rd9D?3k*bq>1R*vvM2cVY>ojqruDcjhY~M zzxvYW9WjdrMso9F)~h5IDAyjHIgYrYE#~-vO8h`AE?&=QCI2U)O@(>oTPHmdD)y_@ zds~8UeB={6m*rkGwr{NKiIm(M@R=A0b^o*7;LAFm7{;L_4ppp8HX=w3e073}^I*^^ zPa}$e$&ykvKh5?gI1xFjo1-{mK@9LiZCn=s=8ZD z)?yUqIS%>SPJuXs{MuBI{md3;xzBP!s+~iCot{_L?p)HX!npGF*=exGx|OG<_d+S* z_GhW@CY(dxcK88$>XJG+>27fcXH#|F&r;8IrzIzUs=p0=)O_099)>>89(c;f`d?cW zfUJ{)#!mp53?RT5TuJ^rGCm>?>o*_6ez3`4sg;|Uj~lwbLO_El--d(i6mcKkXsffhucsDJv7+t@b+4$Hzl*7IktugtfBmb{V>_o8R8#|0 zC!3~pD%PS_+KHY@{SrG-y^_QsGeSZ~be+wj>RM?2^4AtP>8I)ux3uFDdkk#U@~{^g124;c4Z)!2X3!wJEVBg0@z6H7srV z3wFk+wXql|g6jCBv@9u&ilL>M3frR-wa?!7SpATu7k6d6)OW-M1SCp=H&b1i3NEW$ zS~tHOVb3sda1J^Cn0ZuuQTMOMx!!QAOaj&R5^ls;@dy0Rq=bY#Z86!~wQom^S4@Pa zOGW$6j|L8EkzUJE`_py&kp@xj7{B!J52(iQty^#=m>2q6no!G`*c?_W>7ed+iqB(A z_Y~WncAq+T9%Nv;^dr?=ID&pgqYV91Pi7sYMwQ5U#bcQ-dkXj){4E(boDu(*P8p{p z;ctV-f3Gxs`@%!zP*xT!XSiSth$se6sy#*(CZuaa9%tI>`zVobZXi5bgb@wSh_~;+ z5AvR$xc3vO0*4noa^NRYNhCb*&J5=xc-+$fQ*AN}c9MgDlfTC#k?bO;UJcDvSR<;~ewo5@)6nAB<8z7k z8JeN=%t>IQtl)n(fC#$Mpou|*7|`GK4@1Rp9Z`CNG`>Y-xhiFw0|bKLHm?I3xpb+o z;5qZi08hgjHvB1Q)m5wAo!r2%7CEyKnmm*$|NIdrmGM>HPg`ZB?(~HcZV`-~`~Pb~ z#1gX8a-v|k#bAu~CucE&I~|lfbWaX}s+9&EOriIo`^epVU^EwG&4bsR5zv{t${kGC zE81fP6+q}=1oyu%;l{>B;6D_Fzq=YIi(zKya5w%8-I1qi<^~h^Xy8CR5=A3tD$Nn1 zJlBT}Q!Z8@2N3~D7?jyQ0h+-`dC;^y&R}#e(Vy0l^#A99*bx z&eiS=U4TVx;y|5O=qm7=>qw8=%hU6zQwhu(Q=*|xQ~4sH2j9neK||nr|8HW5c@cF> zpWZ%GT1a z(FF=NgU@BIM$N3(A0|c{UhkYcX9E==u!Eu=#pfV*0gR&&ZxMujiukEopUp5i6my&n z@!af%djI=5gz7CR`U#H_zP2@zE^&mv*^V9?BPjG-)I=Dro*dEPoC+p$&7u=`i`Qa> z96q1)B=n-JfF4s(Q&Vfh)Y)@?vGkuece13u7hz=DOp_fdP4>=HKzS1A%pf8nvOAj+ zyul=Jiv-OfVy`vv9slgbR0eoiSO1gx5Wf5WiOFBo)Tugf>VFp;9GnUn5W0GLNNI%_ zWw`Bam}j>jS|da~g}`ozrP@?;_&hJ-uLhbouk2Q2G94TFe{;sJeNY?ztnJkt&RM-M zQmsaw#Zuee?{v_$eCuQE)zu`N zvohGW1v-nXVGmGc&jxwvjxy`4L4N!j1!_aI@EP^w5$KyZVRH*oabuB>_)-4^q+l$F zie?j*B?a8(gQYf!8%rZ-*c2&kmWkjM{97|t)uvV_WhR!!%@OZLUuNcJM;v;AXCL&p z=KDK=tNiaBVr0-!7I$31!NVL@>dO@+1XNs7&A#y{-#51oo{~qc-k?L*?c3nst0LI5 zM?Zoe2R=T23&3le>j)e!#$IEwMXyr7lGyBUL75+ z6*r27BVyCE26lLavhApsz6^x12;l5GUOr91^r74b6?03~|0|usq&zuD+ViDU7-W0js3^!s-JpGf?YG-n z(5`lLbnMB4D;DzT%%1f-&lNd?!z#_lwJ?!4(TH{dL7d;1g@PTP0wTPDQMmoe*d4ls znAfr~fI}QTNJX>bB#f1@w`=WpmIqbb%NBp|QwUi-;L30|kQ+vcjJAE9u*-sdE?+iz#CToiWsSE-DABpXLsOBJ`E%>k3d zgNq#1;`;*=Cu#<;0BkM1cjW%GBCFRDq1LVeP#7a*Rk$#+$y>##GKBWD?C1B!x5 zH`RH1xL3=@%O)mtH{;=HicUz3`uUoVd-c=xXZ6;vwMC4-^KsY|~ zE)OVZYhb!Gr`*{iBK69_;=%7zc*L__y(IvkSB0XKdkx``=&EqCTo(obrukrD8hsJK zU@c)^+QnMz)@Rzno=e@Ao8O{Kd4^CFypNpxU88dto5K#b<0~#=Z5|(s3kSyjO9h0u zK}(F<^)xqJgTXTE;>+S?2Ja$SjYoaD_xO1&W*Ox|zsTJW+Sj+}x1>p~axi`9KQQ=b zrX5|d{16QwGIb{_UIwLZQq@RFrQ{88I7StV`r?Xow-Fzw? z7`(pmm-x92IKnq>e05p+c7&dK$*$HC%^-bV;_~A0{v?UbsF-Ohr$0s3IouJEtCL_G z6llwBSLi}TP&xJVy7L5MGmG}Axq}tam=tE|l7ez56^lICk`FuYd3~m9>H*S}cVA27Lfa?Yrdw#_PfA$3w>3>& z)>7*h18et=u;&GS5Nf!&&&n7zt5tl@Kc6zaZ5&`cl10huDE$rfL%l?bH*|`7U93Ig z3KhGq^xoEd9u_0_{M52s%I9{Y(YQq8PKIIN%pISqlLPIBaiTYF>YcA|PspabSNl3g z*MPPVeyDZba{uM_1a^D+F?U#f??c`2VcP%5bsp#dV)9kbWEuwGlHN3t#(DE5&3$$Y4xb@A=-tj~h_I$PH344A$ z7ZCk;3(P#*_JKd&T!ol2xqm5!?iEprOaUmq1}5UN;P$*3{i&e0#0rg>*Fa@_kx{Vd z&1A3|gx;uo5;B|cWdVC4zxKr43jtmB|7XwzqrMI^l$Z>EsR*B3gA>fI6;}k6FkSQ9_$jlu3)JJyGxY`M&z(DGH808Z z{<`aAeJfehTZV_7OhK711zQ(a=g(5El$8IQ*T6}0Rn}6JpDrBXc1?*wokVh!T^pj?D-40M3`Dujmc|?AQC{J9rR9S z=M(l74=8*C!79D-&FrA56p;64VsCAIVY2DU@5N}Y z+Jl1+H`Pq+oE5E#bH`s{n}&`$4Gy%@KSX41sV&bBI)tLhzD_+c`*3CXIuUK6A~~se zEHd3(4e;<^{N^%W3&ckg0RFj|I`2{^n8%u92b8gwF{_s9;q6e4$Gnh)QGV@|i0!(jm&K{`{6R^?bLxb%3k%t$XD{DgDQ-LtAnJYh zlN=Pen5Hwc!wPSFEJSPQt7}1NPa;3ZXYEc-n&#<$;C}aDxWi(eOkvVyU`CXWzSG`4^rk_xTNUhEv=dEV&@lZ229A1Y2^>a3wChIEuyZgWN9 z#M?bIe(VqU@H{QZi{WIR=lU|Y9y|95R-Z4J2tDo zp<-DgzI^!tbLPZAI5ej;24DUVPz3k9sZdSRCyf$?=8<){H_#yqFKI7d8~B4X(Lmgj z7qcy%BTf;ob6)zj`@RnZ`HWWpgOAeBeN=E)Y2f@Nu<-kd^O1PB$B6MWSIsr{GwO*2 zJ*gHJG}Isj(|S0&#a?@`7SOvIOUb~H_$V^{i@@fz4UjoNyw&oKg~V`Vjhjoo-=KF* z`Bi$r!A!#}jT%YAz{6Cig~&@0Z!&14`V}^>tihD`_vRdW8k(q&rdbLUOH#j}G(lwE zzn^D)MOV8Q^H~k4`?!7-c48lgLR;2eotuO|Hh1r;M*FCxg$i{7@iMg5>DN~YJKNh8 z5%M9pmI%L9GZ@I~)Vub%&-ZiGtGw<*Ew9h${Kt1*Iw4|g7^cp1H9^N2tl@?n*B~zh z)uoqq$$-R{5~ep>tsl-~TZX%Gv<#{*3tD7kqpPpBY{7xjTwavke*7?6`~|a%V+2~z z=6iLe3sy%Agrrt1{#?U6yg9(b$dJAHuC?U1VU5DEw^ud zx2m5Sz`y&zSc?P;q1mKmP+9ap9|07P1+*537z1zy3K^Lun#lFppG-t&&+>O(6a*DF zW}oyT-&+L#pwWc76*(j)mY2B^llgGB%R=_jowHy!r)fS?QHIYuVe93>z?3eJ5(S5YCX8CsD>*Q>h2ST-)n}6$39z(xrUKkCH=V+)pQP&y zO@EI;BS0GGus5qlOU>mMdsed`%B(WT^=pga*V|eiVL{-u+*~%6fXX+Hb6t8>tr~&( z$AVv*-l2ya3%y$Jk60*sUDHHHnLJ#T`K31{iSz@^A6rlgT)bC(d~bp4gp(X1K15X;ZQgK9-#lLe*{1Krx&hU z_?gng`(#fyb0F!3k!zk`nGrY3a#R+}*`du4AaKWSgZSqsB#(fT5)k2$2)aCfJHh+9%kd;QbaV zrAr3ji*JE0Q5Bbyj!R^1#{nl7;AJK1DsqW^il_3hA@Td9&8tO+h)ehRAZ^HfLCltb zJe2yP8rEYgFcLS{p|Wa)*#~lKBepaA7Ck2!kFa>0WHU;Tq>8(4|45}<0CG+Pm0ra! z06f0K=;{WjT3lOA;m@d}IT>X$B1%RRhV8y_gH8G6*0^RV7Q;Qq-?u2p$=Qb=j=r=1 z44iZ(K!HPn*72d=3noN}xDNpy5Q~w%JcB$mxET)YkG&v64A6P)r(_WZ*lro1Y6{Tg zZi>&jLgdGWc(gRCV5qH6$ zoo>1e#9+wS4hhLXDHY={fWfBFff`6!(!l3H7^R-9?~3@*0C{~xhsSd#J%bmNOE)O4 zJ#dv;266@CpT~&u6!pC#r*cEv>}S4a#LBY9jaPmky6T?9(HTwX zZJ|tW8hCS%w2C^+$)uXG1Hhv#oncr5=Z<%z5flLgqFJ3^G99R`oCN^c-B_Er*XopC z0<9A&?{6?`b(0O#>36)wzqazzFg7j}CO&_rAwcVfwN}wtSn7;he_y^$Pp{NR^`7>d zdy2bb;o|oD9o2-UpkY%7B!D8%Gz$WA+!yNLaA^f+i}Gt!J+q{D_xQ4IS^2O);B3;rlcFO7#z1`&D zgJQFvcT2aRWR8RhPcsxBNXlvH(IpY*8(PHhR!RRE+(zVbb!``sZ8GuhXWwh<)R5g~ z|5<$D3<*h5MGn{yfq(kYBL?`~QLX<+DQNJ7|3?Bc>MoVIY(GRWE1-!aIsmTwEa(J0 zhv)w<3=w^Amr?hU8IMkAOz2e>r#5fa7_mLl)^YiBJsLOnj}>Io@`9s0o6cVu&_c>X ztU)Tc#6z!t7Gt`K;N$>TBaWqDNGJymys0pJ;{`|6fc^cQows0}h4{t&Z|-u+5xlT# zWV8b9zY??<4-jP&5=aXH3WxOW&bzW5i6Vvt;y(1^tSPTa!>&1D8`k7z z$k5FUA|;J`=`wWd?fnMwURZh63R`QokXsmaRt;r$=Rd46!O}S%uK&O^@?bZS`k2dw zO@k-B2cx2LpOlvMIgZw+h2zdmOWtQITB=2br58FD*&5c|JG6<*$@12cMXVXhFiL{@XSYSSC?Uzq`&Cv1>(BD;RW6=DOAG5Q!_YT;n@XpW# zHeh4uZ+{J73|2twLV;n25I@skl_7Hyx~^h~WrN&6CXA^MFeDHq9Ac}_B>@a&1PDL$ zMEqi*VV<|;VP=-L=*hT3PfuS6Fb^F$>@^)0=09me_)(2Q8)Wco$9Cv)@e>~=8FWuN$aTI zF^)TYOo!yG*lXbc)w*W!3}9=|F@BRh;3V3 zsq|KE%=LT3Q@zLD2e+jyN*1pAEQn;kD2C&y8OSnI@h+o3hXKE00TDzEqIG@a1w1@~ zXeVHC)2(Gjxj?HO1&3E`DM)nX0DQcTFxjnF;nS;Rb%L?UerKqg7r>B~)*s{gRFhN)K_s*FPj4f)<*hcqf!Ci)M>qvazp z0pUeOZo{89Egd<&U0{ndnkrjPbFO}UK1d9v%Wy~SVi5=}(W^_}&u^(UDH$slAVB9h zmI08POl=1U5{!nnClTfT%`8FaAgcZ^R!SwfaDwHJ+<|Z@KY=^&!1>;5IgG|-wiZTdpA3I1VA|{gDL#x{ShNy6!#K8m`WI(n)SQHJhFu~w7sYvjyfXCH zAB4YLK{@^_(R9{&1Wv$#f8L#rMyrg_rE+`M>VvV_v82kO`%(WKw^?o@umFtOKSn2i zu!uDEpCit0Toi$Ez)~Zok>HGy6Lk>SV>!Bu)&W5l{mVL&E40`KE8Zg9JCWyHeSfWc zgRRh|lR-64r5m!oFy%S1eXWDt=~G9C9%~EO+@ff%qnv2!qmt3-0djr;Y+w7OsbvRK zIALTHjz#FP+Pz^)kuFDZJL;L%*6l89-)_=P@Pjt~T(QaU=d06(6tR!zk@>B{B!;Zh`5 zgXi|&d)}*M_KUGrp_x>^c@}om+V5-)ml`%UpQI>L(yBfOj6sq(G_N0y*D>qrXpoLGDoRWk|^RJ-w`inb+p}-=p5c--Zd+yX!3Ah z{EOu)hU>PwVJ|5=;BVWxL;V;&iEjabvz=J}KoPLF+N)(C2LZi^Js0A6B;>pR#Gli% z+I~QFJkZSb`*=GQgfSJ?FzI){KMrU>EA&4N4GkKN5B5}4U&g*KlUvv1HSOcuo!?P` zsaILVjYSA$rynSlxqFvhVLvF|c&p9)ruX3%okS#1Z;Xe3pH(f^wPVw%N=;C4#g6Jt zEKJ={lrXa@h%|s^-Kli^KOwJ(#V)_S8IIWOxE}*kZt4xKLmGO96uhI3n*%e$nO#Ob zfvjl)=08#Erk&rX6{u5X3&^{Y?A8SxogoqK?meWVNNRi#tGSw!K+x*Y)2RE7tY7}= zk8JH*Y+y-+Si}Q^kJ0B~^$p-qx`?X^V6R~?ww8m<6|3XbBYi1LVGX7aj#^^~rcERS zbB*wX#O-ihx3zl%2F0JyRTkk7F5kNAY(0Dx7-r;xrGKgBehPM6-h2gQ&+D3biS6@r z+9f8Kz+>wS((|RybhzRC3ai+>1bi2RmIDLtLicCUOV!hd_LN2)*?$UsmbnJ=w_NO; z6xp-59m?db?f)u$KEqrL>B-2 z@o**CY(~~%@Z=@sOuwYAR0WT&)OWtD_bwzbYMj5}`}XX4O3Hkf=|RcRT*7y2lFI6y z8IO~hdbz6QLr3=y`fX}mW+aG+US0@)P?K0(crB*f;gRy z?3YfRE5TIl`+5(!?lLt zIoQJC(ZuqfTXe!yk&|*djlY-|A6thN(kQ4>8Ok{}d1ai)b6}NA-#q0DU*I8mf$3!S z58q1P{OAi`80R8cRlcx#0cgd`&I=nN^6+K7|8JG{+oz|IANul{EG^J#mdutr8&*iy`zgB3%}@PSFIR&D~e)O`fM)~Ed=|5ld*w>BVhIbf|E1*nk7Oq@eD^h^Q( zv55ffHn4UI5=1xwF^B>(90mRUcI~khAjWHXu!}XhF2K-oH{+AA~oW!J+P1K9IP@=5j*Zladvt!q*?ypJVvMHL-CO*(|O*-r}@cOJ`_uJMh&q zSX=+&ykC;DHN`}xbEp$}z(2qBr-{B~q|!OQ(Eu9q)aynlfPNSkMrHK>y0d&V%$1|k z@R2{ul%wBzg)tuB$02%;g_}sEDnTElyv#o@fz5UpTAD?Oarq16-EV@mY!O@Ht*>Mf z0|&e7IWWo|$F6Vo8#L|IKnVAEq7xV15y+Gu!!JrK9t(r_Y$1F^17IEb=XnbzuV(0l z?;fOpG$7fe^T2TroH^hzjT-6Qyas&PbV@dzwmE(0ayoI(0dL)^9|>;RrC#g37cTLx z;fk!c-8yu|&EywhX;h**7sHwe2&|P zO06wCwigP*sG~~o-+y$qb?k1GusZssP4D^4!K>-?;eJibUfO0^brGGg%*BICjVI8;?0-1y+1biZ2$64pi74>3Y~q*`f>0o_pP14OpoK z0gyD=$qhnyN$lv>{!w_SW~KWy#Sy!fo>OG;v%xM%yrG%&#_PLK7t8a+j{pU4mfAZT z_%QS+^$>1nR6eA<(fhtTgO4AK*Qpo?`OSNla0OyvD0%yGNCZT~3;+JfkY7e_@Z2LhLC3$EZAlrD)eZ0u9F#ZK1O2C!?EY&R+v z>Xvr|*>xmF-2kkUoACIcBfnI1#|8VsYq5i#?Az_FvFEtno_C)eS_W*+*;Nu8JLZZk zl}j8fcMDZGKC_%`sdl`Q`Q`a#ee==0v*18Cchyr<z9T>t94T zb=3N<^)-8Q9P@xOVM}2|0pTN&BlfMFC6KS<(*YQN4wzInlcRddrDJ?T>m^j2V@q&k z+`)nx?w&04t36E{5;IW5&oKbkg=c-e4E%=G2>>`wAq^on2ExgMFN z{(jZL_F(#Kp6K|%v%w;C`;Zg)hRy!=2bB`jZEMV<%od{O(uWW6P8+yy*zh6sb zlce+DDGq>Uvn8qp8rt)du2vL%0{T`In)%$kbuQD9TCZPt)_Y}L%HXUR63@1(@;`5M z5gfdRi^zc+=g~C@v{#2(tG+6YxL^)5%_<|dy!dosZ`I_d#@H{IAur-H)`L}lBJ{kb z{cEb9?Ui;f{HTB7zgOBGf2-RDU1G^FGr2f~oh@~bAJV)$3?N(6p!RLE+vNQYu#4`= z;3He>7x@cpmzNC;f#K*3pXV^hoF~Oh;N@?6Yo?szCsM&512|SpLAhSs_$S*AnC6O0 zJluwM@+Uxqzc)A4kGTh4dsJ9g{JD1Qg*2VT=G~3BPSkpip|wHa0pU&&F54m-L27m6 z*5Y7;EoqmJVz&ZC!C&G^qVw(IJ(5xe)EC$+sI@TWUAspQ;w z64d_jD1O9UO{7!$#&$5V=6aa$Rwq@?7o0C@M5i2K)E2zH`pa!?Kxk&F7{uDHUiGK& zNE%$LR`BjkcqYuz;N$2666i&Ea*=Mp9Kr60e`{G{Z1ec9YE5k)aQ97*05)Gbp2jHGFu|jOC4#fuNdd zb8=(Z%40kAX6tK#jWGy@5nf*=;g|~8E)N~{q~_Ey0!GyOo=<2n*(HHC$Sd47B0}Lu zY^LV{mE#9)II*bn?2r0Vp5Tl8Ex0j=XTHy2>B!*Z%OS6H`_bLq>OOsC8yn=LDb*rF z|20P7vD=Vd?fsb9cn@#Td+yu*W<>Lra%bhUwv19Sly+r@@%2NmmRWSUXG{*dQn;f@ zgF$)Fr@jDLS>(w2dT8Dhr00!>{OQ{aT8(<1z&W#+N$|)$+?fm(00ErsVXfr31t4C$ zB$&7m!hznZ<^K~FP;wcw`{tA{80t=}7y_2PIm|7-&)i%qQ@n9OHUjItyPMjSVAn`X zzkzXadvThMJGv)#zLcUV`2D(_({3a=3-`v*D|q$0CxuA5dd&m2T97AY9q!HN-K4(+ z83^y`F1bju6V_xdB^0iJTf}VfDnxEEcA}y zkNim9y`>CB{9gh~$s-1}dFArPWXQ~KsxsCoXXb$^v zet}j+Bm2=$>&nHFl~usr3hsQx7A=*!6hW2Lk|`n=j)ozti8UvKGVKB)0^uN!(Xvh8}l}O$o`0gkYn*mXPFcGX-4}EiRD>^23PN&eoGO@jdb}P1XT%$jtD>25|Li5OUXEl_`yGK( zS5^~TrG!)W7-Ov}x>AQb`p5_n`Vz2-8m{rGB;e2>Q%3u0>?7`OHxtx~oatgkojz9! z+LxJb9sXq$YI-WcW65n)sPx{vv$IC`@A1h?0yQmw+gib2UBZvi9GZBwez>fXzk=7d zxa6_eh_){Fd&Nb_;DeP49&k~s{i`8>YGC5t`eC_nso%Styvq9A@{;#9L=WHMzt1c5 zH+WF=#OCbEgKuo%*R{7U3uWKW8FKb&dN`_qj80TPE9>SAg?u*C(1C#um=U-o?vxT; z`BSLJ_ff)-*xhBsf4HTq@V{;a%+vSe{J-vVRyV6=yt{`no-a+`Qjjf$;4G7UmuH2W zXy@VmTY^J1Qca)iMdeyQ#HP}%h$fb^TdM_?oH> zs+D4>rdf7&05;SCrvcRsez-QNf+znp#AFeo$ws}lM@}>3Eqy!r+0&~?weQP^fe!e7 zr!ah*ntXEp!1JAJIN6=yJCsV`$>sf@ekfaY&IeLiyM&R$meGQagl-efA4*7fT^=X( zrSfdqtE25LVfVmo@DwmVk0S}sR!mAK@oYb6spy3a*-+@=@mc&+vidS)i={8#1P_f$ zH7%_CoIG#?OYIjOvWqZ`MuphlEphD3ixRZdZ>D_ zS`dTK)1PxYtA?3}WANk2g`Wg6U8YrQi(r<+Uh(@Mr1E>m2WyrOExC7G) z2^`^2PVxocG^hY>m~a1b2(}t5WEnINfhz!y5E|NRsDUC=rOKs>`DegF)C5vy<=z~| z3MgJ%kOzQ37kaf73t>xvlN@b5SQrV?@A z$T&;O`wfnCQyA$LLs>BNUU1Y5Hq8<+?s&v{LH?%?vU1>I%T9Y0wF7}5`&P@N)gS@1 z7~k&{Sr+w}&nZ3dFkV_5ip2|AOOVq7MuAPTlP&jsgqh_OrxG`;s+0QSv zi9Ap4k{cj$d3c2IsW70B@DA+n*67Yy$8TvcFuEBe2FVv&FX#91Iu%-W|NfW05f?Ub z>R|T0YgO>HVM6Rat#4>kLL0o;S8S6@N4F5dZHGrlSVulsUg?vjp{y&bnydi($8B8Z!M_lB`p5PeHKhHU3mMx zfZ6zFRBHByKosF55&7XUsYCdM$k9fMMsnu8W4gw$3k{FL+_?tFd_ zbKr9t4=3f`Xi*VtU~lc4kGDt4>{75)?CPw8ooAV0$C5X^ai_7~HLQz9DdZ+kR4QL1 zxpX@C1q{TPCd7QIWtzkF1v9l0?9y;4_oyM1*Yl+i2=s7e*Uuqq(nhs|8j0ER3&vBG z3Q21@#(*^mDiyCy5ycaQBBB;!GhSs?`ScQ-i!S9FHS}xpGgSHvRmu1=OiW^@*8g5; zC^j~D75FGOhrvLzVK~ANa6oS~tOm6Na3O&86SP{D8iyEbz(ZCp=GK=!-;ey3d)t5& z3XQJj|2uZi7+iy&b>of-#IaFvRm=1r9+f1Xay3@E^Tv zpj3AMQ2-24wgYa2%K?vvTvDe|aFm(@V6n1L1&sh2pt>A*Lp71xpj|GkaGD>2JD*}w z2)Ja05V7?Y)QomtcEM`+8FZ-|T%549*BziC9|iWMB>>SSzz=qHPO8}cXI*Eq!Al?v z0s&w(RBm)BSiyT1fPpHOK0CV(xP01M3+eq2s~~k$0LG@+-w0o=lSUnX3_?q+0IebF zCxRUZd8n#=R)+hqSL^TpL@_oPeK?^Uz<)C`oovF$|M^eAtAB}T{bxUX1~>MR7U8SA z2mH`CWetF)3ov60(wrW_TsfqzLw1Sclq>_d?mStpS`PyIW^<@0Eo_aPkn(+I0wWZnjp%k-0VZ=fb_xp422cy`Z|21gJbnCtbq3J3`UL&GQf2= ziq?0})X3A1m!S7OjdbB+uu+Q(3b}8X?HmEIVhARzHZE8%!Fq|hHr`hQ>jcu(Ky<9% z+g`RoOi^qwh#CU4XFqC@EDE5I@z75T%yx6L88c1y05w+|kVjq>_FB*?fnDIk5RX zTLi;QG~6MG=ahiwj_p2d#fPha)OErFZrbrX6MROJ@f{xR?LE|Gm~Q6zZT{(Q0!U_d z*t=7BKAR#4&v&PhLh)0g*OoTf5%5sjum3`tS1WAIh}M9_&KPdzscaA)Y2@fcN-+CU z#6iax1wl?ClYzGuax0e#YrOV#Aim1*8Iid20uA*XihK%OT-L@6;4O;%aM>C(-9OYa z@G2Gq3~TEfg%aEkRF~nHXfU9py{IVZ%Sy$(1EpoR4M_N{VBr@de>d#`(-j&?PrmzR z0q?I3QR~`1uU>7uM1K$t%nP^iBPfiALYZV1z(E~h`m8pcw*68y73a{gnk|917a^AY z*dP_4JBkC`kg@r=yqZ1s%PVRtO6%vmnios4)&P(e!QiCbGLc{#mzEL-J|J`Q95R{S z9pnUs8@|2fXm5UHl7RH`N^|M8aRM@WWs88+MBoxj)ES#;KfJ zsV89ANCcI@<%AGi$pwnzwdzr;Vgx(X&!#XM=&& zpiS?h7WE3!bLR-cmn0J? z`NS6EK66pYQ86{?YDnmL)ag%fAx8@gn{25c8!ukhI#lfutF+Cn~oYW|nLrFt3dDeIK5;c-N z1o?A`V$dTc^rq`1>)7@jl2>ZF@hmHwGpQIRe{b*`bB*LwK`9%)Eev`Qc$blBBK(5v zSB94b^ApUk#Sb>_WcdFjejrV~{Lyy)s|Lf%A?K7&js8O;Q&^s^J z*TP4l0~&L&OOGP_Q@T*ADpqt}Y&+@Jb)g&!^$l*2J-z^c?1uYu7#a~2-^2MPN9zls z&$SA@3fum>sV#&^I^QUyyxHh~CcievlWSt4Q-(Sh5B?$x{Z~J}$>1n+zRr5{35?+d zfI_zt1{mZ4A2HjJbj0BwIXHH<4x`AS|)tBV)U_am!#%+GNX z{n&5jm&bB-O|KF|$X*Q~j?PQvGdfTwinSjBKXk*&7ia>^B zX!#SmA6tedftyNA$`~BV@3zZ7#+s3)xcLV~KE6Tpn;Vi4i+}{b*9kH0JF!O^l0#Cc zbFwMwQVM!&j#oJAnML4@{?&^_FE7$c?4bfL=qAm7)x?X_y7r|;zvFr(~A}d$?M!54jveN;4r9a}NM5z6mm@>cS&Cv!_ zyj_Nhvr%A-(f_VUBD0yRYXu3FBp-~uQtyCA(CRQ$rF}M#)hFixANN-VTlx1A+VQ#F zX=K{QI9d2f#FKN|mi%WW9lNIlKQMC?Eq@B&T5FU9Y1{j4A$F#u3cVq&%QxGm7Y)S z!hM_DC>iS4Y?CicZ-kSDcJ;jD3)RG4iI_2&McWTNcSgc%_#-})jIwi&@Wz(uOecs7 z&7UYQdpMoxe`E`0=YUNOKLZ)?Q7@p*MlsRS*0>Pow`>Le=SN;Z>-XHJhxgKWE)cgx zwC_`j^RBoH|p1k4yX0hr|W7Gi?4! zgm?eGgp75#T8Is>4QedX0@j=8J=*A@GqZK~wCATiZ-r607VMo~LdXFmd{La#_D0D& zE;jFJ3z~wy5jLLc#Zjm3=oH^?1E=>t0i>aR*c?gU!h;?~-LtRw{?x2@h*Zm#UK`A- zT6;&mOgcw=N_Ibh*;c|-q+@x%hvuR?Z0Ap(SeTu32r#cApVQHkaDO0 zm5*vhRMKyui!OYm=Fe1`4D_Zzw|qES_u$@sxM2aVa;gaO+kvt=*(MeZ@;z*Nf#k9q-fv{0g!MYtrhpuR^P|<32Fiev zmyEKo#NNlLr8e;VDZV>JVRQd;ZeqWlOML%1ybu9A9Jls6eD{&6RbO11wi=7YJ+m=R<#s2xF6rGGhnF7pZvCPEVZJ8LGPJQfcopIoG!7)z_=(L|C5MuU zim8azNj5@d=nK%++3LLietzmn2$PA@lbc<~hRL27FE$_HJ%Mlm*WE>r2)v zL2)0WGj6%6B87X?IXnubV#`yKa{*^cGKD@ghQPpjp9{rFW2XNp?5agaXv7F{z3x+R zBBNJU|Kvx--uvAU#0BC?KmA^kUwoLpK^~$KibHttUcLW5$ij2da_ojW(j%>%I=i<< z!9#lIO`PTg+*zhj8gG#C1lhC`-;~~y-KURm9((#ClcUboY%c;&$ZUj8TKgHdFcX$e6Rz(a!N8%om!^=aW<% znujU2OE5a6y=s)KFITrk<4Ed_&2Qt6ym9i;<#+MxrybNKlpIP=N^J3CeQ`bMj6}PZ zdbkqx#-29yK-Rs>RsF>@JkVN_kz1Jh3432R5hw5WyJkojvZq=pq&yY(6f{2KeHn5g zr=G(YZmnt4W2n?-x&Og+5dz<17+@`19!tZz^LpP+|3QuQ8SG zoRU4}d-G!;IpCSoswSPdo-TAHSO%@@wu?GuVA~jdS$X?`${(Efcd`t#g?%e#;WXWC zxa9z2t+~m2y2Ut?E#H%kP$AB#uYOB@dYtG?R|QbPD50W!G}mIkG<$UF$7Iw)G91Y< zK4#NHLdB20&GOX4S(vouwVcw?IrO6AY7U8qV(ts8IwXKNuM8s)JohZatWhWl|lgDbI-mQ!3>;J52yasR?<6GJKZn(jqOI2{~7fh4w)P8!!j z6LcU}AY#M5lkxiwl3g1tBrId^EI<(A9gFLq8w}Cn^M>!|j{gYD^()}KSK50@O<{ZS zF6!YNfJ=6xwLp89KAuMWGT9;2ffYE~$-rB91LsQViXmukJy}7ApA1L&UMF}TkxBGH z`nk`8&Q=hovO?*8w4VRu>zAh4oq@dF*`F!NfPKCib)9^#41-Q7s6f^d4HFsaunNny zpqqYaT{iyuhqh__^zt8vPBB5qV-)wXnco5?G!X-mDO76(RiFe?E;N-%-1J+03L2Ya zWI}JCw9w?_-AK&n^hYeCbA13PJ~ZBYfpwYj>;XnDJem$r^C*+zVo1QGW60cbDOxb`Tz!y zAn;BE@Kcqc3p%XcgGTx_z3bR>Wf(3Wh>6&aNlFONghPhW#Vq~miq0oBFt#)D-~=MB z#JKoxX2}U?ARk`cGI9r$E%^<-`*(>QP#;l4w)7A4<3k|IQ)WS+2`ivv$xyGpSth&I zjP!OA!2MnN0XmPvn#W(jcxmDXg_7v`Ebt;Cz{R@Hb4nO^ZXAG1)*BG;TxJKTv(_4q zb|X2L=(cYUKZMZNKGsx*l<~e~$YR=I)&axOKlOVgTxE=W;AC3643XsC`{QpI_V&vm z7Vx@WhAgpSzS=|;>S6Hc8@p<1P#`WEN<-^^cYk|XX(Q2TXJuS`*$i9bWd`S`Hx~ZX z!!O>@izkEdYR5I@)f=cin_P1nQvq&%-55kS?~uB~D!U8o^vzQB_NX#+AzyL60x9X& z44WOE08T0?%az%~Ov^CFCyYFxxADmaBcEhwNSa(9EZc}}fp+ROD7mO>qU3|{q~UX1 zOCln)C6`BQaf(|NYg7|ajv(h^qzyP%fa=7#Z4#&eyd?!3;S!HT#;Vj*n3RiNc%X`^#=5)x~hJG*A z#f!K&ZbR?Vo}AU9e^q%urZJ@6HRtNoO zr;jURyZ}Mh{@$_$={Erb#u#Qb#`AA=0RCO+%?4zzxpWa|Hkf3vv}NA&gbrXUsn8wq zLYXYMBK6w!^C-9t&(^L(7cM%!ThFeukRz#6tabkhEI273b#2c;K--Yo){DhEsIJ#z zUfDS=E%`*OFYd?i!F=4qbvV8T;z7=h(o-AJTPg(6!&@~7bZ8o_p=L0?P>@3{yAP6M zTKBokbWf3lmeUZ^^R4JLzrl~o_tH+Oi%&V?IlP6g?v;zc!A7@|Hx?k&yKq6s5dWzj z@PYrKC^$0J6nK%Jy~B@Te#1z?St0YJP; z`5N5H!#huH289MsqXP&6sU_zKsn#bd7)cwcdiT_mlNfbCn(c? zfaT5C8BsmZBd)BQB^Lo1Pk&qC&i(fipg7(P@El1XZ8wG~cDYdw0sh$kDDO(+q1@Z> zgBfEVkqC!59Ub9>L?khzEay~+7!gfPj%4j1yJ1?TMZ4WtvPGhrJ+8m4UG+#AV5cC zP_~jzlN=L8VCPs53=;D8i(miYH_NYZUO3)$y4=!T03$|vkDn0{E zvlB=?B_m5Ca}4!DrUkf+%HHRinio$zoIMsZ-+^`#pPEMG0)hEP;}KWd_#*}EC|95U zd^{*?bg4+Ra8xo7JCjcAkeH-lY%c&0vL-$&1WH1bK$a&!xv&sviQw? z|A?0gZF{M@Z#EqagIvfxs7m2EY~rY|t8vOsnDE8t20%|6mY%b(Q`bgRnkv@-N`f+US%loW8nu~{h(;*aY1#5JOZHIY$ z*Mf1>c1YRL)$P$}XSM|h!Aon!XIEgy`ATn-MKtt)Q4=!O5N#K(zg|qkj!hmpCG=n;rsbOoH^5--S;L3 zNm_tAg_x7>!XhM*ig@>#eTAJ?e_Z|OxNk=+A%?L-Hm(xm9!M^69hfgWa>w{wObJDD zf*n+=%Oab;Asl8q07MDsp(B?jcWglRWqzh)m$3hg%Qh0gAG{Hol9 zM&z%@Bk9G60D2NdHwS?Q=z@w5CNq3coK^z*!SpUnLb{IG^3a{wYSS6fEwo{n4Pgnr zjKxuj{Kax{i|Z6ujw^o5qY-#Iwb;FgqbT}e`4`CIq6Q`kZO%WW(**p!b5s2rP20XX)`a7KTm26fwloCr7E@EXV zUn@*#L9;~DzPXs8SwutK=tbqsTb&6M3$oRzZ6*voBity=Tk#0(a6Si2d5?u)*`8L0 zn`~N26~PWW2Q%Ofk9;a)gp+8_=;jfNeX9Vp1%@m%{&e%g*o9VAF<4rK-wT4~ONlT$N054z93VHDo=*to-Hq9h@9~rtyLkrB= zXyf?X_$}%OBs};}Y>~X<;F2#OsefGI=z`#ye$%(ZLq6u=G7aiRV^0v=ifGWfrxf%P zw5DL4M>H)sms#VMrWpi*i(a8)?y?7>5_0gUIp( zoB%6UFDeib_(rp>KXOy|K(5NdLrcPUPbXm5)DVg2{|LaK*C(GSj>f^gF+DNn0^{t;-{6ne-dV7IGDg!f8DNUnOG5kEMvop%nzLD|51~^AqauUDNIO8Ud(o z^)*>1EY%NG#~d3Ny}0T!&$1&ADajpdgr>&n@5*FrN2)bNBJD*7JBbs( z3OvO=aNW*eS&awPdbPXH4QGU{yhrT~;l2TIrK>%&DukU~PQfgy*a5XwyX}T&S{zd9 zB`}x}JFf}o75wfSiywerA2GkD)G|rdC&o80n2hFvU>~*C=e}f=Lc{48o!hV~;&`L} z3gs6LUtcVLWKB!ic#id;f5-I=+A2Zd)`^_5qeAw4<9Vz-IoF?W5I#j=cPPpIqH*QZ zn5kW`@nGGUkmVFrSF8U zc&19>bY`la;76n0Xf9P=Kjz#ud&JfK19E0tYmt*C%ma^-VE=)X_odyMJb>kS0#zFY z=i?dW)Z79yu@_63wL}$h3AWy4(*QSi%MT_>xUF&?{HC()_3i}|aYD>V zN&TdQbo?95%y2qeD@ll>Bz8rHw3ihz%ZrfpJQRDR&nYEDBERUvCFW=CH2arTjHC7N zXsyvb<5z(R=@p5K@6T~_Uk`Z)33P20g5-Q+BzB_>hzulakL1OP9^-mDkq(9GWnNm% zCGvPz@#p7l1lJ3ku_Iv=tTa7C^yQ@H5-v{$&D@Eo;V{XT%Prd;{dVTC_f(~_2sTl~ z?N--W35~A@QY@-$yNcP&VIOz(xc2PGNK^=)_~UJN#MkW0W=e(cjzxg1zgN=w;<9rW z*{HG^7!4h03E({l(_%OWjF%kZ95!E+ z;AuV5*Co01`TyDbJJ=*EOw3C$x2XM9#x1LAIx2oDw&sWU1BMu)T(MiJI-?V-0 z6Q#}?;aV4Z8yAC5SyN606FM}>=heON#k_*kJGPvb$-8(7^4LfyCRnaxlfw01xlQeW z4wXW2Li=&3i8u*U6=Z*qO7N&_QxdaPfd-~Sq4OJ$IQ35g`E>PcOKBvnD%lgxBwZME z8rX4Lo!M{Y4QjKG(@igr@{4Xf&`nn$aF!U0FPI}fdot;9r0cB4*@k>j54a~Z<=c5V zRFkPiL)=w+*(Ys-3w@2VZdzPrakxF+Y+{H!%BjgG`M<;%kME}OqG8lp~&^2He zx>TU-_IBn4;3I(Wp0g8Sw?g+it3Ltf73*FA@XwzP?`&*!>a8VvM!fm69ot@;3yjJ7 zJ@8XW2^O)fX|O%Te-`kFZriT-$%H^pOFcVugSM~)Hkd{k76Tq8Q zD`2+#JvH!}YL^U!OmCxs$Z&@mJg_QM1!NXS#L+K;`JtGX17`e~oj9rDBq@D$Fpe4XqADdRWI|*PA zxTS z34`V4q1b099d#M8?VIBFXS{p}N6QdVToUj5-2;oMotgUaJ7lXgJ?~_I5ZxL!QNAx{ z2iF&A{a%9vuQeEI6x1y#iL|Qx$H$iYKUu8d?f>n&{{Q*srkZuU?1AjbyS|Qv=+6FX LWw-yaC58Skgq<$1 literal 0 HcmV?d00001 diff --git a/benchmarks/results/smoke_postpush_20260423_123452/charts/membench_comparison.png b/benchmarks/results/smoke_postpush_20260423_123452/charts/membench_comparison.png new file mode 100644 index 0000000000000000000000000000000000000000..30a6b2575088134a38bfbd7efd972152367f5e81 GIT binary patch literal 33079 zcmeFZ2T;`Ow=LMBpaOzQXp~?&h-45YNK!cJQ52M%5fBlHP0pYalxQMI6eZ`JGc>KB zfMm%*L}HU;1G7K8|Noo0^KQNSs-|jcs^(OkYDM}NzOeV&YpwmgyeEJ22<1si3tx;IzGwwT+3T zg%F>h0G}w&>BqLVRyJ4o`OW|R8+?}5#{4x}cFk~=!&d)j*kCYBk?8+@x9pz3!(co= z-TM2QvSZZDz~S3+EU-(VOkP`Gc1nu8wO14zfd) zZnD!|8L|VVcCv&Z);e;m_R&Oz*cjHUwCGVGr|K z>DH6($kLO}H~VpMrl4DQB-&w+)4D$`t1_ai=t(D6M~V_hdc|t*NShr0k-SIWxZ0OC zCX-@is@%70uS}=YO?9S)_4Xzy#0r@8WM5=G)#uiu@9Vuf+m|%)Bl-TP?1ctqUw{5e zQ4(pXC4nuhtlMR-gkxLsaqEZHPd9^G^G(`2f|{eneND3}gh>m8Q~qXk2T!>km6zK1 zxtxAd#3CvvICeV5W$t}5bL@C6X+nDUurHhQ<4yVW{I=VZ8V$s6SUuNcjdjeH<;Q}g zHh*^{QXDxMPi%*`XQq*uub%(?DL9d+b?dx*6jQUr(pOu(h46m2Xvfi)BX1R?I9Hq( z-v5fRT;5(A^37|$!p+Jo{mC(QL1=3xuf>mL?dMwsA6AEc!TiVH&(0Q&_%J69T_eed zjbF*B@YrmRt&K3vd|&CGmRb6w$6?5=4qljXV?)PjJY-Vjc$Q9@YR0l;@8eH_wLgqw zSl>a-I@cbXeFTax{Dq{yl%KVdK*0)j+GU-V35J?Jwp z;w>gtZc#qzObyxL+v$Dy_AaIS#(3DX6s06aTjF3vy<4hEnyLVdzhc&9leYM0nL1^m zj%ITO8^0p4q_wJDo#CD^J$JFHmKVp)QC_kAeQ$KZZmNTmNy^pn%CFS4%-iN%CRMEV zJ#h~P%blny-{0BUV*4gycS*)I z?pw2JQ)2f0IKpw_Ilt)^$+dpP*lTlku-?NqqaD{re5q75O%jd=oA&0L)OhSH#Xe(S z*&{-+x?-$aJANA1S8f33RgZm+dZ+-nnyxNNFAUvuyj z{nuFOs$rNqz7@@zr`(p$W(hs|8g5(pCMZj{!nwwC-~QVY1lS@)pC{V0_3NHZ#ClZL zk5PHVjuAh`6K&=;+*dl)sN8-fMOinV?ORNe-ZptUHoD|r72?fWwAjdvEr&(TK3JE# z3FGf~1LD#yr?dFiVGUEd zZ%!uJM(=EIVJ}(qi<$?UUtoK}Ky$H6+veLd6;+Ld;UMw33+wRfN?NB}W^$f|@@oAm zB6wclJ7rb9kNne(z}x1V>tn%GL06`dNlp|rEX7yFsq~WEwpJ%~v&!a5pYd0ba)*6S zxnE!sHdC)@fDP4Z^Y=kE#|OQ-uD@a!iXA})`t-~&*s@z`fdv=q}DgJ{}lKjR| ztHyVnR6A5UaV~-ST}~6xVYSKY*9KiC)?T*6{bd|&r}`S>Fh5*FPRAjCXM#z@ zLc0b+jYc+H2-|PdBFXN~p;_=Gq>`@BEblI5_~U`w;p z17!|1N(z!H^RNKoTev7}Ja)H+sD$B53kUZxN|vbDgiToplU?SLSw`!?~2NIacAsd^`;aLEk(m;HRcL5(^^P+0*sJ zHdyWvWCQEeV|Jqx$GZ7vSQf;Vr#iU{Mmp8>c-NQP6(AVGI-_KB9De%juE@ZpQ|=+l zl3XM=3kci2{4v}+zeb7YV}y2m7#T2?>!beHYnLWsT=?9=+;=u6mRp|2$@(&~z-&#r zX!gT3b^aO-CNvNg%XQ7ch<4x$7k8K=+hVc)d}c#{yV8JM z7bmFQq!JNA{}28skA;?bK_}(;JK^^Xv0*Bk@?@J7B!qdgK3J@F1 zd8EJZ0pFbc)y!@m? z{lWINgRJic$JSkO+J$oiW&GI+N%GM$VSGAmi&~ZyDi8a=N6{?FljBL3k#@DEFM_ewxokCJ_8+ zQS8O`aJUmj9Ua--Cj9*kck#F*{{9oqZnThOKV6SWiZ`Ys%mnDd^Lo!Q6Ep&FL5(fqryY3OIBY63Gw3`F^*^VlFYB6n;7+D0R24ij0YEk z=^nm&m{HKx$WP2~mrscKBz|)pPnTr71in@KJLiZV zBtnA$(|1(Y#@y&*?>$_fADMIumR$XZ&iCD#!$*z0NBSWus<)aTZVVVii^yMDT<-nGaLTK5|>@!42~>uzC; ztCE@>4xBO4KApJ4*-`Vr7GWP>$l%)Q?`nnY!v4Yr@xK(Eo`a|y_2_W0&~d_v2py z9kbq|-*M$yEXx;=Th#GumfD(eU3aKIb~AqNYplnP6G~g{l?!(81c|dH_X>EPE{yuA zL?9?XWj~N>)YQQ)syc-~EuhK>$mOg#_f8q_^n{9|i%G27b+=Rd9$aUK$|IE9yDpe% zhcpy#rGtI+uc-qPb`Sfp$Jo+sf@_Wo=?l=xw~cE;(R~$?uc9H z`2!hWiHhkD%OC-?EutJ5YS_cbeSJ*SccF|?FTtBI&RaO^$DMg-$bHLvefT+TEyC*^ z-r}nK4MTP;4mb$Adc)zhYibd-ePO;WR2bC*WLGf{N)(HyM-T{Br9b6C!lm>Xkarlz zx`l>PDg1pbZ0rDQh*Or})h~C=76x62-s5*&Z<57L5SktBfXjhqHb0M==7 zqv?quZdu|p*|L?CIDmWlPRI8*ir$i2Zk44SC*0xj7yYH(49Qz_${qLU7ZVwy^dlz{ zpX6$f;l_CFP(tm1iQM!c^Yd0w8w(@9F~Jy*oelP&jrT6AfWlNxN@rh^^`X8+6l5BF zl9s`hUx9hWus2rIizm{&KSo-Va+Z@8o2Uk)RVByI(SmpI>(b@-JL5l2rM>tQq_6B)fTF}6KWPccZzy@qs2q_ zccph4KaKMHE5SG|Hz9MZqQ6J=+PJ40YjvJYkw{P;NsC)}&fDGcR>vLKdt&i>Q( z&K7ZLErKe3v=T?=-^j@E<;9r9pk24lMqvZdF_=xV&DwS>P=rzJynNxYG~-W9f9W|- zCQRD+uWk35W+f1RLEfqOe8Rq|?l4ExtJ3YAmR*v9BP6tT$@&&f)vXockaQH3ejD$d z$B5@R?5_vU28~V>57_s40}Ma29Xe_`MfBKpNmUJ0`LM8iVZCo?M|EV445M*BFP9mE zxu;DE-csLi7=C&%5p$+lJ0h|o|1X&E%w=Dux`;YjX$f)@c5hqUKn{UTYcxkN+yquO zq$IDNL7O&_QM%t{jnyMco=Sew`N+Y^#_iC2;0+m463nZG=!G( z6cIT4{!xRgOlf!5pnA-faQylDrs6znnjj?O8x4iOO3ks1vaL+V8(N839^0nQ42&KM z3DHI6cpQnZ^!G=_B0^&o(?u+8Ju0*4*L55Q>`ZDPPu$M4P$Dy3UELts);F-(u~JM)-2G2)WRo6;P#<5Q+`J~mJW)F_&bi4)7;K5L|5 zz2fD(GL`V&5DKB$4m&X)tCHL0M!Zrj>Tc`6FC_SVQOBL=#!t|5pQF9<`-AIsJ)Gdz zfKIC%Zd|`ekSw-JZh>vgCXmh$Dl<-fr<}>e$V7;#@&0Qk%J7-Fja(^Brs)CVD;-L5 z5}BwlX~}|OchZF+c)Bx9ZF6SSfkR~S&h?X*9^acvWR`w=M5JH9uKk-^m|@Ylw%ur` zmW9S>{F$6w{+-ORiT=d9k+!=FHbGU-nYd8(b-h9Q0ERI`^BT)hV6c)et@Ya|h)yMc zYRmPIJcwzGq`ZoO|NfW6l@OHx`n`wY|IOe37l+iDK0}fHcxAeq*5=pO!vNt@35YR# zbw-*rFPjYS87^k?$XiDiQ%v?^WT@O}iL2}cy3xBlqt_dUFlv~R#;PxGIn?Zx&`&Si z%+M;LVfe=6iT?J#&AOq7L0=3>`tafO*GLimiv4nlP`}092s}CI=JXdL={&j4`?yK( zl5ExitJhgpZJ#GSw?0}|1xvsiNE$B~Up_Vc4}1Jx>Z^^V317>~jq~0fS2rl8g^ZiO zR>AccQq}aNZVc^>!BvJuC_=~Usu}{BeSMBF`-A{nP?Do8q^K{UtU_o$we<+l8Z+ygR=_50YJle@Z?}>3b~a_TdX2-X2;Zy#=OWLGK0hWz==+U4qU1S`3`*V3WoRvpM~I z9hZ7^j~9J)Sdcn!M3?&J#_khFO*Gb??Di$tZMu%Up4#+ zhCN)u$yyZFKjI8!qp#S>j5P$xuFVb90gH9)*SE-`o))u?(XmOpL?IH3v1{HMn@sy3(6tPn2lH|JJ3u-*~lYCW2yT^-=|_ekxYXEb|yVR zQ{}x8q8D;r{@n`9<2`XIm(^WSft;_X~ICGg;CF>pFcABs_u2HNIqIXv+sLlORHUcT@E zVEj0UCi;-M6xL4eFZm5@AuSZUu9Otr8phGtU(QE*0mKv|$EoQ$8KFR33n1x)s;<*G z{>e08C9J&k_A=r+vu6wY5=|@NUL%0?{%JWQ)jhhiT2$rjzP;&bij?!X^LAh*crkE8m=OW~5s0Z%mV-u1Wh;i06xm7u~hp!|;L zzd|?Q{o)*mgClS~sxVz=6Ut1+@iJm16@o96yB5VGK32cp8~-Xg0(srrUTzE(!*E{@ zl>Zl}DQMZE$OwoOH7gph=d|US>(a73Qq*rPCkmtowuh19yW;{L+n$qiQGg?e=+JhH zBOzyp4`>cs(&+D@Dv#9D0WFyj4@?FC1u_-(82HcE(<#JreL;0?l?Sy3%1p{DB0z`% zMRkx~Y^WGFw%F+=cqj1MBWg^fh-FOHqrb5a;Qc(wxyB5zZGcfEy-iEz`8936T~_MU zSO&gQ@+%sgKXiYM4*{P$ivwu zAr-tF(d#I+o9-yA9nn*U;=~GAJ8!s7;zWM20sF{!m|hI+?nfs12ELm6)T~rxKi|l; zxln8R!?m2JGq7EdP4N~tRX&j+ZQ{eL{&l;9thVOn#Fm%kb9 zE8DulIzKi?mbu>=aFtsEIN}ABtVjTppg~;s4!jd1H-^VNbXe5smU8jOi#|c$0UL*& zf1v;6JvkFx_jH}LiVMCI@Z!|WFP=589}y^6g~V~+0rhQGeP*YC;J z=S=q}BFwLD*Qv5(c&1@C$Hn4$lM0nQa1r0CzrOjC>*(9^j9W-|O4;u&3flf|?bt5T zuk$fc^uPYYzR$dUYd@{Tz0pSlBv?B9N;LftnSfX~sgczgaXV-K4#i*A2o z75jH6>sm`}CImcoNLJlC4k44_qQd(09MU7Z|8b{WsF;(c@*p`0N7;30T(6B+H+KLb zC21T;#>*gn7{6}elYj92!?ouEPdbvxAN>rZv+YMk{|d?Q^j^)Qv4$ju!&gVmDsrOz zKVA1{G|{dyX;0)m%2x@b`Unufv`x1y7>B9&%LN7$a1fEHP@%R%1lpF}+cQUI0jdhk zN3+_4?+>74qFy~WL8}`V)2O7zs%|miC$!su$xw)u8Y%3vkQ;bV1RF^J+tGI7$j#+7#|B7c(>ML7`(h; z4y07v&0w}T;IvG%T<26m_JEM50c=P{C{ZisfpO;~H^^z^^RWZ6@K)R~1o&v8g(f4~ zWT~tEK%g|oxGsL7ztAvM#njIm$b^wv{CqB@mxzk+btpvZAXKr&I8UX}kMh1`x-GR( zPp6-jQAjyP2v(SgfSg0Q|KO36^q~nLx2ctZLgL?_(sI|B1%Bxu|mrF@ye?sw}M=@i*p zsn+DDdD4b{#c=gNqonj6ZF<2D@k4%+rn1E)g&AeK16?o8!mR*z-T;j z@0%Z~tFtJZEmV%xN`DfNrS6U0UkwkLDI-((l8z&B;>7-T03K!aZU*z%wEa~;r@PKG z#V)|mF#xe&V%4dl<}E%~{HecBRb_8cdXh`nbv{+uOpySrssA-x5Gl=aBibX4#mVx~ zS5gczib2=O`ez-B2yM=4|4vT?=A=me5JZph6%RkN4P=^Pum>3Gb3*7*{F53;#PlsV z$l6{YarrXT0B+WJvW~@DR3n=Eb!y#u0Dr}D8$G6h>sDx#;2!ezdpV%;sRv)=xLzc{ z?)?m4qnwVfT;Cu}^zzPZ(Ga6XEc0{XD_I)O14yLHf;cZKdwc~hxr ziGuWDKTAM2NUk@o8$GrMN8h~#DV)D^mEk_ikfr`;K&Ag1{6wzAf-PFTw^RPv0-=a& z_D4Qw>JpX6$l$jG@<5)5lc@Dpk;^3=|ATAW0n;w;01uXM!kQax%*a6`FdQ!; z{P1ZK8jb|zN9OtGw%S&h+(hl(YxqA8K%WO|)r(K3H2$U7DrhMJz^k0?GyWFAfHWLR zHwy&&PKsM0_z(U8syj!m0Yip0*=ZJ8L=Q>>tAD)6vhc?v6P`-6!+@E{`dkRenWt4a zRGRE6yoKlJ(_1T3`mj^pYoI)f zts^)NaKyS^48&v8_HeFs$qnj@S*25{wp9@9s6az7cD@UkqpIbEanvy=M(IoKXHPZ+ zu`(kf5fzHd?#thAhRKO1i*^D&qvV<=y))#ru-LSOaZo>wDsY6(HRoF6Q0*Oh@Nfu8 zT#KfahI;2Ri>91-jWhe0E@d%qU^*1>kF zxnO#X>M^pb@KaCf-#_w<1-iM}1u0I$-~S)B!rUXzH|^$APE+N*6E1LuhaV+}-sMK_ zEU)oSz~ygyQ61yMpb~NzLx3>416d)7))YVkHRw%uP7UsoT6TFlo%CE6@YyKPL2>B| z8Qcfz z{|KxD8ecFbC`Mb(^yG|yfrADuz!r(6JBR|TgLHHpX|Szj4n$=fuz(z)T3cvy=ANhU z$PqC@DZh;Ky{ZvewPCQNFlpA(g9@_QUA^?F&ak;Iw0iE{yLS}n=h;im;uGBj{uxlP z-#4F1!$?C#Wc^qYE|$%MA9P<}!0%QPzkj;<7&conEaBgS6)s3tPa&WpgalrRJ7axF zw?i(LIJnD5{9AcSz^+`mM1OP2^qd_r{;;AKi4^0mRP5 zU|j&tAIT;t$hk@5{2qq^dh@O9fXEhYg~XEyUC=?7#cb#L#iDGQ#YTW!Zq_?z5jjOWi(1fG$WU>fk5xY18 z5siIE#DlsPVGAp8gCrc~qHuv=aFa6AqH22;(dQj$0vr?%k+EhE&e2Mfva8GYv-4ti zTp(!a&A6O|nw@gWz>|?^Sh3P2C&>)_X7?oz8586ZAu9tj!$lpDLF4kQe;4Z~X?Q8@ z1pf-yH};NGsqLN#Fr8FT)$7a@R>IoShovZAGlpOg#K1_~X70@5vn;l)*Vx(UsH%~_ zprfJ~fS5iMt#`~J|I6=H=1u%nQLFybtFVFdS?U^@m>AuSZeL@tgQ26Z?*pBx3A@z; zaQieXvmR2dUT#GJb84~$M0Zt(xj8UH&aecl+rXJ2O$oeKm@e*p+xSjd6@VRbpu--$ zHV@ObzSUc}+H1;hr46pI_ClbmGlr2ZCU+KG8h84x*w3gVTNh~)aU2;Qz)mktKl&D- zg3#16q?3ODxI_+>zsqZz5ZkXG^Fat;Rf4+h8C<36b0HFGGO|GbY?P$zgA!}0WeliL z&V_!oAQ3>Kg3Yb=a2q6WDjHxLZD+o4#Fkg%N^H;zvi}lZup9BlGCJwzxB)$>(03z< zMdN2N3cwjzFv2y%HP5ldgJ8vZSGg;O^qF31smQi|08l2eJ&Atn(Y*fsH_3;Ozeq^B z54Ti&Z+5Fo-7xU%79!-hV<5Wlm_aFZy(=1ma~ve65hw_F+Y&^Y#pWSRdLvo@A>5SQ ztZXF5s2p<#2sr{^>04Hd%0k6_E!BpdW%K+qpm@Uy|9JSe7{bdvIFEvcN z(%*9(!m5?8!77mWR&&oW#<#zp2G4!Y5dVmt`v9%zuqSzQ5d=1Gz{w)YmsK7y4lmL- zIoXWIC785+xaPbIMR}ub^SUhOT^{p5A?jVA;r$0x;jKTv7qQf(bTkM*K9ncita1It z*v`bWqNsJ!Xb&#Og>S{-fZjCyH*=PD)2MBYu zL1&#Pt!7V$N%i91)K>d(7sGYc0=*7-SyLp{YQps z>t#}E_YHsi>8u{QN-`9UppzwBpvcXUdAqBCDL!C5;0^_y#*|dm&Zg;W{+}P;t&VYN zvoaZ{gIi%`to-BiTl?<`lD4jJKNCG%8!WvO9%>l&b=_dZU|jQ-b?4ih(zJD(fVZmk zCycR=K=0$R)}EnY*z&E=j+{gMX{|?^u14 zST_rubY1RpNr$YQB14mIallMU>sTUpq2Qng_2wS6N3i&w<7>Duq0(#fRiz$3CRh3> zGnH>HtWVdmDpx8j!bmhR-jw%_t2>NzC2qaAU+BW!>uS5#-6!QvhdB~1Huta=$KxQFZs6t(vEc;XTVL=DZUcSi4&W>u_e zQihDC7t5V(R=cNoRZPFu>pG2+>Kpab3}u%Ir@1Q$t!eA4J(3O@Dy9t|aNq+<#ZU)B+&N&B@6vhJTcjlc{a_HyFRy*FY^x#W4ZaR9GUO`sEuuXA=c9d<5R%%r>5;tx zxP@k}wDdRDG3s&Q<6qRsJ*kd)C@&D~fpy`}&H(U`BJIG^v4pf*cekh!sOy4meQ_Xt z$L{2y`1pg-=9#D%0qmoZ&POdEca2ouzp-r_i>c@8n_H~hkLY>v1tdX)ON>f z@uFjvnYmUbWYrKvC0W1?mm=xQV*O9*`F`+UDUJ8T4rcxj7Kr5>anzq*zDZE_<-~Mf zk>yljd8evYLJoCbp*eG?xB3e^JcC;NL_TN|dm#?xOOzm~0VTLQLtfU}*fvy~0;c`k zxF~L>p=S;utBTzC#(;fH2U{cnH7$8~K!TavhSmR1yDe6}#-`nY@bv!JjChV$esNC$ z5*}!gdobe7mDMsm;0^1LzP%YiIJE(2u=36vqF@^PcGhMKWh6V!_-^m+7G~YS#dyMt zOHs2)h}yZiL77<##8Ox(p`rW?g@vG%rfEh2qo}Ztvzm^bihD`6x)*)fVcHWH|BB}J zagd|KG(4C?sdXU4aIGI4D_5eR3g%4LI&K_=yYU#3Fy&FhkO7p?{61QgjE7#j_D_Fd zy0%pf`J~1XCV|s|Ve634lUmq=zB3U8Ex36e$qH%*#+j#k1-u>qdly{HvZ~75>WtCZ zVf&E*r3Z@h#!E*ZS&RlD)xzXg1k)UB>f;9|0TSVO@aztmvQ3XpjOS!7bT{JXD=zBpG4e>n;M%M%EW(l?JIPyajL zXV0F||HKd5fboe2NS`?}^Wc#ozOtF)0Mkw&tef8gON`ZrhV`T_IOLYi{0;`Zi~dt@ zhl3`S)XL@KA(&hSOnDs&R6|mLA?SZIIKZI#!mbiwasC~*VFv?apXx3SG=h>NB7T!7 zt1nQvApfxAOb$VDrLX$w{v`rnK8F#AU|-Z)q5%kRnAiEz(gHKpr|4#plPXvVp_Lkg z?8uade}H8_JjR&%4l)ZIUwc`jmrg^1aI{!s=-L2vk(wiTbZS-9bQtD}$AYXawN%dR zEoDZ5pV>)i%SySdXRNnB2Azvv5Lr+(hb|c5%a$(+5|28Fgv}KW3r7o(UdZGu@c(dHS25GWBl? z2`^iMz=QQ75d;~n*UBe4}Uu;aF?tIzega5NlFK__$qL(_${ZX_M zz|RRcEW3Y;k<;iTdeV1_uU7%iEBEG^plT!*bm^CUa$0(WPr5Us2ge+}sM10oZXl}> zP?gYuA~m@k=gW3m=%=cWdC`Dyg*4RO>=wa!N5YJ&MW71trg)v;9fzv!lKkiY(d={K z5!fsbAugZJW(Fuu{i%+W>xGEolmJf{!)AYw*st(Q_QTLWQ498b#_-cWfi2|^o6ffw zxKiN(ObollkXhchHg#8|p1>N?duptiz`2}LT*gy<@X6*C{>W!czDvEUv+l?M3`~wK&FSfkk=;NeUC6R&LNc{ z;-b-Q_gGZQP@lYX!FIOyasaEiumljkaUd^EZUY6)7iNDCx7T2$JiNh4Q0(tt9>Qdc zu2xx!&{97*!DQXj9wXFLA!hT-t3`TOg4Y4Wi4JJ}$c;P@m%7`m$^RTmF7)94gH}p^ zWF&%q;F+)n3&IOpfJ(n^5x$6Dv77q&U2PxPRrGJv>mUZS{mp7*2lNmnChgF^)9~Y~ z6w5O|guHI$3dLEKhL{N2!h;rI%bs7YI>P$J1HB?bHpyy0Rffa?yW}^8B z#5i-rXYOSgNfyw<_u7+kBth1(7X_zLyO7C`T%%;LxI)fv#~<52}l8NAfSAK$l*ZZ9o5V(V5mI?E*)YnQsgF2 z*iU!yy7?@KJq-C1edqy5xg(#G;=O!Gu}Q7J0qN36IxinW-X784AD%1rQH&P-Y z?(D5ua!U}531oX&i!vE}tRC^hW??$^hmqNYa*_r_U@JthZfbzQD5Q^alTzZHTN6as z!GF@?=hx@zz$oyhwg!j^@4Y50pw7Kb1^+^=WO9<0)V({4VO3tJ_(zRvoUa0A18RCu z3s%qWGb+AAF#K+%&F3S{-UPjTI)Eg^-nJ@_UHqmL>i(K^0^MH{I-(>IAC+<>SYq*? z9>^56fKXLOaUNTJLyCu-Kp{#r2K$6dVv7;Ol`rjb2s82z^J>+g;)cd zeYXz{RQ7nNmJh;(oPUcW^Tos_pd!W3SZ<<0azTsK*3TmRlLGZ?4iNd0{HkDOx;`ze zZ{_vT`L`^@ieLpji#x2uk&TI11qRRWU;$Gs#%9@lyWAb`!pW!W03!S(n+OQJBplL( z)&Wh$q_P6L8=JaFska2lOfJq?Uyx@86bQ<{*C0*NUk$xZJw2KipyKZ>i9}RH<#oOh zyb##=Qfzm#yGoJ4Abr#SscE`8k6C{)ZiBM(78fu+j(`YzcOu}a;U!QaY+YUL z5!fyU23EE!`A<5*8-?kFBBO;2o-e-SswK!08<1Fa)D^Bw#^+18uFs#HC>{0Vt_2mQ z4iV1uhj%|Z27#MUh?ig!`j^iW$j!fNsrc^@1>^m@zj=GyPrnh+>C9e`kgdKQMzw2A zz)zJj4dXqt9m#dmy?EGmt_!Y{+GBU#rT#_=@;%j@XW5SHVOpp5S2MLTWc@-;kHwaJ z$8{{ge&>`(%Vy9y-!;?qVPlo}z*yoi#gyKas={qY9yCyzcVjPtUgb0x*T4bKN*cPC z@aUWJ5%u0x(LwEI(#BF9D$7C*RdjC|s11u+3_jk*l)f&wco zKYzpD3|Q_R193w;eHNE&%c08ag@kE5--tla(bHG9fX*7<{{?@n58HxG;{=T; z=2S2d5MN^~zk^3k1T57-xif=;1Qw;lC281HRBo$+GXX7Jsx%EQ>abd6L;vW>x4(lq zi@Xc}iqB8QO7H*=AC%wtqghrDR)NNfAHNLL+v^tbIlXc!^IPU$3OMHU+E%r-494|u zA#0~D`%600zkyKU%ayWc57O~6>>g?RIak1g(xmd2qU?n=%2s>@ODlcqt}KC2CT{kHHhUyxup5rP&7LogvKC8l{kF8wOr(D}lIL-qBf_ zDtP!1Z1bz+{tPzA*Ydj6)ajVNsg6bgOUGG(jamPT!`_KcO8@gm zo*7qU+5~A7%cwpCCY4Kqa_LiL$b)Aa4Vr>eiR)^U-(L8AaN>nfcUPv4IAY)Rq3XF_ zN#{*nkI^Iyu0r>>m$QB$*3f68E>;A9^jFdNqprZ&STIAro+pCnNCS*(r-6>R zLuHShs85g>*jw#Gf{5&2zLBvQU*qGn4S4J13NdJ-3gKYoaVCz-B>-RiwYfP8ks+H_ zW6#ZrQKtZnEP^&1)lSMQg&_QSBN^|T6?0u77-_BEtES-R@*yw$W9~0>5~`jW6S4s0 zv=0nfgJ{P1K5#wxQ_$_%ARz|AEtypiwJ&ArY9sTE4+LkooJ@~~6UOlmEWjqD4Z(w`EM^q^tD9{>R;eX-KmcmhOdWJk=V;&SVy3FT3*N1@aPGSgC&3xZc7Tw zA3e@-9j zUbCi6{*#P%(~&|A*WOu%A9EACL{Tz~a`}~MUeQD<%(&%eeFc|vo;8U4bjU9N8U=Fc zhLmPu1o}n~oDy{P53w)#BBj1sm1h*P^_z?uWvzer*b5=y@v4~m_fk2<%il5vM86U@ zyYFQ<5d?@^!7?i+J7rYAMlQM=iRgP@B>GzJSdIiUEdJuQzwjHs!d0wFp6}3h8=G(Z{2jSop6+di{|qZHoVO1P%=#VU8+&NL zL0aOHR?c} z_DSSM71llWGPhZY$plD1(fG}D&CE6l7WeA8I6N13QkBC@iOQzyJ~~B1=kv9V#^*;(EfbHfWH?A)C@Q?C{LR1v9pB&;60}*AbX2C8_&ur4 zRAG>=u;<|?@)$ypWD&K?>JL7%))d*-D?{#UvOU|aO0})}b#;acRYzC*+z6z?K`l$R zhVr%hlpDg+x%%#IB$t)YG_N0d6JB9{XYrD}WnP8h3Eu`u7VQ|@O;E0#q*#-~a%WO`LQ z|JbrG;59hdbPd+Ac72Y|$!n_~!wr4XcC2q`x-U;+{miB-Rv0HGRV&6_H&%Ymch+0h zuJ)N7E1dq$>pH(+%$d%ud#w1ceZz||NEVy1?to2Fq4&JFe2CLzqd;WnFz`bu^Y*IO z|Kyn4IYf&-{^cL_3mY*WcFL(!Tf~mowjuJk(_hzDit@MH`78NyJ=QFkK_{OAnrlM= z>DjUs&UQ--?t+-8jXvVY_axnarZ?yEOizl;5A3?KL|n395}qyJq1;Y?(v^N<)2yhI zY+5U`SK^X++Xa%uY?#2x#N?!9W7vOS^;yj<-Ko5nXLN_?TBzu%-YbHN@pZl6^ALB2 zvbgP#*_iUa_02xRI(f;W+YIi&@~g8l?XP~kZfco;nI9Lqk#tzrplARDZ(>jsm5VAk zM1^Ug(&EBteWF-V!ibu#&QLzl=`3{hfD=V97_R#!Z!PMlD%^slguFd&vK1RMU7p#Q zAHrpM|9Cri*rv*jKfq<%!?>GQ;_yH6ncPJGtjhO=RT2iaJyrujCLe^3SbX!y(JCLg|?s>Xli%%P?~wvPvb6{}rkCQ+4{%$dwPPGbos64gC{mxD~O zZmTNaGppDG_pi52Tl(ajay_27Bf-1OML&?*u~Cw_UUrM-RYWQJyF@pcWtC&bw-?1;E`NTl%L-Hr9U`PNqBrLo=d zpQ+n(&W|6j&osT%5A4)pd@pKfLRJaZ(vaec&L?S!Yy2GYG#;d!;lEkxbY0z2G8M-E zxOsRIrAkDR3cJjKPHeettVUXi?uG|+Y>L% z%U>HC{Zbm9MAt1oL8Tr@rV350S8NXz>{ycUjbHz7Ud}&fxd15^c`9j#rMZISb5T0; z6YU_!a!(q(2CN zUM%?U-7u_{HmaUe@1GXj+mA7#Z%t`4;0`%iMns2~trnc(xJO5T(amb|PCHQtJ3i zUY~J8uH$ei%mly7NBCVT-1R%m2pcOx{;_+@!DhC zepeQ5rMSCNK2jBda`(<1uR8Tb^FNoVxT_cT)P zkPp)-NW88sX6iz^+FnlwkgDqI;B%@4yDQU;0H$N8@$X_RF}I0NZe$3UkK>hqIBhFy zo*(puqJqQQL+~LB_+}h|B;a1IL|1;GpLz%52z*-N8frm-BNTphLspeQMphGWuUpap z7r@`XVKDxHKJ6ex4b_KzNIL+|q%2ZCa$TcbXU4<0mo10L_g?o-yYmY z%kEGEctPcq^0|=dY$fXpSOZ7ryHbU83{N@nV|`{7 zV$4o%WdXMm=dw1paqH@+sOJI^aRzX3N^#U2BE2z;dLf4}#$Ur<4^DSy=}k%DtY7^G zV=iaA*jUNv*BY$0)wN<6JEf!Hd0?0Jg*+_<&dt=X=4K!RrSv7L-xb1ZcBrEJyD^TB}Mf8Xqc zj9D+QXF|u5RkPT{&u#~x?9fpToCjQips54FJFK?)`g;kd#j|Pq{ab{~&u)!$KWnrg zyBc!ec#t+^;D9Gr+!vw|v-}~?1T}uy6Sr00FdFzVkt_p|QUc1*5 zk)k*Q_mX_Sn*N`sG)@+wne${&fT1_U_V!#BaRw3=+am>KlJdS{glw-J#VQ%0G2Y&;NPXo2Cg}YliB~p>w7dexn-emRrBNYqy zYI^6jIh~){AgFvf=!Yz*;UH_go<+?~OcIV(nscDXkq@H4iQ&->2|w@`v=3L*nj}Rt zLc8)&)T%^3)d{C!p^oP}k;1&+e!D<36;|vKhh>LExfi#F!Vbn?G2_FB z4-a0u+@CaLx&hQo^H&a_*mwJ`pY-%oGp^W@c;spHpf%n>!WcOBL2{*wOX*s`|0jV8 zzn67XUjI+4tN)5o|6deu|3wn}@6!AK|JPxU{D1dwy#MzfJ`JGGIvDF}!R&VmY21i; z^Z^5|FYrks=BU-Z4*bZq;OV3T9a8DXUDQVH$snIV;R#ZiE7I!MfkKxB41^B(dknIu z`_}rxSRLwzU<0umbt@`AQbL;518-#$9Z%_<^|5SF?0qNOlYBvG|2JrPQirTNK=tI+PXsXdAw%tB(2OW~0lIW% zgK@_joRsPPy?e?8w=#x_gM@ftY{?K@xDL#nP2&;B;Br!KK2T)vP^5XG#&5{};Sj?` zq4`M$`Xp;X!>ClB*aEj|+s3FUYLzuWveh3`C9-3KXTb}Yw0ClQT{{oFr^x`@jG@!I zc&w|CD_5n1Hv=651P*X%I1BAG>a>l6BNR-wk&~{r3iJ^#f4L*z2DWK`E_6 zkg!5j)dsg;!jn@l>6nL^R8}n9mPkF(dE5;`Vk@{<@}+`WupAu-ebr`F?VLo`(`?kB z)L8(2kjOzmTa3L$mU^R7NHhz-WTMzB;`4?|2uKg*eN5(n8#$|yE?tM#Cer>&Z6}W- z>qo}`YUjkHKEyCd!t;ZDgk}VKdvCve{5P~}YdGV7*ut8du>w~-|B78DOu+IWXab6B zFBv?&*K|6Hy#%8I24S#}?R8=oAV!pp;squGb`Q33oxpS)QR*RlIAxYdz0zxPSM3Kah=;?CA+2 zWq4aQ3d+dNb8#<*NP6m=0|&yE3qct4(^8Cltco;!e(y+j!%=>1{U#_5me5=@0t5;F zRr|2SHEWmeDL&6i0ss&D#2egMbLV>nlePZKaF;m4FMrGWarP-@p~vmT=g+|NPq}p7 zna2$2v$Q7fBj$Vr=7n=f}#IDjM}^U|i^ zv3bVES90fp8Lh1QNQxf>q)lC@{}6nVLvg=*LV3YKHDyA`1L}< zBnfqBW>oJSD2Wo?Pi2>D*t_dXGpsV)Irce*Z@@b$73_a}E2!3y!a#V6&JHHMV1nX} zsUEFQE^TXk>36%QcqgQ`^l1WZz_)MZ{ee0NcXTa4c3-VwT)!To@8N*MXRr% zYThl<_=iGGkbFIr_Y!Fmquec%-BD)Ud-R0}P&Q~yqF){90;6YXi*XsMATcu68$Lqc z0&S46(3MJusS|BPj%E88DcxP2Qx2JUOrkRhUJ-V@sX(PrKltf=w8%yAOZ3xydWBR!G^60tIXamK6 z&%$gsAJgn-1IJM7>HUs=LjmJCRgC^Qu3`nKNtF$u!o*|&3ZYATA(TB~UXAass8^j$ z-#X9Eh->330GcKi7@Ha_ho*Fqq6DLVT3vTBHE+-P%L6s9(aR?80OJa5NGOe4opvUPt^wJNR_~^xOH6YG%0+GkMF`tNfbp#@t z&|S99D0MAd-Dv$Wk_L+#!)Y&%ZVWMSa^|;6$F8KK- z`W1r+-5rb*l$PF**87sf4r#23!2iVx0%JZJLci8SKb!Yffsik8uBQKJhhWU@M<-?l*)ep zlql?E3|XX0`&<$1E@YO2E*AW`tv0KFu@(G$5SZ0Ui!*Jg{)R|_b*`Wef&LSF7Yi?6E`UKlwdV@^7i#8|Q>d7ga|@lfEx1L^nk3`g%*~^4PC< z$I$GEEU&RCAendbOL#LO@->3N!H+7+$q5X{Z6Z#z$A9=8ld$gVKt|PG05>=ofkB2T z$ov`}cfk_11g1oYXW|V#x~)9FTUaFEjtNbDa|nLZjbwO2@YES~<)#NSX?8vD4vVJg zS>n|u;8hb5wRmLuzHg@@OavHy%K79yBN^dC4=SFnzTEQNqYnnD9 zQ#QEMEk3Ww$mhQw?LAZkYq96@$w!7{nGn$2BAeLbo0q?un_Etsh-wz%%If_2^=_v~ z*PzfL%SSd8$$dk8f1BVDp;!Ch^cVq_l_?&`t-qDw+XjFUe0mFOjX zpHZq5{v8PWSO1Fl_#fK(#~QM^s{$wr(eSn+-jgke4fxl-Smld8Wf1ieTPyQ`B~z(g;|5(_p?Z#1c^tes7;FbS^R72tAv+z7Brk2l=lw5!JtQIzH=J z4|*yw5OQ;>TYp~p>wZ*8WAhOxgC+RcDB=7I&{blfQ(ltl|M14q>jRPY$PJL7{b@yq zY$$?)n-|=??HPAi!?0pi0Cb43|FZ)*<6hDZnX1OD1`55@I#0O8w$44_m(?$|egf&;-hyzSZjF+hjlYL3wF1!V_9OsU$BVpvF zvM(qseU!hpZ80S$*dUqorQk9ewWo=1i}Ic#wS%x=@!f^7u`uYRuZ;+spl zjBLi3>*ORmNP12)N2^Hl(X%7ben0x7D0L-`P+liUq}sBPf%YgJ>b!WC@JDKr zEt~;5YE~SeUo&vgHJj2rs`amO1DEz~0B1 zE}1ktR?x`vI=O6xVKBiSLEtZywlQKCmSzDQCbl`&!ja=RDyDmngmN;}&vkorQ7gHcbr5JhGnkqgr zyzJviXH3qOrdaCZx<$L-(KON?12kYa1tO zUXX4>CG4?9_$XZ*aKt)BB&&Hn`4xV6b63XpD#*gAk8(8UI7kMPd?tbIL4qR@Yf1L* za)@=-e!!QPKO`N4F42wCfElutmvlaTUeY4nliNN2aVWVx_Mj2;MHcfLoN`!EM$sKS z@hU4K%-?1;(kz=rukHcx=vgxp_rU0+Va(@%);}8t`FE<+m4GImt3z{` zY0!EEDxG{-^$X!WbI8(&{3C)@{b>z{yXl-n2O9AKv-KTEei$u^?CYpS%UtYM2T`sB zi7QMyYcj-#6&t-L7vHI+YbK@AyEGyN=jyU21g89BI{abf^7V=;CJ)7OAH`}>Ps-R;K>Z`RYkrGe3{J#wm zqk4xSpfW}4_T`l;oX&LGbBHv}<4NCDC-eZy0-j(;=i=FVQ(N*M##jGSXzQY2%C%36 zAGyaq`EC}wRrou#=nx zT7KGd=XKHQT!o&g0wDqFfMeZAtv+V6!hR3b&tc{T-*cCt-q#oa^oa61CUfJHFx%-PjP=AwZZqH8 zYX-skh7gD6PdgYtX>a+J>FCh(q7K6^JhiH~P@;eg1f*!ls`-lLLm8~RH1W?4HDrF< z!p9hI?bu!Ka|wL4BW}4hwwi~P=<(d#h#w^P;lCSjU4&l!9lX6-o5ENwk%V^XY1?Eo znOMYV&}^);o?=j`-O?-75$1n5Ck*LCCROxsm?_yvP24W$-qP`X5OwQeJo}fV7sDCf zCj+lbUp?MPP5jctb!x?K%tamyk2Q_GMY#8>J@d7fB%06(kCqg(OHg>a7~_;4SWYpF zu=HLfuxw*14(6^#Vw3w?hdys9!)USy)w|-z8lnPnjchALI+~ zgSm+u;kUU*fN`|BVYg{Upe*P#xbMh<+x+@ObeXBcMT{O15ZOX?=lQyjBRW*%TTG3} z=Ni^yKefO3I&yn%9cF!vq&PAEhGY6L?imk%2QkP}#QWZ0Cp~lWpaqFTVfK*LBX-df zd*Lxmcn3l4TXSO9@vvD|)~UOJ73YzUBRFi49vK0;UT`ARKGU7Jh-~&~!$n%>$o2UP zACnUxZJ7`xEAD@k13qwG_|CEU1uZa(V8?D`384R4d@I?sW{p?$cqS=;a_`OViY}Nw zjw9a=GBWFV4@hA7Xiwz!6J*Rv-!cN@*GBlK@Uw~*6bY}vWmvXlI%6_5Mz3My2oDn; z7BE9ui4?}<pwypTg?4n$hYM57MwIPG3gs@Dn);XTSvGNJJVtT;k(FAPvmfdsDfX5IL@y z4AZ<9(lxCvq7qyLzwn5=z(`XZ;>H*Id= zLMiwhZw#Xc@OC6lg9#Px=ig|k16HZsiuwzn=V5D{4%rDWde-`l#Gvgg+C?OfRS-qe z%$hwOk?d(R3^|PFS$b3kGk-FCEla>XG3fco`p;lbu&$rw5NMK>yJt}-tj2@iCSu@Z zse4bM-)k{+&_I3J#u1C5&(lOAMDi{LS5g+z%ErzSiXu&vc@|UWBul>^H z4XEe!rTe)UmB*t!L6@x|+XDjWSmA9hrs@tQbij)(D0gFP*klKy=fUj-6tlzh&A@ zpCQYcn+Baej|Zq{6Hp#YTsS<6*_sj>89$Z@=R2Y6WILq}6Q&!T>5zuH?ts*iC^D8< zNSb0y%R|8@_FRE4i%)%W#EF(ZXe*<-lK~?(yXpUNCy&xaate#H3bE9v3^p5%!LBbQhcS!!=r zfm~<`nz~xr2ZUaWlpt^_q;jo^nurQPqt=rg3290?rA_*t0%Wks-3iMzODEfe z4a^ziwbL?LjZft5rrbH}1IB^Dv4ISJwM{sx_YJ<>#d!kJZ|S%#lt@9cU^)M~DXhXA z(~YvL%JL2-rg%tdL^y5VC3csbwKS{X%UOJ!9^#&N*pxKK&@i%8hC*<7P=R-Yz;hy^ z5$QTrI)^;wELv&|BJa-C{Z%er5!{7m>%`!`!HW#w!8dXZ#~Bsz1)yXLsWTtoBu?7L zor9rq<-1wQzY%Q})$uN>Ew?F}Z@qYqe}u}TZY=P~_l4mfPs|D2c)uRy)>)XLGU0Yr z@i*Q>z_|D2Q^GlSCnXaH16As4^i7?tygCd!x;$__8>Vm-~6{qa^qHk)Kod(N!0#>2^o9O z94`9s6yy6!gQMigM6LQXb6GuJfM;?<&!9@8*ijXUjTPI3c!{JI^5w!wS@N6Rxq}bF zrNij1|fL9l5;yUZYfgMxV%49~{$n>*H16|XD7jiZbgL+P9+==b(bp>3a; zu#rGFpMH7p(iQ8=$Qy(9G-ZRO;v3=x0%jSC9g_Jry?hAtuqCHB2E+IKpYbSYN!wx9 z`R2l9RPE@!>ICGLMY>!sV6lI4zKAn;H_t@8k~4q8ce_l9FhoV*Jp|{1WOh9}!dRfA zGIw|IPv4r;tWqA&Z(FU^dFJwo>w8KEa@W4*2sY6a_rL!?_$bG&Bj##w4ZCX9~fY4c7;x$=#r9DOKXe`&RBpNre z3s4`to2bXyeC;C^yDzqx@~xd`yAua&arA}!WShCeqnMl1M$?TiEOQa1!BsZ-QPBKf z)5w|S*pV3)#lx^)E7dI?*5ucEI)bS#G)@UJ0gJ0)bQBXp;L1%%SEWfq*g;YsThH+i z^7+3M8s=hjq_`0q<_TrBF2_kQ_Wf97q`hb)YkW{FZ3Y#6U2O$&Zi3)M80!~_Ji?y| zcc$$}5%%%bB@U8xWT*j}gdmo0uq>Pc-E0PkAPNJVx%Jt1=i0hkUk*nL#)W{Woe81E zvmxT(f5v5Oz3mBSHvm4!?x%owI*9UkjPe;(17B{@7Ga*OPHds ziRn##-9aA1b4Y*FHcaG#SiVIy+45tKWE{v%SITMngpsBwjlzho@g)qSER4-ARgf_mY&ECC4TeR}jJdrs583-~} z&j)~_tIslCr5|Q9SDG{7tKIV>)Kr9SoLe@Ccr_H2@F-bOE)a7!mFK&sn802K#2Zl` zVyd$4>4;LLbi#zV&@ggNuhB7o|6uPz z)%!&5(-nPaZf+w|G%nya-zY}KmaOj+@tG&e)_ZoFl(G09;Zt-|9RboVK;)Tdn zmZPvC;k|6$$ic2atC`+j3;A}OU(1;*^j6~Li#uUc9`hy#DY}==ox3iuqTn-ko(4D8 zGPiKo0YRIXIXQPEZ9?UenvAm#iydD!mT6daL~axlKXCJPnuF0xcYWgzOxU&~B{jTG zSE2{TGI{SuW7ny^-cR1I`QB;X$A{q0xcLkr`YN(qtRd4;UQhbhpElt;GmNJRZWDh-Pe$3w kFf%Xy@e%(" + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "example_count": 0, + "measured": false, + "metadata": {}, + "metrics": {}, + "mode": "raw", + "notes": [ + "limit_per_category=1", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": null, + "reference_kind": "historical", + "runtime_seconds": null, + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "blocked", + "system_name": "mempalace" +} \ No newline at end of file diff --git a/benchmarks/results/smoke_postpush_20260423_123452/runs/convomem_new_brainctl_cmd.json b/benchmarks/results/smoke_postpush_20260423_123452/runs/convomem_new_brainctl_cmd.json new file mode 100644 index 0000000..c2c7c04 --- /dev/null +++ b/benchmarks/results/smoke_postpush_20260423_123452/runs/convomem_new_brainctl_cmd.json @@ -0,0 +1,29 @@ +{ + "artifacts": { + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\convomem_new_brainctl_cmd.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" + }, + "benchmark": "convomem", + "caveats": [ + "Blocked while loading ConvoMem evidence data: HTTP Error 404: Not Found" + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "example_count": 0, + "measured": true, + "metadata": {}, + "metrics": {}, + "mode": "cmd", + "notes": [ + "limit_per_category=1", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": null, + "reference_kind": "measured", + "rows": [], + "runtime_seconds": null, + "series_name": "new_brainctl", + "source_path": null, + "status": "blocked", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/smoke_postpush_20260423_123452/runs/convomem_old_brainctl_cmd.json b/benchmarks/results/smoke_postpush_20260423_123452/runs/convomem_old_brainctl_cmd.json new file mode 100644 index 0000000..72c554f --- /dev/null +++ b/benchmarks/results/smoke_postpush_20260423_123452/runs/convomem_old_brainctl_cmd.json @@ -0,0 +1,29 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\convomem_old_brainctl_cmd.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" + }, + "benchmark": "convomem", + "caveats": [ + "Blocked while loading ConvoMem evidence data: " + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "example_count": 0, + "measured": false, + "metadata": {}, + "metrics": {}, + "mode": "cmd", + "notes": [ + "limit_per_category=1", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": null, + "reference_kind": "historical", + "runtime_seconds": null, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "blocked", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/smoke_postpush_20260423_123452/runs/locomo_mempalace_raw_session.json b/benchmarks/results/smoke_postpush_20260423_123452/runs/locomo_mempalace_raw_session.json new file mode 100644 index 0000000..3280184 --- /dev/null +++ b/benchmarks/results/smoke_postpush_20260423_123452/runs/locomo_mempalace_raw_session.json @@ -0,0 +1,38 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\locomo_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\locomo_mempalace_raw_session.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" + }, + "benchmark": "locomo", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "measured": false, + "metadata": {}, + "metrics": { + "avg_recall": 0.6028, + "cat_1_recall": 0.5899, + "cat_2_recall": 0.6921, + "cat_3_recall": 0.4604, + "cat_4_recall": 0.5809, + "cat_5_recall": 0.6188, + "perfect_rate": 0.5534, + "top_k": 10, + "zero_rate": 0.3499 + }, + "mode": "raw_session", + "notes": [ + "granularity=session", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": 0.6028, + "reference_kind": "historical", + "runtime_seconds": 2106.411, + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "mempalace" +} \ No newline at end of file diff --git a/benchmarks/results/smoke_postpush_20260423_123452/runs/locomo_new_brainctl_cmd_session.json b/benchmarks/results/smoke_postpush_20260423_123452/runs/locomo_new_brainctl_cmd_session.json new file mode 100644 index 0000000..658b086 --- /dev/null +++ b/benchmarks/results/smoke_postpush_20260423_123452/runs/locomo_new_brainctl_cmd_session.json @@ -0,0 +1,4455 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\locomo_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\locomo_new_brainctl_cmd_session.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" + }, + "benchmark": "locomo", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 199, + "measured": true, + "metadata": {}, + "metrics": { + "avg_recall": 0.9413, + "cat_1_recall": 0.8641, + "cat_2_recall": 1.0, + "cat_3_recall": 0.7436, + "cat_4_recall": 0.9571, + "cat_5_recall": 0.9787, + "perfect_rate": 0.9196, + "top_k": 10, + "zero_rate": 0.0402 + }, + "mode": "cmd_session", + "notes": [ + "granularity=session", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": 0.9413, + "reference_kind": "measured", + "rows": [ + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_1" + ], + "question": "When did Caroline go to the LGBTQ support group?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_10", + "session_13", + "session_12", + "session_4", + "session_14", + "session_5", + "session_7", + "session_11", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_1" + ], + "question": "When did Melanie paint a sunrise?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_14", + "session_13", + "session_8", + "session_11", + "session_16", + "session_17", + "session_9", + "session_18", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_1" + ], + "question": "What fields would Caroline be likely to pursue in her educaton?", + "recall": 0.0, + "retrieved_ids": [ + "session_7", + "session_13", + "session_18", + "session_10", + "session_4", + "session_14", + "session_9", + "session_17", + "session_5", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_2" + ], + "question": "What did Caroline research?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_1", + "session_2", + "session_8", + "session_14", + "session_13", + "session_18", + "session_15", + "session_5", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1" + ], + "question": "What is Caroline's identity?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_11", + "session_16", + "session_17", + "session_3", + "session_1", + "session_8", + "session_9", + "session_12", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_2" + ], + "question": "When did Melanie run a charity race?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_7", + "session_8", + "session_14", + "session_13", + "session_18", + "session_15", + "session_1", + "session_5", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_2" + ], + "question": "When is Melanie planning on going camping?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_2", + "session_16", + "session_12", + "session_7", + "session_17", + "session_14", + "session_18", + "session_9", + "session_6" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_2", + "session_3" + ], + "question": "What is Caroline's relationship status?", + "recall": 0.0, + "retrieved_ids": [ + "session_16", + "session_1", + "session_8", + "session_9", + "session_12", + "session_7", + "session_5", + "session_14", + "session_18", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_3" + ], + "question": "When did Caroline give a speech at a school?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_13", + "session_18", + "session_5", + "session_14", + "session_10", + "session_7", + "session_8", + "session_19", + "session_2" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_3" + ], + "question": "When did Caroline meet up with her friends, family, and mentors?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_7", + "session_9", + "session_2", + "session_16", + "session_5", + "session_17", + "session_3", + "session_8", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_3" + ], + "question": "How long has Caroline had her current group of friends for?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_7", + "session_1", + "session_4", + "session_12", + "session_13", + "session_16", + "session_3", + "session_6", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_3", + "session_4" + ], + "question": "Where did Caroline move from 4 years ago?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_8", + "session_19", + "session_4", + "session_16", + "session_15", + "session_7", + "session_17", + "session_9", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_4" + ], + "question": "How long ago was Caroline's 18th birthday?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_11", + "session_16", + "session_15", + "session_3", + "session_17", + "session_8", + "session_9", + "session_7", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_4" + ], + "question": "What career path has Caroline decided to persue?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_7", + "session_14", + "session_1", + "session_3", + "session_16", + "session_15", + "session_8", + "session_9", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_3", + "session_4" + ], + "question": "Would Caroline still want to pursue counseling as a career if she hadn't received support growing up?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_7", + "session_17", + "session_1", + "session_18", + "session_6", + "session_2", + "session_10", + "session_5", + "session_3" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_5", + "session_9" + ], + "question": "What activities does Melanie partake in?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_15", + "session_8", + "session_18", + "session_1", + "session_4", + "session_13", + "session_9", + "session_19", + "session_17" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_5" + ], + "question": "When did Melanie sign up for a pottery class?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_16", + "session_14", + "session_10", + "session_12", + "session_17", + "session_8", + "session_18", + "session_11", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_5" + ], + "question": "When is Caroline going to the transgender conference?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_7", + "session_17", + "session_1", + "session_14", + "session_9", + "session_3", + "session_4", + "session_13", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_4", + "session_6", + "session_8" + ], + "question": "Where has Melanie camped?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_18", + "session_9", + "session_6", + "session_2", + "session_4", + "session_16", + "session_8", + "session_1", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_4", + "session_6" + ], + "question": "What do Melanie's kids like?", + "recall": 0.5, + "retrieved_ids": [ + "session_9", + "session_10", + "session_15", + "session_18", + "session_8", + "session_17", + "session_1", + "session_16", + "session_4", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_6" + ], + "question": "When did Melanie go to the museum?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_13", + "session_1", + "session_14", + "session_5", + "session_11", + "session_18", + "session_8", + "session_7", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_6" + ], + "question": "When did Caroline have a picnic?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_8", + "session_14", + "session_13", + "session_18", + "session_15", + "session_1", + "session_5", + "session_11", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_6" + ], + "question": "Would Caroline likely have Dr. Seuss books on her bookshelf?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_18", + "session_10", + "session_6", + "session_17", + "session_4", + "session_14", + "session_9", + "session_5", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_6", + "session_7" + ], + "question": "What books has Melanie read?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_7", + "session_17", + "session_2", + "session_8", + "session_18", + "session_1", + "session_4", + "session_13", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_5", + "session_7" + ], + "question": "What does Melanie do to destress?", + "recall": 0.5, + "retrieved_ids": [ + "session_7", + "session_8", + "session_18", + "session_1", + "session_4", + "session_13", + "session_9", + "session_19", + "session_17", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_7" + ], + "question": "When did Caroline go to the LGBTQ conference?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_7", + "session_14", + "session_1", + "session_13", + "session_11", + "session_10", + "session_9", + "session_4", + "session_3" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_7" + ], + "question": "When did Melanie read the book \"nothing is impossible\"?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_7", + "session_17", + "session_2", + "session_8", + "session_14", + "session_13", + "session_18", + "session_15", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_7" + ], + "question": "Would Caroline pursue writing as a career option?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_1", + "session_4", + "session_18", + "session_8", + "session_9", + "session_12", + "session_5", + "session_14", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_8" + ], + "question": "When did Caroline go to the adoption meeting?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_2", + "session_17", + "session_19", + "session_13", + "session_10", + "session_5", + "session_15", + "session_7", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_8" + ], + "question": "When did Melanie go to the pottery workshop?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_4", + "session_5", + "session_16", + "session_12", + "session_17", + "session_14", + "session_13", + "session_1", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [], + "question": "Would Melanie be considered a member of the LGBTQ community?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_5", + "session_14", + "session_11", + "session_9", + "session_3", + "session_7", + "session_10", + "session_15", + "session_16" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_4" + ], + "question": "When did Melanie go camping in June?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_18", + "session_9", + "session_6", + "session_2", + "session_4", + "session_16", + "session_8", + "session_13", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_3", + "session_5", + "session_8" + ], + "question": "What LGBTQ+ events has Caroline participated in?", + "recall": 0.75, + "retrieved_ids": [ + "session_9", + "session_10", + "session_3", + "session_2", + "session_11", + "session_7", + "session_17", + "session_5", + "session_14", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_5" + ], + "question": "When did Caroline go to a pride parade during the summer?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_10", + "session_11", + "session_5", + "session_18", + "session_16", + "session_12", + "session_2", + "session_15", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_3", + "session_9" + ], + "question": "What events has Caroline participated in to help children?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_9", + "session_10", + "session_3", + "session_2", + "session_11", + "session_7", + "session_17", + "session_19", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_9" + ], + "question": "When did Melanie go camping in July?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_18", + "session_9", + "session_6", + "session_2", + "session_4", + "session_16", + "session_8", + "session_13", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_9" + ], + "question": "When did Caroline join a mentorship program?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_10", + "session_3", + "session_8", + "session_14", + "session_13", + "session_18", + "session_15", + "session_1", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_8" + ], + "question": "What did Melanie paint recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_17", + "session_11", + "session_12", + "session_18", + "session_10", + "session_14", + "session_1", + "session_8", + "session_16" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_3", + "session_6", + "session_8", + "session_9" + ], + "question": "What activities has Melanie done with her family?", + "recall": 0.4, + "retrieved_ids": [ + "session_10", + "session_5", + "session_15", + "session_7", + "session_1", + "session_6", + "session_13", + "session_16", + "session_17", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_5", + "session_9" + ], + "question": "In what ways is Caroline participating in the LGBTQ community?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_11", + "session_9", + "session_15", + "session_14", + "session_3", + "session_7", + "session_10", + "session_16", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_6" + ], + "question": "How many times has Melanie gone to the beach in 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_3", + "session_14", + "session_6", + "session_4", + "session_7", + "session_17", + "session_8", + "session_18", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_10" + ], + "question": "When did Caroline join a new activist group?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_1", + "session_9", + "session_13", + "session_12", + "session_4", + "session_3", + "session_8", + "session_14", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_10" + ], + "question": "Would Melanie be more interested in going to a national park or a theme park?", + "recall": 0.0, + "retrieved_ids": [ + "session_11", + "session_18", + "session_15", + "session_16", + "session_7", + "session_14", + "session_5", + "session_4", + "session_3", + "session_6" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_11", + "session_9" + ], + "question": "What kind of art does Caroline make?", + "recall": 0.5, + "retrieved_ids": [ + "session_2", + "session_4", + "session_3", + "session_5", + "session_6", + "session_11", + "session_10", + "session_15", + "session_14", + "session_16" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_11" + ], + "question": "When is Melanie's daughter's birthday?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_4", + "session_8", + "session_18", + "session_1", + "session_13", + "session_9", + "session_19", + "session_17", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_11" + ], + "question": "When did Caroline attend a pride parade in August?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_5", + "session_11", + "session_10", + "session_8", + "session_9", + "session_14", + "session_12", + "session_17", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [], + "question": "Would Melanie be considered an ally to the transgender community?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_18", + "session_17", + "session_1", + "session_14", + "session_9", + "session_5", + "session_11", + "session_15", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_12", + "session_3" + ], + "question": "Who supports Caroline when she has a negative experience?", + "recall": 0.5, + "retrieved_ids": [ + "session_4", + "session_10", + "session_12", + "session_19", + "session_9", + "session_5", + "session_7", + "session_15", + "session_11", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_12", + "session_5", + "session_8" + ], + "question": "What types of pottery have Melanie and her kids made?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_10", + "session_15", + "session_7", + "session_5", + "session_16", + "session_12", + "session_17", + "session_8", + "session_2" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_12" + ], + "question": "When did Caroline and Melanie go to a pride fesetival together?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_14", + "session_5", + "session_11", + "session_12", + "session_10", + "session_17", + "session_8", + "session_18", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_12" + ], + "question": "What would Caroline's political leaning likely be?", + "recall": 0.0, + "retrieved_ids": [ + "session_18", + "session_4", + "session_14", + "session_9", + "session_17", + "session_5", + "session_10", + "session_13", + "session_7", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_13", + "session_8" + ], + "question": "What has Melanie painted?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_13", + "session_9", + "session_14", + "session_17", + "session_16", + "session_11", + "session_8", + "session_12", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_13", + "session_7" + ], + "question": "What are Melanie's pets' names?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_7", + "session_8", + "session_18", + "session_1", + "session_4", + "session_9", + "session_19", + "session_17", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_13" + ], + "question": "When did Caroline apply to adoption agencies?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_2", + "session_19", + "session_17", + "session_8", + "session_14", + "session_18", + "session_15", + "session_1", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_13" + ], + "question": "When did Caroline draw a self-portrait?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_14", + "session_16", + "session_11", + "session_17", + "session_7", + "session_8", + "session_19", + "session_2", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_14", + "session_8" + ], + "question": "What subject have Caroline and Melanie both painted?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_9", + "session_19", + "session_13", + "session_12", + "session_16", + "session_15", + "session_3", + "session_1", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_14", + "session_4" + ], + "question": "What symbols are important to Caroline?", + "recall": 0.5, + "retrieved_ids": [ + "session_14", + "session_7", + "session_2", + "session_11", + "session_6", + "session_18", + "session_15", + "session_3", + "session_8", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_14" + ], + "question": "When did Caroline encounter people on a hike and have a negative experience?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_4", + "session_16", + "session_8", + "session_14", + "session_11", + "session_5", + "session_15", + "session_18", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_14" + ], + "question": "When did Melanie make a plate in pottery class?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_5", + "session_16", + "session_12", + "session_17", + "session_8", + "session_15", + "session_7", + "session_10", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_12", + "session_14" + ], + "question": "Would Caroline be considered religious?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_18", + "session_1", + "session_8", + "session_9", + "session_7", + "session_5", + "session_14", + "session_4", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_15", + "session_2" + ], + "question": "What instruments does Melanie play?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_6", + "session_2", + "session_3", + "session_8", + "session_18", + "session_1", + "session_4", + "session_13", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_11", + "session_15" + ], + "question": "What musical artists/bands has Melanie seen?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_11", + "session_17", + "session_14", + "session_8", + "session_18", + "session_1", + "session_4", + "session_13", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_15" + ], + "question": "When did Melanie go to the park?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_16", + "session_13", + "session_1", + "session_14", + "session_5", + "session_11", + "session_18", + "session_8", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_15" + ], + "question": "When is Caroline's youth center putting on a talent show?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_9", + "session_11", + "session_14", + "session_5", + "session_19", + "session_12", + "session_10", + "session_16", + "session_17" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_15" + ], + "question": "Would Melanie likely enjoy the song \"The Four Seasons\" by Vivaldi?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_15", + "session_11", + "session_8", + "session_10", + "session_4", + "session_14", + "session_9", + "session_17", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_11", + "session_16" + ], + "question": "What are some changes Caroline has faced during her transition journey?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_16", + "session_3", + "session_8", + "session_7", + "session_18", + "session_11", + "session_14", + "session_9", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_16" + ], + "question": "What does Melanie do with her family on hikes?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_7", + "session_12", + "session_4", + "session_16", + "session_8", + "session_14", + "session_19", + "session_2", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_16" + ], + "question": "When did Caroline go biking with friends?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_6", + "session_3", + "session_8", + "session_19", + "session_2", + "session_12", + "session_4", + "session_17", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_16" + ], + "question": "How long has Melanie been practicing art?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_3", + "session_15", + "session_17", + "session_1", + "session_6", + "session_19", + "session_4", + "session_8", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_13", + "session_16", + "session_7" + ], + "question": "What personality traits might Melanie say Caroline has?", + "recall": 0.6667, + "retrieved_ids": [ + "session_10", + "session_16", + "session_15", + "session_8", + "session_14", + "session_1", + "session_18", + "session_9", + "session_4", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_15", + "session_17" + ], + "question": "What transgender-specific events has Caroline attended?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_15", + "session_9", + "session_17", + "session_3", + "session_1", + "session_14", + "session_5", + "session_10", + "session_2" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_17", + "session_7" + ], + "question": "What book did Melanie read from Caroline's suggestion?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_7", + "session_17", + "session_2", + "session_8", + "session_14", + "session_13", + "session_18", + "session_15", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_17" + ], + "question": "When did Melanie's friend adopt a child?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_2", + "session_17", + "session_19", + "session_8", + "session_6", + "session_16", + "session_3", + "session_12", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_17" + ], + "question": "When did Melanie get hurt?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_8", + "session_5", + "session_13", + "session_1", + "session_16", + "session_18", + "session_14", + "session_15", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_18" + ], + "question": "When did Melanie's family go on a roadtrip?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_10", + "session_7", + "session_17", + "session_4", + "session_3", + "session_13", + "session_2", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_18" + ], + "question": "How many children does Melanie have?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_10", + "session_3", + "session_18", + "session_1", + "session_4", + "session_13", + "session_9", + "session_19", + "session_17" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_18" + ], + "question": "When did Melanie go on a hike after the roadtrip?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_4", + "session_14", + "session_12", + "session_16", + "session_8", + "session_2", + "session_9", + "session_1", + "session_3" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_18" + ], + "question": "Would Melanie go on another roadtrip soon?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_1", + "session_9", + "session_13", + "session_11", + "session_12", + "session_16", + "session_8", + "session_4", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_19", + "session_7" + ], + "question": "What items has Melanie bought?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_4", + "session_8", + "session_18", + "session_1", + "session_13", + "session_9", + "session_17", + "session_15", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_19" + ], + "question": "When did Caroline pass the adoption interview?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_3", + "session_2", + "session_17", + "session_13", + "session_8", + "session_14", + "session_18", + "session_15", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_19" + ], + "question": "When did Melanie buy the figurines?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_8", + "session_14", + "session_13", + "session_18", + "session_15", + "session_1", + "session_5", + "session_11", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_19" + ], + "question": "Would Caroline want to move back to her home country soon?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_19", + "session_18", + "session_8", + "session_1", + "session_4", + "session_7", + "session_10", + "session_2", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What did the charity race raise awareness for?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_7", + "session_3", + "session_8", + "session_14", + "session_13", + "session_18", + "session_15", + "session_1", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What did Melanie realize after the charity race?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_7", + "session_8", + "session_18", + "session_9", + "session_1", + "session_4", + "session_3", + "session_14", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "How does Melanie prioritize self-care?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_9", + "session_13", + "session_4", + "session_1", + "session_7", + "session_11", + "session_17", + "session_19", + "session_16" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What are Caroline's plans for the summer?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_2", + "session_10", + "session_11", + "session_15", + "session_16", + "session_7", + "session_17", + "session_14", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What type of individuals does the adoption agency Caroline is considering support?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_11", + "session_19", + "session_13", + "session_17", + "session_15", + "session_14", + "session_8", + "session_1", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "Why did Caroline choose the adoption agency?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_2", + "session_13", + "session_17", + "session_16", + "session_8", + "session_14", + "session_18", + "session_15", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What is Caroline excited about in the adoption process?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_2", + "session_19", + "session_13", + "session_8", + "session_5", + "session_16", + "session_1", + "session_9", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What does Melanie think about Caroline's decision to adopt?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_2", + "session_17", + "session_19", + "session_8", + "session_12", + "session_4", + "session_5", + "session_16", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "How long have Mel and her husband been married?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_10", + "session_7", + "session_8", + "session_16", + "session_15", + "session_17", + "session_1", + "session_6", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What does Caroline's necklace symbolize?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_14", + "session_1", + "session_8", + "session_9", + "session_12", + "session_7", + "session_5", + "session_18", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What country is Caroline's grandma from?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_3", + "session_1", + "session_8", + "session_9", + "session_12", + "session_7", + "session_5", + "session_14", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What was grandma's gift to Caroline?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_19", + "session_1", + "session_8", + "session_9", + "session_12", + "session_7", + "session_5", + "session_14", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What is Melanie's hand-painted bowl a reminder of?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_8", + "session_5", + "session_12", + "session_16", + "session_11", + "session_14", + "session_1", + "session_13", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What did Melanie and her family do while camping?", + "recall": 0.0, + "retrieved_ids": [ + "session_10", + "session_7", + "session_16", + "session_8", + "session_17", + "session_13", + "session_18", + "session_9", + "session_6", + "session_2" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What kind of counseling and mental health services is Caroline interested in pursuing?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_7", + "session_11", + "session_5", + "session_6", + "session_1", + "session_2", + "session_8", + "session_3", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What workshop did Caroline attend recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_4", + "session_8", + "session_17", + "session_11", + "session_12", + "session_18", + "session_10", + "session_14", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What was discussed in the LGBTQ+ counseling workshop?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_8", + "session_7", + "session_1", + "session_5", + "session_6", + "session_9", + "session_11", + "session_14", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What motivated Caroline to pursue counseling?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_7", + "session_5", + "session_1", + "session_6", + "session_3", + "session_18", + "session_14", + "session_8", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What kind of place does Caroline want to create for people?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_14", + "session_17", + "session_2", + "session_3", + "session_5", + "session_6", + "session_11", + "session_10", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_5" + ], + "question": "Did Melanie make the black and white bowl in the photo?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_16", + "session_12", + "session_4", + "session_6", + "session_13", + "session_3", + "session_8", + "session_14", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_6" + ], + "question": "What kind of books does Caroline have in her library?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_7", + "session_10", + "session_17", + "session_2", + "session_4", + "session_3", + "session_5", + "session_11", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_6" + ], + "question": "What was Melanie's favorite book from her childhood?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_7", + "session_10", + "session_17", + "session_8", + "session_14", + "session_18", + "session_1", + "session_4", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What book did Caroline recommend to Melanie?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_17", + "session_6", + "session_8", + "session_14", + "session_13", + "session_18", + "session_15", + "session_1", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What did Caroline take away from the book \"Becoming Nicole\"?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_13", + "session_6", + "session_17", + "session_2", + "session_1", + "session_14", + "session_5", + "session_4", + "session_3" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What are the new shoes that Melanie got used for?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_17", + "session_6", + "session_15", + "session_9", + "session_4", + "session_1", + "session_16", + "session_19", + "session_2" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What is Melanie's reason for getting into running?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_2", + "session_16", + "session_18", + "session_4", + "session_10", + "session_15", + "session_14", + "session_6", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What does Melanie say running has been great for?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_10", + "session_2", + "session_6", + "session_17", + "session_15", + "session_9", + "session_8", + "session_3", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What did Mel and her kids make during the pottery workshop?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_16", + "session_10", + "session_4", + "session_7", + "session_18", + "session_5", + "session_12", + "session_17", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What kind of pot did Mel and her kids make with clay?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_10", + "session_5", + "session_7", + "session_2", + "session_4", + "session_3", + "session_6", + "session_11", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What creative project do Mel and her kids do together besides pottery?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_12", + "session_14", + "session_10", + "session_7", + "session_5", + "session_16", + "session_17", + "session_15", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What did Mel and her kids paint in their latest project in July 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_10", + "session_7", + "session_12", + "session_15", + "session_14", + "session_17", + "session_1", + "session_13", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What did Caroline see at the council meeting for adoption?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_2", + "session_17", + "session_19", + "session_13", + "session_10", + "session_5", + "session_15", + "session_7", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What do sunflowers represent according to Caroline?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_11", + "session_14", + "session_1", + "session_9", + "session_12", + "session_7", + "session_5", + "session_18", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "Why are flowers important to Melanie?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_14", + "session_7", + "session_2", + "session_11", + "session_6", + "session_18", + "session_15", + "session_3", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_9" + ], + "question": "What inspired Caroline's painting for the art show?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_16", + "session_9", + "session_17", + "session_11", + "session_8", + "session_12", + "session_13", + "session_4", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "How often does Melanie go to the beach with her kids?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_7", + "session_14", + "session_6", + "session_1", + "session_9", + "session_11", + "session_2", + "session_18", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "What did Melanie and her family see during their camping trip last year?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_18", + "session_8", + "session_7", + "session_16", + "session_12", + "session_17", + "session_4", + "session_9", + "session_6" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "How did Melanie feel while watching the meteor shower?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_8", + "session_17", + "session_13", + "session_16", + "session_14", + "session_1", + "session_5", + "session_15", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_11" + ], + "question": "Whose birthday did Melanie celebrate recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_4", + "session_17", + "session_10", + "session_14", + "session_8", + "session_13", + "session_12", + "session_18", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_11" + ], + "question": "Who performed at the concert at Melanie's daughter's birthday?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_4", + "session_8", + "session_18", + "session_1", + "session_13", + "session_9", + "session_19", + "session_17", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_12" + ], + "question": "Why did Melanie choose to use colors and patterns in her pottery project?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_16", + "session_19", + "session_10", + "session_4", + "session_14", + "session_2", + "session_8", + "session_7", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "What pet does Caroline have?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_13", + "session_1", + "session_8", + "session_9", + "session_12", + "session_5", + "session_14", + "session_18", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "What pets does Melanie have?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_13", + "session_8", + "session_18", + "session_1", + "session_4", + "session_9", + "session_19", + "session_17", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "Where did Oliver hide his bone once?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_10", + "session_6", + "session_7", + "session_11", + "session_8", + "session_14", + "session_18", + "session_15", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "What activity did Caroline used to do with her dad?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_10", + "session_5", + "session_15", + "session_7", + "session_8", + "session_14", + "session_18", + "session_1", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_14" + ], + "question": "What did Caroline make for a local church?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_8", + "session_15", + "session_5", + "session_7", + "session_10", + "session_16", + "session_18", + "session_17", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_14" + ], + "question": "What did Caroline find in her neighborhood during her walk?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_7", + "session_10", + "session_16", + "session_18", + "session_8", + "session_19", + "session_9", + "session_11", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "Which song motivates Caroline to be courageous?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_3", + "session_11", + "session_14", + "session_1", + "session_8", + "session_5", + "session_18", + "session_4", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "Which classical musicians does Melanie enjoy listening to?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_15", + "session_8", + "session_18", + "session_10", + "session_1", + "session_4", + "session_13", + "session_9", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "Who is Melanie a fan of in terms of modern music?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_5", + "session_11", + "session_8", + "session_18", + "session_1", + "session_4", + "session_13", + "session_9", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "How long has Melanie been creating art?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_3", + "session_15", + "session_17", + "session_1", + "session_6", + "session_19", + "session_4", + "session_8", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "What precautionary sign did Melanie see at the caf\u00e9?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_5", + "session_10", + "session_15", + "session_14", + "session_1", + "session_11", + "session_18", + "session_8", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What advice does Caroline give for getting started with adoption?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_13", + "session_2", + "session_19", + "session_8", + "session_15", + "session_12", + "session_18", + "session_4", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What setback did Melanie face in October 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_9", + "session_10", + "session_15", + "session_3", + "session_8", + "session_14", + "session_13", + "session_18", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What does Melanie do to keep herself busy during her pottery break?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_8", + "session_16", + "session_10", + "session_14", + "session_2", + "session_7", + "session_18", + "session_5", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What painting did Melanie show to Caroline on October 13, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_13", + "session_8", + "session_17", + "session_16", + "session_11", + "session_9", + "session_1", + "session_15", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What kind of painting did Caroline share with Melanie on October 13, 2023?", + "recall": 0.0, + "retrieved_ids": [ + "session_2", + "session_4", + "session_3", + "session_5", + "session_6", + "session_11", + "session_10", + "session_15", + "session_14", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What was the poetry reading that Caroline attended about?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_13", + "session_6", + "session_7", + "session_2", + "session_4", + "session_15", + "session_3", + "session_12", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What did the posters at the poetry reading say?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_10", + "session_6", + "session_7", + "session_2", + "session_8", + "session_14", + "session_13", + "session_18", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What does Caroline's drawing symbolize for her?", + "recall": 0.0, + "retrieved_ids": [ + "session_14", + "session_10", + "session_16", + "session_7", + "session_1", + "session_8", + "session_9", + "session_12", + "session_5", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "How do Melanie and Caroline describe their journey through life together?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_8", + "session_4", + "session_17", + "session_14", + "session_19", + "session_7", + "session_15", + "session_10", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What happened to Melanie's son on their road trip?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_10", + "session_12", + "session_17", + "session_7", + "session_6", + "session_14", + "session_2", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "How did Melanie's son handle the accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_16", + "session_8", + "session_14", + "session_13", + "session_15", + "session_1", + "session_5", + "session_11", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "How did Melanie feel about her family after the accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_10", + "session_7", + "session_2", + "session_9", + "session_1", + "session_4", + "session_3", + "session_14", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "How did Melanie's children handle the accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_16", + "session_14", + "session_13", + "session_15", + "session_1", + "session_5", + "session_11", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "How did Melanie feel after the accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_2", + "session_9", + "session_1", + "session_4", + "session_3", + "session_14", + "session_13", + "session_8", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What was Melanie's reaction to her children enjoying the Grand Canyon?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_10", + "session_3", + "session_7", + "session_1", + "session_4", + "session_13", + "session_9", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What do Melanie's family give her?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_7", + "session_3", + "session_19", + "session_2", + "session_18", + "session_6", + "session_4", + "session_8", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "How did Melanie feel about her family supporting her?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_7", + "session_8", + "session_17", + "session_16", + "session_3", + "session_2", + "session_15", + "session_18", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What did Melanie do after the road trip to relax?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_17", + "session_1", + "session_10", + "session_14", + "session_12", + "session_15", + "session_2", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_2" + ], + "question": "What did Caroline realize after her charity race?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_7", + "session_10", + "session_8", + "session_18", + "session_9", + "session_1", + "session_4", + "session_3", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_2" + ], + "question": "What are Melanie's plans for the summer with respect to adoption?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_2", + "session_12", + "session_17", + "session_10", + "session_13", + "session_8", + "session_11", + "session_15", + "session_16" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_2" + ], + "question": "What type of individuals does the adoption agency Melanie is considering support?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_11", + "session_19", + "session_13", + "session_17", + "session_15", + "session_14", + "session_8", + "session_1", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_2" + ], + "question": "Why did Melanie choose the adoption agency?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_2", + "session_13", + "session_17", + "session_16", + "session_8", + "session_14", + "session_18", + "session_15", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_2" + ], + "question": "What is Melanie excited about in her adoption process?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_2", + "session_19", + "session_10", + "session_7", + "session_13", + "session_8", + "session_5", + "session_16", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What does Melanie's necklace symbolize?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_14", + "session_8", + "session_18", + "session_1", + "session_13", + "session_9", + "session_19", + "session_17", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What country is Melanie's grandma from?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_3", + "session_8", + "session_18", + "session_1", + "session_13", + "session_9", + "session_19", + "session_17", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What was grandma's gift to Melanie?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_19", + "session_8", + "session_18", + "session_1", + "session_13", + "session_9", + "session_17", + "session_15", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What was grandpa's gift to Caroline?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_4", + "session_1", + "session_8", + "session_9", + "session_12", + "session_7", + "session_5", + "session_14", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What is Caroline's hand-painted bowl a reminder of?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_8", + "session_5", + "session_12", + "session_16", + "session_11", + "session_14", + "session_1", + "session_9", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What did Caroline and her family do while camping?", + "recall": 0.0, + "retrieved_ids": [ + "session_10", + "session_7", + "session_16", + "session_8", + "session_17", + "session_13", + "session_18", + "session_9", + "session_6", + "session_2" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What kind of counseling and mental health services is Melanie interested in pursuing?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_7", + "session_11", + "session_5", + "session_6", + "session_1", + "session_2", + "session_8", + "session_3", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What kind of counseling workshop did Melanie attend recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_13", + "session_8", + "session_5", + "session_6", + "session_7", + "session_1", + "session_11", + "session_10", + "session_17" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What motivated Melanie to pursue counseling?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_7", + "session_5", + "session_1", + "session_6", + "session_3", + "session_18", + "session_14", + "session_8", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What kind of place does Melanie want to create for people?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_14", + "session_17", + "session_2", + "session_3", + "session_5", + "session_6", + "session_11", + "session_10", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_5" + ], + "question": "Did Caroline make the black and white bowl in the photo?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_16", + "session_12", + "session_4", + "session_6", + "session_13", + "session_3", + "session_8", + "session_14", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_7" + ], + "question": "What are the new shoes that Caroline got used for?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_17", + "session_6", + "session_15", + "session_9", + "session_4", + "session_1", + "session_16", + "session_19", + "session_2" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_7" + ], + "question": "What is Caroline's reason for getting into running?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_2", + "session_16", + "session_18", + "session_4", + "session_10", + "session_15", + "session_14", + "session_5", + "session_6" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_7" + ], + "question": "What does Caroline say running has been great for?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_10", + "session_2", + "session_6", + "session_17", + "session_15", + "session_9", + "session_8", + "session_3", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_8" + ], + "question": "What did Melanie see at the council meeting for adoption?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_2", + "session_17", + "session_19", + "session_13", + "session_10", + "session_5", + "session_15", + "session_7", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_9" + ], + "question": "What inspired Melanie's painting for the art show?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_16", + "session_9", + "session_17", + "session_11", + "session_8", + "session_12", + "session_13", + "session_4", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_9" + ], + "question": "What inspired Caroline's sculpture for the art show?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_16", + "session_9", + "session_17", + "session_11", + "session_12", + "session_8", + "session_13", + "session_5", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_10" + ], + "question": "How often does Caroline go to the beach with her kids?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_7", + "session_14", + "session_6", + "session_1", + "session_9", + "session_11", + "session_2", + "session_18", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_10" + ], + "question": "What did Caroline and her family see during their camping trip last year?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_18", + "session_8", + "session_7", + "session_16", + "session_12", + "session_17", + "session_4", + "session_9", + "session_6" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_10" + ], + "question": "How did Caroline feel while watching the meteor shower?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_8", + "session_17", + "session_13", + "session_16", + "session_14", + "session_1", + "session_5", + "session_15", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_12" + ], + "question": "Why did Caroline choose to use colors and patterns in her pottery project?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_16", + "session_19", + "session_10", + "session_4", + "session_14", + "session_2", + "session_8", + "session_7", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_13" + ], + "question": "Is Oscar Melanie's pet?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_7", + "session_8", + "session_18", + "session_1", + "session_4", + "session_9", + "session_19", + "session_17", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_13" + ], + "question": "Where did Oscar hide his bone once?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_10", + "session_6", + "session_11", + "session_8", + "session_14", + "session_18", + "session_15", + "session_1", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_13" + ], + "question": "What activity did Melanie used to do with her dad?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_10", + "session_5", + "session_15", + "session_7", + "session_8", + "session_14", + "session_18", + "session_1", + "session_11" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_14" + ], + "question": "What did Melanie make for a local church?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_8", + "session_15", + "session_5", + "session_7", + "session_10", + "session_16", + "session_18", + "session_17", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_14" + ], + "question": "What did Melanie find in her neighborhood during her walk?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_7", + "session_10", + "session_16", + "session_18", + "session_8", + "session_19", + "session_9", + "session_11", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_15" + ], + "question": "Which song motivates Melanie to be courageous?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_3", + "session_11", + "session_14", + "session_1", + "session_8", + "session_5", + "session_18", + "session_4", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_15" + ], + "question": "What type of instrument does Caroline play?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_6", + "session_14", + "session_2", + "session_3", + "session_1", + "session_8", + "session_9", + "session_12", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_15" + ], + "question": "Which classical musicians does Caroline enjoy listening to?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_15", + "session_8", + "session_18", + "session_10", + "session_1", + "session_9", + "session_12", + "session_7", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_15" + ], + "question": "Who is Caroline a fan of in terms of modern music?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_5", + "session_11", + "session_1", + "session_8", + "session_9", + "session_12", + "session_7", + "session_14", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_16" + ], + "question": "What precautionary sign did Caroline see at the caf\u00e9?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_5", + "session_10", + "session_15", + "session_14", + "session_1", + "session_11", + "session_18", + "session_8", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_17" + ], + "question": "What setback did Caroline face recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_10", + "session_9", + "session_15", + "session_3", + "session_13", + "session_11", + "session_12", + "session_18", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_17" + ], + "question": "What does Caroline do to keep herself busy during her pottery break?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_8", + "session_16", + "session_10", + "session_14", + "session_2", + "session_7", + "session_18", + "session_5", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_17" + ], + "question": "What was the poetry reading that Melanie attended about?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_13", + "session_6", + "session_7", + "session_2", + "session_4", + "session_15", + "session_3", + "session_11", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "What happened to Caroline's son on their road trip?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_10", + "session_12", + "session_17", + "session_7", + "session_6", + "session_14", + "session_2", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "How did Caroline's son handle the accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_16", + "session_8", + "session_14", + "session_13", + "session_15", + "session_1", + "session_5", + "session_11", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "How did Caroline feel about her family after the accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_10", + "session_7", + "session_2", + "session_9", + "session_1", + "session_4", + "session_3", + "session_14", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "How did Caroline's children handle the accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_16", + "session_14", + "session_13", + "session_15", + "session_1", + "session_5", + "session_11", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "How did Caroline feel after the accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_2", + "session_9", + "session_1", + "session_4", + "session_3", + "session_14", + "session_13", + "session_8", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "What was Caroline's reaction to her children enjoying the Grand Canyon?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_10", + "session_3", + "session_7", + "session_1", + "session_9", + "session_12", + "session_5", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "What did Caroline do after the road trip to relax?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_17", + "session_1", + "session_10", + "session_14", + "session_12", + "session_15", + "session_2", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "What does Caroline love most about camping with her family?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_18", + "session_7", + "session_14", + "session_9", + "session_6", + "session_2", + "session_4", + "session_16", + "session_8" + ], + "sample_id": "conv-26" + } + ], + "runtime_seconds": 5.736, + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/smoke_postpush_20260423_123452/runs/locomo_old_brainctl_cmd_session.json b/benchmarks/results/smoke_postpush_20260423_123452/runs/locomo_old_brainctl_cmd_session.json new file mode 100644 index 0000000..bbe862c --- /dev/null +++ b/benchmarks/results/smoke_postpush_20260423_123452/runs/locomo_old_brainctl_cmd_session.json @@ -0,0 +1,38 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\locomo_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\locomo_old_brainctl_cmd_session.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" + }, + "benchmark": "locomo", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "measured": false, + "metadata": {}, + "metrics": { + "avg_recall": 0.9217, + "cat_1_recall": 0.7614, + "cat_2_recall": 0.92, + "cat_3_recall": 0.6616, + "cat_4_recall": 0.9738, + "cat_5_recall": 0.9821, + "perfect_rate": 0.8817, + "top_k": 10, + "zero_rate": 0.0438 + }, + "mode": "cmd_session", + "notes": [ + "granularity=session", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": 0.9217, + "reference_kind": "historical", + "runtime_seconds": 445.74, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_mempalace_raw_session.json b/benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_mempalace_raw_session.json new file mode 100644 index 0000000..6191fa1 --- /dev/null +++ b/benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_mempalace_raw_session.json @@ -0,0 +1,33 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\longmemeval_mempalace_raw_session.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "measured": false, + "metadata": {}, + "metrics": { + "ndcg_at_10": 0.8948, + "ndcg_at_5": 0.893, + "r_at_10": 0.983, + "r_at_5": 0.966 + }, + "mode": "raw_session", + "notes": [ + "top_k=10", + "Runs MemPalace benchmark module raw session retrieval logic directly." + ], + "primary_metric": "r_at_5", + "primary_metric_value": 0.966, + "reference_kind": "historical", + "runtime_seconds": 695.36, + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "mempalace" +} \ No newline at end of file diff --git a/benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_new_brainctl_brain.json b/benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_new_brainctl_brain.json new file mode 100644 index 0000000..7f89cbf --- /dev/null +++ b/benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_new_brainctl_brain.json @@ -0,0 +1,56 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\longmemeval_new_brainctl_brain.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 1, + "measured": true, + "metadata": {}, + "metrics": { + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0 + }, + "mode": "brain", + "notes": [ + "top_k=10", + "Legacy 470-question session-level slice." + ], + "primary_metric": "r_at_5", + "primary_metric_value": 1.0, + "reference_kind": "measured", + "rows": [ + { + "answer_session_ids": [ + "answer_280352e9" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e47becba", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_280352e9", + "02bd2b90_3", + "sharegpt_Cr2tc1f_0", + "f6859b48_2", + "ultrachat_214101", + "5dac7cc2_1" + ] + } + ], + "runtime_seconds": 2.644, + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_new_brainctl_cmd.json b/benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_new_brainctl_cmd.json new file mode 100644 index 0000000..86be500 --- /dev/null +++ b/benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_new_brainctl_cmd.json @@ -0,0 +1,56 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\longmemeval_new_brainctl_cmd.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 1, + "measured": true, + "metadata": {}, + "metrics": { + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0 + }, + "mode": "cmd", + "notes": [ + "top_k=10", + "Legacy 470-question session-level slice." + ], + "primary_metric": "r_at_5", + "primary_metric_value": 1.0, + "reference_kind": "measured", + "rows": [ + { + "answer_session_ids": [ + "answer_280352e9" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e47becba", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_280352e9", + "02bd2b90_3", + "sharegpt_Cr2tc1f_0", + "f6859b48_2", + "ultrachat_214101", + "5dac7cc2_1" + ] + } + ], + "runtime_seconds": 2.577, + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_old_brainctl_brain.json b/benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_old_brainctl_brain.json new file mode 100644 index 0000000..ef92691 --- /dev/null +++ b/benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_old_brainctl_brain.json @@ -0,0 +1,32 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\longmemeval_old_brainctl_brain.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "measured": false, + "metadata": {}, + "metrics": { + "ndcg_at_10": 0.9253, + "ndcg_at_5": 0.9204, + "r_at_10": 0.9894, + "r_at_5": 0.9681 + }, + "mode": "brain", + "notes": [ + "top_k=10" + ], + "primary_metric": "r_at_5", + "primary_metric_value": 0.9681, + "reference_kind": "historical", + "runtime_seconds": 85.439, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_old_brainctl_cmd.json b/benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_old_brainctl_cmd.json new file mode 100644 index 0000000..594d9a6 --- /dev/null +++ b/benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_old_brainctl_cmd.json @@ -0,0 +1,32 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\longmemeval_old_brainctl_cmd.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "measured": false, + "metadata": {}, + "metrics": { + "ndcg_at_10": 0.9247, + "ndcg_at_5": 0.9206, + "r_at_10": 0.9894, + "r_at_5": 0.9702 + }, + "mode": "cmd", + "notes": [ + "top_k=10" + ], + "primary_metric": "r_at_5", + "primary_metric_value": 0.9702, + "reference_kind": "historical", + "runtime_seconds": 130.863, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/smoke_postpush_20260423_123452/runs/membench_mempalace_raw_turn.json b/benchmarks/results/smoke_postpush_20260423_123452/runs/membench_mempalace_raw_turn.json new file mode 100644 index 0000000..edb0934 --- /dev/null +++ b/benchmarks/results/smoke_postpush_20260423_123452/runs/membench_mempalace_raw_turn.json @@ -0,0 +1,35 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\membench_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\membench_mempalace_raw_turn.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" + }, + "benchmark": "membench", + "caveats": [ + "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 200, + "measured": false, + "metadata": {}, + "metrics": { + "hit_at_5": 0.885, + "simple_hit_at_5": 0.885, + "top_k": 5 + }, + "mode": "raw_turn", + "notes": [ + "FirstAgent slice only", + "turn-level retrieval", + "topic=all" + ], + "primary_metric": "hit_at_5", + "primary_metric_value": 0.885, + "reference_kind": "historical", + "runtime_seconds": 804.35, + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "partial", + "system_name": "mempalace" +} \ No newline at end of file diff --git a/benchmarks/results/smoke_postpush_20260423_123452/runs/membench_new_brainctl_cmd_turn.json b/benchmarks/results/smoke_postpush_20260423_123452/runs/membench_new_brainctl_cmd_turn.json new file mode 100644 index 0000000..4c6d1ae --- /dev/null +++ b/benchmarks/results/smoke_postpush_20260423_123452/runs/membench_new_brainctl_cmd_turn.json @@ -0,0 +1,54 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\membench_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\membench_new_brainctl_cmd_turn.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" + }, + "benchmark": "membench", + "caveats": [ + "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 1, + "measured": true, + "metadata": {}, + "metrics": { + "hit_at_5": 1.0, + "simple_hit_at_5": 1.0, + "top_k": 5 + }, + "mode": "cmd_turn", + "notes": [ + "FirstAgent slice only", + "turn-level retrieval", + "topic=all" + ], + "primary_metric": "hit_at_5", + "primary_metric_value": 1.0, + "reference_kind": "measured", + "rows": [ + { + "category": "simple", + "hit_at_k": true, + "question": "What is the name of my niece's company?", + "retrieved_ids": [ + "simple_roles_0|sid=119|g=119|s=5|t=18", + "simple_roles_0|sid=101|g=101|s=5|t=0", + "simple_roles_0|sid=135|g=135|s=6|t=12", + "simple_roles_0|sid=11|g=11|s=0|t=11", + "simple_roles_0|sid=25|g=25|s=1|t=5" + ], + "target_ids": [ + "119" + ], + "tid": 0, + "topic": "roles" + } + ], + "runtime_seconds": 0.567, + "series_name": "new_brainctl", + "source_path": null, + "status": "partial", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/smoke_postpush_20260423_123452/runs/membench_old_brainctl_cmd_turn.json b/benchmarks/results/smoke_postpush_20260423_123452/runs/membench_old_brainctl_cmd_turn.json new file mode 100644 index 0000000..070c2db --- /dev/null +++ b/benchmarks/results/smoke_postpush_20260423_123452/runs/membench_old_brainctl_cmd_turn.json @@ -0,0 +1,35 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\membench_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\membench_old_brainctl_cmd_turn.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" + }, + "benchmark": "membench", + "caveats": [ + "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 200, + "measured": false, + "metadata": {}, + "metrics": { + "hit_at_5": 0.93, + "simple_hit_at_5": 0.93, + "top_k": 5 + }, + "mode": "cmd_turn", + "notes": [ + "FirstAgent slice only", + "turn-level retrieval", + "topic=all" + ], + "primary_metric": "hit_at_5", + "primary_metric_value": 0.93, + "reference_kind": "historical", + "runtime_seconds": 140.592, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "partial", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/smoke_postpush_20260423_123452/summary.csv b/benchmarks/results/smoke_postpush_20260423_123452/summary.csv new file mode 100644 index 0000000..766000a --- /dev/null +++ b/benchmarks/results/smoke_postpush_20260423_123452/summary.csv @@ -0,0 +1,15 @@ +benchmark,series_name,system_name,mode,reference_kind,status,example_count,primary_metric,primary_metric_value,runtime_seconds,dataset_path,source_path,notes,caveats,avg_recall,cat_1_recall,cat_2_recall,cat_3_recall,cat_4_recall,cat_5_recall,hit_at_5,ndcg_at_10,ndcg_at_5,perfect_rate,r_at_10,r_at_5,simple_hit_at_5,top_k,zero_rate +longmemeval,old_brainctl,brainctl,brain,historical,full_same_machine,470,r_at_5,0.9681,85.439,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,top_k=10,,,,,,,,,0.9253,0.9204,,0.9894,0.9681,,, +longmemeval,old_brainctl,brainctl,cmd,historical,full_same_machine,470,r_at_5,0.9702,130.863,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,top_k=10,,,,,,,,,0.9247,0.9206,,0.9894,0.9702,,, +longmemeval,mempalace,mempalace,raw_session,historical,full_same_machine,470,r_at_5,0.966,695.36,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,top_k=10 | Runs MemPalace benchmark module raw session retrieval logic directly.,,,,,,,,,0.8948,0.893,,0.983,0.966,,, +locomo,old_brainctl,brainctl,cmd_session,historical,full_same_machine,1986,avg_recall,0.9217,445.74,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,granularity=session | top_k=10,,0.9217,0.7614,0.92,0.6616,0.9738,0.9821,,,,0.8817,,,,10,0.0438 +locomo,mempalace,mempalace,raw_session,historical,full_same_machine,1986,avg_recall,0.6028,2106.411,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,granularity=session | top_k=10,,0.6028,0.5899,0.6921,0.4604,0.5809,0.6188,,,,0.5534,,,,10,0.3499 +membench,old_brainctl,brainctl,cmd_turn,historical,partial,200,hit_at_5,0.93,140.592,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,FirstAgent slice only | turn-level retrieval | topic=all,MemBench comparison is partial because ThirdAgent and noise-extended slices are not included.,,,,,,,0.93,,,,,,0.93,5, +membench,mempalace,mempalace,raw_turn,historical,partial,200,hit_at_5,0.885,804.35,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,FirstAgent slice only | turn-level retrieval | topic=all,MemBench comparison is partial because ThirdAgent and noise-extended slices are not included.,,,,,,,0.885,,,,,,0.885,5, +convomem,old_brainctl,brainctl,cmd,historical,blocked,0,avg_recall,,,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,limit_per_category=1 | top_k=10,Blocked while loading ConvoMem evidence data: ,,,,,,,,,,,,,,, +convomem,mempalace,mempalace,raw,historical,blocked,0,avg_recall,,,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,limit_per_category=1 | top_k=10,Blocked while loading ConvoMem evidence data: ,,,,,,,,,,,,,,, +longmemeval,new_brainctl,brainctl,brain,measured,full_same_machine,1,r_at_5,1.0,2.644,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,,top_k=10 | Legacy 470-question session-level slice.,,,,,,,,,1.0,1.0,,1.0,1.0,,, +longmemeval,new_brainctl,brainctl,cmd,measured,full_same_machine,1,r_at_5,1.0,2.577,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,,top_k=10 | Legacy 470-question session-level slice.,,,,,,,,,1.0,1.0,,1.0,1.0,,, +locomo,new_brainctl,brainctl,cmd_session,measured,full_same_machine,199,avg_recall,0.9413,5.736,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,,granularity=session | top_k=10,,0.9413,0.8641,1.0,0.7436,0.9571,0.9787,,,,0.9196,,,,10,0.0402 +membench,new_brainctl,brainctl,cmd_turn,measured,partial,1,hit_at_5,1.0,0.567,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,,FirstAgent slice only | turn-level retrieval | topic=all,MemBench comparison is partial because ThirdAgent and noise-extended slices are not included.,,,,,,,1.0,,,,,,1.0,5, +convomem,new_brainctl,brainctl,cmd,measured,blocked,0,avg_recall,,,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache,,limit_per_category=1 | top_k=10,Blocked while loading ConvoMem evidence data: HTTP Error 404: Not Found,,,,,,,,,,,,,,, diff --git a/benchmarks/results/smoke_postpush_20260423_123452/summary.json b/benchmarks/results/smoke_postpush_20260423_123452/summary.json new file mode 100644 index 0000000..e7f5f05 --- /dev/null +++ b/benchmarks/results/smoke_postpush_20260423_123452/summary.json @@ -0,0 +1,509 @@ +{ + "generated_at_utc": "2026-04-23T12:35:06.799013+00:00", + "metadata": { + "argv": [ + "benchmarks\\compare_memory_engines.py", + "--label", + "smoke_postpush", + "--longmemeval-limit", + "1", + "--locomo-limit", + "1", + "--membench-limit", + "1", + "--convomem-limit-per-category", + "1" + ], + "cwd": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl", + "datasets": { + "convomem_cache": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "locomo_data": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "longmemeval_data": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "membench_data": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent" + }, + "generated_at_utc": "2026-04-23T12:35:06.703127+00:00", + "git_commit": "b9a75f6b975c5330e3d35a2627a5e132b2e06e93", + "historical_summary_mode": "recovered", + "historical_summary_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "machine": "AMD64", + "platform": "Windows-11-10.0.26200-SP0", + "processor": "Intel64 Family 6 Model 189 Stepping 1, GenuineIntel", + "python_version": "3.14.4 (tags/v3.14.4:23116f9, Apr 7 2026, 14:10:54) [MSC v.1944 64 bit (AMD64)]" + }, + "notes": [ + "Historical old-BrainCTL and MemPalace series come from the recovered 2026-04-18 comparison bundle.", + "New BrainCTL series are rerun in the current checked-out repo using the legacy benchmark definitions.", + "MemBench remains intentionally partial because the legacy comparison only covered the FirstAgent slice.", + "ConvoMem remains a coverage/status benchmark here; it has no dedicated comparison chart in the legacy chart pack." + ], + "runs": [ + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\longmemeval_old_brainctl_brain.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "measured": false, + "metadata": {}, + "metrics": { + "ndcg_at_10": 0.9253, + "ndcg_at_5": 0.9204, + "r_at_10": 0.9894, + "r_at_5": 0.9681 + }, + "mode": "brain", + "notes": [ + "top_k=10" + ], + "primary_metric": "r_at_5", + "primary_metric_value": 0.9681, + "reference_kind": "historical", + "runtime_seconds": 85.439, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\longmemeval_old_brainctl_cmd.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "measured": false, + "metadata": {}, + "metrics": { + "ndcg_at_10": 0.9247, + "ndcg_at_5": 0.9206, + "r_at_10": 0.9894, + "r_at_5": 0.9702 + }, + "mode": "cmd", + "notes": [ + "top_k=10" + ], + "primary_metric": "r_at_5", + "primary_metric_value": 0.9702, + "reference_kind": "historical", + "runtime_seconds": 130.863, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\longmemeval_mempalace_raw_session.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "measured": false, + "metadata": {}, + "metrics": { + "ndcg_at_10": 0.8948, + "ndcg_at_5": 0.893, + "r_at_10": 0.983, + "r_at_5": 0.966 + }, + "mode": "raw_session", + "notes": [ + "top_k=10", + "Runs MemPalace benchmark module raw session retrieval logic directly." + ], + "primary_metric": "r_at_5", + "primary_metric_value": 0.966, + "reference_kind": "historical", + "runtime_seconds": 695.36, + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "mempalace" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\locomo_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\locomo_old_brainctl_cmd_session.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" + }, + "benchmark": "locomo", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "measured": false, + "metadata": {}, + "metrics": { + "avg_recall": 0.9217, + "cat_1_recall": 0.7614, + "cat_2_recall": 0.92, + "cat_3_recall": 0.6616, + "cat_4_recall": 0.9738, + "cat_5_recall": 0.9821, + "perfect_rate": 0.8817, + "top_k": 10, + "zero_rate": 0.0438 + }, + "mode": "cmd_session", + "notes": [ + "granularity=session", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": 0.9217, + "reference_kind": "historical", + "runtime_seconds": 445.74, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\locomo_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\locomo_mempalace_raw_session.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" + }, + "benchmark": "locomo", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "measured": false, + "metadata": {}, + "metrics": { + "avg_recall": 0.6028, + "cat_1_recall": 0.5899, + "cat_2_recall": 0.6921, + "cat_3_recall": 0.4604, + "cat_4_recall": 0.5809, + "cat_5_recall": 0.6188, + "perfect_rate": 0.5534, + "top_k": 10, + "zero_rate": 0.3499 + }, + "mode": "raw_session", + "notes": [ + "granularity=session", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": 0.6028, + "reference_kind": "historical", + "runtime_seconds": 2106.411, + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "mempalace" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\membench_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\membench_old_brainctl_cmd_turn.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" + }, + "benchmark": "membench", + "caveats": [ + "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 200, + "measured": false, + "metadata": {}, + "metrics": { + "hit_at_5": 0.93, + "simple_hit_at_5": 0.93, + "top_k": 5 + }, + "mode": "cmd_turn", + "notes": [ + "FirstAgent slice only", + "turn-level retrieval", + "topic=all" + ], + "primary_metric": "hit_at_5", + "primary_metric_value": 0.93, + "reference_kind": "historical", + "runtime_seconds": 140.592, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "partial", + "system_name": "brainctl" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\membench_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\membench_mempalace_raw_turn.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" + }, + "benchmark": "membench", + "caveats": [ + "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 200, + "measured": false, + "metadata": {}, + "metrics": { + "hit_at_5": 0.885, + "simple_hit_at_5": 0.885, + "top_k": 5 + }, + "mode": "raw_turn", + "notes": [ + "FirstAgent slice only", + "turn-level retrieval", + "topic=all" + ], + "primary_metric": "hit_at_5", + "primary_metric_value": 0.885, + "reference_kind": "historical", + "runtime_seconds": 804.35, + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "partial", + "system_name": "mempalace" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\convomem_old_brainctl_cmd.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" + }, + "benchmark": "convomem", + "caveats": [ + "Blocked while loading ConvoMem evidence data: " + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "example_count": 0, + "measured": false, + "metadata": {}, + "metrics": {}, + "mode": "cmd", + "notes": [ + "limit_per_category=1", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": null, + "reference_kind": "historical", + "runtime_seconds": null, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "blocked", + "system_name": "brainctl" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\convomem_mempalace_raw.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" + }, + "benchmark": "convomem", + "caveats": [ + "Blocked while loading ConvoMem evidence data: " + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "example_count": 0, + "measured": false, + "metadata": {}, + "metrics": {}, + "mode": "raw", + "notes": [ + "limit_per_category=1", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": null, + "reference_kind": "historical", + "runtime_seconds": null, + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "blocked", + "system_name": "mempalace" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\longmemeval_new_brainctl_brain.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 1, + "measured": true, + "metadata": {}, + "metrics": { + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0 + }, + "mode": "brain", + "notes": [ + "top_k=10", + "Legacy 470-question session-level slice." + ], + "primary_metric": "r_at_5", + "primary_metric_value": 1.0, + "reference_kind": "measured", + "runtime_seconds": 2.644, + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\longmemeval_new_brainctl_cmd.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 1, + "measured": true, + "metadata": {}, + "metrics": { + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0 + }, + "mode": "cmd", + "notes": [ + "top_k=10", + "Legacy 470-question session-level slice." + ], + "primary_metric": "r_at_5", + "primary_metric_value": 1.0, + "reference_kind": "measured", + "runtime_seconds": 2.577, + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\locomo_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\locomo_new_brainctl_cmd_session.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" + }, + "benchmark": "locomo", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 199, + "measured": true, + "metadata": {}, + "metrics": { + "avg_recall": 0.9413, + "cat_1_recall": 0.8641, + "cat_2_recall": 1.0, + "cat_3_recall": 0.7436, + "cat_4_recall": 0.9571, + "cat_5_recall": 0.9787, + "perfect_rate": 0.9196, + "top_k": 10, + "zero_rate": 0.0402 + }, + "mode": "cmd_session", + "notes": [ + "granularity=session", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": 0.9413, + "reference_kind": "measured", + "runtime_seconds": 5.736, + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\membench_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\membench_new_brainctl_cmd_turn.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" + }, + "benchmark": "membench", + "caveats": [ + "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 1, + "measured": true, + "metadata": {}, + "metrics": { + "hit_at_5": 1.0, + "simple_hit_at_5": 1.0, + "top_k": 5 + }, + "mode": "cmd_turn", + "notes": [ + "FirstAgent slice only", + "turn-level retrieval", + "topic=all" + ], + "primary_metric": "hit_at_5", + "primary_metric_value": 1.0, + "reference_kind": "measured", + "runtime_seconds": 0.567, + "series_name": "new_brainctl", + "source_path": null, + "status": "partial", + "system_name": "brainctl" + }, + { + "artifacts": { + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\convomem_new_brainctl_cmd.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" + }, + "benchmark": "convomem", + "caveats": [ + "Blocked while loading ConvoMem evidence data: HTTP Error 404: Not Found" + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "example_count": 0, + "measured": true, + "metadata": {}, + "metrics": {}, + "mode": "cmd", + "notes": [ + "limit_per_category=1", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": null, + "reference_kind": "measured", + "runtime_seconds": null, + "series_name": "new_brainctl", + "source_path": null, + "status": "blocked", + "system_name": "brainctl" + } + ] +} \ No newline at end of file From 9368f288279ae1d3395e37e89972038b74d66e3a Mon Sep 17 00:00:00 2001 From: Mario Jack Vela Date: Thu, 23 Apr 2026 10:34:13 -0500 Subject: [PATCH 05/13] Add shared second-stage reranker and benchmark tooling --- benchmarks/analyze_benchmark_failures.py | 77 + benchmarks/brainctl_retrieval.py | 48 +- benchmarks/build_hard_negatives.py | 245 ++ benchmarks/convomem_bench.py | 82 +- benchmarks/train_tiny_reranker.py | 240 ++ pyproject.toml | 2 +- src/agentmemory/_impl.py | 40 + src/agentmemory/retrieval/__init__.py | 6 + src/agentmemory/retrieval/diagnostics.py | 8 + src/agentmemory/retrieval/feature_builder.py | 375 ++++ src/agentmemory/retrieval/judge.py | 87 + src/agentmemory/retrieval/mlp_reranker.py | 129 ++ .../retrieval/models/tiny_mlp_v1.json | 1963 +++++++++++++++++ src/agentmemory/retrieval/second_stage.py | 247 +++ tests/test_convomem_bench.py | 52 + tests/test_second_stage_reranker.py | 177 ++ 16 files changed, 3740 insertions(+), 38 deletions(-) create mode 100644 benchmarks/analyze_benchmark_failures.py create mode 100644 benchmarks/build_hard_negatives.py create mode 100644 benchmarks/train_tiny_reranker.py create mode 100644 src/agentmemory/retrieval/feature_builder.py create mode 100644 src/agentmemory/retrieval/judge.py create mode 100644 src/agentmemory/retrieval/mlp_reranker.py create mode 100644 src/agentmemory/retrieval/models/tiny_mlp_v1.json create mode 100644 src/agentmemory/retrieval/second_stage.py create mode 100644 tests/test_convomem_bench.py create mode 100644 tests/test_second_stage_reranker.py diff --git a/benchmarks/analyze_benchmark_failures.py b/benchmarks/analyze_benchmark_failures.py new file mode 100644 index 0000000..37ebbad --- /dev/null +++ b/benchmarks/analyze_benchmark_failures.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import argparse +import json +import sys +from collections import Counter +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent +REPO_ROOT = ROOT.parent +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + + +def _latest_bundle() -> Path: + bundles = sorted((ROOT / "results").glob("seq_full_compare_final_*"), reverse=True) + if not bundles: + raise FileNotFoundError("No seq_full_compare_final_* bundle found under benchmarks/results/") + return bundles[0] + + +def _load_rows(path: Path) -> list[dict]: + payload = json.loads(path.read_text(encoding="utf-8")) + return list(payload.get("rows") or []) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Summarize current LongMemEval/LoCoMo/MemBench failure slices.") + parser.add_argument("--bundle-dir", type=Path, default=None) + parser.add_argument("--output", type=Path, default=None) + args = parser.parse_args() + + bundle_dir = args.bundle_dir or _latest_bundle() + long_rows = _load_rows(bundle_dir / "runs" / "longmemeval_new_brainctl_cmd.json") + locomo_rows = _load_rows(bundle_dir / "runs" / "locomo_new_brainctl_cmd_session.json") + membench_rows = _load_rows(bundle_dir / "runs" / "membench_new_brainctl_cmd_turn.json") + + long_fail_r5 = [row for row in long_rows if float(row.get("r_at_5", 1.0) or 1.0) < 1.0] + long_fail_ndcg = [row for row in long_rows if float(row.get("ndcg_at_5", 1.0) or 1.0) < 1.0] + locomo_nonperfect = [row for row in locomo_rows if float(row.get("recall", 1.0) or 1.0) < 1.0] + locomo_zero = [row for row in locomo_rows if float(row.get("recall", 0.0) or 0.0) == 0.0] + membench_miss = [ + row for row in membench_rows + if not row.get("top_turn_ids") and float(row.get("hit_at_5", 1.0) or 1.0) < 1.0 + ] + + payload = { + "bundle_dir": str(bundle_dir), + "longmemeval": { + "total": len(long_rows), + "fail_r_at_5": len(long_fail_r5), + "fail_ndcg_at_5": len(long_fail_ndcg), + "by_question_type": dict(Counter(str(row.get("question_type")) for row in long_fail_ndcg).most_common()), + }, + "locomo": { + "total": len(locomo_rows), + "nonperfect": len(locomo_nonperfect), + "zero_recall": len(locomo_zero), + "by_category": dict(Counter(str(row.get("category_name")) for row in locomo_nonperfect).most_common()), + }, + "membench": { + "total": len(membench_rows), + "misses": len(membench_miss), + }, + } + + text = json.dumps(payload, indent=2, sort_keys=True) + print(text) + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(text, encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/brainctl_retrieval.py b/benchmarks/brainctl_retrieval.py index b295105..e8052b0 100644 --- a/benchmarks/brainctl_retrieval.py +++ b/benchmarks/brainctl_retrieval.py @@ -77,7 +77,7 @@ def _search_brain(db_path: Path, query: str, top_k: int) -> list[dict]: brain.close() -def _search_cmd(db_path: Path, query: str, top_k: int) -> list[dict]: +def _search_cmd(db_path: Path, query: str, top_k: int, *, debug: bool = False) -> list[dict]: import agentmemory._impl as _impl _impl.DB_PATH = db_path @@ -101,7 +101,7 @@ def _search_cmd(db_path: Path, query: str, top_k: int) -> list[dict]: format="json", oneline=False, verbose=False, - debug=False, + debug=debug, ) payload = _impl.cmd_search(args, db=None, db_path=str(db_path)) memories = list((payload or {}).get("memories") or []) @@ -177,6 +177,34 @@ def rank_seeded_documents( shutil.rmtree(work_dir, ignore_errors=True) +def search_seeded_documents( + query: str, + seeded: SeededCorpus, + *, + pipeline: str = "cmd", + top_k: int = 10, + debug: bool = False, +) -> list[dict]: + work_dir = Path(tempfile.mkdtemp(prefix="brainctl-legacy-query-")) + db_path = work_dir / "brain.db" + try: + shutil.copy2(seeded.template_db_path, db_path) + if pipeline == "brain": + results = _search_brain(db_path, query, top_k) + elif pipeline == "cmd": + results = _search_cmd(db_path, query, top_k, debug=debug) + else: + raise ValueError(f"Unknown pipeline {pipeline!r}") + out: list[dict] = [] + for result in results: + row = dict(result) + row["doc_id"] = seeded.rowid_to_doc_id.get(int(result["id"]), "") + out.append(row) + return out + finally: + shutil.rmtree(work_dir, ignore_errors=True) + + def rank_documents( query: str, documents: Iterable[tuple[str, str]], @@ -190,3 +218,19 @@ def rank_documents( return rank_seeded_documents(query, seeded, pipeline=pipeline, top_k=top_k) finally: seeded.cleanup() + + +def rank_documents_with_rows( + query: str, + documents: Iterable[tuple[str, str]], + *, + pipeline: str = "cmd", + top_k: int = 10, + category: str = "benchmark", + debug: bool = False, +) -> list[dict]: + seeded = seed_documents(documents, category=category) + try: + return search_seeded_documents(query, seeded, pipeline=pipeline, top_k=top_k, debug=debug) + finally: + seeded.cleanup() diff --git a/benchmarks/build_hard_negatives.py b/benchmarks/build_hard_negatives.py new file mode 100644 index 0000000..7e86bfa --- /dev/null +++ b/benchmarks/build_hard_negatives.py @@ -0,0 +1,245 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +from collections import Counter +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parent +REPO_ROOT = ROOT.parent +SRC = REPO_ROOT / "src" +for _path in (REPO_ROOT, SRC): + if str(_path) not in sys.path: + sys.path.insert(0, str(_path)) + +from benchmarks.brainctl_retrieval import rank_documents_with_rows +from benchmarks.datasets import resolve_dataset_paths +from benchmarks.locomo_bench import _build_corpus as _locomo_build_corpus +from benchmarks.locomo_bench import _load_samples as _locomo_load_samples +from benchmarks.longmemeval_bench import load_entries as _load_longmemeval_entries +from benchmarks.longmemeval_bench import session_document as _longmemeval_session_document +from agentmemory.retrieval.feature_builder import FEATURE_ORDER_V1, FEATURE_VERSION_V1, build_features, vectorize_features +from agentmemory.retrieval.query_planner import plan_query + + +def _latest_bundle() -> Path: + candidates = sorted((ROOT / "results").glob("seq_full_compare_final_*"), reverse=True) + if not candidates: + raise FileNotFoundError("No seq_full_compare_final_* benchmark bundle found under benchmarks/results/") + return candidates[0] + + +def _stable_split(key: str) -> str: + value = int(hashlib.sha1(key.encode("utf-8")).hexdigest()[:8], 16) + return "heldout" if value % 5 == 0 else "train" + + +def _read_run_rows(path: Path) -> list[dict[str, Any]]: + payload = json.loads(path.read_text(encoding="utf-8")) + return list(payload.get("rows") or []) + + +def _serialize_feature_vector(feature_dict: dict[str, float]) -> list[float]: + vector = vectorize_features(feature_dict, feature_version=FEATURE_VERSION_V1) + if hasattr(vector, "tolist"): + return [float(value) for value in vector.tolist()] + return [float(value) for value in vector] + + +def _record_for_candidate( + *, + benchmark: str, + query_id: str, + query: str, + split: str, + gold_doc_ids: list[str], + candidate: dict[str, Any], + rank: int, + query_label: str, +) -> dict[str, Any]: + plan = plan_query(query, requested_tables=["memories"]) + candidate = dict(candidate) + candidate["bucket"] = "memories" + candidate["type"] = "memory" + candidate["_stage_position"] = rank + features = build_features(query, plan, candidate) + doc_id = str(candidate.get("doc_id") or "") + return { + "benchmark": benchmark, + "query_id": query_id, + "query": query, + "split": split, + "query_label": query_label, + "gold_doc_ids": gold_doc_ids, + "candidate_doc_id": doc_id, + "label": 1 if doc_id in set(gold_doc_ids) else 0, + "rank": rank, + "source": candidate.get("source"), + "base_score": candidate.get("pre_second_stage_score", candidate.get("final_score")), + "retrieval_score": candidate.get("retrieval_score"), + "rrf_score": candidate.get("rrf_score"), + "final_score": candidate.get("final_score"), + "feature_version": FEATURE_VERSION_V1, + "feature_order": FEATURE_ORDER_V1, + "feature_dict": features, + "feature_vector": _serialize_feature_vector(features), + "candidate_excerpt": ( + candidate.get("content") + or candidate.get("summary") + or candidate.get("title") + or candidate.get("goal") + or "" + )[:800], + } + + +def build_longmemeval_records(bundle_dir: Path, dataset_path: Path, *, top_k: int) -> tuple[list[dict[str, Any]], dict[str, Any]]: + run_rows = _read_run_rows(bundle_dir / "runs" / "longmemeval_new_brainctl_cmd.json") + entries = {entry.question_id: entry for entry in _load_longmemeval_entries(dataset_path)} + records: list[dict[str, Any]] = [] + selected = 0 + skipped_no_window = 0 + + for row in run_rows: + if row.get("r_at_5", 1.0) >= 1.0 and row.get("ndcg_at_5", 1.0) >= 1.0: + continue + entry = entries.get(str(row["question_id"])) + if entry is None: + continue + docs = [ + ( + session_id, + _longmemeval_session_document(session_id, session_date, turns), + ) + for session_id, session_date, turns in zip( + entry.haystack_session_ids, + entry.haystack_dates, + entry.haystack_sessions, + ) + ] + ranked = rank_documents_with_rows(entry.question, docs, pipeline="cmd", top_k=top_k, debug=True) + gold_ids = list(entry.answer_session_ids) + if not any(str(candidate.get("doc_id") or "") in set(gold_ids) for candidate in ranked): + skipped_no_window += 1 + continue + selected += 1 + split = _stable_split(entry.question_id) + for rank, candidate in enumerate(ranked[:top_k]): + records.append( + _record_for_candidate( + benchmark="longmemeval", + query_id=entry.question_id, + query=entry.question, + split=split, + gold_doc_ids=gold_ids, + candidate=candidate, + rank=rank, + query_label=entry.question_type, + ) + ) + + return records, { + "selected_queries": selected, + "skipped_no_gold_in_window": skipped_no_window, + } + + +def build_locomo_records(bundle_dir: Path, dataset_path: Path, *, top_k: int) -> tuple[list[dict[str, Any]], dict[str, Any]]: + run_rows = _read_run_rows(bundle_dir / "runs" / "locomo_new_brainctl_cmd_session.json") + samples = {str(sample.get("sample_id")): sample for sample in _locomo_load_samples(dataset_path)} + records: list[dict[str, Any]] = [] + selected = 0 + skipped_no_window = 0 + + for idx, row in enumerate(run_rows): + if float(row.get("recall", 1.0) or 1.0) >= 1.0: + continue + sample = samples.get(str(row["sample_id"])) + if sample is None: + continue + sessions = [] + session_num = 1 + while True: + key = f"session_{session_num}" + date_key = f"session_{session_num}_date_time" + if key not in sample["conversation"]: + break + sessions.append( + { + "session_num": session_num, + "date": sample["conversation"].get(date_key, ""), + "dialogs": sample["conversation"][key], + } + ) + session_num += 1 + docs = _locomo_build_corpus(sessions, granularity="session") + ranked = rank_documents_with_rows(str(row["question"]), docs, pipeline="cmd", top_k=top_k, debug=True) + gold_ids = [str(value) for value in row.get("evidence_ids", [])] + if not any(str(candidate.get("doc_id") or "") in set(gold_ids) for candidate in ranked): + skipped_no_window += 1 + continue + query_id = hashlib.sha1(f"{row['sample_id']}|{row['question']}|{idx}".encode("utf-8")).hexdigest()[:12] + split = _stable_split(query_id) + selected += 1 + for rank, candidate in enumerate(ranked[:top_k]): + records.append( + _record_for_candidate( + benchmark="locomo", + query_id=query_id, + query=str(row["question"]), + split=split, + gold_doc_ids=gold_ids, + candidate=candidate, + rank=rank, + query_label=str(row.get("category_name") or row.get("category") or "unknown"), + ) + ) + return records, { + "selected_queries": selected, + "skipped_no_gold_in_window": skipped_no_window, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description="Build LongMemEval + LoCoMo hard-negative reranker data.") + parser.add_argument("--bundle-dir", type=Path, default=None, help="Legacy comparison bundle directory.") + parser.add_argument("--output", type=Path, default=ROOT / "training_data" / "hard_negatives_v1.jsonl") + parser.add_argument("--summary", type=Path, default=ROOT / "training_data" / "hard_negatives_v1_summary.json") + parser.add_argument("--top-k", type=int, default=10) + args = parser.parse_args() + + bundle_dir = args.bundle_dir or _latest_bundle() + dataset_paths = resolve_dataset_paths() + if dataset_paths.longmemeval_data is None or dataset_paths.locomo_data is None: + raise FileNotFoundError("LongMemEval or LoCoMo dataset path is unavailable on this machine.") + + long_records, long_summary = build_longmemeval_records(bundle_dir, dataset_paths.longmemeval_data, top_k=args.top_k) + locomo_records, locomo_summary = build_locomo_records(bundle_dir, dataset_paths.locomo_data, top_k=args.top_k) + records = long_records + locomo_records + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open("w", encoding="utf-8") as handle: + for record in records: + handle.write(json.dumps(record, ensure_ascii=True) + "\n") + + split_counts = Counter(record["split"] for record in records) + label_counts = Counter(record["label"] for record in records) + summary = { + "bundle_dir": str(bundle_dir), + "output": str(args.output), + "record_count": len(records), + "split_counts": dict(split_counts), + "label_counts": dict(label_counts), + "longmemeval": long_summary, + "locomo": locomo_summary, + } + args.summary.parent.mkdir(parents=True, exist_ok=True) + args.summary.write_text(json.dumps(summary, indent=2, sort_keys=True), encoding="utf-8") + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/convomem_bench.py b/benchmarks/convomem_bench.py index 9b4f406..7e9a68a 100644 --- a/benchmarks/convomem_bench.py +++ b/benchmarks/convomem_bench.py @@ -2,13 +2,14 @@ import json import time +import urllib.error import urllib.request from collections import defaultdict from pathlib import Path from typing import Any from benchmarks.brainctl_retrieval import rank_documents -from benchmarks.framework import BenchmarkRunResult, BLOCKED, PARTIAL +from benchmarks.framework import BenchmarkRunResult, BLOCKED, FULL_SAME_MACHINE, PARTIAL HF_BASE = "https://huggingface.co/datasets/Salesforce/ConvoMem/resolve/main/core_benchmark/evidence_questions" @@ -32,10 +33,22 @@ def _download_json(url: str, path: Path) -> Any: path.parent.mkdir(parents=True, exist_ok=True) if path.exists(): return _read_json(path) - with urllib.request.urlopen(url, timeout=30) as response: - payload = response.read().decode("utf-8") - path.write_text(payload, encoding="utf-8") - return json.loads(payload) + last_error: Exception | None = None + req = urllib.request.Request(url, headers={"User-Agent": "brainctl-convomem-bench/1.0"}) + for _attempt in range(3): + try: + with urllib.request.urlopen(req, timeout=30) as response: # noqa: S310 - remote dataset fetch + payload = response.read().decode("utf-8") + path.write_text(payload, encoding="utf-8") + return json.loads(payload) + except (urllib.error.URLError, TimeoutError, OSError, ValueError, json.JSONDecodeError) as exc: + last_error = exc + time.sleep(0.4) + if path.exists(): + return _read_json(path) + if path.exists(): + return _read_json(path) + raise last_error or RuntimeError(f"Failed to download {url}") def _discover_files(category: str, cache_dir: Path) -> list[str]: @@ -55,14 +68,25 @@ def load_evidence_items( categories: list[str], limit_per_category: int, cache_dir: Path, -) -> list[dict[str, Any]]: +) -> tuple[list[dict[str, Any]], list[str], list[str]]: items: list[dict[str, Any]] = [] + caveats: list[str] = [] + loaded_categories: list[str] = [] for category in categories: loaded = 0 - for subpath in _discover_files(category, cache_dir): + try: + subpaths = _discover_files(category, cache_dir) + except Exception as exc: + caveats.append(f"{category}: discover failed: {exc!s}") + continue + for subpath in subpaths: cache_path = cache_dir / category / subpath.replace("/", "_") url = f"{HF_BASE}/{category}/{subpath}" - payload = _download_json(url, cache_path) + try: + payload = _download_json(url, cache_path) + except Exception as exc: + caveats.append(f"{category}: item load failed for {subpath}: {exc!s}") + continue for item in payload.get("evidence_items", []): item["_category_key"] = category items.append(item) @@ -71,7 +95,11 @@ def load_evidence_items( break if loaded >= limit_per_category: break - return items + if loaded > 0: + loaded_categories.append(category) + else: + caveats.append(f"{category}: no evidence items loaded") + return items, caveats, loaded_categories def _message_docs(item: dict[str, Any]) -> list[tuple[str, str]]: @@ -134,28 +162,12 @@ def run_brainctl_convomem( ) return run, [] - try: - items = load_evidence_items( - categories=categories or list(CATEGORIES.keys()), - limit_per_category=limit_per_category, - cache_dir=cache_dir, - ) - except Exception as exc: - run = BenchmarkRunResult( - benchmark="convomem", - system_name="brainctl", - mode=pipeline, - status=BLOCKED, - example_count=0, - metrics={}, - primary_metric="avg_recall", - primary_metric_value=None, - dataset_path=str(cache_dir), - notes=[f"limit_per_category={limit_per_category}", f"top_k={top_k}"], - series_name="new_brainctl", - caveats=[f"Blocked while loading ConvoMem evidence data: {exc!s}"], - ) - return run, [] + requested_categories = categories or list(CATEGORIES.keys()) + items, caveats, loaded_categories = load_evidence_items( + categories=requested_categories, + limit_per_category=limit_per_category, + cache_dir=cache_dir, + ) if not items: run = BenchmarkRunResult( @@ -170,7 +182,7 @@ def run_brainctl_convomem( dataset_path=str(cache_dir), notes=[f"limit_per_category={limit_per_category}", f"top_k={top_k}"], series_name="new_brainctl", - caveats=["Blocked because no ConvoMem evidence items could be loaded for the requested categories."], + caveats=caveats or ["Blocked because no ConvoMem evidence items could be loaded for the requested categories."], ) return run, [] @@ -212,15 +224,15 @@ def run_brainctl_convomem( benchmark="convomem", system_name="brainctl", mode=pipeline, - status=PARTIAL, + status=FULL_SAME_MACHINE if len(loaded_categories) == len(requested_categories) else PARTIAL, example_count=example_count, metrics=metrics, primary_metric="avg_recall", primary_metric_value=avg_recall, runtime_seconds=runtime_seconds, dataset_path=str(cache_dir), - notes=[f"categories={len(categories or list(CATEGORIES.keys()))}", f"limit_per_category={limit_per_category}", f"top_k={top_k}"], - caveats=["ConvoMem comparison is partial because it uses a bounded same-machine sample, not the full benchmark."], + notes=[f"categories={len(requested_categories)}", f"limit_per_category={limit_per_category}", f"top_k={top_k}"], + caveats=(caveats + ["ConvoMem comparison is partial because it uses a bounded same-machine sample, not the full benchmark."]) if len(loaded_categories) != len(requested_categories) else caveats, series_name="new_brainctl", ) return run, rows diff --git a/benchmarks/train_tiny_reranker.py b/benchmarks/train_tiny_reranker.py new file mode 100644 index 0000000..dd7603c --- /dev/null +++ b/benchmarks/train_tiny_reranker.py @@ -0,0 +1,240 @@ +from __future__ import annotations + +import argparse +import json +import sys +from collections import defaultdict +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import numpy as np + +ROOT = Path(__file__).resolve().parent +REPO_ROOT = ROOT.parent +SRC = REPO_ROOT / "src" +for _path in (REPO_ROOT, SRC): + if str(_path) not in sys.path: + sys.path.insert(0, str(_path)) + +from agentmemory.retrieval.feature_builder import FEATURE_ORDER_V1, FEATURE_VERSION_V1 +from agentmemory.retrieval.mlp_reranker import DEFAULT_MODEL_PATH + + +@dataclass(slots=True) +class TrainConfig: + epochs: int = 40 + lr: float = 0.01 + l2: float = 1e-4 + seed: int = 42 + hidden1: int = 32 + hidden2: int = 16 + + +def _load_records(path: Path) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + with path.open("r", encoding="utf-8") as handle: + for line in handle: + line = line.strip() + if not line: + continue + rows.append(json.loads(line)) + return rows + + +def _group_query_metrics(records: list[dict[str, Any]], scores_by_key: dict[tuple[str, str], float], *, k: int = 5) -> dict[str, float]: + grouped: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) + for record in records: + grouped[(str(record["benchmark"]), str(record["query_id"]))].append(record) + + long_ndcgs: list[float] = [] + locomo_perfect: list[float] = [] + for (benchmark, _query_id), items in grouped.items(): + ranked = sorted(items, key=lambda row: scores_by_key[(str(row["query_id"]), str(row["candidate_doc_id"]))], reverse=True) + top = ranked[:k] + labels = [int(row["label"]) for row in top] + if benchmark == "longmemeval": + dcg = 0.0 + for idx, label in enumerate(labels): + dcg += float(label) / np.log2(idx + 2) + ideal_labels = sorted((int(row["label"]) for row in items), reverse=True)[:k] + ideal_dcg = 0.0 + for idx, label in enumerate(ideal_labels): + ideal_dcg += float(label) / np.log2(idx + 2) + long_ndcgs.append((dcg / ideal_dcg) if ideal_dcg > 0 else 0.0) + elif benchmark == "locomo": + positives = sum(int(row["label"]) for row in items) + if positives <= 0: + continue + top_positive = sum(int(row["label"]) for row in top) + locomo_perfect.append(1.0 if top_positive == positives else 0.0) + return { + "heldout_longmemeval_ndcg_at_5": round(float(np.mean(long_ndcgs)) if long_ndcgs else 0.0, 4), + "heldout_locomo_perfect_rate_at_5": round(float(np.mean(locomo_perfect)) if locomo_perfect else 0.0, 4), + } + + +def _init_params(rng: np.random.Generator, input_dim: int, config: TrainConfig) -> dict[str, np.ndarray]: + return { + "w1": rng.normal(0.0, 0.12, size=(input_dim, config.hidden1)), + "b1": np.zeros(config.hidden1, dtype=float), + "w2": rng.normal(0.0, 0.12, size=(config.hidden1, config.hidden2)), + "b2": np.zeros(config.hidden2, dtype=float), + "w3": rng.normal(0.0, 0.12, size=(config.hidden2, 1)), + "b3": np.zeros(1, dtype=float), + } + + +def _forward(x: np.ndarray, params: dict[str, np.ndarray]) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + z1 = x @ params["w1"] + params["b1"] + h1 = np.maximum(0.0, z1) + z2 = h1 @ params["w2"] + params["b2"] + h2 = np.maximum(0.0, z2) + logits = h2 @ params["w3"] + params["b3"] + probs = 1.0 / (1.0 + np.exp(-logits)) + return z1, h1, z2, h2, probs + + +def _clone_params(params: dict[str, np.ndarray]) -> dict[str, np.ndarray]: + return {key: np.array(value, copy=True) for key, value in params.items()} + + +def _train_model( + train_x: np.ndarray, + train_y: np.ndarray, + config: TrainConfig, + *, + initial_params: dict[str, np.ndarray] | None = None, +) -> dict[str, np.ndarray]: + rng = np.random.default_rng(config.seed) + params = _clone_params(initial_params) if initial_params is not None else _init_params(rng, train_x.shape[1], config) + batch_size = min(128, len(train_x)) + for _epoch in range(config.epochs): + order = rng.permutation(len(train_x)) + for start in range(0, len(train_x), batch_size): + batch_idx = order[start:start + batch_size] + x = train_x[batch_idx] + y = train_y[batch_idx].reshape(-1, 1) + z1, h1, z2, h2, probs = _forward(x, params) + diff = probs - y + grad_w3 = (h2.T @ diff) / len(x) + config.l2 * params["w3"] + grad_b3 = diff.mean(axis=0) + grad_h2 = diff @ params["w3"].T + grad_z2 = grad_h2 * (z2 > 0) + grad_w2 = (h1.T @ grad_z2) / len(x) + config.l2 * params["w2"] + grad_b2 = grad_z2.mean(axis=0) + grad_h1 = grad_z2 @ params["w2"].T + grad_z1 = grad_h1 * (z1 > 0) + grad_w1 = (x.T @ grad_z1) / len(x) + config.l2 * params["w1"] + grad_b1 = grad_z1.mean(axis=0) + params["w3"] -= config.lr * grad_w3 + params["b3"] -= config.lr * grad_b3 + params["w2"] -= config.lr * grad_w2 + params["b2"] -= config.lr * grad_b2 + params["w1"] -= config.lr * grad_w1 + params["b1"] -= config.lr * grad_b1 + return params + + +def main() -> int: + parser = argparse.ArgumentParser(description="Train the tiny shared second-stage MLP reranker.") + parser.add_argument("--data", type=Path, default=ROOT / "training_data" / "hard_negatives_v1.jsonl") + parser.add_argument("--report", type=Path, default=ROOT / "training_data" / "tiny_mlp_v1_report.json") + parser.add_argument("--model-out", type=Path, default=DEFAULT_MODEL_PATH) + parser.add_argument("--epochs", type=int, default=40) + parser.add_argument("--lr", type=float, default=0.01) + parser.add_argument("--seed", type=int, default=42) + args = parser.parse_args() + + records = _load_records(args.data) + records = [row for row in records if list(row.get("feature_order") or []) == FEATURE_ORDER_V1] + if not records: + raise ValueError(f"No usable training rows found in {args.data}") + + train_records = [row for row in records if row.get("split") != "heldout"] + heldout_records = [row for row in records if row.get("split") == "heldout"] + train_x = np.asarray([row["feature_vector"] for row in train_records], dtype=float) + train_y = np.asarray([float(row["label"]) for row in train_records], dtype=float) + heldout_x = np.asarray([row["feature_vector"] for row in heldout_records], dtype=float) if heldout_records else np.zeros((0, len(FEATURE_ORDER_V1))) + + norm_mean = train_x.mean(axis=0) + norm_std = train_x.std(axis=0) + safe_std = np.where(norm_std == 0.0, 1.0, norm_std) + train_x_norm = (train_x - norm_mean) / safe_std + heldout_x_norm = (heldout_x - norm_mean) / safe_std if len(heldout_x) else heldout_x + + config = TrainConfig(epochs=args.epochs, lr=args.lr, seed=args.seed) + params = _train_model(train_x_norm, train_y, config) + _, _, _, _, train_probs = _forward(train_x_norm, params) + heldout_probs = np.zeros((len(heldout_x_norm), 1), dtype=float) + if len(heldout_x_norm): + _, _, _, _, heldout_probs = _forward(heldout_x_norm, params) + + def _scores(rows: list[dict[str, Any]], probs: np.ndarray) -> dict[tuple[str, str], float]: + return { + (str(row["query_id"]), str(row["candidate_doc_id"])): float(prob) + for row, prob in zip(rows, probs.reshape(-1)) + } + + train_metrics = _group_query_metrics(train_records, _scores(train_records, train_probs)) + heldout_metrics = _group_query_metrics(heldout_records, _scores(heldout_records, heldout_probs)) + + # One additional epoch on LongMemEval-only hard negatives if heldout nDCG stays flat or improves. + long_only = [row for row in train_records if row.get("benchmark") == "longmemeval"] + long_applied = False + if long_only: + long_x = np.asarray([row["feature_vector"] for row in long_only], dtype=float) + long_y = np.asarray([float(row["label"]) for row in long_only], dtype=float) + long_x_norm = (long_x - norm_mean) / safe_std + extra_config = TrainConfig(epochs=1, lr=args.lr, seed=args.seed) + extra_params = _train_model(long_x_norm, long_y, extra_config, initial_params=params) + if len(heldout_x_norm): + _, _, _, _, extra_probs = _forward(heldout_x_norm, extra_params) + extra_metrics = _group_query_metrics(heldout_records, _scores(heldout_records, extra_probs)) + if extra_metrics["heldout_longmemeval_ndcg_at_5"] >= heldout_metrics["heldout_longmemeval_ndcg_at_5"]: + params = extra_params + heldout_probs = extra_probs + heldout_metrics = extra_metrics + long_applied = True + + model_payload = { + "feature_version": FEATURE_VERSION_V1, + "feature_order": FEATURE_ORDER_V1, + "norm_mean": [round(float(v), 8) for v in norm_mean.tolist()], + "norm_std": [round(float(v if v != 0 else 1.0), 8) for v in safe_std.tolist()], + "w1": np.asarray(params["w1"], dtype=float).T.round(8).tolist(), + "b1": np.asarray(params["b1"], dtype=float).round(8).tolist(), + "w2": np.asarray(params["w2"], dtype=float).T.round(8).tolist(), + "b2": np.asarray(params["b2"], dtype=float).round(8).tolist(), + "w3": np.asarray(params["w3"], dtype=float).T.round(8).tolist(), + "b3": np.asarray(params["b3"], dtype=float).round(8).tolist(), + "metadata": { + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "source_data": str(args.data), + "train_records": len(train_records), + "heldout_records": len(heldout_records), + "longmemeval_extra_epoch_applied": long_applied, + }, + } + + args.model_out.parent.mkdir(parents=True, exist_ok=True) + args.model_out.write_text(json.dumps(model_payload, indent=2), encoding="utf-8") + + report = { + "data": str(args.data), + "model_out": str(args.model_out), + "train_records": len(train_records), + "heldout_records": len(heldout_records), + "train_metrics": train_metrics, + "heldout_metrics": heldout_metrics, + "longmemeval_extra_epoch_applied": long_applied, + } + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8") + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pyproject.toml b/pyproject.toml index a4a3dbd..142fadf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -151,4 +151,4 @@ Documentation = "https://github.com/TSchonleber/brainctl/blob/main/docs/AGENT_ON where = ["src"] [tool.setuptools.package-data] -agentmemory = ["db/*.sql"] +agentmemory = ["db/*.sql", "retrieval/models/*.json"] diff --git a/src/agentmemory/_impl.py b/src/agentmemory/_impl.py index bd8ecc1..5868f51 100644 --- a/src/agentmemory/_impl.py +++ b/src/agentmemory/_impl.py @@ -7663,6 +7663,23 @@ def _normalize_bucket_scores(bucket_name): limit=limit, ) + _second_stage_debug = None + try: + from agentmemory.retrieval.second_stage import ( + SecondStageConfig as _SecondStageConfig, + rerank_bucketed_results as _rerank_bucketed_results, + ) + + _second_stage_config = _SecondStageConfig.from_args(args) + results, _second_stage_debug = _rerank_bucketed_results( + query, + _query_plan, + results, + config=_second_stage_config, + ) + except Exception as exc: + _debug_skips["second_stage.skipped"] = f"{type(exc).__name__}: {exc}" + if db_vec: db_vec.close() @@ -7710,6 +7727,12 @@ def _normalize_bucket_scores(bucket_name): except Exception as exc: _debug_skips["answerability.skipped"] = f"{type(exc).__name__}: {exc}" + if (_second_stage_debug or {}).get("enabled"): + for key in ("procedures", "memories", "events", "context", "entities", "decisions"): + rows = list(results.get(key) or []) + rows.sort(key=lambda item: item.get("final_score", 0.0), reverse=True) + results[key] = rows[:limit] + total = sum(len(v) for v in results.values()) tokens_out = _estimate_tokens(results) log_access(db, args.agent or "unknown", "search", query=query, result_count=total, tokens_consumed=tokens_out) @@ -7870,6 +7893,7 @@ def _normalize_bucket_scores(bucket_name): query_plan=_query_plan_dict or {}, procedure_debug=_procedure_debug, answerability=_answerability, + second_stage=_second_stage_debug, top_candidates=_top_candidates, ) except Exception as exc: @@ -17258,6 +17282,22 @@ def build_parser(): srch.add_argument("--rerank-budget-ms", type=float, default=None, metavar="MS", help="Strict latency budget for cross-encoder rerank (per-call and rolling p95). " "Defaults to env BRAINCTL_CE_P95_BUDGET_MS or 350.") + srch.add_argument("--no-second-stage", action="store_true", default=False, + help="Disable the shared deterministic second-stage reranker.") + srch.add_argument("--no-second-stage-model", action="store_true", default=False, + help="Run the second-stage reranker without the tiny MLP residual model.") + srch.add_argument("--second-stage-top-n", type=int, default=None, metavar="N", + help="Combined top-N candidate window for the shared second-stage reranker. " + "Defaults to env BRAINCTL_SECOND_STAGE_TOP_N or 10.") + srch.add_argument("--second-stage-model-path", default=None, metavar="PATH", + help="Override the tiny MLP JSON artifact used by the shared second-stage reranker.") + srch.add_argument("--judge-rerank", nargs="?", const="ollama", default=None, metavar="PROVIDER", + help="Enable the optional top-5 judge reranker with the given provider " + "(default when passed without value: ollama).") + srch.add_argument("--judge-model", default="llama3.2:3b", metavar="MODEL", + help="Model name for the optional judge reranker (provider-specific).") + srch.add_argument("--judge-top-k", type=int, default=5, metavar="N", + help="Top-K candidates sent to the optional judge reranker (max recommended: 5).") srch.add_argument("--rollout-mode", choices=["on", "off", "canary"], default=None, help="Top-heavy retrieval rollout mode override. " "Defaults to env BRAINCTL_TOPHEAVY_ROLLOUT_MODE or on.") diff --git a/src/agentmemory/retrieval/__init__.py b/src/agentmemory/retrieval/__init__.py index 716a6fe..e2fb1d7 100644 --- a/src/agentmemory/retrieval/__init__.py +++ b/src/agentmemory/retrieval/__init__.py @@ -5,14 +5,20 @@ from .diagnostics import build_debug_payload from .evidence_graph import expand_procedure_evidence from .late_reranker import rerank_procedure_candidates +from .mlp_reranker import TinyMLPModel from .query_planner import QueryPlan, plan_query +from .second_stage import SecondStageConfig, rerank_bucketed_results, rerank_top_candidates __all__ = [ "QueryPlan", + "SecondStageConfig", + "TinyMLPModel", "assess_answerability", "build_debug_payload", "expand_procedure_evidence", "generate_procedure_candidates", "plan_query", "rerank_procedure_candidates", + "rerank_bucketed_results", + "rerank_top_candidates", ] diff --git a/src/agentmemory/retrieval/diagnostics.py b/src/agentmemory/retrieval/diagnostics.py index 1a16749..a051ccc 100644 --- a/src/agentmemory/retrieval/diagnostics.py +++ b/src/agentmemory/retrieval/diagnostics.py @@ -10,6 +10,7 @@ def build_debug_payload( query_plan: dict[str, Any], procedure_debug: dict[str, Any] | None, answerability: dict[str, Any] | None, + second_stage: dict[str, Any] | None = None, top_candidates: list[dict[str, Any]] | None = None, ) -> dict[str, Any]: payload: dict[str, Any] = { @@ -17,6 +18,8 @@ def build_debug_payload( } if procedure_debug: payload["procedures"] = procedure_debug + if second_stage: + payload["second_stage"] = second_stage if answerability: payload["answerability"] = answerability if top_candidates is not None: @@ -25,7 +28,12 @@ def build_debug_payload( "type": cand.get("type"), "id": cand.get("id"), "final_score": cand.get("final_score"), + "pre_second_stage_score": cand.get("pre_second_stage_score"), + "second_stage_heuristic": cand.get("second_stage_heuristic"), + "second_stage_mlp": cand.get("second_stage_mlp"), + "second_stage_judge": cand.get("second_stage_judge"), "why_retrieved": cand.get("why_retrieved"), + "feature_summary": cand.get("second_stage_features"), "text": cand.get("content") or cand.get("summary") or cand.get("title") or cand.get("goal") or cand.get("name"), } for cand in top_candidates[:5] diff --git a/src/agentmemory/retrieval/feature_builder.py b/src/agentmemory/retrieval/feature_builder.py new file mode 100644 index 0000000..acad4cf --- /dev/null +++ b/src/agentmemory/retrieval/feature_builder.py @@ -0,0 +1,375 @@ +"""Feature extraction for the shared second-stage reranker.""" + +from __future__ import annotations + +import json +import math +import re +from datetime import datetime, timezone +from typing import Any, Iterable + +try: # pragma: no cover - numpy is optional at import time + import numpy as _np +except Exception: # pragma: no cover + _np = None + +_STOPWORDS = { + "a", "an", "and", "are", "as", "at", "be", "by", "did", "do", "does", "for", + "from", "has", "have", "how", "i", "in", "is", "it", "its", "of", + "on", "or", "that", "the", "to", "was", "we", "what", "when", "where", + "which", "who", "why", "will", "with", "you", +} +_LOW_SIGNAL_TOKENS = { + "summary", "history", "timeline", "recent", "today", "yesterday", "tomorrow", + "game", "issue", "problem", "thing", "stuff", "update", +} +_DATE_RE = re.compile( + r"\b(?:\d{4}-\d{2}-\d{2}|\d{1,2}/\d{1,2}(?:/\d{2,4})?|" + r"jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|jun(?:e)?|" + r"jul(?:y)?|aug(?:ust)?|sep(?:tember)?|oct(?:ober)?|nov(?:ember)?|" + r"dec(?:ember)?)\b", + re.IGNORECASE, +) +_TEMPORAL_RE = re.compile( + r"\b(yesterday|today|tomorrow|when|before|after|during|timeline|history|recent|latest|first|last)\b", + re.IGNORECASE, +) +_SESSION_RE = re.compile(r"\bsession[_ :#-]*(\d+)\b", re.IGNORECASE) +_DIALOG_RE = re.compile(r"\bD(\d+):", re.IGNORECASE) +_ENTITY_RE = re.compile(r"\b[A-Z][A-Za-z0-9_.:-]+\b") + +FEATURE_VERSION_V1 = "v1" +FEATURE_ORDER_V1 = [ + "base_score", + "retrieval_score", + "rrf_score", + "confidence", + "query_overlap", + "informative_overlap", + "tfidf_cosine", + "exact_phrase", + "entity_overlap", + "alias_overlap", + "query_temporal", + "candidate_temporal", + "temporal_anchor_overlap", + "query_session_hint", + "candidate_session_hint", + "session_gap_score", + "intent_bucket_fit", + "source_keyword", + "source_semantic", + "source_both", + "source_graph", + "bucket_memories", + "bucket_events", + "bucket_entities", + "bucket_procedures", + "bucket_decisions", + "candidate_age_score", + "support_evidence_score", + "status_active", + "status_stale", + "status_needs_review", + "position_score", + "neighbor_margin", + "query_length_score", + "candidate_length_score", + "procedural_candidate", +] + + +def _normalize_token(token: str) -> str: + tok = re.sub(r"[^a-z0-9]+", "", (token or "").lower()) + if len(tok) <= 2 or tok in _STOPWORDS: + return "" + if tok.endswith("ies") and len(tok) > 4: + tok = tok[:-3] + "y" + elif tok.endswith("ed") and len(tok) > 4: + tok = tok[:-2] + elif tok.endswith("es") and len(tok) > 4: + tok = tok[:-2] + elif tok.endswith("s") and len(tok) > 3: + tok = tok[:-1] + return tok + + +def _token_set(text: str) -> set[str]: + return { + token + for part in re.split(r"\s+", text or "") + if (token := _normalize_token(part)) + } + + +def _informative_tokens(text: str) -> set[str]: + return {token for token in _token_set(text) if token not in _LOW_SIGNAL_TOKENS} + + +def _safe_float(value: Any, default: float = 0.0) -> float: + try: + return float(value) + except (TypeError, ValueError): + return default + + +def candidate_text(candidate: dict[str, Any]) -> str: + parts: list[str] = [] + for key in ( + "content", "summary", "title", "goal", "description", "search_text", + "name", "compiled_truth", "why_retrieved", + ): + value = candidate.get(key) + if value: + parts.append(str(value)) + for key in ("observations", "aliases", "supporting_evidence"): + value = candidate.get(key) + if not value: + continue + if isinstance(value, str): + parts.append(value) + else: + try: + parts.append(json.dumps(value, ensure_ascii=True)) + except Exception: + parts.append(str(value)) + return " ".join(parts) + + +def _alias_values(candidate: dict[str, Any]) -> list[str]: + raw = candidate.get("aliases") + if not raw: + return [] + if isinstance(raw, list): + return [str(value) for value in raw if value] + if isinstance(raw, str): + try: + parsed = json.loads(raw) + except Exception: + return [raw] + if isinstance(parsed, list): + return [str(value) for value in parsed if value] + return [raw] + return [str(raw)] + + +def _entity_terms(text: str) -> set[str]: + return { + match.group(0).lower() + for match in _ENTITY_RE.finditer(text or "") + if len(match.group(0)) > 2 + } + + +def _intent_bucket_preference(plan: Any, bucket: str) -> float: + if plan is None: + return 0.5 + tables = list(getattr(plan, "candidate_tables", []) or []) + if not tables: + return 0.5 + try: + position = tables.index(bucket) + except ValueError: + return 0.2 + return max(0.2, 1.0 - (position * 0.12)) + + +def _source_flags(candidate: dict[str, Any]) -> tuple[float, float, float, float]: + source = str(candidate.get("source") or "").lower() + return ( + 1.0 if source in {"keyword", "procedure_fts", "intent_entity", "intent_decision"} else 0.0, + 1.0 if source == "semantic" else 0.0, + 1.0 if source == "both" else 0.0, + 1.0 if source == "graph" else 0.0, + ) + + +def _bucket_flags(bucket: str) -> tuple[float, float, float, float, float]: + return ( + 1.0 if bucket == "memories" else 0.0, + 1.0 if bucket == "events" else 0.0, + 1.0 if bucket == "entities" else 0.0, + 1.0 if bucket == "procedures" else 0.0, + 1.0 if bucket == "decisions" else 0.0, + ) + + +def _temporal_anchor_overlap(query: str, text: str) -> float: + query_dates = {match.group(0).lower() for match in _DATE_RE.finditer(query or "")} + cand_dates = {match.group(0).lower() for match in _DATE_RE.finditer(text or "")} + if not query_dates: + return 0.0 + return len(query_dates & cand_dates) / len(query_dates) + + +def _extract_session_hints(text: str) -> list[int]: + hints = [int(match.group(1)) for match in _SESSION_RE.finditer(text or "")] + hints.extend(int(match.group(1)) for match in _DIALOG_RE.finditer(text or "")) + return hints + + +def _session_gap_score(query: str, candidate_text_value: str) -> tuple[float, float, float]: + query_sessions = _extract_session_hints(query) + candidate_sessions = _extract_session_hints(candidate_text_value) + if not query_sessions: + return 0.0, 0.0, 0.0 + if not candidate_sessions: + return 1.0, 0.0, 0.0 + gap = min(abs(q - c) for q in query_sessions for c in candidate_sessions) + return 1.0 / (1.0 + gap), 1.0, 1.0 + + +def _parse_iso_timestamp(value: Any) -> datetime | None: + if not value: + return None + try: + text = str(value).replace("Z", "+00:00") + dt = datetime.fromisoformat(text) + except Exception: + return None + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc) + + +def _age_score(candidate: dict[str, Any]) -> float: + when = _parse_iso_timestamp(candidate.get("created_at")) or _parse_iso_timestamp(candidate.get("updated_at")) + if when is None: + return 0.5 + age_days = max((datetime.now(timezone.utc) - when).total_seconds() / 86400.0, 0.0) + return 1.0 / (1.0 + age_days / 30.0) + + +def _tfidf_cosine(query: str, text: str) -> float: + q_tokens = list(_informative_tokens(query)) + c_tokens = list(_informative_tokens(text)) + if not q_tokens or not c_tokens: + return 0.0 + docs = [q_tokens, c_tokens] + vocab = sorted({token for doc in docs for token in doc}) + if not vocab: + return 0.0 + doc_freq: dict[str, int] = {} + for token in vocab: + doc_freq[token] = sum(1 for doc in docs if token in doc) + n_docs = len(docs) + + def _weights(tokens: Iterable[str]) -> dict[str, float]: + counts: dict[str, int] = {} + for token in tokens: + counts[token] = counts.get(token, 0) + 1 + if not counts: + return {} + max_tf = max(counts.values()) or 1 + weights: dict[str, float] = {} + for token, count in counts.items(): + tf = count / max_tf + idf = math.log((1 + n_docs) / (1 + doc_freq[token])) + 1.0 + weights[token] = tf * idf + return weights + + q_weights = _weights(q_tokens) + c_weights = _weights(c_tokens) + dot = sum(q_weights.get(token, 0.0) * c_weights.get(token, 0.0) for token in vocab) + q_norm = math.sqrt(sum(value * value for value in q_weights.values())) + c_norm = math.sqrt(sum(value * value for value in c_weights.values())) + if q_norm == 0.0 or c_norm == 0.0: + return 0.0 + return dot / (q_norm * c_norm) + + +def build_features( + query: str, + plan: Any, + candidate: dict[str, Any], + *, + neighbors: dict[str, Any] | None = None, +) -> dict[str, float]: + """Build numeric features for a candidate row.""" + + bucket = str(candidate.get("bucket") or candidate.get("type") or "memories") + text = candidate_text(candidate) + query_tokens = _token_set(query) + query_informative = _informative_tokens(query) + cand_tokens = _token_set(text) + cand_informative = _informative_tokens(text) + query_overlap = len(query_tokens & cand_tokens) / max(len(query_tokens), 1) if query_tokens else 0.0 + informative_overlap = ( + len(query_informative & cand_informative) / max(len(query_informative), 1) + if query_informative else query_overlap + ) + exact_phrase = 1.0 if query and len(query.strip()) >= 4 and query.lower().strip() in text.lower() else 0.0 + query_entities = _entity_terms(query) | {term.lower() for term in getattr(plan, "target_entities", []) or []} + cand_entities = _entity_terms(text) + entity_overlap = len(query_entities & cand_entities) / max(len(query_entities), 1) if query_entities else 0.0 + aliases = {alias.lower() for alias in _alias_values(candidate) if len(alias) > 2} + alias_overlap = len(query_entities & aliases) / max(len(query_entities), 1) if query_entities and aliases else 0.0 + query_temporal = 1.0 if _TEMPORAL_RE.search(query or "") else 0.0 + candidate_temporal = 1.0 if _TEMPORAL_RE.search(text or "") or _DATE_RE.search(text or "") else 0.0 + temporal_anchor_overlap = _temporal_anchor_overlap(query, text) + session_gap_score, query_session_hint, candidate_session_hint = _session_gap_score(query, text) + source_keyword, source_semantic, source_both, source_graph = _source_flags(candidate) + bucket_memories, bucket_events, bucket_entities, bucket_procedures, bucket_decisions = _bucket_flags(bucket) + status = str(candidate.get("status") or "").lower() + position = max(int(candidate.get("_stage_position") or 0), 0) + prev_score = _safe_float((neighbors or {}).get("prev_score")) + next_score = _safe_float((neighbors or {}).get("next_score")) + current_score = _safe_float(candidate.get("final_score") or candidate.get("retrieval_score")) + neighbor_margin = max(current_score - prev_score, current_score - next_score, 0.0) + confidence = _safe_float(candidate.get("confidence"), 0.5) + support_evidence_score = min(len(candidate.get("supporting_evidence") or []) / 3.0, 1.0) + features = { + "base_score": current_score, + "retrieval_score": _safe_float(candidate.get("retrieval_score"), current_score), + "rrf_score": _safe_float(candidate.get("rrf_score")), + "confidence": confidence, + "query_overlap": query_overlap, + "informative_overlap": informative_overlap, + "tfidf_cosine": _tfidf_cosine(query, text), + "exact_phrase": exact_phrase, + "entity_overlap": entity_overlap, + "alias_overlap": alias_overlap, + "query_temporal": query_temporal, + "candidate_temporal": candidate_temporal, + "temporal_anchor_overlap": temporal_anchor_overlap, + "query_session_hint": query_session_hint, + "candidate_session_hint": candidate_session_hint, + "session_gap_score": session_gap_score, + "intent_bucket_fit": _intent_bucket_preference(plan, bucket), + "source_keyword": source_keyword, + "source_semantic": source_semantic, + "source_both": source_both, + "source_graph": source_graph, + "bucket_memories": bucket_memories, + "bucket_events": bucket_events, + "bucket_entities": bucket_entities, + "bucket_procedures": bucket_procedures, + "bucket_decisions": bucket_decisions, + "candidate_age_score": _age_score(candidate), + "support_evidence_score": support_evidence_score, + "status_active": 1.0 if status in {"", "active"} else 0.0, + "status_stale": 1.0 if status in {"stale", "superseded", "retired"} else 0.0, + "status_needs_review": 1.0 if status == "needs_review" else 0.0, + "position_score": 1.0 / (1.0 + position), + "neighbor_margin": neighbor_margin, + "query_length_score": min(len(query_informative) / 8.0, 1.0), + "candidate_length_score": min(len(cand_informative) / 64.0, 1.0), + "procedural_candidate": 1.0 if bucket == "procedures" else 0.0, + } + return {name: round(float(features.get(name, 0.0)), 6) for name in FEATURE_ORDER_V1} + + +def vectorize_features( + feature_dict: dict[str, float], + *, + feature_version: str = FEATURE_VERSION_V1, +): + """Return a numeric feature vector in canonical order.""" + + if feature_version != FEATURE_VERSION_V1: + raise ValueError(f"Unsupported feature version: {feature_version}") + values = [float(feature_dict.get(name, 0.0)) for name in FEATURE_ORDER_V1] + if _np is not None: + return _np.asarray(values, dtype=float) + return values + diff --git a/src/agentmemory/retrieval/judge.py b/src/agentmemory/retrieval/judge.py new file mode 100644 index 0000000..31ea7cb --- /dev/null +++ b/src/agentmemory/retrieval/judge.py @@ -0,0 +1,87 @@ +"""Optional local judge reranker for top candidates.""" + +from __future__ import annotations + +import json +import re +import urllib.error +import urllib.request +from dataclasses import dataclass +from typing import Any + + +@dataclass(slots=True) +class JudgeConfig: + enabled: bool = False + provider: str = "ollama" + model: str = "llama3.2:3b" + top_k: int = 5 + timeout_s: float = 6.0 + url: str = "http://localhost:11434/api/generate" + + +def _coerce_score(value: str) -> float: + match = re.search(r"(-?\d+(?:\.\d+)?)", value or "") + if not match: + return 0.0 + try: + score = float(match.group(1)) + except (TypeError, ValueError): + return 0.0 + return max(0.0, min(score, 1.0)) + + +def _candidate_synopsis(candidate: dict[str, Any]) -> str: + for key in ("content", "summary", "title", "goal", "description", "search_text", "name", "compiled_truth"): + value = candidate.get(key) + if value: + text = str(value).strip() + return text[:1200] + return "" + + +def _judge_with_ollama(query: str, candidates: list[dict[str, Any]], config: JudgeConfig) -> list[float]: + scores: list[float] = [] + for candidate in candidates[: config.top_k]: + prompt = ( + "You are a retrieval judge. Score relevance from 0.0 to 1.0.\n" + "Return only the numeric score.\n\n" + f"Query: {query}\n\n" + f"Candidate: {_candidate_synopsis(candidate)}\n" + ) + payload = json.dumps( + { + "model": config.model, + "prompt": prompt, + "stream": False, + "options": {"temperature": 0}, + } + ).encode("utf-8") + req = urllib.request.Request( + config.url, + data=payload, + headers={"Content-Type": "application/json"}, + ) + try: + with urllib.request.urlopen(req, timeout=config.timeout_s) as resp: # noqa: S310 - local optional service + body = json.loads(resp.read().decode("utf-8")) + scores.append(_coerce_score(str(body.get("response") or ""))) + except (urllib.error.URLError, TimeoutError, OSError, ValueError, json.JSONDecodeError): + return [] + return scores + + +def judge_candidates( + query: str, + candidates: list[dict[str, Any]], + config: JudgeConfig | None = None, +) -> list[float]: + """Return optional judge scores for the top candidates.""" + + cfg = config or JudgeConfig() + if not cfg.enabled or not candidates: + return [] + if cfg.provider == "ollama": + return _judge_with_ollama(query, candidates, cfg) + return [] + diff --git a/src/agentmemory/retrieval/mlp_reranker.py b/src/agentmemory/retrieval/mlp_reranker.py new file mode 100644 index 0000000..a23b6c7 --- /dev/null +++ b/src/agentmemory/retrieval/mlp_reranker.py @@ -0,0 +1,129 @@ +"""Tiny MLP reranker inference loaded from a JSON artifact.""" + +from __future__ import annotations + +import json +import math +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +try: # pragma: no cover - numpy is optional at import time + import numpy as _np +except Exception: # pragma: no cover + _np = None + +from agentmemory.retrieval.feature_builder import FEATURE_ORDER_V1, FEATURE_VERSION_V1 + +DEFAULT_MODEL_PATH = Path(__file__).resolve().parent / "models" / "tiny_mlp_v1.json" + + +def _relu(value: float) -> float: + return value if value > 0.0 else 0.0 + + +def _sigmoid(value: float) -> float: + if value >= 0: + z = math.exp(-value) + return 1.0 / (1.0 + z) + z = math.exp(value) + return z / (1.0 + z) + + +@dataclass(slots=True) +class TinyMLPModel: + feature_version: str + feature_order: list[str] + norm_mean: list[float] + norm_std: list[float] + w1: list[list[float]] + b1: list[float] + w2: list[list[float]] + b2: list[float] + w3: list[list[float]] + b3: list[float] + metadata: dict[str, Any] + + @classmethod + def load(cls, path: str | Path | None = None) -> "TinyMLPModel": + model_path = Path(path) if path is not None else DEFAULT_MODEL_PATH + payload = json.loads(model_path.read_text(encoding="utf-8")) + return cls( + feature_version=str(payload["feature_version"]), + feature_order=list(payload["feature_order"]), + norm_mean=[float(v) for v in payload["norm_mean"]], + norm_std=[float(v) for v in payload["norm_std"]], + w1=[[float(v) for v in row] for row in payload["w1"]], + b1=[float(v) for v in payload["b1"]], + w2=[[float(v) for v in row] for row in payload["w2"]], + b2=[float(v) for v in payload["b2"]], + w3=[[float(v) for v in row] for row in payload["w3"]], + b3=[float(v) for v in payload["b3"]], + metadata=dict(payload.get("metadata") or {}), + ) + + @classmethod + def try_load(cls, path: str | Path | None = None) -> "TinyMLPModel | None": + try: + model_path = Path(path) if path is not None else DEFAULT_MODEL_PATH + if not model_path.exists(): + return None + return cls.load(model_path) + except Exception: + return None + + def _normalize(self, feature_matrix): + if _np is not None: + matrix = _np.asarray(feature_matrix, dtype=float) + mean = _np.asarray(self.norm_mean, dtype=float) + std = _np.asarray(self.norm_std, dtype=float) + safe_std = _np.where(std == 0.0, 1.0, std) + return (matrix - mean) / safe_std + rows: list[list[float]] = [] + for row in feature_matrix: + rows.append([ + (float(value) - self.norm_mean[idx]) / (self.norm_std[idx] if self.norm_std[idx] not in (0.0, 0) else 1.0) + for idx, value in enumerate(row) + ]) + return rows + + def score(self, feature_matrix) -> list[float]: + if self.feature_version != FEATURE_VERSION_V1: + raise ValueError(f"Unsupported feature version: {self.feature_version}") + if self.feature_order != FEATURE_ORDER_V1: + raise ValueError("Feature order mismatch between runtime and model artifact") + if _np is not None: + x = self._normalize(feature_matrix) + w1 = _np.asarray(self.w1, dtype=float) + b1 = _np.asarray(self.b1, dtype=float) + w2 = _np.asarray(self.w2, dtype=float) + b2 = _np.asarray(self.b2, dtype=float) + w3 = _np.asarray(self.w3, dtype=float) + b3 = _np.asarray(self.b3, dtype=float) + h1 = _np.maximum(0.0, x @ w1.T + b1) + h2 = _np.maximum(0.0, h1 @ w2.T + b2) + logits = h2 @ w3.T + b3 + logits = _np.clip(logits.reshape(-1), -30.0, 30.0) + probs = 1.0 / (1.0 + _np.exp(-logits)) + return [float(v) for v in probs.tolist()] + + x_rows = self._normalize(feature_matrix) + outputs: list[float] = [] + for row in x_rows: + h1: list[float] = [] + for bias, weights in zip(self.b1, self.w1): + total = bias + for value, weight in zip(row, weights): + total += value * weight + h1.append(_relu(total)) + h2: list[float] = [] + for bias, weights in zip(self.b2, self.w2): + total = bias + for value, weight in zip(h1, weights): + total += value * weight + h2.append(_relu(total)) + total = self.b3[0] if self.b3 else 0.0 + for value, weight in zip(h2, self.w3[0]): + total += value * weight + outputs.append(_sigmoid(total)) + return outputs diff --git a/src/agentmemory/retrieval/models/tiny_mlp_v1.json b/src/agentmemory/retrieval/models/tiny_mlp_v1.json new file mode 100644 index 0000000..d9339b1 --- /dev/null +++ b/src/agentmemory/retrieval/models/tiny_mlp_v1.json @@ -0,0 +1,1963 @@ +{ + "feature_version": "v1", + "feature_order": [ + "base_score", + "retrieval_score", + "rrf_score", + "confidence", + "query_overlap", + "informative_overlap", + "tfidf_cosine", + "exact_phrase", + "entity_overlap", + "alias_overlap", + "query_temporal", + "candidate_temporal", + "temporal_anchor_overlap", + "query_session_hint", + "candidate_session_hint", + "session_gap_score", + "intent_bucket_fit", + "source_keyword", + "source_semantic", + "source_both", + "source_graph", + "bucket_memories", + "bucket_events", + "bucket_entities", + "bucket_procedures", + "bucket_decisions", + "candidate_age_score", + "support_evidence_score", + "status_active", + "status_stale", + "status_needs_review", + "position_score", + "neighbor_margin", + "query_length_score", + "candidate_length_score", + "procedural_candidate" + ], + "norm_mean": [ + 0.36849625, + 0.99348148, + 0.0, + 1.0, + 0.4938599, + 0.49430492, + 0.04392525, + 0.0, + 0.61979308, + 0.0, + 0.16296296, + 0.99259259, + 0.0054321, + 0.0, + 0.0, + 0.0, + 1.0, + 1.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.99999999, + 0.0, + 1.0, + 0.0, + 0.0, + 0.29330117, + 0.36849625, + 0.70833333, + 1.0, + 0.0 + ], + "norm_std": [ + 0.05354046, + 0.06219823, + 1.0, + 1.0, + 0.21382714, + 0.21424068, + 0.02311982, + 1.0, + 0.40672457, + 1.0, + 0.36933188, + 0.08574694, + 0.07350232, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 9e-08, + 1.0, + 1.0, + 1.0, + 1.0, + 0.26323239, + 0.05354046, + 0.21390693, + 1.0, + 1.0 + ], + "w1": [ + [ + 0.02177844, + -0.07193969, + 0.08529199, + -0.15862129, + -0.09098113, + -0.12999588, + -0.05729463, + 0.01085175, + 0.05169438, + -0.03503787, + 0.09383065, + -0.07432615, + -0.19292686, + 0.13062156, + 0.13366458, + 0.02675225, + 0.01192958, + -0.03038564, + 0.16023862, + -0.20214085, + -0.17921269, + -0.08560957, + 0.1258194, + 0.09327172, + -0.15348158, + -0.10133985, + -0.13932081, + -0.08316691, + -0.08106961, + -0.02785429, + 0.19389943, + 0.21056218, + -0.26755475, + 0.01688236, + 0.07833868, + 0.05400312 + ], + [ + -0.10343042, + -0.09575113, + 0.09514009, + -0.11959222, + -0.09066836, + -0.06638923, + 0.18933333, + 0.02736969, + -0.03274888, + -0.01241056, + -0.02156421, + 0.09551152, + -0.09345788, + 0.15921929, + 0.0459755, + 0.1718745, + -0.0103254, + -0.01908983, + 0.14965714, + 0.06452237, + 0.08384934, + -0.15791742, + -0.11703931, + 0.09677836, + 0.20168746, + -0.12995975, + -0.04624062, + 0.09965932, + -0.10341123, + 0.00907972, + 0.01571308, + 0.17058243, + -0.02241735, + 0.03512919, + -0.03318253, + 0.00037278 + ], + [ + 0.0642239, + 0.06982343, + -0.04181994, + 0.04794188, + -0.07005416, + 0.23251009, + -0.14985486, + 0.30190149, + -0.03752951, + -0.03021773, + 0.14500495, + 0.13014826, + 0.03973439, + -0.00829119, + -0.01572633, + 0.01097532, + 0.09483536, + 0.0243908, + -0.03028248, + -0.12403669, + 0.24616288, + 0.10023208, + -0.10919831, + -0.19425891, + 0.20734525, + 0.04132584, + -0.0940105, + -0.16095685, + 0.0574279, + 0.00072153, + -0.12020348, + -0.03085636, + -0.28722799, + 0.02216413, + -0.0714844, + -0.0899194 + ], + [ + 0.13118822, + 0.1337805, + -0.05544633, + -0.10858721, + -0.14589973, + -0.07945286, + -0.09265881, + 0.22507568, + -0.15118757, + 0.01829566, + 0.0909163, + 0.02625966, + 0.07050119, + 0.16232525, + 0.04182604, + 0.06964817, + 0.04133068, + -0.12094605, + 0.04358631, + 0.02821487, + 0.02062189, + 0.04189496, + 0.06698252, + -0.26949781, + 0.16300098, + 0.04548417, + -0.10417425, + -0.04880166, + -0.18415786, + 0.05376409, + -0.01315878, + 0.05443086, + 0.02271785, + 0.04402513, + 0.00102002, + -0.02828122 + ], + [ + -0.24327827, + -0.01684738, + 0.10289051, + -0.04535016, + 0.06692471, + 0.09167366, + 0.07103091, + -0.10232298, + -0.03434823, + 0.17646483, + 0.2228408, + 0.02837795, + 0.03022375, + 0.01104805, + 0.23397008, + -0.00680958, + 0.08014725, + 0.08476709, + -0.28900359, + -0.17073763, + -0.04045284, + 0.28572735, + -0.02656714, + 0.1201317, + 0.0306058, + 0.15437982, + -0.04271535, + -0.0701395, + 0.04673276, + 0.13974646, + -0.00427052, + 0.08513194, + -0.00986107, + -0.01625741, + 0.09533017, + -0.02208679 + ], + [ + -0.16956082, + -0.10382103, + -0.02294167, + 0.15580656, + -0.03805346, + -0.11968453, + -0.05477229, + -0.0344637, + -0.04372018, + -0.3077998, + 0.08593909, + 0.03396054, + -0.17074, + -0.10042279, + 0.24907646, + -0.02043568, + -0.08255125, + 0.07946848, + -0.13867197, + 0.05352402, + -0.01702936, + 0.05039002, + 0.07764791, + 0.14243483, + 0.16197019, + 0.13191218, + 0.17275909, + -0.00558686, + 0.01229842, + 0.19755942, + -0.16366307, + 0.00674143, + 0.11198238, + 0.18715383, + 0.02162967, + -0.03239765 + ], + [ + 0.00953365, + -0.10173179, + -0.15298335, + -0.04272403, + -0.18319722, + 0.10486989, + -0.00590679, + -0.17549946, + 0.02991358, + -0.02840365, + -0.19181707, + -0.10702026, + -0.27758428, + -0.07128191, + 0.00832035, + -0.09347739, + 0.10766817, + 0.04617467, + -0.03523067, + -0.09672926, + 0.07378306, + 0.04649429, + -0.00163656, + -0.12239555, + 0.00144545, + -0.01585718, + 0.22103, + 0.03344207, + -0.01768166, + 0.03713038, + -0.03067999, + 0.0129347, + 0.13662668, + -0.05644484, + -0.07867567, + 0.21242347 + ], + [ + -0.0526237, + 0.07980607, + -0.13590651, + 0.08844463, + -0.1370018, + 0.03096751, + -0.03026132, + -0.07083899, + -0.07846376, + 0.02116779, + -0.00760873, + -0.01764576, + 0.00036318, + -0.17754947, + 0.01921044, + 0.05160277, + 0.195346, + 0.06674082, + -0.12857275, + -0.15381661, + -0.20754627, + -0.0200184, + 0.08414518, + -0.2230359, + 0.02431993, + -0.14920814, + -0.04045595, + -0.12086893, + 0.19046986, + 0.07069987, + -0.08900545, + 0.08625775, + 0.00961814, + 0.14759828, + 0.14705998, + -0.01180588 + ], + [ + -0.0074478, + 0.08981802, + -0.11026291, + -0.11196166, + 0.03119422, + -0.02510156, + 0.03547326, + 0.0378481, + 0.05626221, + 0.03549631, + -0.14343475, + -0.02005277, + -0.0570597, + -0.10650713, + 0.12906529, + -0.10211837, + -0.11634264, + 0.03554715, + 0.08567213, + 0.08560301, + 0.01971413, + 0.09794971, + -0.12412904, + 0.01187649, + -0.13113169, + -0.03826802, + 0.22906095, + 0.08685771, + -0.0746166, + -0.13801442, + 0.11085119, + -0.01044795, + -0.06543746, + -0.07875183, + 0.18937969, + -0.0292486 + ], + [ + -0.10190371, + 0.06559258, + 0.0596207, + -0.02463656, + 0.10112172, + -0.00437765, + 0.17012416, + 0.14460884, + 0.12201063, + -0.04460088, + -0.06418587, + 0.04558447, + 0.05304697, + -0.04293422, + -0.10141369, + 0.07981857, + -0.10645458, + 0.24405084, + 0.23952056, + 0.02897859, + -0.04682537, + 0.07496171, + -0.00144922, + 0.11162834, + 0.04760814, + 0.02605088, + 0.00494431, + 0.00755539, + 0.24707587, + -0.01053839, + 0.00415074, + -0.09459292, + 0.05857703, + -0.00995291, + 0.05930841, + -0.25153132 + ], + [ + 0.09972278, + -0.08282613, + 0.01708003, + -0.11392891, + 0.23335189, + -0.08444921, + 0.00740774, + -0.08743347, + 0.01416104, + -0.21067027, + 0.18806838, + 0.12040872, + 0.08108939, + 0.09636783, + 0.03994259, + 0.13015021, + 0.16019044, + -0.01044454, + -0.14110244, + -0.07362956, + 0.22159558, + 0.15010985, + -0.02526645, + 0.21557182, + 0.00724162, + -0.0242343, + 0.21084726, + -0.22688671, + -0.02703428, + 0.11276176, + -0.03391354, + -0.05684184, + -0.06904587, + -0.12956661, + 0.11676405, + -0.10723548 + ], + [ + 0.09389682, + 0.02886594, + 0.08280465, + -0.04065766, + 0.35082627, + 0.05511858, + 0.08129052, + -0.0784468, + 0.04146949, + 0.039334, + 0.07200802, + -0.12680236, + 0.00768413, + 0.20635882, + -0.00310153, + 0.04395532, + -0.02294643, + -0.03682615, + -0.10043061, + 0.17402883, + -0.02088721, + -0.06251829, + -0.14581254, + 0.06191564, + -0.15621715, + -0.06930244, + -0.01021498, + -0.23487896, + -0.15314286, + 0.10384902, + -0.01273359, + 0.06828616, + 0.00045514, + 0.05379671, + 0.1489388, + -0.03154839 + ], + [ + -0.00094139, + 0.01470014, + -0.05123716, + 0.10077176, + 0.04048439, + -0.06372807, + -0.25247454, + -0.25750802, + 0.00095556, + 0.20714796, + -0.08690899, + -0.01423565, + 0.08755366, + -0.1657545, + 0.03764462, + -0.03432763, + 0.16834961, + -0.09036482, + 0.02823553, + -0.05284409, + 0.20001706, + -0.0522151, + -0.187496, + -0.04457715, + -0.00613968, + 0.03032655, + 0.01586607, + -0.00147736, + 0.00838492, + 0.02537674, + 0.02675732, + -0.00583073, + 0.10939987, + 0.17992156, + 0.13552421, + -0.08224909 + ], + [ + 0.12570181, + 0.02853048, + 0.01901246, + -0.20714438, + -0.130671, + -0.15886863, + -0.01481054, + -0.01950729, + 0.00357921, + -0.18394432, + -0.13069432, + 0.17988674, + 0.02125287, + 0.04710881, + -0.09993957, + 0.05444064, + -0.05306994, + -0.1237914, + 0.19320889, + 0.00385043, + -0.13236322, + -0.05745519, + 0.08223666, + -0.10710635, + -0.00956136, + -0.06043581, + 0.04612928, + -0.02649893, + -0.12906589, + 0.10629848, + 0.07396983, + 0.03079797, + -0.07636891, + -0.01138118, + 0.07364424, + 0.16569767 + ], + [ + 0.04869343, + 0.1046794, + 0.07502229, + 0.05209712, + -0.2621759, + -0.15999226, + -0.10735316, + -0.12740727, + -0.09509378, + 0.10359232, + 0.00427048, + -0.08854962, + 0.06334275, + -0.12478451, + -0.19062473, + -0.03701681, + 0.17449254, + -0.14924002, + -0.14659004, + 0.03224874, + 0.07042552, + 0.09483483, + -0.04208705, + 0.00137327, + 0.21556782, + -0.07531918, + 0.38136987, + -0.01240239, + -0.09015235, + 0.05885391, + -0.11988787, + 0.07654247, + 0.0892475, + -0.02000153, + 0.0717552, + -0.01977973 + ], + [ + -0.1083893, + 0.02719483, + -0.03709757, + 0.02850982, + 0.02631509, + 0.01493138, + -0.1513492, + -0.06349164, + 0.03565267, + -0.03939753, + -0.14579311, + -0.10147141, + -0.08450011, + 0.05692681, + -0.24859712, + 0.11219304, + 0.01576809, + -0.10658669, + 0.02986501, + -0.07431182, + 0.03830324, + 0.17968864, + -0.12259429, + -0.03588854, + 0.1072362, + 0.03735017, + 0.10208074, + -0.00335572, + 0.04761318, + 0.14394359, + -0.12490967, + -0.04507281, + 0.07589468, + -0.13383145, + 0.06231894, + 0.15451331 + ], + [ + 0.02752007, + 0.08370549, + 0.05477758, + -0.07125188, + -0.11679037, + 0.17016251, + -0.12249453, + -0.10515524, + -0.01257553, + -0.00735416, + -0.09000362, + 0.02636512, + -0.02333946, + -0.01572022, + -0.13399937, + -0.21962659, + 0.03096742, + -0.00847616, + 0.2184145, + 0.05649979, + -0.10421822, + -0.05502525, + -0.01153401, + -0.12172936, + 0.00137256, + -0.04820791, + -0.08842372, + 0.02704967, + 0.06662673, + 0.03470064, + 0.13247577, + 0.07859481, + -0.13120231, + -0.03510298, + -0.13159535, + 0.00739977 + ], + [ + -0.11715609, + 0.00822256, + -0.07937974, + -0.17341471, + -0.05175816, + 0.01727163, + -0.04206952, + -0.01130419, + 0.24723567, + -0.1262661, + 0.0391111, + 0.10093862, + 0.02531153, + -0.21956659, + -0.05500543, + -0.04024684, + 0.18764472, + 0.04008947, + -0.1980829, + -0.06397288, + 0.02127377, + -0.05093988, + 0.13527471, + 0.24569166, + 0.02983517, + 0.02927358, + 0.1158615, + 0.1136444, + -0.07461183, + -0.04265619, + -0.04944842, + 0.04651935, + 0.1146271, + -0.19102864, + 0.08403239, + 0.00446445 + ], + [ + 0.10293369, + 0.03828861, + -0.04353828, + 0.00864994, + -0.08211438, + -0.02234312, + 0.10757668, + -0.21079098, + 0.20608842, + -0.04010878, + 0.12875975, + 0.00602635, + 0.08550691, + 0.11132359, + -0.03516023, + -0.23874292, + -0.04338438, + 0.00613308, + -0.15362887, + -0.04936465, + 0.14540815, + 0.03766489, + -0.27351152, + 0.21408165, + 0.00530205, + 0.03276328, + -0.04586856, + -0.13324635, + 0.11841198, + 0.04027488, + -0.16991108, + -0.17515843, + 0.08136158, + -0.12583403, + -0.16259245, + -0.01064868 + ], + [ + -0.00308159, + 0.07603885, + -0.04577892, + -0.06349803, + -0.01373614, + 0.03722561, + -0.15666096, + -0.17593156, + 0.04519183, + 0.15590445, + 0.07792015, + 0.15950125, + -0.04732885, + -0.07255313, + 0.23231738, + -0.17929126, + -0.11286161, + -0.09180482, + -0.05079991, + 0.16341136, + -0.03882987, + -0.02947231, + -0.17948048, + 0.13623767, + -0.02433393, + -0.13664304, + -0.04970314, + -0.14054563, + 0.13881109, + -0.35144387, + 0.05322308, + 0.17538562, + 0.01512884, + 0.03023331, + -0.09529026, + 0.00045526 + ], + [ + -0.02934886, + -0.17325203, + -0.14340794, + 0.02790309, + 0.12247526, + 0.15109986, + 0.00325818, + 0.25534439, + -0.09834673, + 0.06987341, + -0.28257702, + 0.10428134, + 0.05454213, + -0.06402659, + 0.13263336, + 0.16355761, + -0.05379289, + 0.10795228, + -0.0624302, + -0.12478956, + -0.2029042, + 0.11417254, + -0.11067474, + -0.11043058, + -0.12980725, + -0.05771158, + 0.11078876, + -0.131114, + 0.17224473, + 0.04591657, + 0.05556841, + -0.07345686, + -0.09859937, + 0.16663537, + 0.15632663, + 0.206122 + ], + [ + -0.08873752, + -0.04147731, + 0.0583989, + 0.00262056, + 0.01176597, + 0.01928817, + -0.06495876, + -0.15439079, + -0.13864633, + 0.20774294, + 0.03922907, + 0.10119926, + -0.03146056, + -0.12828726, + -0.11537626, + 0.10735272, + 0.05424497, + 0.08867213, + 0.09744908, + -0.28934637, + -0.00210618, + -0.2700385, + 0.17522805, + 0.10253596, + -0.0181145, + 0.17242164, + 0.00483053, + 0.03464758, + 0.06348852, + -0.43752637, + -0.18356702, + -0.20068415, + -0.11011686, + 0.33622902, + 0.10076121, + -0.27817325 + ], + [ + 0.15155031, + -0.05549479, + -0.05629185, + 0.19208915, + -0.0134966, + -0.04365763, + -0.03407944, + -0.13152914, + 0.14272008, + 0.14119804, + 0.00945079, + 0.06634315, + -0.01380649, + -0.07846321, + 0.04169803, + -0.08628179, + -0.18776956, + -0.01914541, + 0.02898041, + 0.19318741, + -0.10822073, + -0.09914044, + 0.0338885, + 0.07670552, + -0.08947388, + -0.13935935, + 0.03329264, + 0.14929706, + 0.16350562, + -0.20668184, + 0.02751999, + -0.12479607, + -0.05953754, + -0.07092188, + 0.17837235, + -0.24003014 + ], + [ + -0.02644152, + -0.08330372, + 0.00149832, + -0.0287041, + -0.13026478, + 0.12660931, + 0.11838335, + 0.22028705, + 0.02935224, + 0.05265632, + 0.04729529, + 0.28026789, + 0.09425452, + 0.05131364, + -0.04881779, + -0.1801838, + 0.07644703, + -0.07829926, + -0.21285768, + 0.30572149, + -0.04105434, + -0.09382923, + 0.09201851, + 0.05307113, + -0.14994082, + -0.25383425, + -0.00430674, + -0.05172725, + -0.22554983, + 0.05417717, + 0.08821288, + -0.13333532, + 0.05715924, + 0.17794845, + -0.03257984, + -0.06513834 + ], + [ + -0.03763227, + -0.02805238, + 0.05765229, + -0.12274026, + -0.18755951, + 0.06468117, + 0.07465772, + 0.34838258, + 0.06263381, + 0.20913679, + 0.19566679, + -0.02418976, + -0.23811086, + -0.02269463, + -0.03410159, + -0.35551337, + -0.06461074, + 0.06576878, + 0.06180924, + -0.04860086, + -0.0097842, + -0.27826263, + -0.13673082, + 0.14986284, + 0.06130694, + -0.22327691, + 0.04011037, + -0.30000981, + -0.03812411, + 0.05726646, + 0.04489732, + 0.0138041, + -0.10086703, + 0.02488461, + -0.13818043, + 0.00174222 + ], + [ + -0.03870747, + 0.1780725, + 0.05354908, + 0.02149916, + -0.0561415, + 0.18642364, + 0.16015055, + -0.14049706, + -0.20490198, + 0.05264511, + -0.24185259, + -0.2398624, + -0.15462673, + 0.03941393, + 0.02222469, + -0.06517728, + 0.13764843, + 0.02254224, + -0.06925984, + -0.23227024, + -0.20454588, + -0.11556182, + -0.13425741, + 0.07619522, + 0.04692139, + 0.00349093, + 0.0499055, + -0.20434689, + -0.10397334, + -0.13940123, + 0.07578873, + -0.02571452, + 0.05598873, + -0.01920406, + -0.0288338, + 0.08275897 + ], + [ + 0.07509929, + -0.09929031, + 0.07979457, + 0.02638253, + 0.00624973, + 0.03474035, + -0.00952976, + -0.04416129, + 0.10674796, + 0.09929432, + -0.06322341, + 0.19062965, + -0.17624465, + 0.04340253, + 0.07425248, + 0.29026194, + -0.28712541, + -0.17366287, + 0.15283475, + -0.03723398, + -0.1937536, + -0.1097477, + 0.05370289, + 0.08874427, + -0.21426623, + 0.00370767, + -0.23369716, + -0.09989975, + 0.01429785, + -0.08539698, + -0.1684033, + -0.0308824, + -0.13765079, + 0.04781285, + 0.01223584, + 0.05673375 + ], + [ + 0.04353096, + 0.11603282, + -0.01181061, + 0.16299702, + 0.21199367, + -0.1468536, + -0.08343194, + 0.04096016, + 0.05488327, + -0.0355655, + 0.06949034, + -0.0573693, + -0.27979257, + 0.1583769, + -0.04068469, + 0.05215236, + -0.09432684, + -0.00815235, + -0.07526179, + -0.03432454, + 0.0578106, + -0.02411695, + 0.0069884, + 0.07637932, + -0.01471263, + -0.01410417, + 0.07986198, + -0.06688418, + -0.06852955, + 0.16435854, + 0.03969912, + 0.07632237, + -0.02476636, + 0.00716701, + 0.00947252, + -0.04607969 + ], + [ + 0.04872011, + -0.20494439, + -0.05076294, + 0.10014853, + 0.0158735, + -0.16382666, + -0.02681615, + 0.20730954, + -0.13361573, + 0.00798033, + -0.18994007, + 0.01297251, + -0.08139196, + -0.04110773, + 0.12757961, + -0.06710524, + -0.20224528, + 0.03142396, + -0.07634442, + -0.02277612, + -0.06268567, + 0.13346952, + 0.06580607, + 0.04086852, + 0.11940689, + 0.14560854, + -0.11736164, + -0.04897494, + -0.01992604, + -0.05804605, + -0.03629088, + 0.04156788, + 0.01237345, + 0.01270757, + 0.13595367, + 0.12220615 + ], + [ + 0.0546879, + -0.03929968, + -0.00956, + 0.04279683, + 0.12060208, + 0.20085032, + 0.03208466, + -0.11834625, + -0.05396534, + -0.08363673, + 0.17618768, + 0.15732041, + 0.08848845, + -0.17710831, + -0.13694396, + 0.05577353, + -0.09908341, + -0.10789354, + 0.06489379, + -0.13352015, + -0.30757026, + -0.02938826, + -0.02250595, + -0.2138949, + 0.12702464, + -0.32053268, + -0.16530099, + 0.00462674, + 0.2257149, + 0.26897635, + -0.05789735, + 0.03540943, + 0.29624866, + -0.05108691, + -0.04332441, + 0.12354349 + ], + [ + 0.25838144, + 0.02045325, + -0.20234917, + 0.17548277, + -0.05953706, + 0.20726814, + 0.02193118, + -0.0294143, + 0.03626955, + 0.11867327, + 0.03986568, + -0.19253432, + -0.03482367, + 0.12798386, + 0.0007602, + -0.18719385, + 0.02970069, + 0.02276647, + 0.09149196, + 0.06950235, + 0.09412037, + -0.12361764, + 0.03335566, + 0.01002804, + 0.1230208, + 0.04748133, + -0.00518659, + -0.03738536, + -0.0203532, + -0.00023023, + 0.10439911, + 0.04344824, + -0.16362102, + -0.17766666, + 0.04221681, + 0.02207265 + ], + [ + -0.05071466, + 0.07009577, + -0.17354118, + -0.1425593, + -0.14303103, + -0.02268008, + -0.13131811, + 0.09322018, + 0.00673806, + -0.14130498, + 0.09519986, + -0.1535287, + 0.02259053, + -0.03975285, + 0.31151923, + -0.03565573, + -0.02149328, + -0.17446578, + 0.05373714, + 0.06290018, + 0.03266323, + -0.00683012, + 0.01896202, + -0.06669987, + 0.00466656, + 0.18725139, + 0.00836021, + 0.12583931, + 0.04962296, + 0.04893267, + 0.17739813, + 0.07590366, + 0.1745511, + -0.18428964, + -0.11855853, + 0.11544419 + ] + ], + "b1": [ + 0.01215218, + -0.02405024, + 0.02332759, + -0.01931053, + 0.00310449, + 0.0178361, + 0.00745933, + 0.01784208, + 0.00653888, + 0.00435611, + 0.00587579, + 0.0097813, + 0.01016966, + 0.02578757, + 0.00167515, + -8.193e-05, + 0.02436347, + -0.00372971, + 0.03452251, + 0.00149435, + 0.01774672, + 0.00285032, + -0.00124859, + 0.01308786, + -0.0122056, + -0.01149606, + -0.0203385, + -0.00081575, + 0.00071594, + 0.00319573, + 0.00893007, + -0.00196122 + ], + "w2": [ + [ + 0.02740235, + -0.05076654, + -0.00312784, + -0.1608918, + -0.2166921, + -0.14159529, + -0.02300082, + -0.00726687, + 0.14023843, + -0.19117808, + -0.08276512, + -0.0123104, + -0.12449349, + -0.04698411, + 0.06514045, + -0.06826449, + -0.00356702, + 0.0168091, + -0.08438088, + 0.27522479, + -0.00258487, + 0.03806313, + -0.02839343, + -0.15215063, + 0.08966034, + -0.07161907, + 0.08245703, + -0.01364006, + -0.00978226, + 0.3489985, + 0.01569161, + -0.02382035 + ], + [ + -0.0639759, + -0.04225679, + -0.16380494, + -0.05554519, + -0.26979957, + 0.01335445, + -0.18686381, + -0.01192395, + -0.25982722, + -0.07931886, + 0.04844313, + 0.08898588, + 0.27917977, + 0.112701, + 0.02552941, + 0.05692458, + 0.06272948, + -0.1091249, + 0.04324377, + 0.01117187, + -0.2703177, + 0.02631778, + 0.06005873, + -0.32961964, + -0.20106192, + -0.0435366, + -0.1898426, + -0.19431066, + 0.1382359, + -0.21310048, + -0.00204322, + -0.08267993 + ], + [ + 0.08309151, + -0.13800807, + -0.03830937, + 0.04991511, + -0.14347499, + 0.03920532, + 0.11599362, + -0.10049625, + -0.24153865, + -0.13881208, + 0.06781136, + -0.22524932, + 0.03914358, + -0.20786233, + 0.0349127, + -0.22440376, + -0.05056412, + 0.09885717, + -0.16687383, + 0.21386208, + -0.0228775, + -0.07410541, + -0.2527007, + -0.0281445, + -0.11157569, + -0.16500284, + 0.00211684, + -0.28114996, + 0.15117333, + 0.03838075, + 0.03394321, + -0.06075656 + ], + [ + 0.02875412, + -0.22452424, + 0.05091956, + -0.00157297, + 0.16084491, + -0.08254621, + 0.18790229, + 0.08666138, + 0.10277764, + -0.11234781, + 0.22170382, + -0.13154063, + -0.08208117, + 0.11455946, + 0.19369475, + 0.23421899, + 0.2722542, + 0.11886162, + 0.17908568, + 0.04427845, + -0.01629876, + 0.01661952, + 0.00175824, + -0.02021142, + 0.19034692, + -0.07012939, + -0.02144, + 0.11337975, + -0.16314259, + -0.09478562, + 0.13309197, + 0.16074607 + ], + [ + -0.00173931, + -0.01527242, + 0.07291196, + -0.07956235, + -0.00584083, + 0.00153189, + -0.0964356, + 0.07717989, + 0.00640036, + 0.00556372, + -0.02504018, + 0.10424449, + 0.1089349, + -0.0467027, + 0.14523547, + 0.07111062, + -0.18841923, + 0.07722889, + -0.01839827, + 0.11500113, + 0.24993109, + -0.07734934, + 0.23848894, + -0.05166054, + -0.03263521, + -0.08845007, + 0.03991133, + -0.11841842, + -0.03881195, + 0.06823234, + -0.21234976, + 0.06337744 + ], + [ + 0.1896967, + -0.11438067, + -0.01345407, + -0.07697002, + 0.15728876, + 0.0753857, + 0.04423694, + 0.07391768, + 0.03724821, + -0.24537233, + 0.04900044, + 0.03112069, + -0.01063814, + 0.05093553, + -0.33762841, + -0.01221162, + 0.15456577, + 0.09962478, + 0.04406098, + -0.08612326, + -0.16884556, + -0.11960671, + 0.02409531, + 0.13257575, + 0.10690759, + 0.0984879, + 0.00719562, + -0.05426996, + 0.17153521, + 0.0057244, + 0.12173967, + -0.04004198 + ], + [ + -0.2935767, + -0.00997258, + 0.18419565, + -0.08466296, + 0.04934941, + 0.07531426, + -0.08611727, + 0.18498668, + -0.15788079, + 0.07488553, + -0.1775369, + 0.04851853, + -0.03006078, + -0.13452092, + -0.03737682, + 0.07029607, + 0.16427307, + -0.03117217, + 0.28070396, + -0.10280042, + -0.04532835, + -0.03975688, + 0.00941286, + 0.14464547, + 0.0289707, + 0.09859632, + -0.17400148, + -0.16655471, + 0.06165844, + 0.09050512, + 0.00132185, + -0.02778914 + ], + [ + -0.14723535, + -0.18146009, + 0.13306341, + -0.08678992, + 0.0994116, + 0.21550012, + -0.06367392, + 0.18977638, + 0.01126556, + 0.10272909, + 0.14978667, + 0.22048578, + 0.08076494, + 0.17775731, + 0.00493304, + -0.15312211, + -0.09584724, + -0.09616919, + 0.19892324, + 0.0556904, + 0.22608941, + -0.08194049, + 0.08011246, + -0.00748817, + 0.00339772, + -0.00118562, + -0.14300438, + -0.16947718, + 0.04024083, + 0.13811735, + -0.01978202, + -0.17910409 + ], + [ + -0.13974919, + -0.09055365, + -0.02949788, + 0.27520278, + -0.10761603, + 0.15537565, + 0.09355607, + -0.04369603, + 0.24119312, + -0.02970721, + -0.09095577, + 0.05557385, + -0.15388525, + 0.15114121, + -0.20771044, + 0.14321111, + 0.26817474, + -0.10315218, + -0.03591523, + 0.1185384, + 0.05504627, + 0.04214964, + 0.10827614, + -0.25537298, + 0.16475888, + -0.04644768, + 0.00446995, + -0.0210026, + -0.02442843, + 0.02250936, + -0.17633985, + 0.04741798 + ], + [ + -0.04485926, + 0.01738657, + 0.08919131, + 0.03490676, + 0.06289554, + -0.12019968, + -0.00428636, + 0.04420618, + 0.09873289, + -0.29704451, + -0.06652041, + 0.08122147, + 0.04266871, + 0.11497824, + 0.01138465, + -0.03892815, + 0.11868044, + -0.19576926, + -0.07093067, + 0.00497217, + 0.06703986, + -0.13902237, + -0.23959185, + 0.07410588, + -0.02846486, + 0.01628364, + 0.12160802, + -0.08690999, + -0.04744886, + 0.1376722, + 0.06167737, + -0.21113702 + ], + [ + -0.01454235, + 0.05088008, + 0.23823154, + 0.11678192, + 0.09078839, + 0.21746072, + 0.21165065, + -0.06710502, + 0.02344842, + -0.11889344, + -0.00205321, + 0.32502418, + 0.23573088, + -0.17244292, + 0.16527799, + -0.01845815, + 0.12000328, + 0.03095096, + -0.03835036, + -0.15264269, + 0.02213253, + 0.10946772, + -0.19894242, + 0.07061435, + -0.20404351, + -0.09203702, + -0.26131119, + 0.04609703, + 0.04504379, + -0.28318949, + -0.28871293, + 0.09547239 + ], + [ + -0.17712978, + 0.05075438, + 0.23604771, + -0.05783474, + -0.07098208, + -0.06341745, + -0.04773828, + 0.0037519, + -0.04165603, + 0.15909509, + -0.18394281, + -0.02715359, + 0.05584253, + 0.10433933, + -0.12933205, + -0.0482846, + 0.00630398, + 0.00427443, + 0.07647934, + -0.16471008, + 0.04445176, + -0.13745028, + 0.03679401, + 0.04010788, + -0.09661267, + -0.13009088, + 0.05016106, + 0.05286186, + -0.02683352, + 0.19041431, + 0.08399934, + 0.02040794 + ], + [ + -0.12575682, + -0.16192032, + -0.15060651, + -0.0273118, + 0.06304916, + 0.11897971, + 0.11831066, + -0.07307611, + -0.04830444, + -0.33329721, + 0.02476922, + -0.0959515, + -0.04621725, + 0.04627493, + 0.03466157, + -0.0717781, + -0.02657949, + -0.05491454, + -0.01444727, + -0.17204116, + 0.08983893, + -0.07483061, + -0.08787621, + 0.11554217, + 0.05958444, + -0.11276958, + 0.13979152, + 0.03802697, + -0.00940856, + 0.04367635, + -0.17935266, + 0.2820885 + ], + [ + -0.10319162, + 0.10274566, + -0.11110572, + -0.08557924, + 0.06820238, + 0.00820877, + 0.01210727, + 0.19439169, + -0.25519655, + -0.04120656, + 0.11649499, + 0.05363248, + -0.06074082, + -0.19251751, + -0.10984914, + 0.08338429, + -0.02499488, + -0.35414388, + -0.04710524, + -0.2114185, + -0.09693611, + -0.07492991, + 0.15742115, + -0.13955111, + 0.04744345, + -0.07572633, + -0.01830112, + 0.02913674, + 0.00938497, + 0.13167229, + 0.00749638, + -0.13201764 + ], + [ + 0.31943014, + -0.08432617, + 0.18718318, + -0.30415333, + -0.06108028, + -0.02845825, + 0.10473676, + 0.05487655, + 0.03984443, + -0.1352787, + 0.03846703, + 0.05445135, + -0.03997273, + 0.01116839, + -0.06130443, + 0.04329849, + 0.05326606, + -0.13862572, + 0.18475981, + 0.01571757, + 0.11940737, + 0.19283411, + -0.02825378, + 0.24871839, + -0.17500225, + -0.12879848, + 0.07831164, + 0.19050277, + -0.06442317, + -0.14526286, + -0.00910779, + -0.00386253 + ], + [ + 0.22544379, + 0.07889532, + -0.12363103, + -0.15969646, + -0.13417809, + -0.21123119, + -0.04925414, + -0.02236964, + 0.10643368, + 0.0985539, + -0.18887526, + -0.06206391, + -0.02642684, + 0.05165471, + 0.18773697, + 0.08646466, + 0.17181541, + 0.14106746, + -0.0094216, + 0.13638593, + -0.16254755, + 0.04849888, + -0.22739913, + -0.11432125, + -0.08325791, + 0.02325804, + -0.080957, + -0.12700495, + 0.06934157, + -0.09850965, + 0.14362632, + 0.2211805 + ] + ], + "b2": [ + -0.01472544, + 0.00844073, + -0.00039875, + 0.0600412, + -0.00916065, + 0.04614733, + 0.00974107, + 0.1414663, + 0.00809017, + -0.00268618, + 0.04562834, + 0.06281594, + -0.01474848, + 0.00139086, + 0.08759203, + 0.03757449 + ], + "w3": [ + [ + 0.14591786, + -0.08587059, + 0.02409859, + -0.17396011, + -0.00671001, + -0.12531235, + -0.03221764, + -0.25423137, + -0.03881778, + 0.05542406, + -0.14071863, + -0.20701404, + 0.11415409, + 0.04921243, + -0.17700276, + -0.10803946 + ] + ], + "b3": [ + -0.9166933 + ], + "metadata": { + "generated_at_utc": "2026-04-23T15:22:07.379646+00:00", + "source_data": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\training_data\\hard_negatives_v1.jsonl", + "train_records": 2025, + "heldout_records": 560, + "longmemeval_extra_epoch_applied": true + } +} \ No newline at end of file diff --git a/src/agentmemory/retrieval/second_stage.py b/src/agentmemory/retrieval/second_stage.py new file mode 100644 index 0000000..c7a1d53 --- /dev/null +++ b/src/agentmemory/retrieval/second_stage.py @@ -0,0 +1,247 @@ +"""Shared second-stage reranking across retrieval buckets.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from typing import Any + +from agentmemory.retrieval.feature_builder import ( + FEATURE_VERSION_V1, + build_features, + vectorize_features, +) +from agentmemory.retrieval.judge import JudgeConfig, judge_candidates +from agentmemory.retrieval.mlp_reranker import DEFAULT_MODEL_PATH, TinyMLPModel + +_BUCKET_TYPE_MAP = { + "procedures": "procedure", + "memories": "memory", + "events": "event", + "context": "context", + "entities": "entity", + "decisions": "decision", +} + + +@dataclass(slots=True) +class SecondStageCandidate: + bucket: str + original_index: int + row: dict[str, Any] + + +@dataclass(slots=True) +class SecondStageConfig: + enabled: bool = True + top_n: int = 10 + heuristic_weight: float = 0.62 + mlp_weight: float = 0.28 + judge_weight: float = 0.10 + model_path: str | None = None + model_enabled: bool = True + judge: JudgeConfig = field(default_factory=JudgeConfig) + + @classmethod + def from_args(cls, args: Any) -> "SecondStageConfig": + judge_enabled = bool(getattr(args, "judge_rerank", None)) + judge_provider = str(getattr(args, "judge_rerank", "ollama") or "ollama") + judge_model = str(getattr(args, "judge_model", "llama3.2:3b") or "llama3.2:3b") + top_n = getattr(args, "second_stage_top_n", None) + if top_n is None: + try: + top_n = int(os.environ.get("BRAINCTL_SECOND_STAGE_TOP_N", "10")) + except (TypeError, ValueError): + top_n = 10 + return cls( + enabled=not bool(getattr(args, "no_second_stage", False)) and not bool(getattr(args, "benchmark", False)), + top_n=max(int(top_n or 10), 1), + model_enabled=not bool(getattr(args, "no_second_stage_model", False)), + model_path=getattr(args, "second_stage_model_path", None), + judge=JudgeConfig( + enabled=judge_enabled, + provider=judge_provider, + model=judge_model, + top_k=max(min(int(getattr(args, "judge_top_k", 5) or 5), 5), 1), + ), + ) + + +def _heuristic_score(plan: Any, features: dict[str, float]) -> float: + intent = str(getattr(plan, "normalized_intent", "factual") or "factual") + score = ( + features["base_score"] * 0.24 + + features["informative_overlap"] * 0.23 + + features["tfidf_cosine"] * 0.20 + + features["query_overlap"] * 0.07 + + features["intent_bucket_fit"] * 0.08 + + features["entity_overlap"] * 0.06 + + features["alias_overlap"] * 0.04 + + features["exact_phrase"] * 0.05 + + features["support_evidence_score"] * 0.03 + ) + if features["query_temporal"] > 0: + score += ( + features["candidate_temporal"] * 0.04 + + features["temporal_anchor_overlap"] * 0.08 + + features["session_gap_score"] * 0.06 + ) + if intent in {"temporal", "decision"}: + score += features["bucket_events"] * 0.04 + features["bucket_decisions"] * 0.03 + if intent in {"procedural", "troubleshooting"}: + score += features["bucket_procedures"] * 0.06 + features["procedural_candidate"] * 0.04 + if intent == "factual": + score += features["bucket_memories"] * 0.05 + features["bucket_entities"] * 0.04 + score -= features["bucket_procedures"] * 0.04 + if features["source_graph"] > 0: + score -= 0.08 + if features["status_stale"] > 0: + score -= 0.12 + if features["status_needs_review"] > 0: + score -= 0.08 + return max(min(score, 1.0), 0.0) + + +def rerank_top_candidates( + query: str, + plan: Any, + candidates: list[dict[str, Any]], + config: SecondStageConfig | None = None, +) -> tuple[list[dict[str, Any]], dict[str, Any]]: + """Rerank a flat candidate list using heuristic + tiny MLP + optional judge.""" + + cfg = config or SecondStageConfig() + if not cfg.enabled or not candidates: + return candidates, {"enabled": False} + + head = [dict(candidate) for candidate in candidates[: cfg.top_n]] + tail = [dict(candidate) for candidate in candidates[cfg.top_n :]] + for idx, candidate in enumerate(head): + candidate["_stage_position"] = idx + candidate.setdefault("bucket", candidate.get("type") or "memories") + + feature_rows: list[dict[str, float]] = [] + for idx, candidate in enumerate(head): + prev_score = head[idx - 1].get("final_score") if idx > 0 else None + next_score = head[idx + 1].get("final_score") if idx + 1 < len(head) else None + features = build_features( + query, + plan, + candidate, + neighbors={"prev_score": prev_score, "next_score": next_score}, + ) + feature_rows.append(features) + + heuristic_scores = [_heuristic_score(plan, features) for features in feature_rows] + + model = TinyMLPModel.try_load(cfg.model_path or DEFAULT_MODEL_PATH) if cfg.model_enabled else None + if model is not None: + feature_matrix = [vectorize_features(features, feature_version=FEATURE_VERSION_V1) for features in feature_rows] + mlp_scores = model.score(feature_matrix) + else: + mlp_scores = [0.0] * len(head) + + judge_scores = judge_candidates(query, head, cfg.judge) + if judge_scores and len(judge_scores) < len(head): + judge_scores = list(judge_scores) + [0.0] * (len(head) - len(judge_scores)) + elif not judge_scores: + judge_scores = [0.0] * len(head) + + debug_candidates: list[dict[str, Any]] = [] + for candidate, features, heuristic_score, mlp_score, judge_score in zip( + head, + feature_rows, + heuristic_scores, + mlp_scores, + judge_scores, + ): + pre_score = float(candidate.get("final_score") or candidate.get("retrieval_score") or 0.0) + final_score = ( + pre_score * max(0.0, 1.0 - cfg.heuristic_weight - cfg.mlp_weight - cfg.judge_weight) + + heuristic_score * cfg.heuristic_weight + + float(mlp_score) * cfg.mlp_weight + + float(judge_score) * cfg.judge_weight + ) + candidate["pre_second_stage_score"] = round(pre_score, 8) + candidate["second_stage_heuristic"] = round(heuristic_score, 6) + candidate["second_stage_mlp"] = round(float(mlp_score), 6) + candidate["second_stage_judge"] = round(float(judge_score), 6) + candidate["second_stage_features"] = { + key: features[key] + for key in ( + "informative_overlap", + "tfidf_cosine", + "entity_overlap", + "temporal_anchor_overlap", + "intent_bucket_fit", + "session_gap_score", + ) + } + candidate["final_score"] = round(final_score, 8) + debug_candidates.append( + { + "bucket": candidate.get("bucket"), + "id": candidate.get("id"), + "pre_score": round(pre_score, 6), + "heuristic": round(heuristic_score, 6), + "mlp": round(float(mlp_score), 6), + "judge": round(float(judge_score), 6), + "final": round(final_score, 6), + "features": candidate["second_stage_features"], + } + ) + + head.sort(key=lambda item: item.get("final_score", 0.0), reverse=True) + reranked = head + tail + debug = { + "enabled": True, + "top_n": cfg.top_n, + "model_enabled": cfg.model_enabled, + "model_path": str(cfg.model_path or DEFAULT_MODEL_PATH), + "model_loaded": model is not None, + "judge_enabled": cfg.judge.enabled, + "candidates": debug_candidates, + } + return reranked, debug + + +def rerank_bucketed_results( + query: str, + plan: Any, + buckets: dict[str, list[dict[str, Any]]], + config: SecondStageConfig | None = None, +) -> tuple[dict[str, list[dict[str, Any]]], dict[str, Any]]: + """Apply second-stage reranking to the combined head across all buckets.""" + + cfg = config or SecondStageConfig() + if not cfg.enabled: + return buckets, {"enabled": False} + + ordered: list[SecondStageCandidate] = [] + for bucket_name in ("procedures", "memories", "events", "context", "entities", "decisions"): + rows = buckets.get(bucket_name) or [] + for idx, row in enumerate(rows): + candidate = dict(row) + candidate["bucket"] = bucket_name + candidate["type"] = str(candidate.get("type") or _BUCKET_TYPE_MAP.get(bucket_name, bucket_name)) + ordered.append(SecondStageCandidate(bucket_name, idx, candidate)) + ordered.sort(key=lambda item: item.row.get("final_score", 0.0), reverse=True) + + reranked_rows, debug = rerank_top_candidates( + query, + plan, + [item.row for item in ordered], + config=cfg, + ) + scored: dict[tuple[str, Any], dict[str, Any]] = {} + for row in reranked_rows: + scored[(str(row.get("bucket") or "memories"), row.get("id"))] = row + + updated: dict[str, list[dict[str, Any]]] = {name: [] for name in buckets} + for bucket_name, rows in buckets.items(): + updated_rows: list[dict[str, Any]] = [] + for row in rows or []: + updated_rows.append(scored.get((bucket_name, row.get("id")), row)) + updated_rows.sort(key=lambda item: item.get("final_score", 0.0), reverse=True) + updated[bucket_name] = updated_rows + return updated, debug diff --git a/tests/test_convomem_bench.py b/tests/test_convomem_bench.py new file mode 100644 index 0000000..0c1de2c --- /dev/null +++ b/tests/test_convomem_bench.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from pathlib import Path +from unittest.mock import patch + +from benchmarks.convomem_bench import run_brainctl_convomem +from benchmarks.framework import PARTIAL + + +def test_convomem_degrades_to_partial_when_one_category_fails(tmp_path: Path): + cache_dir = tmp_path / "convomem_cache" + + def _fake_discover(category: str, _cache_dir: Path): + if category == "user_evidence": + raise OSError("connection reset") + return ["sample.json"] + + def _fake_download(url: str, path: Path): + path.parent.mkdir(parents=True, exist_ok=True) + if "sample.json" not in url: + return [{"path": "assistant_facts_evidence/sample.json"}] + return { + "evidence_items": [ + { + "question": "What does the assistant know?", + "message_evidences": [{"text": "The assistant knows the deployment plan."}], + "conversations": [ + { + "messages": [ + {"text": "The assistant knows the deployment plan."}, + {"text": "Unrelated chatter."}, + ] + } + ], + } + ] + } + + with patch("benchmarks.convomem_bench._discover_files", side_effect=_fake_discover): + with patch("benchmarks.convomem_bench._download_json", side_effect=_fake_download): + run, rows = run_brainctl_convomem( + categories=["assistant_facts_evidence", "user_evidence"], + limit_per_category=1, + top_k=5, + cache_dir=cache_dir, + ) + + assert run.status == PARTIAL + assert run.example_count == 1 + assert rows + assert any("user_evidence" in caveat for caveat in run.caveats) + diff --git a/tests/test_second_stage_reranker.py b/tests/test_second_stage_reranker.py new file mode 100644 index 0000000..4c80da0 --- /dev/null +++ b/tests/test_second_stage_reranker.py @@ -0,0 +1,177 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from agentmemory.retrieval.judge import JudgeConfig, judge_candidates +from agentmemory.retrieval.mlp_reranker import TinyMLPModel +from agentmemory.retrieval.query_planner import plan_query +from agentmemory.retrieval.second_stage import SecondStageConfig, rerank_bucketed_results, rerank_top_candidates + + +def _temp_model(path: Path) -> Path: + payload = { + "feature_version": "v1", + "feature_order": [ + "base_score", "retrieval_score", "rrf_score", "confidence", "query_overlap", + "informative_overlap", "tfidf_cosine", "exact_phrase", "entity_overlap", + "alias_overlap", "query_temporal", "candidate_temporal", "temporal_anchor_overlap", + "query_session_hint", "candidate_session_hint", "session_gap_score", "intent_bucket_fit", + "source_keyword", "source_semantic", "source_both", "source_graph", "bucket_memories", + "bucket_events", "bucket_entities", "bucket_procedures", "bucket_decisions", + "candidate_age_score", "support_evidence_score", "status_active", "status_stale", + "status_needs_review", "position_score", "neighbor_margin", "query_length_score", + "candidate_length_score", "procedural_candidate", + ], + "norm_mean": [0.0] * 36, + "norm_std": [1.0] * 36, + "w1": [[0.0] * 36 for _ in range(32)], + "b1": [0.0] * 32, + "w2": [[0.0] * 32 for _ in range(16)], + "b2": [0.0] * 16, + "w3": [[0.0] * 16], + "b3": [0.0], + "metadata": {"test": True}, + } + # Make one hidden path look at informative overlap and cosine similarity. + payload["w1"][0][5] = 1.2 + payload["w1"][0][6] = 1.2 + payload["w2"][0][0] = 1.0 + payload["w3"][0][0] = 1.0 + path.write_text(json.dumps(payload), encoding="utf-8") + return path + + +def test_tiny_mlp_load_and_score(tmp_path: Path): + model_path = _temp_model(tmp_path / "tiny.json") + model = TinyMLPModel.load(model_path) + scores = model.score( + [ + [0.0, 0.0, 0.0, 0.0, 0.2, 0.9, 0.9, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0] * 36, + ] + ) + assert len(scores) == 2 + assert scores[0] > scores[1] + + +def test_second_stage_promotes_exact_match(tmp_path: Path): + plan = plan_query("When did Caroline go to the LGBTQ support group?", requested_tables=["memories"]) + model_path = _temp_model(tmp_path / "tiny.json") + candidates = [ + { + "id": 1, + "bucket": "memories", + "type": "memory", + "content": "Caroline mentioned a cooking class during session_7.", + "final_score": 0.85, + "retrieval_score": 0.85, + "source": "both", + "confidence": 0.9, + }, + { + "id": 2, + "bucket": "memories", + "type": "memory", + "content": "Session ID: session_1\nCaroline went to the LGBTQ support group on January 12.", + "final_score": 0.78, + "retrieval_score": 0.78, + "source": "keyword", + "confidence": 0.9, + }, + ] + reranked, debug = rerank_top_candidates( + "When did Caroline go to the LGBTQ support group?", + plan, + candidates, + config=SecondStageConfig(top_n=2, model_path=str(model_path)), + ) + assert reranked[0]["id"] == 2 + assert reranked[0]["pre_second_stage_score"] == 0.78 + assert debug["enabled"] is True + assert debug["model_loaded"] is True + + +def test_bucketed_rerank_preserves_bucket_membership(tmp_path: Path): + plan = plan_query("How do I roll back a bad release?", requested_tables=["procedures", "memories"]) + model_path = _temp_model(tmp_path / "tiny.json") + buckets = { + "procedures": [ + { + "id": 9, + "title": "Rollback release", + "goal": "Restore service after a bad release", + "final_score": 0.74, + "retrieval_score": 0.74, + "source": "procedure_fts", + "status": "active", + } + ], + "memories": [ + { + "id": 10, + "content": "We chose SQLite because it is easy to operate.", + "final_score": 0.83, + "retrieval_score": 0.83, + "source": "both", + } + ], + "events": [], + "context": [], + "entities": [], + "decisions": [], + } + updated, _debug = rerank_bucketed_results( + "How do I roll back a bad release?", + plan, + buckets, + config=SecondStageConfig(top_n=2, model_path=str(model_path)), + ) + assert updated["procedures"][0]["id"] == 9 + assert "pre_second_stage_score" in updated["procedures"][0] + assert updated["memories"][0]["id"] == 10 + + +def test_bucketed_rerank_disabled_is_noop(): + plan = plan_query("Who owns the consolidation daemon?", requested_tables=["entities", "memories"]) + buckets = { + "procedures": [], + "memories": [ + { + "id": 21, + "type": "memory", + "content": "Bob owns the consolidation daemon and dream cycles.", + "final_score": 0.83, + } + ], + "events": [], + "context": [], + "entities": [ + { + "id": 2, + "type": "entity", + "name": "Bob", + "final_score": 0.91, + } + ], + "decisions": [], + } + updated, debug = rerank_bucketed_results( + "Who owns the consolidation daemon?", + plan, + buckets, + config=SecondStageConfig(enabled=False), + ) + assert updated is buckets + assert updated["entities"][0]["type"] == "entity" + assert updated["memories"][0]["type"] == "memory" + assert debug == {"enabled": False} + + +def test_judge_disabled_returns_empty(): + scores = judge_candidates( + "What is SQLite?", + [{"content": "SQLite is an embedded database."}], + JudgeConfig(enabled=False), + ) + assert scores == [] From 34e55c4baadab3057495b855e33fadb524d60531 Mon Sep 17 00:00:00 2001 From: Mario Jack Vela Date: Thu, 23 Apr 2026 13:06:06 -0500 Subject: [PATCH 06/13] Add opt-in long-context probe reranker --- .../README.md | 34 + .../charts/aggregate_primary_metrics.png | Bin 0 -> 72174 bytes .../charts/coverage_status.png | Bin 0 -> 38111 bytes .../charts/locomo_comparison.png | Bin 0 -> 43814 bytes .../charts/longmemeval_comparison.png | Bin 0 -> 48858 bytes .../charts/membench_comparison.png | Bin 0 -> 33854 bytes .../comparison_table.csv | 65 + .../comparison_table.json | 834 + .../metadata.json | 24 + .../runs/convomem_mempalace_raw.json | 29 + .../runs/convomem_new_brainctl_cmd.json | 106 + .../runs/convomem_old_brainctl_cmd.json | 29 + .../runs/locomo_mempalace_raw_session.json | 38 + .../runs/locomo_new_brainctl_cmd_session.json | 44071 ++++++++++++++++ .../runs/locomo_old_brainctl_cmd_session.json | 38 + .../longmemeval_mempalace_raw_session.json | 33 + .../runs/longmemeval_new_brainctl_brain.json | 12197 +++++ .../runs/longmemeval_new_brainctl_cmd.json | 12197 +++++ .../runs/longmemeval_old_brainctl_brain.json | 32 + .../runs/longmemeval_old_brainctl_cmd.json | 32 + .../runs/membench_mempalace_raw_turn.json | 35 + .../runs/membench_new_brainctl_cmd_turn.json | 3369 ++ .../runs/membench_old_brainctl_cmd_turn.json | 35 + .../summary.csv | 15 + .../summary.json | 514 + src/agentmemory/retrieval/__init__.py | 2 + src/agentmemory/retrieval/feature_builder.py | 33 +- src/agentmemory/retrieval/long_context.py | 434 + src/agentmemory/retrieval/second_stage.py | 32 + tests/test_long_context_explorer.py | 102 + 30 files changed, 74328 insertions(+), 2 deletions(-) create mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/README.md create mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/charts/aggregate_primary_metrics.png create mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/charts/coverage_status.png create mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/charts/locomo_comparison.png create mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/charts/longmemeval_comparison.png create mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/charts/membench_comparison.png create mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/comparison_table.csv create mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/comparison_table.json create mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/metadata.json create mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/convomem_mempalace_raw.json create mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/convomem_new_brainctl_cmd.json create mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/convomem_old_brainctl_cmd.json create mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/locomo_mempalace_raw_session.json create mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/locomo_new_brainctl_cmd_session.json create mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/locomo_old_brainctl_cmd_session.json create mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_mempalace_raw_session.json create mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_new_brainctl_brain.json create mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_new_brainctl_cmd.json create mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_old_brainctl_brain.json create mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_old_brainctl_cmd.json create mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/membench_mempalace_raw_turn.json create mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/membench_new_brainctl_cmd_turn.json create mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/membench_old_brainctl_cmd_turn.json create mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/summary.csv create mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/summary.json create mode 100644 src/agentmemory/retrieval/long_context.py create mode 100644 tests/test_long_context_explorer.py diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/README.md b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/README.md new file mode 100644 index 0000000..60209ed --- /dev/null +++ b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/README.md @@ -0,0 +1,34 @@ +# Legacy BrainCTL vs MemPalace comparison bundle + +- Current repo commit: `9368f288279ae1d3395e37e89972038b74d66e3a` +- Historical reference source: `C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json` +- Historical source mode: `recovered summary bundle` +- Command: `benchmarks\compare_memory_engines.py --label rlm_srlm_probe_refresh --membench-limit 200` + +## Datasets + +- LongMemEval: `C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json` +- LoCoMo: `C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json` +- MemBench FirstAgent: `C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent` +- ConvoMem cache: `C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache` + +## What is measured now + +- New BrainCTL reruns: LongMemEval `brain` and `cmd`, LoCoMo `cmd_session`, MemBench FirstAgent `cmd_turn`, and ConvoMem `cmd` coverage/status. +- Old BrainCTL and MemPalace are frozen historical reference series loaded from the recovered 2026-04-18 bundle. + +## Blocked or partial runs + +- membench old_brainctl cmd_turn: MemBench comparison is partial because ThirdAgent and noise-extended slices are not included. +- membench mempalace raw_turn: MemBench comparison is partial because ThirdAgent and noise-extended slices are not included. +- convomem old_brainctl cmd: Blocked while loading ConvoMem evidence data: +- convomem mempalace raw: Blocked while loading ConvoMem evidence data: +- membench new_brainctl cmd_turn: MemBench comparison is partial because ThirdAgent and noise-extended slices are not included. +- convomem new_brainctl cmd: changing_evidence: discover failed: HTTP Error 404: Not Found | ConvoMem comparison is partial because it uses a bounded same-machine sample, not the full benchmark. + +## Output files + +- `summary.json` and `summary.csv`: all series in one table. +- `comparison_table.json` and `comparison_table.csv`: long-form metric rows. +- `runs/*.json`: per-run payloads. +- `charts/*.png`: regenerated charts with old BrainCTL, new BrainCTL, and MemPalace together. diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/charts/aggregate_primary_metrics.png b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/charts/aggregate_primary_metrics.png new file mode 100644 index 0000000000000000000000000000000000000000..5364ba6d7f748d064a7bef10c7b29d8802e33a5f GIT binary patch literal 72174 zcmdRWbyQVdyDzq&Ac~5Dl$6rlp`y}Y(6vRn8);Awu|T>(5Co*8r3FN~Vbdk;rX)78 ziTfL_vM(kCZ4z2Jh_|{VDc6fMKqLKfONIQ6k;2puklfHXf$th-j z_>|M*Dct@a@}Q$-Szc)G5_BD#8r>mCobMQ@=(7qNZU7eVI{7K-v=j)rQ8zn zKmK*p`0$xm*MlR3htI7E{`u?2#J7$e{yOFb9@*j3v5Uvs51-5p(;e;*3B@&3R1=WpB}+Te}@4^lWiRRy0A-c9NrRcN>#ktF8P+so`CYd^F#$*K54G&hE2jCt^tP z@Sc(*!@4YUhKML^u*j_Km5)=N|HJs$!8Ij{_3S#V^_@s7_#pVk>MQ+%`@#` z!Y3f$A|;Ghj1i#mK1mqlgHO`h%f=0r_xb8 zY3rj6!HnS*?l_t0=1BRe-+4xD`paWgxle1~+)!0M#;LB{VusB2Y$hhSPF{7!Jzqj6 zngC~x{3H=7g?23KNP#~@3|M-O!&4)S5S(%-4Ned_AdMLlRSt? z`HM9P(S06#-9E}HGW*_4O;tBHw;zLpngpb5=Ge^4TP*qVeubxRYUHP;4WaeWUzBCB z^Zj}oIU?@c-S5`B6vaayCCfzOegsgVI+LVnFEJ^7i1*l*UK}np+Sn7Ci}8QVnDDsH zpCZ_wLRcnCtC;R#;$6d7m8t25;JP8p&ZLGjZC0g@3(bX*Tn4r`hRzA&#}_JG<#^&H zs45*q$2qgOyD=KRi`#frGK$XBY_ZKW@8z6n3RkSynkHY}DX*{cKGibOgPRrXotQO~ zVEj_%^e1t6HMvfS+pzgG|F}VYMLP{+t zXQGaFXlZM8rnaf6$z0fS=$r9-o2kaoN1yH!HhiG*-YW1Gd*!~p78uQMRk2*LtwYYG z?-#+zAfS~*IB!xxsjk}(*E)J9Pj@1?U8s6-51VhuW}pa zkDx~DwgvV!Cjw3{WiRbM{B&QsA&8z{Jy(BvWjflbk-c)~X;F~aJmycl=N@PFdYSWT zlwAb9>}8Yw+~gP$XRhwuFg9(brLtB3buA1=cWt)gd_|>VycjF3q<_fop7iJq!xzW# zn_V+*JwMu6Y-hBpt{3nrzLq^PGwe}{g2iIW%m%zFu!0>A&+m2>GUdErtV~v(vcZWJ zmTJceIb;m(uC|FA_V?!*Wj0-A)4H-VV3w`EZ&5tE$7;@+x4gd2i_s{uq}YM$c&hF- z3FA=3+M{%p%rQpAxa0A{&)B}fck>lVf1tj->`O>)F6JLdD>+c6STDP`v(>U^flP&d z23PESRZ|%28HE@Dn{T-E+ukQ52k}W5&Q`#j9$t0XSR5Wpk`9it5Xa5uJ&~MUH4vPJ z`SxOUvO#b451mZN4X25p@2ABie9z{%TBa+fU*4JT=Z>y(+1qipju&&EA5mrB7EsC5 zFj#fFXWub1gvQvk#fery#ULswuUM=W^JvQbI^7y;CR!ON=%GKin!V9em$Fe*v{e1_ zjOHl&<&CA$a|(giU#gFmK3h5+u-d{usuo`Zqa?)h^TSeFcExt%;uUVA)@yFRe&Cnl z`b!+ltOxQEL5X<*7vqT5uJ^)dUn=xZ_CIU*39*5-juXS zAL136RWeRzJxF*lj4L$jZJoA}sGvMYA#^c@M&e}jMsM#N%5&FXgpta9EpT|kZFBq; zUw|D3ok?1u_^U2}Y96NU>Yl&ban1Z{tm1bn^im-&@N(*GL6qNmVNbD8Cei09tUm~1#( zWYdciXPO;J*>%ohS5~+N{jYK}Jem9h4uBUY*%+yWWfs+zPZF4^@NMA$;I8n zyj&r4q%L>guq8@R1m?<3*@)D(X#a<85|tjiFD7c=8lk6JqI-`|=u!w+e=o~+h)%rs z_GDWWZ&Qr0=&Nl3*R@$)(K<5QLD$M8xyVkRe6n}Hoh#k8Y?nisRhw}Sdgu7m^9&1| z+18xG*st5OY~QPg*3vxDz7gY5yi&Qp+vdjLQ;w?NkPM=eesSs?rTK`Fkca!ZSv$Rc z!-au@aes=H^;BGYQ;eWpGfuNEJ6pbq^qN)qdf56IHo}DUp*b5ON|7r&i=|5%8+y#f zy}kBu5xid!QIM6HbTMFzW}%yThqI_ZjT3GPHzWHwC?pxU%i@2-D~`+H&AOc|s%v}i z1Rh@2LC0e7+Jkm8RVKmf!U(F)VeGjwx%_jEBzOwy^x`0d?J2-#ZjGyo4F`V*I zJPc0$%@gOyZ^068(^2`I-C;5w48!}Gl;*>isnZ86;vO}ZGPfi|g4W8L1) zGuO*b6u(Y0`dDMw5@VwfC(>Wia!!1Q*wX#auiBQ9#!%*0+lkakHWNS9YdXHp2u|Ye zZ6sgX*PSD!53^W1?E8hR;#KlQ2RoyE^w0O?lAA8w z>?zeMdCa?=43n;Mo>k|=P8Nr;A6+~1HMRZE-0u3Bk6*8Cu@sWtG83>;A1N?XEv@v0 zTF!n>6yU>mX8F}$P2ucK5uAES?l`PrB=*?ud|sQmWwyI|!M?4%$-dTtw&~mC_*|^* z?qsl{92NR26b{4fyCn@ZaCn_U$eyEjAR5~yzMtQ} z>VU&8pvBrdIx;o+yBGyC;N4MigT{mkDLMKrx`%ho0!}jV-q^X4hTV%3Q)hLyC%P zDm3yGM2m&r4MSkVGbly2<88-8U!8 zg>J?x`0zdbak}(&9}EgPtzP@!FZ07rN45_xS zum+#fU<)H&`0<*=u`X?}CqtdPx-OCC!y-yPlB?G6d+KP4Rj@YTU6tUMh=IN{eSO>m5LmD+GB>jng>>a`G)9RI>zLly| zR1s#e#DBKeG2^WX74fiaHp(zF5;;$6ub;KyEkn1FYOdw|Truc`;&mr8@7?W}F&Ui? z&zGyRqK14#9rZT?MVxJ=!`Whscd>(3(QS=>BYIe|@b1s@(Mb*%wBfJYyISpbdRtn# zWmIc>LgU7AWmapuBl#weSLuWOH5v80*%bh5yZX6A9>>GG#gg#+uKzW`B*3trtI-#@ z`k&$permeVHxL9^{8gdr^4RTf4uDj6s#XEupac-`Kd7CeTuUZWKD%RL%;&WHBvRD1 za0f6NLrA_)zK4egnm|S~G{N7mdCPCUn9zBI=Lv#Ii(x{dqGk29N*(1BnT4`%X0oZC zFE`Y?Q}yBmt(FzO$9jSN>U7HkU57lw7P*294;n7JX}PVnF2zg?0zAUZ@UF&=^#pQ$ z@$$mMu%G7npaSiMOV`*w?irzRfs9xpO6-Pvb77pMwiSl-T;Qb+v(b^9*mcUh_2M-E zJCEOs>HR$W`mDQXzpKJE=h0bqvetNUs|ufJZX^HhbXE4w4Jg8`k|%%#I!zg4j%Q|c zy;b2gWj;w5&!8B$Q%&7=;n&^p^50dd=A?zclq4v3ua^0?k!@lzy^4jF!=^{C5T0TU z&}KfW+^2H!p>r7-lgWCbN*J4FVRMO==Y7xd>Ju2gj!TiN=lku*SIX{Ptd^V{uX%Mr zyTe)D0pNUYujIy(>30Ye7x=bV1bW?Wq~6@jR6lv-MJsBOfjFS>k0iZd#ouiIK`F9j+Zj}E}`Z%w^V~%wE;_@d|N4xUWaMh;qx)wNH=XtV~5xIO@4SJcJ&z z0W>;|eMRLlo6D&Y=els7qJKskcfaoEOh->X{Rh*bZ#i3uGL<*F(9|05Ew{8>)@C2R zqf)KZV$u5cb*s56>^A4_4)D5J8ibtt8Q9<94P{bh;(iz|>bbkF`i`=f%VxC10V5}T zCR0c&S6}+wiFd6@$|Yf8XTuy%=R8jyT}+U872^`4SLc84+um%)Cp5J(>Y1)`hFZrO zj&SS5%c&;(OsO9HZ5{L#WngnP1-9*|v&KzVRK)@YC?y7G%HHJ9luY#!hoMkq@7m~- zgPANx%GBRnW;Jci!3Lnkx`F13{uLK~aD@@vv8dH;YmF7A?V%btKcK2Cp(UDybw}%@ z3xzrT`6Ugoy>_@9y7{zAN8&xh&fL_mYy0nKhwA8>et45@zKW@Dy{f#f%V?3`ZdU-X z#jX@OvLoY#{QM_d3q_-H$=O(M6$Fh1Z`=!4_2GmIg>TT+bJliu$ok|$TV_IDp?W8AiQ~dRuFb^#lxv`+2>2qCfjTTl z$=(#Dp1FNSHr#WqLz2@&i}1W=zOinmWr5N+(*j-JvsZr%umA38(lQG*glmdYBQ%RF zW1CNU|GP6;c7<#9xF=c0O?QR;tr@D)!Ta5c{9K{|4%csM=GQhs_0mzB8v&G0(r(r^ z1zex3N^@EJFAP<+{L&5zLpnZxRrvC#yIfkw|jxH||F# zWB=&r=*_lI%qEv|lZPAND=jQnVL+nTFT~s%jVBWV3_K(klby)kd`Z1-<`{*$lW<`lNGUf?AWn_1Wu5f%;O1Q7|S|jo9YwISB0#k#dzs-8H zBE#rt=2kgCCAdPBr($C|c$9FPmu?B28Qro_T|sm$App}H$^F^F*LOvC?zq+JOk2^Q z2d#emGPG5VeTv|owu*|1Y2NI%giJI+B)3r*%(*COshZAqbx+pzueuT>0~pftVeGWm zO$#mr^Yo!P%Qi+_r)2|RTt~FbEl4%zTqV5i3AY_`70Z2+f|KDA&KM@%GV1(@(nK2% zQtaNj*H-MIt8Y>V>`;dd(VFm{(qt<2RFn_QC^5d2{XqR(zagkr$=Vc8pFQba1Cw!b zLJHQ)pOxp63bIEBV-S}9ZC%qhwP;WjrqE$k(ebE{=8Ok-B9fJb7 zZ5zO}UBabJ`p_f&H}>z zD$A_TORP*bd+*|Q)#^nSwf40(G?t?^${MB=Ue2jYh!AV-XRva6Awh2Q-G_YMw+1Ue z`#Q<5!S7q|Aqt+Cl(5TMs>7XZD6n?Z#EpLEX`3BD=SzkWilKm)ap2Ry|8?d;jQX6b z>?&qefRSbhmA#T?jsYR2{l(1Fs{C3U;)76W#lKy{S)5so!&l3ICi=Hj(06+ zNBwdfC))}Qnl5uk>PQ(hh6DmdX*bY=>+a>6aN-oa{yM!$XA*;!R?gFx-C5evdA|F& z_&48d=*`d>oxmv63rs_uoSkhP;iXT0;G;($zKW zzTmtv`8r84K91y26{o(hRqmn;`UX3cXc8kDFaHxfZXVeA$n;l#g;3C9FRjee>-!&X z&+$rL5#-=#?~yqC%krE0Ao{mm0k_Zno0R@PqM4s>6D)mMdK$E1qqew6GrWf+j0)HO zfLzggFb;kqM`u+(__pv{jf8+))Clq>e*E(*AII~Gii+suqm%%S-n(+3yA$IdZ_kdH zX^U^FeM>68d6wF+vak5rQrvowpq=4s@^H5M2cLPK#>=C2=$lOqnOKTc;GH0PSq2cu zk~B#C3^#XXJ7__R4g&BRy%AB-+xc0($r;cdJpAyEtTFewNF*!7YH;b-kNu`OlKt*D z{)Gs3`+U-T-?nWX*G>A1CH+n(1+pc$H;|)jfO86`9WPz@EQV zdFPI=u0}FG_4G?OtMiBc!`*lPMfc&k)IhGmTVOAeAW%;(3wGh=YR@Ao{f6MP#h5`6 z$?ZM|ITyB}p0&BImMeq#CKJCC4i3Rd1lnMZukRl+HZuW0e#m2kw5{hD9Z=y zH9dgIEJ{Q`hsV<+@cO{}%gAzthh+PUW0cGs91j3rF*vVIhu(dCLHfkJdC2DiGg5snmi(2kL1@#u@v%}5~!(|7M2fR8>(xN|}}PNbMlHmrH=sZN1u&lwz# z)o6wG@9s2OP)35mmWbM`XS#7iia|b#4kR=F0CAAIryu1TKb(WQIRxMR5=SJBW5w7G z?*r{fHfRb1>5LYxXUI^oo$UVJ?o`JoNk^*Dyp zr+s;+L0HAW!@2B(spJ*zO%#PsTe5+QH{98;HD)%!?-uy;<4}%i4wJk#tfY zs2|~?feG_i40Ipwz4zuK=j}ykJBU7DJ6RuS{N4fdE-v7tB*ySAO$AVt0Wcw(r`x$8J6_vgZ3j2oa$(1W}} z^nQJ;%DWSYv2-X?=;rozl-}Y{@#B+(Wcsb%alKzXcK%3i?X0#W?C0?QaY`XbsnnuBw4DC`C#8e=Osi=oukfdGe$lwNM$v0q6T_u*FA2aayBjR*p zhI(ES95>nkK7{xAvuvI6B-^P*hBYEHSbtm~m2S;`3aHNPlHTDn?|ZVC9@=qvV6+&- z%g?>5swj7Z#U($G@*@dMMW5;D=ot0`3rdD22~y{U$Miad=IR(M*~-RTSDlh*TLh;B z8L#O@Q%}hN$^sRBvxw~5B`4cK)F@D~J23aYyLZ5xm1OoVcq#E&E>iBB8GgJnh{kd~ znPl9C??l`-mtW$dL25HGNuV*SUsgAX7=~_Z{9cv$g@kY07m%JlyZ`J;QIz+xlm((; z?%T#B8^&U2PE%tvwXnbnLRdK9_{i1C8<|OaK)N%>^E$1yG;MDXmP?RdenaUcNM7c^7vV1cwnNaj-t4rv_Xm2MWxlugw|A3r+i#yPzFvNmzw)>fPr)LU(W)I->bw zcuxgodu-J}D=KuQT1d$l&%^5J6j{c?WE5Wk(I}6--1x?|j*12R!Ola+HST@>X-Y=B z>1H(ksRcn-ta+*hZC)izn0_rQI2!ezPpv8k+WOB!(+XBjSBdy7T|OJKX=1?%fZj!w%~m9S?;RkF3jL-S#JIWR8&+qRWc$#T-W-n z5KESZGqy5K4rV`CG2eI3!v*9q?`u~Y)hV%;c3J;@iK)5UbOL}`_s}8$RWMd&U*fv( z*Mb=2nfCVfxPZz&z}Ua|4wdSzSs9t%K1`Q{Fjlp%-{o#yVb>AiR*=-M|4_B&P~m`4 z`DX13XTMTQimGqodh#95Za5nyzUovh1c2w7uw(P~5(vqW#2U=`Rzu(ZbpFKuI|g2< zx)<)8Q=WNSd?A{Js#D-6Hk3p64#^lg^`XSi04l422O&2f-+%Gqg?`s(`Oi!xaMWn) zwR>C!jdWlF1YKs;kb-YDSea}<%vxf?E zG5)Z&G>RhQF{VC2aFIi~;~b^PhsP-jk$5g(e?0p7QI_-G8&Y;QFi(lC ztgP}B+Jq0T_t2S3?aNY#k8(}K>A_#>RBsxZ<6BEF6T*?O2Gu@ef%5t%;8&jq)V<9J zEyLNgO^;e+dZ5Pygm}E-_^sm79HF8_UK4XuV^Ybu0YWc7jjfXLjS46>H&wG@z)!25 z$3Qn>co)a1=UXNRC7!pc$amhQ6^2TmL8|*}F&KsbNJ1YBf|30Y*chG7uOFPZ*I=BR z4?qOoBBR1`lQH&6I&2#vK*oL^zc{F~JYC^pyFZNdjYPp=-|Z2Wz#ApjglT ze<-sc@(50$xi%KsQq8+LXpjHa+KFlZCB}e2vJ+#e@5%WagBca+U}46H9E@?<1$Y^M zAedgN=MHh9-^2VduexX+lb{e5^+#ZfT)+AFa|kduq*bB%QkC1?{Fo0JzA)1WQ&*dxS7(@Q}vOY)~L$n1K^_TK?FbjY0tiV@4YKotFfE znNC$z73IFjKUN;`t(s3DJ;)-XJu)(~GUB=@0v3E#_XXz;7Uk6D zpNU7A8oW1w*C75Z;sz-PvqmOsNYrBK^jx4iJ9Nm(`M0IU!e zErBrAurOGJG`)nd(K6@9Q9LHs!Ke(T^4Po$W-f(kxxpwvXWk+Sm0MQBB|U!?3jAhP zK8pc51XcWaPu{DfpsB-bO*vxp_fWrg+1AZ|JW56>Uo-Rp8q^cDK?7qy5$6^C=nb>s zl7g0H^-C;j99o+ha2e?Je!k-@0!+QMtX8moA9~_k9}qY;D3QUSGzpg}9-RL7$Os=| zN_lU6O}O^-poeg~hr|ELE!}~aULVRnS?w|kFrfjYohbk!pB3|_PMsR?uX*V3yE}*6 zDO)}7KATS2?6TP5yfV7(uSd1$@7eC^ip`2wllkq)e}1K@I)Z(*RbG=M3~96%+`hkMgF>5b+$ z(;CS-c#EHQD)MFc`%qXYEsGUqQ?A8!FToXCnThv|dVL1eBNjlbXH+Q;z#*O!feh1l zhhb~X)$tV|mxB3~M;C+CVUx$Z>?||Svjx7aAFVE`Wjo%+5@H!Lb53+jF)N&H_TVvFTsKm^x^#}t& z&Snx56C+w+w;0GrNaa;t(-5Uhjh=bc^aLbBBLAcp zT+%?89#Wt&%Degq09F@1I|M#P_ZQ`+;{igP+Ru-kNJ3S5*h2NcLM|L5$sAPoo}3Z! zDd=Uxq+o`^BC3(2Oby1r?-2j^vs~Z>dYO6v*akgclz#yZs@%m!yWtj!IQOm`d^6s& zON;^n0%{_+z%`arHdP}#ZK`alH-%-Y=g_0-V4!-01%iV!k?|_ zt`nqD&RcFwHjoH9{0=FOWmKemO(pJ;A3*4}arN(qJMkL4)?d6rLHYQIHngcNaGV$+ z1{4M>2N6_16;vKOMO2>>?_sLJ3;yUM5a5}>JqQLE7mYrnVKWTo&=qp=!vp=nN7;3( z1S)&N9tFSUTJ8pogs%RF$DLJTKPoQTBGNeA@}GXS@m z-Ui`=s71h}3BHmZ@*B1Q#7}Sh0Y}4mdo3;U%ERmlD7R^P$NWUxtxWcxxmOOCI5d@w z9(?H2Pq4}*U9-wa6OR5f0H<_I@_F*YPas0`mUbYM*u_GjkXqp8;8=dy(*fYv-~Y0-e(D;#8I^498{viHaIM5r4kp158Jk$GU335OHQKTlK#g>>)@L@t z9IxqWf3v3PE<8N5F_Y6DKT$)10p^i~<2vMd0SNnnqEJvZFp_4Hy~iBSIyiK24|N6FY1bKlM{juWs+%uvgzE^&i`5Cq`Y zPLpv>*yfkd=H{l)V0vwR)I+H@NA{xM;m zO>!Ptg9wQ}6w~prQzuVGy{(1N+jRtgy2>D{lXw?GBuy~*bU?%r_!l7y)aR-B&f)11 z4?4-6@rM#5ZEd}Lh8<}C*O3`ab|qgE=yp$^RU!=ld{~VvR3+OvDzOkKftn}JQOa%Z z5OnrFFVybn4f?d^;bP-cad}+~U^2hJE(a%YI}I%Xg-!!XdT!hLXL-`|4g;m7mhpyS{PytsUASxsn5 z9~^2rtC6yx8xOA|_IC(7N{2xq<_4Uwos%cP;zxOg^3ouJBFYY&7CqFz_klUk8Pu4R zQmG*w!M|HuK)MPSQU|aFG#JQUQ6L^C(&pAARc_zD&1KR_mxijWaNa>eaip z12GEON)#mQUG<{Cbhvoq!FRc^v;gH}x(>7LG;@79pYT6}(#;MfIRwH}QPgUT9Wj7n z=SRvlAznDqt_BeL#&GGgaKKitf$j)`XCZ6=`J-u(S!Y5_9hGMYgbuXg#68(S><E@CgR+oPos(-g>d^#=)8ETs~^-bS?9TMc>y1d`V!2 z=68mmV@5l`$OR&X0x;33XsZg&s=2pdQr&}p8dc8#eV2u6or&SI95TK$F?&VOHmMoH zgHilemfFzMwMd*GMJLOonq_`x=_{x%j8Mfx1H`s(3<<^lu!7jrbcM!Mi-g_SQv125 zE~1{0Aei3szXIJa?IYT5W@^eHi~{gGl2im@&Z+zh^q>OEY#L8UuF3VRf+C#+dO`yB z85oIX4tc&?WJzXtv3wR9X1x3dCpJXNDlle~&!wg*px+qM;8Pm5tPUZACZ*vooO(a& zYhIm0eX9hF8xeMyEZ6sJe5h|fMEl;LoeHiC=x=6_@mrb?7$J$zh-{$Bxr$D3gMU{+%+vVSS>>#Fz=|3g};TwhJd6Id(F_jBvXKw1x)>cM#U| zs_qaFZQMG+BjTUzX#>-0zG%v0;+vw^%d5icCIoEFvk3fk)Jz zqbKQ8S7A5fFL5Lv+f{gvQBjG8xsi8G1+fxHZXLzD_86F`rA=elIg|O{G{|UM>m6^u z#bT7ELBtF^$#Ln^ZV0a%+M{_OyI3vfDT8rgK5Yk<4w!vmsg-MW;Ti-!0!xoqB5x%o z|8Ti#0x(g((D+ADSkQz87W!C#D1yH1kx@K7(ACJoa38sUbGem9{7FDSNXnE${K>$T!W?I!CfQHqC zwmlq&P#;q^#1QGAt`YJ-|HL{FtuZ(G7S1Q9Ww+{TErruY`N7zny91Y70Vcfs`a-g7 zIMPxQs^v2I_8{fZvS2w}^72fIMaO?xPRJkp=70&qFOnE2cP(=JQ%ky%dlpPLtqn|X zn@HSOe@am%fL?if!rCSF3_fUr1vt;Le47KVMQxhug?*5d>1iORU>))wO9_UFWe|nc ztmz=_@M!oSR1zAMt`(W~$o%xDU@o?u;tD>xI8e|e;t$wf8ZfDVX1U9{KGdkrN>5KD zB8*f_rF%dkZz;qI862GjB~Tg?BrVthh9nrk=`CiRvTul&TUK7)A|$$mOe>g^?8pB_ zNAM-0U;?_sr~H#lF8s#BYM)#o>39`VNfkIPIrizL3S6tw9W$vI5NjXr4uQg9e5xI) z1vgF&lmwH2hd@9MXKCr9Dc1 zlSbJ)<_ewv^De?+TlBgeXyidjfO@VM$4{6TFXH?(j0CKLWlyr!cLScZKk~O&Rlm8M zMh-kjpo;xdKXYO&SA*68|0GWNKj{iiytRdu39UazjW_P%EyT?t8zP>X0fJWg>p^6z zEhB!>w=)bhhm`8SvEjjz1>(ok2h#@c$Nm4zHj43%#nDQ3ByA1ysMHswG&zNPI`AOAbWt5jM`PxHVmyw2mj@tv+q65C1|+Uape@}* zRg~d}vg?!?ADyb)+m?k@eEGK+FchBW4hT5=isX~~3oW9H%OSieXg%iTw$i|esD{on zvCk$!pwHE!#|jyBZVU#q)#sZ7w${# zGTu|rIGZ4BZ7|!OfS@7FE)tA}d?xqc>@J9OCfY?1@JIukLDGgyd3cPu9dD;J-TLtJ zS3?*(iVa{%F!ahOYc)-jPY8guG!S0D;dqVPsg*V}AdzNtR6o-f@vM2>R6v+Vrmx*u z!cd7Q5YLYTD>0ozlm&PD-`&#p-;-UE2sk8yPFpbD0+QULLJRv&lZ#3IP`Lr!)3mFk zW@H|bIY)0;i0 zl}&Q#3tYID8kASA7-s<|FtLlr9&od#0Ya$eOV7sUDt%6l3BfG)FVp??^F7+-;B&tR ze+a9hRc8tXabPi*6X_{hkhX7~c5!VwXue*3uqTKz4bEhG8)CAd0Qida&mx%uP=NaR zdU0D@<&))k$LL_6SRww&>w<{&5}L20eqeIBg&%jPQA( zFjt~wChxtyjMPz+fP|*l%tM{jT67W8H1s!a)GLVtff$J3mae2Z%B3py6B&0qxUEGi-lNdl+Od+J7~+;kU$1yfNyULsdIr%vy(sHD>s) zula9ou==Npd6*dZdxig_ty{kYjfxJY0N(XZYeS17f!nK^S=Q^s%LJePg#rUcp@Th8 z_aLWF3)sMRGiGjg?Ux_uP!YHom?su-wC?jl45Flqrm&yN@Eu4pHtWd#CU}TY9^vs) zlc;eb4CjqGp#NUDPp5diq*hl~gAwWpL`}f#;Kfcf|K4B)WmE6O$&-!k2@(cS*K8pc zC>_C(aPj7&yNJjJvLGotJNql_%+eA&XdFDfpdf}GJwZ?}N##xl`k-!pU5Vv7L8Oog zXxEN+=>*gWNLkqyO6`MXl)tUFR zWq0qlOfC*ns~4N+m=BiS-V^hX%i6Q69pX1DoPU(8L1;2s98ISfSZ#Q2HdC{x5z^ce z>(ywFCfH5%0qPt=IotrB=X&hA6g#hehWyLDC_W1&l*d-GU|r$-XeH`?03|E90Kp)& zzXdxVQp?IC&NCEq0|n{8iHBXzULqzY&Ps8#wzkf1J7AX9u{dw;lG?eAMc4iw38!Qj zBie=GQdOjG^IHyaD3^o4RxwoJJ}8$}1UttN`IkkrQ0@Yq6mi;aOT74~PX;g$ZW>3V zgr{cS#)3_m$g4tVDEaOR%{?7vw$OBy@YN$m^4~MU_l~`iN-w0a7_nM^le{vRF&D(F zsB=@bO4V|-liI6PbI79`*di_#%Iu)t9rIi>FLR zO0>T@sOK7n&Q-DnaZNGF41L?@C!O;?wPA2;biX0!gq=b6SDKaek5m+7nAC@#5K(b@ zNS&!3N(l>+odNC|@p~C|6lfInen56xA;Q4p;UReK100g(Dd<$M*(nL)FGAuy|7ZH8DsXJqEr(rJ6j}y8_q-a- zN12ag%Wb{k10y$>GcWB8(N(eZAFs=&qN{4?bjqA^4X3zX)oDP2bPIesi30YHH_M

I0gaY8-Cg4zxV4zUNRUuUX_JfNuXZkI@Fw|MF+> z7f>1HuVL0xrS0-sjSNk1!+yQ!#u0ua<9rsb^S5f6IDkQ@D9nTMTdXu)EIwMwI zKfeSf6<_b5CZ&2;7L9W6pJR$=y>By>M|r4 zVYg`m42J>4Rsu^Dk!^wn3Ka+@hsTB_tjjLQ7(RA)o$I8Z14})X<8k_X@!hq@cBK#n zIbWFbwf)}_Ict)O3K1MM8WTOY+-f`TC4^U!*%u$HT2Eha=$h_#owh@FlgVga37ElB18jw@A2U9I`eD*CU zBhpABhgBp01E9dB$%f!$sle-oM{A%d$%4j@?D&01nZ1~&+DpV`(J9an$Xyh3PJsCp z(L6s!3?u%4;3CU>`(9aD+2jOexxq63Zjxd^fIjx!%^+S)qGJ1-4~SsS|3+Kgz+pF= zYQLZPWe#5*44WCHA<~l!KOcR(b>#7--La};h%tcwF-1O_0S=vK&@B1`q=KSmvLT6q z1dRS*v3!`dvIEiedpUTi0c$_Wt&rWYQxgH=e9tpfu9~T70azoE$p&e6V3CFkS*%}T zdA|{zKK%jMchoJ-^!Qijs3h!k`dMYawj^x>Ia0 ze~D#+p)I6WHZ&~HNTFex&}{P0CrPfcQmV;U=O-o&e#1VN`GEpiXoRM%{jfVtA8|U& z7l9*!SVZ5Ah4@;k3DN2;7H<}mvmWirXDwMU?PxL_|5x(Bz~7YKe*>B6rHmF-m$k$ z|5??jZ~Nl8)*Fl8OP375(Otf=xH3w#)8eRz{9Ih&kFE?|j26Cs5Wf67Y5pT9^-tjS zpW6j*t%n%nq>51bFRw;0LvrmOsWj@SfBCGxRDuMbhQsYJ+K}`DTY<8kZMstm4l^6@BM&0 zVc4@#cU1&hgC$1-HJ~4$v%?^-8CnXgtQIj~`AqO6SbT4@W>H{gcQ+Z3Z4h`edG5g0 z^FTOrXxOr#9IX(YkL$I_?Q}n$NCT%4RUT4&2xy}A*_q}3U^^=}mC{rai zy+Zbf)Y09ZMPHg@#H1k{NM?xx+R^|t^#*i9o*Bf~KoV?7k^#&EN!T!b-V_H$d?Rd= zxB-?q&x|@%N}QQ!0E9N|QctbH2J2*yK&JOfvZ1*}uZmiYh(r&<_J>|WPZxK{mT$p^ z8zg*c*z7V4+scYT06J!L$j1Wjfy~s2qZ+j${QQ(^jWuW|br$_4ou7%NhQUp21 zmim^ojSQDge!{X&VHY%m=*{?UqB>M5#XVR&DR(3)c315B%RDmWV)oKuSN*8@J}_sJ zYmk9?7{sW^c8OVqXS~?Al#IuWBXdTJmMLY*v_HewgTk?@@462uUtR zHa5bg`x z&k`^0m1My^A-I6lJ{GjVPX-It?B`e5I<#tVuyXO95+MpZ$PAW8n-G!V38I^WgG>i< zO|CO4vx4xd7_)5Zm!5VIA#;4&WSHG~Zzz)u_$GyxfWO1Qf&6_~5`2nl9_ITxrKVX0 zD}wM5`6WeKCq^GMeO8R5MmWrL=Ad+*3YA8U5wJGXE!`gic9rT~jIuydc~A4zrK)j4 z#3BEspGvT%T49lc0+e1B*sH6OL>cY7fjBr8;%hOl+}vE`npQtv>MKMD3}}yLgz=%t zCth{RI2@eM4;DlzU+Bolm-oS5cl605R@RhPJF2RQfk86Mf{o7?vwCwnzvyVf-hNhp zm&{Gf^mfY9sm6hxrm2zg(({&;4inq^KebchTGqv`a!X@liYI@rdrl4YcNs<&=CnW2 zW`4ulUJl^^pSl}R35(IzI-2^%C+w8eTEoj%6MH6Q2EBYld&3@}1~Yw>!8EYT?7wcDYPC$3G9y+8)K_VgV@c;qQf71#&Zk~ee)RB0_#u{ zQ5!T|U^9y{5+Fko$P3JdZE+UclCQd9Q>ZiK0p#Ni7D_YSKLMDu(7w=*C8s&|)OF?_ z?t$jEf@Z6Qew;8xz#CJJLmM0iQjq|){@XXH8PE7c_r7+JB-{1IiaHs-iRrJb@!|C`)3%LUq8mvy(MHKnL^hK3p0&j<0&w}<~1L56ZkEL zY{yQXeM^$8TnEw-ziysN{1Pb1TKR-;u&?&O?3JT0$SdA1KIpuN8NVzLCesJ$ z&Qkkp&_ek11Rz@7vJO7T{W!N1&B^XFmY<*Hsfc?HWCoiLe7zsWvMDcSyn`AHco)eP zjV7Zgal3x+ZK(C^71)x+5xlF;u+vqSNW!@d8W2QG*pT~g6qKuoV(y8$Z!1B7=sJ>@ zgS2Vn$N=PP{kyk>?55wftPfCa1As$Y930q~OUUu+31jlp>)`k{f=R5!M}!*PXE@$n zc&QKq+_m*K_PJuQZeLqnsUke7gkAbd7Bd_7&g%b-gXlEJ$0-70Vyq)@F8Qe zN^GaznADtibal<|)WGrXLy(FKwwKz8yhXAe(9?|;1P`-18@J97!SXx-8%v))c2B>Z zS2&ppw}k+C_@|uxGjLM56SVC$p4|0S7=iTd|0bsCwNo~Vx~#!P6MQ=S`_hVfTL8WL zNvfM_-%{jfI2 zEd}|r7_qItNVrxeBxK_^Ki9?R6LQ-`4`}1R&b(>S+ zs7VB<2A}#p(S?H$Rg1{|aui&n=J2C|CR+(TzqXTa!wnGh;TAa4Z+$>yYy>(fuNhEi zF{q*W;-Ce%-#W}DG#v$Dg7>jEx$Z-IGdsv z9|m%D^zRwijwf{OmS^7|y@n{>(w&_BXM!%9ihv;yEnQ}1vW2~1bpMu^aW{aoV4spU za)ZW(FYkp7&}$8bzp5*VC^%-W%SYu0-L$)$rdgSwl&xM@;trCpTx#Z#6~M0sFe{J^ z#PI3qbGsS3d14H20o^yiZvPfXmku1n5OhY|R^>^q@I$~pf70(AwC(7Rt{`nAyA)T4 z=%s`1B5_Ul%ph1C!uAyhCp@NQ^Fm_JpTe@?!Ze6z_rJv!-WB+l2+^MGRjQaJ{V~sA zOtA*YqqSYd6S$1==7C>C(?vVkC(bN?NnfqiQax9d$wE~f?zy-bInTB^NIdYbJ{@JP z_TJ$0%KpU$_*tdBXSiC_>23$Ns^8$>y2;fX>vAWbPPC`l$%29b6kAc!RVs^nOv zQ$cNp8|6qO0hSGd9o^hlBO$i{j*=z_5Yr)l4$~yJ%+D)|NFme-;zBUw-tS}-pdA)4*qU4bM*kQ-B66LsmHPXUJ>LA3~zf{0Y7me|%LiQQr76VgZ! ztty8QEXheTwKAhNN_B`$A{rFjqTwcp#-Wf7uMbCgT%Fq`ye81@(}pAxyu9%8Dj`|b z2(d7T5zuTP`3AVw3hr75*9UJbON7A-0ta7k#a(iwDFv z7&s8GqQIOP2b9Pm6)|vwU}8iMLQ;ei`;TlHHQygUyluFr)VpHxPByfAoYN}}ia5YH8#qwMti6>rh8{f!KZbedr;D0KBq(lpM zrr}lYK(4!0#n#WYutJ~1P#NBsiAds6A>=kp+u+2*NG+?;Ltg9=&K#1ak$uZRYCgap zG{Y<59$RqRE&`G}R6_PrkRms%o+ZUUd)7_YAs_qIzaALSB=+A zyf-A`x6!V{^SCn>r6l4WZmQy}D@%Flzle0!uAV4k${HbbTqhnP_~&_+?o-RW?pGrbLqz;k>Lx!FLuMfTGHuC0%yYwbsLPLTaH z`*tmo{mo%%bD=zc%k0b&Srl$NZW)W6?mk#$ zV=rNeBX=&D3RV>33?>W%r4mU|Zu87E1WWF1W7rXo6-kOFEZRX3LlA;=5PcFF+Jh|? zOd$C3_Iw0QCj{0!a?2w`V(QE_kmY`;4)+a0_7$=Jd02g3=#SpN#L8Buyeo(fGG&5d znE1-dN@wl4ANJs9{k{8Iue6+6k;_Y-G?c8&#b!2o0{!62gqX%K`Tcd8bF#ftO;K>S zgryJVt~<$(2Mt#YbKHW&H8s?EmMT$rL*?Xjb~Bp|brc%X1#hgRa@^EglCguA^f%j= zCR-Ntb)Fob7Qt z0VQDk^-DHkrUwf4P$i*AK)vKAd1Pvxnf)H&_=Rks|2JoZXQ>LJsj?MrTa)D*Bf)UrrFzH&=)rAIBq|;(lW*(+1Rq0vs{;vrdDlAMiXzyXRdnq|pG4)2c2ft! zzp1rd)rIQ zr*dmM)6Gt8d5ncW`fPr$LG|O4Z@88}(M-oGaJQq1EJ-c3!(cbUcJ|vOx#Bk$_!swg ziGNjj#n|kaPK@)A}+D*Lg>G1XcyvvPt$Q68^Pw3x< zgsY>_8c1$ovd`rY9APFUqn_d8OBC>m*Ceh@{NAsm0aEL?G#U^E6aY2; z9QvN#O(m?5#}rm-LLt?`Znsf1sl-cyOI)m$2iJsH*jeiY&yzXSP?9uB*CzoaOAvke z0_^VPDSusHA*+uy${sDk@}d9JW{BL9?Im%Kqs2D6sA6|aSd!HLqVCPZsovYZVa-Xi zT`EO`C_>3li4-Ln+PlnSDVc{9GAvYvW|bnTkd&|pYng|lk||S`c}ivr$&mT^e4~Bs z>$$J@e(&cv-s5-}?Q&=XrjnGeD&DO!@zjv~7OzKc#Jc>0`#ZkLz>| zl3j9JM+>y#tLoYY-C`6f?pU6ks;^tYrNbw1kq1wDbpp@vXSKTCQQHoaNR{`+A|~e1cqskLV1jQihqjSW`@cEwUf zBnc;cnd=tR=yNC1IUrgVt3X$gkX&C>B&^;1bnk_ig)NG!KZ{j_-uaW+j^HeEP;|4rA%y7=FHav=+7 z-Vr|zw>Os53I1)A(|U92GI0yRDkqE-Ml{HIWPr1%rsb+0h90kO-JOi1u#uhxeq8$e zFk1(}9HnRbweWrnd3S<>{w=~-(ek}h+S!h|v5fb&@A|zL@<-g`ZQI|v#`nZe&Iyxw zm$j5xl%KCC-BY3il=%4Qz|W)SiXLs&if_5A=s&yVZAL?w)r41^wNIw*;#i9YP*Tv0&JFG6j@ErUckaw$BL2`Y;)ojli1N*=`kK3{ zIJP8LX4pjHo`j;=_ltMZiuC=wcAMXneAHm;8R^ZE_D3w!Iy~NSw86w-0v+(UrhCdsH+~P-cIx_yhGvYOVgqGxt(DYFuVLu2Ygc5O-JCban8e5jzDsWbNv1eEfL~NKz-Lq)QdlN`pYLPY`9lmOZ4n7fNb?1Kw?HJT(Fk;{~b&X4{qVb&|blXSjr?KiGcR{Tj z5!&$^DjTs75!cLRqD)qPG>)cN0WD@$oi(JDyfh&M?6Ue1mlL!jyAy#6@F2P7?s4f6 zIh3_CSqzxv^Dsat9i?`GtY$ZJLwDnk&OHyu8bF)P=Q8&tV7APM@?*V|l4)6(ztfV( zpN6P`-x|y{!k$~NiZ9r#b4PdONs?>;4~;J@>j66_aU}CzViZ?B2iwB?2~^>84B<6T zZX%W-@CO|{fg^l^XDnvMW}D-`6?LzP!k%OoG{t+p$LF|@23)J+fXmNfkKBZD=y|`k zwejHzcq251e=x$+)!*dJ{g}lW?wpsiwN2l^Dt|^R$tDC*d#Yev&!#ubnC{PUrn~)1 zPOl^X?X&N3uRCkZ`2)q=dw!mGaE!N@cYeOq0?6}44>}*!o<*HKd$tn&C!~a1k^6Wq zK>9OU%g3EP#HNepr|UuNbnp3=LfmJ@7-2~mQ7$SQs}~IL{0Jl_#+JZgOXzIF`?T<> zigeAzF5)~#&FP2X7mu~JY|C97i<=2n|Nd{R{=MEruiF$GhqSlYxq>z3KP~ohV2Bb) zq|Y7&45Y%%H+?(k67`(Cv#5Y&Y>1TqhSLI%F#6L?8%|qO5am)h2sZYRV8x=-kiay;OplQDgxXqvf%PBA902K4L{0$T@6+3UB36xl4{3mO{AkrEvqchiB zdbd4iTC?Nc_REJqz)NC0K?OJqaeDuoE&Z?Rn6SYQFsfNb7d85p=o;*CQvvt2-fF^6 zx$F7G86;&RAt0XMUOfhk1MX^+*u@fSpAGS%!qUrfO3=lYNmk>ThTLj{)s-ybI5*7% z(-6=F1piI4^aa1D`&*2yk}r%UD6QidO>!Q0_|$&7LB1nt`g6-@lFD5j1Va78eJGvT zMa>oOfE!c9&8(yr2Q&(I^oNe-PY<^HDm@leKnME*pjAIfH_rJJ_C?|~elRqqLfzLo z>iy0@@l`_Mi{2^edl~7e`P^f!iB>;4I1G+~@;>|Jxa_y|bq`-S`!5S~rSg|Xd><=) zZt19fFFIZDnsuvEDa#)YB2zg*-9F&Dh#r)Hnhn-l?%`Ce7hoFzI3R{YmQ z{L@K{C-SC247P){W=|)0kJ81jlv|r-K5|7KUCEHg(|8o$^ zZT6-KmHG9{pEU5rTl3SRe88g}?usI~yEhHtT*n#)^&rw&$_T59Xj_t|LVb&9*LqMo}630qIqx z_u|1I;RG);&x45HO4Q<@_u)6csr$=Oh)C#Hdq11Uq_4I*Z~!UpcI!d3FQNAU*gof| zI;YkyQsb!aDE)=pyY3UuWyDQO-b7ne(dt_hQdFGfd%rhqavuIkZq;vvh9f*~3A8wq z`t|XJb*TqM9PR`WUwS|Iz0bb-2>as)TkH!h&)zR{v`H2XGt&j`O%g3mK!6JYaLji! zWM=Mn8u--HT$AaRNwcb}elm>2oPf3vSY1j%kg+aY_=ZciLO}1_-il2@&eLE09u73? z)sUCHfF-Nd9<-2BQVL{^M7+I2KW$I#5KdWAxjTwOuCg8!(~L$!0;>w2(xZ(aY3_mz z(M+sx7l-S`7Th;x4YP0xEL-ndnRlmLfxB#QRdPM)b}xE^PJT1u_HD~M?@L;{j!=eG zIx17H|5P6P&@-VPqscGGS^ki%twus1g=jg9x33uKkAD_-CTA)89~Fo; z^%x>492n;FIjz$+Rm2!LT#CZ9{s7o4F+qNSO26KzOEA8O=y;&5u&8e%E;J~N1BbX+ zSMuD22d(H9^elWL7L_}z(j4QfRQxG@?-sNsc(tt4^iG?;C~rqh-p%q6cN#GHj&vas zOP$gU3d|4K;}PQSgiBmMoJl-s5sqTPBs!VZUyvw52x>yK@j$`e!~d^Z4Y*7VtcwOa zzl1rXz#_6deO)W^a4Ib#y8wavu3-%}RtnfSEry(AW0dC^-3)`Gsej_FaF|lnjr%b& z?`(zxn?I|bHC+DFNr z*)VNa`$26}G}VMeghYp5ceKg_Q(i|z?WNFEpB&te3{q3fAK??GTa&|5l5ya*j+=Tq z!?>zU7n{Bk``^qSh5qAZ=XZQ8q&e&_0HQ7HS=4z`IO8sK!k>VcZioo;n*U)Pk1;e` z1TN%vQ743!+81QCK7blc$6x2rx52Kdn0X&DWD;|JDMSz?dWe-yr+Y6#G#B`Mt!#67 zHOi}VDEjF-iOX5O)_Ya=-E-MNxxsKe&C$qwe|hu2FbP`Xh_sa)03<@xOe)isJ)r5ZPQ-V z3?16J+3A4o^8mieQj3tRHx9{YKUQXVFM4_T(OK*J2cF*jQwr^0N;g`{R&5>Y_N|-O zee}Wx3O`0(J7fsN01`$&a5*9M5Y}6t6J; zZ044g-8H4}8@-|C(`0(;miNfSv zqAso8<3qI<4`jw63=%_5%drtxq|@ftLE7bCcGI#OS3ms;8P8;!zON*zRM_M2RkcbgrTlrOT6usQ*FrG&B3>>7LP{hz|o-}KXx;X z5FU0)&-%4jf~vawtBH@JVgg3LC8J7=Xl*NPSDHo5yO4=6Wq*ChueLi_HTXfo;H902 zp>X8uP2q1wC9Aha{1Yiz*QvpdD!O|UP2Hpc32b4v*{8odA&YnSAVA|tU=-gtw#3)A zwP2+5V~fJN&ryQ0xv+A)-_ZU(08qS0daj+wLF%XSitr45vAc}UNs@_BkvQbER6(|d zK-8Tbcgu_XY$P#|zMSW6R1e7bH)l&ibj_`m_3h8 zl6GKXW&3ZbD9S&{4$FNg;2-dZ4a8Xlekl{@K?cARD!N_~_>@S!6*Vb->+y3Gfk4w9 zKPfWRWB*VYb6|C zV*iN>p0Q0NgDCTC@lL0Yx7PNH3?hJbZUYYEhrz)cwP+^y5MiF}QHbIm`%L~Oc->>r zL>261Y;?jL{M&rjc9ZBv6TQwOKIJ4m{PJD(-Qs3nKGAqg%3j`H%qgw}-v-~*Si@Ao zDDZwwKJFi?PF4@*_pc!7k*{Z~;g4OHpGYCuLWplpT_{EF7gxTwU)ou#wq>GQC{vZ? z3H39_)ZmvKH#mC>UfX2<&!wdwQx>%o+gF6bQ>AW#1NyEotBlI0854K91@DqTX-Z0t zchU6i$Ff`}na5q7Aczw-gW5IgOg1Ix6WtghWLCi7x4UV^l@EH=HkZd^H7M760LTmH z7VR5(tS*O>S^eurJ1wCZeG$vGO+pGzVyB4gf;z<*;#{{Mx`$_d)3G853SA+@8tjXX z6GxBjhmh&X7`p(j=ksn#a~g~gSUMvyVY&2hPWx&>?N8`&TAMt3|exY z)6OO3bQE$KAh7%**m6STh5}x{U)|>UrLVJb&F=v;Nw#UaLbIqU)x9`73ow{>jk>_0 z*AKk-DwEEKLV+lUpYHxGn`McWa9>(wD09_h5M{2imQh#D!}9q)8c&rOoJoJXkm{(7yuFlf#@cw5p;5Aae0VsC;)K=$XhRJ z)f$6F@*9XR@eo@A65(MYj5mXdAi@-fUMp==lT@6>d5`{d0v^&2@jzgs6RO31n3TPr z-DB!VdSE0Tc|I`aw6kdEOc}qSINRi{o)U5yNg^SC zfFQ|IBpzOxWesC)S#{9z>IaaBa-10b&+hQ@Ex}{ z_;DrpDJ-D(U&Iu9UrcchuH)H&2YaGQ$suxeX`yzrMKF# z>(zn9GM{ar-eL45gJgZ_e=+2cCl4OMl@Js-j{l6!_FblpcoBc&8UJML4aJpId{c@k z7dw=K>fQkOf(Hbj;}}|VBOnpcYc}uqa9v@TkAGKS=eHvv@(y#a>v+;Q?#pvu4^f_? zxB6x2Msr{{BNS@f{xyYv;K$$~aUQ4EwC}6?s4uhdy$5Rbk)$*@$d)pD(c`yjXKEro zVtm{ho@m ze;vj)DtudRh2JMD7I=&2JPE_yM>42RS~g%eh9i?+LBa@l<({F9G+ofV(2}zZ0aPUI zH9+p9K)78dk%162s<%G<>5%5k+3nDLpCtt|ETN(Z7F~GO^0et-xJsja+vg3Y$oUuW zTDUA^RW9Jx-7$2$#7xYTEJSSQc-e1tN+UphGnr{&{Rzw!rH2(j%*jjm&szs(KEYc( zPwNw7E8&H$Qxa(hg@guk_dz|Jc7-X}C-uwm0pa(Fv`goSWLhxRS$bR8_BvOiPN|x3(I?Hl*V+>k)UsdYgH1jhada>`x`TGJ8}r= zYJvdR4oBvj)Z5WDv1h!I#?Tmc=Xb?iesfOyGU=Z(Fb@&v`!1ZY-a1n-^AOA=nH(^; zxSTNsodAhJB5o0K^?o2=61X@3S-!vO+fV_*Z~~JxS^NpO=Y>HH_+r)I$M(vZQo2$6 zdfSbw_HdvRrxL_04-vLTc!a2jpV5!{;|Wj`(o5kXRsfPP^YKnisQfY#r1t(M$8mgJ zn~7!w-Bzw~5QP^{ia*Tg3d97Qo=N)I<$AJ~=>{0rw_>7vX*~!wfy5g2munb&sQmf& zn>eq)LOo6ODkAnJ+`9Ql1pUK|0u4HB^@mmq>syQ-&hfzg{-{q&wvG5`QU9#`rZ8df z-eTBu6Spk7v>l~m-2~wY!az@94@WD*Na?{l@5kTE`Sh=kY>VGUM4Yp=sr&W8?#!J# z_fZSpqc8G_kXVDe`+98i60ZG~xl8Y2ex675AYoZ z<;{n0d_sT5<2wE`gn7$mof?XV8hltp;RyijC@_~*uPJws4PVXJV)1iv(xHDqi#7XaRa@&d-m;y%B@57+ye2VC;)&bwY#}n1g!cJ9FoF}yE!BZ09Gk6nU7 zm!2^EMm}eyZOt`s1s~V#!~s{M^k9v+4GBUZL;lN5(Z<&_glZ-K`0misJD zpYCot7k9w&=4sYolHs9!+9WB`cR3f7ZaCj~iOmLR?fp#o5^|p#iII7%_ ztO>nL*E*+E=zdQ`>+bz=iNr+{eJ+@m$Ld_w$4J7(U3^JLBKQ)d2>!Nstq6&t($@@_ z80a|pD@%P>)yKKfs~YlSEjLuvuNY0SmHRXt9XfoQb(4>2@6XYUSf%yr0(NzZkFE_H zE)y;rm}oUKO)tGEd8x$sOMbu25zF%nu4~7@8Q;j$;oCo4zM)~PCreM-w{BjHPcav7 z!i0H&KXX&B&f6_hHJ6Ld&d{bY%O=$`TpoM3>)OZYvfV*kBw2_lt2}=X)UNFH=quGR z${Jp&Bk@&}#XU!|`EKH<$%n#MnQhsU*qcb`2Ex4Cm)%TlV7S4pw=_0OoLOVpQI~E~ zvY7}K1hp^DK4uvihveFYb3TEuB(|-s7>HVLE}6LB1UAzTC$2{|@vMmEYzKXnG|}$A z+V4DMucBY}O?bKs72&>p@GOfCFgSbIOrgZ3D(q;zMd`PzDRGX?PJuO@DNh1QKlSKI zejApWER9y%yv2nUG&DWM*mLq@>#b+`xa0Ck!nIReO+1#VnPL&h-rmuSvbKW9jcC|3FHCZM3ZP@bv9^X`Vo^*bBBA?K3ECFAW{=43K#5-FutBcNV<&XrK)1 z(e7Y@oX7=AZ@pA%ji$BfOfNWd?bh43Z#mTE&1KfD zpl+VRB`#!={A#w%Lu7l(P4(CcEHS;qeLx6;+m}uh#DJYZgttxkz-aXsg+bAuJAe|B zbQ0B00_n6$L(7;r|FaV;qBso;60bYe7{dC+s7cAZ+sm5&f`XEr;K&s;uJM9%`$^u3 z&1HN({+nY~`5#k@B%+A89Jg1=1KfKMZ@^q*=A1vgJ>OsOVZ%9Pe_hA^V^xbXY{wYN z-*5OAuOExaymPdnUaIg+xw|HJqU*sI2j95nl0sZ|2gki=$dKjhtcqc|PZgmpfnrkD zhz1hL^e7gdHL#vpn_M=2>}zH0nK*n_8=qc`nxa&v0nU9BQN?YYz(0Sbq$Ah(4?%5J zVxh`7T0F!(`o!c-9a*$A7JDn_#Ji|7Hwv7OWV)B;gw=rTEEUY^b_>#5<Cbzw(6yixf#f1Q|kZm%>3k>cZt_w#CidqA>Y*T#H*=k(m!=Z`uC=e zY9;f8e7UP|HR`VDK$+R`B@cGcp6{|~IjgG4yOB-j>AL$ydS`hB$}C0QOE>1}dqx-~ zCltAl{Zi>M(??nZ8AhmV^o|43n~W=2NW2j5dJ-x3`pVL^+q&c)%wXVweX3WLDNjvS z^`VzA%eZC2hzh7uk92nEuG|DJUL5y52=(_V!ucE=>trO&K0aU-u^5>9S|=vNuIm2` z2KO92RqC$6*YHIjs{r={xAFeC53)v*-@muPZzK5bY!aIDLvbO+YCvPf>d~$%L<5WYOkW)|9IbQ!-t7d(+b+xc~uf|K4!+{tRavy2ssr4-xranU2#Vzgw z^{R8sIXB36oT*qeaF_CcPEp2jSIB&mxp57V&vA*G{F#I>UM$+7Eu5|HE9f@?#JMaDmbql~ClFGTAF;sy(Ggl*^8po!^41 zOApnYYX5*ee1P7c*iYIS9ziPp$3J^dPM_IH!u|v9=s;4jHGx4}TdYtp&^BCsK;o={ z33Vf5XC%S{cG(u->}~LPSN%g{EZ;0Virv>#Eh1q~$th)>SspJoCC~TTB)gN1xgCRw zF8gD1gGT&69+(=B zmmMzOspY)2t4l6hdr7?Dta$Y)^RWQ*4uLnuLCgf4(s0|feAVT)IWJCyY;N*1SUetc zrIPOqulJC{)TC5ZieozKP5RdZ{}uKhZCS2XJpGmQV1}ZV+Rs~|CV}y$O4snQitah{ zei0p7CRJbmOOdwDipKI+=|Ln(0Z*ha=GMcSyVk2in5N_&xhXy@Cf@b(>5$Vj*X^N# zhN8vg!(Qs`^1$7HywJYwkS%BsXA_+)q?eE;JH|E>8W-{mh4AK= z{o0uWgo_y|Z9lxjfM1LE(lo&EHvdC!!{wt8+jzKu`+Vttfq0Q2p}_vuKFu*Vb0fE- z(}Sl}&a)|s(Zb^P?PE$XVzjmk_z5kdTR@u|XJ2kx)*otAvy@~-rwkx! z{20-B6g=6x`%4NKs(iBeOZaBOw!cOXzlGx=n|k2)P62DnuF(I=0YtldnIpVOQeA2u z!jhId?6j;FE!*va@rFo4PTs?gM3s}XqAt_a+oWB`E3stam@(UuhkzB$oo;wXxh5bN zz4XN{!96=_j;M7llT<*w4DkRG1Gj zNX^A<%q>utbZk2y{6B~EKPwd^!^P`3(0-3K6swRo37>@PL%FXGdlbmdA$X|*T>ICF zlYJ&mT!O^J0NayDo95Ma5;#gy`S8Cd!eoI!{SA{6`z8Fp)Q=ZW9H#zHfcm5ZdqR`u z^k(=Y3T%(UQ{l<3*FIBN4Dsk>lNXFIS$dP=No374z@{dS} zw0*M%px6C|FCu!+ZwFkX+R|WRC@E`9!3<>!8bV|7%yi7s$3MSV%pc0Ov`-Z#TPpWp>M~i$_sq$fi5wE*l`Um}}^53?JLGB1;y!Y_H& z{dQDsUj;lAuhs?Y|-!9#48#P5ezt8gEa zbIE%T3|OCt+yniAsDXPslPLOGwCA*7KJ`lV$yiDkEP!N3`OOnVw-e?zIUV{HB7|Ul z1?S4P-n1WgNVGjc+YfaDYxJxkNr~8vPGblgTon0?R!)%wMt~k_xjxR2deINZ@L3;#Eq=@@A&5vt z!-z>c&h*UrO{+;7EczVD?=wLpEHNf5PjL`-y|}V&2h4GGs(0ZI+*TZ6D%Sv_bpNPm zC&yZRsDFhlK3#t@vu|M}gfT$+%V7jz-x###{2@e7TOw}&F%_?0FwU5SufLz*OO6U4 z+8&F79`?vaPRai>U$O7*>htTQ9(}QSc^=Zk4XXsXS`%0T07-scmu`E&++@hVND=nl?`O!6mbfGx7vzF)xSdD zxzNdlVtuivw(9lYsfvQnl4PJ_=fSQ7A>E{nh&mPZ$7!Vq+%DUa6RYv~!E8iSzxg@vgpMSdn zOsq6cQZRTkuXw3^j)^5@<{wxt(UZ^q1GDz|d#PPtC#~^~CdAC3s0;|jjV>{E?_Yr` zhc1l{nEKzE83>RNIoo8(V@@$M9$5Y2*6!1}VyyS7V&>)DD|5CA?jcg>h#;~PRttt? z2=W@{7HYGSxB$W@R7JZ@O+uouR%7w+WdO@L1%iPXZYL>M5I)_6a20S<&W@r$iI`+8 zzaY8eJGdzW_Yf1N5NBM}bK~^ybJ0*|-T*YDVUP~V;OGdVzkoh3uXBcG~8X2*c;~a_s3J5(@tVKbv4UbLnXQyC#*g>;r-m!pq^(d|N_k>^9!Lraz8gB_u~~2E%0kVfpWJ8oSHg zl|;|taVJ?mU70|;`EjI*ogP@g?Q+UV4ktT_j%>N{cRhN(9d{gw=K;m}l7n^(i%JWS zd`&7+6?BTp;UpFw6l>4Fe#Gz$%Tqk}nBNV;jnHTBQY#@s{PZBU)jLf76&n0r1$^`A zj#BOjvK#`n|T#-(Fy zOK;+Q&O?PEOmJ(0l#<-S_bNEOZKozFW%e`h^s^kJSpy_j315dFCZ{wT9hkE0H(#1A zB7GKkf8R0A<`aA$q98w@-A4fwDSfi&o&8?)|1a88Gu}VaLHEc?1=AbigeoTU_l=lf zX!!uW(UW{k4gYlqVrv>7Ho@*+MnVRlI#)#K>)@Ic{SJ}*Gq@swj5`%RZ6{ z@-q?v%mQ94YY?1r7YLp@YCj*LR-tpxCl(&Crmcgo4eogS@y8!zZIRezGaPh8CUaj% zZ{IIMAJC%+Kwqs)*K~5~TExtBdY%vgDOQKfUR*h$AC!(*!)MnV7M9ilN zW5Y}+r(+Zt5(%F%kZHUN83rN+ffC2!pJ5`gZ zX?RZN0}&&EV;QsOW!G8XX;59;SGTM*%`VlZEADI|f0>&ek056{-}&arhGl%T%y@NI z^O!c`1VPAASFtAg3M2mTL6@$P6YCh*i;YwpEyHJlzrs4<8V(?};-6a=wH2&hTUj+^ zr0;cGCqoD-Rh{Wu>FpeC8xxaP4u?U;AzT z*~odfhBkeM_H6wg8yT!7*@mHBET1k+^U59_RmQTOx@S2lPs47@e`0Y0oy}nGcxQ&k zy!8J)&b z-24cJvqhX+nV)Qpshr=lEx(pG#65GniAeHCej7#uN}``KgV#^jp`H(bIMjC`Dx#LQ zeht8E))*FzHxju)Do4W+MQx&IkJ5O7N|hK`xnju~SuR<=p0Z!s(??u+(=eEJ&3RO) z_d9^N0##7;v#7EZE*fp<7w{KGKK9lhI{VxRicug-{x-rZQQ(_ z3L%OK|CbriXcCODIolkjTcmHY=^Bk>TO)S4st7Me=O^qeFI|=GvB2iVYy*KIH9T`+ zP{XgV;!L1=d*!Z^G3Q)LHDlzC(MC0NG)&r2D-7dRGM`+xVwu?N7t!*D7855m;WWIH zY?)-Z7LNWDGXYzXF(^eD={JI_*4sd4`f{XIllO2m%-3hb0oPg&rv%;wWsO>iTim@; z-_X7mzx@eqk$ro{I$rabvxsVrDz8Z2dWzx4O)>;A{^!H!WP~EeTRAv5c)N7ZoT3Fs z-yJ%fC78M9r@_3~;~$kE&zPKWjh3vc$!gBBBe+xSexg`|u)D{Q^PVLOz4vgXRo3VBUW`aNsX{e4_jA#LTkg*AMHRje#mh0&FKQP#stsJed#3HEelAy(?z$p)RmrG}$3=8DKO$N4dA$B1Qk6xz z<3#DlnN7tU^NOKBxQq5;g>>ql*#Y#e;sG#zCS8TuG#88<(U-k+OIH5|J$#ue3kiEy zC1{l<&ehfY?g<{`5qR&kDKbc~$sSpt*l?7e@K^b0whFQO2Z)X+P~sMVDgC1K_zIqjyL|7qprsY5OlAmYDv z?}*HzNz3~I4G%Dh8No#RsJ^bcQ!aBn&Sk}n%%$zd_+pD)<0nT^V3WTV%h1}p?Eg2| z3Y$57Sr39%wVFix)MGQeWG%nf<*J^qt^gZO1XhOwPEbyA(Fc8*bPkJh0;F?NrVxD)Z|EW;cJdm_+uv4ePxkfHk zsl4I)$a=BwCKr8uoZyso@PDKlrCFl)^^m0m^aa!(u7cCQRo8G;F+4UGL+;K_19-iA zllbpe{O%foV4?yXZ8M7#rwbK-vv~Zc(!;QOrj`4F7ft_kW2x`PSv0SD0nOEeHn*vD zv;I!K*L}Q_$DK>k-r`W&j_JS4vN3i+F$Pn`YyQ7QI6OlNGprB`%XRKqqRT&inNv5c zc%1(WyMAvn@;q77w@?3Y(jVG3`v`8$G$`1&@+oJO!8#|GjwycTq`yC5au{brp zy)1Op3KAT);E5uPO4q;db=^Bl^ezn>2;PxIB8;+nhpcBFGa*lw(d&HAY!2Fj*55L| znC0H~OQ?r?xTjGNnawsO{K03&)*fAmUrWbvQH@)ViNqh917Oz5qEmbJnU_aABw$0I z=HM26`0Xjs;`;F_uX7&lc*O*XLx)TyhZtju_W!trKB!P?2-cQ2{`yY2Kz599PSNm` zSK;5ab8$R{IW#3fGzFv712zLB3Y;6Ry-?(s9+RHg8+$L?yE-eTt;#x+8JB-w?_5Cq z+1M)@s}efJIxExloa7uo_d46xv?n&uJb$!RTc_G<^?&o%3duT5JsKO$o?;t0Ejj+lxVRYvj!V{@2YALMlP7J_l^&gBcD zp;Ju_0(N1AlRp(J!?JslnwJib0muL0NU5LxnvXWJ?27|=s|_VP6_hY)O(X*Rmkr= z`rUPC&_ABZbU1J^Bz7F~CI;L{ILV$F6+9}L*SpAi3LE)2Ly~m5NaiQ z1Ksi-z}>TpP7d}!E!D*p+6K&@?~PJZ^p)zLqnEPwaEK=+MLv}p92Z!jUAOI$UWSFj zgSRQpuk7vn8fS6->{evNDLbcj$K}_?;fBa}vP;idEjV6X?|R|dV0&FfT_yxFiIj+a z>V1J&V7ahMH`dTI(qm|#{txNMV=f5CSi$9t`9cR-zfbA|`TcrRx`c+uN>Y$i#C^%l zNqIsj=hoWhtSJ=4BaRt$U1v_M zsb^mp1le4a_)mIMk7IKKg>R>6_?1(`>&hbzvj&AB(JuON?GGQHp51s%sdX32l+Qa^ zFX~;ke$Oe1TAQn51y>t_^vkBWR4;FM3iVVq9*DIIgAN~bM8{NTsTMIkyKG!Bw-~32 zz3kv+k6^CC=i)06bmy4%j8pGgggU#l&21}UJ+7lWaWD8Waa*mN^C`Y)Ef+nFQH!7s z4gE`b=ZYSmMw-~2!S8IX6i541PO&@oj-PdtsWKsv-7yrm>IAAzEye6i$SJ*59~NB% zR=lChoSe3L0#7t_zav~0@p!$5NQW8Hzot^LMKSqfd}dK8?aJ=(uRUXRY5MRWRQ z_9=BVc{WE4ObB&!-m#l8K+)x$Gnd7M>hwf48!FKx=Axj(gR=+nsj=INtrVLVrH=Z46}2n@c_d|gi+>N{rdSd!zTqV#Ts zP3*kH@I65-AulW394l)WVe;I8Q|0~%wj&}s^skbEeUq9FuhrUCJO6lkKKe@4aJHY$ zy4rK5Wk;y6F78$IduvIjngG^T4w9}L^kB5$=(hFe3vww{{uaKP?|DQsnL<1ZBcu(` z$)<)cxiKp1(8VZJJK?Q;Xlf?6=>GG$tihCdp-y=b%c6yL{S{OJSWWLeRo1ZWxQRvO zn@tB3w*aNX;9^;`n%R#Fd=nXtsreK=r<5t<2v^rP#>Qi2Jsd+F-+a11uQsl6{g(GM zht6oP9CQ`1qXb+^hd*_1?O8Y1tOMyB%I}jP>LB$5r-Snfu9yg8J-Mk@WMzOkbYe@8 zNHxfa#@P8pItJAL*6NndWfMrrn8!7Z#=3j>Z`^wM@lVA~rx1@`YiGeBwL0nUm-&aJ z$NOYSeAN|7Qh8+1)8&6!3OwHpp}>&M@k<6d{gw61vSr{PK-UnmX7JKQ(=$0x=|av z5X~;LgkIEGr54gJm&n-xzKEwxHPlp|6mG=3_Z*k4c;VQ&fqJ8fFzf*~y?{$@4DWlT zGWjXPfJ3YRCds>kWA?GBPbG_{R61)b!n})?Xj-Cl;Csm4a#eh`1*U?-0W)Fkp^UcXm@hkvkNh8 zu!~hcb8HQeDKedY>pV@^AARVAa3dndQaW|!6-LlvwCNE24I^m3;&+)@W6Fsn3 zTSF!q4>7O~$c3+ij?=Y%BklfER<0b^dlPuA=+2YO`8dm4yuJt|m7YEOxIk&ssvXN) zJ9gi=t-YCOrF-n}Z9Yu;I~SMh?_2eZ`6?7m4*xhue<8GfDoD&ZGsN?%fy`d`DPzZ>gLRvH@0n~=_3E$rQPw_CzD(p z7Wz}f$MX6LHwfo0B8v-?JLT16=NmWT2=Fb${u2F5P4hUxp^9M;4-x-OaM-+_SByL2 z0!6nxo{kF0NU+qY;8y?Q;DE*?0W_$@6aDOQG1Zw}V=OkN3U^tw*&jTWrc~=Yb!>Hp z+$3~|#xkwJjPf|_maCQ?+p+&|YZ}Yf9uY+rgf7KmxfYU@haSz@b?_i z#G>tcX}hj8hvyfk-_y&lQRgCBEK76Bo*mX7fS>i}hF#x(8$L z`=4tB%jSCg+Y9(EM@Cc$A&Rm2=nb^{nt)`eSd?m2g`Nt!#S9j*+XPrr>GEZ6tDPePf+ z{?+LBr@K!1o7R%~iq$SNzhWWl-$_j}M-gOkU9Rw07*#XFBEh_EDH{hg4hZwRUxkt^ zsGE*9*@_dAhoIuXX-4C2JAULS4Yi{+s|7M*-hr1hhmnGUkoz-H)BBNm1^2h!3_BiF zAB|PmoJh?jd$Mjj~sE^~U^JT>~fTY@GGt4bKa-7UFQY zV~p{{-JdU%Ic_`F85z@kPo66lD_SejrS5>VvW?z@jFL{=3CxsJYO3tF`g>8AxG=AEoZ!1IOFsW1z)L_;vr2Mv#sZG&WD0l(aG z#Y*B(wnoIw$qgiP3J76)#MemG*Iy&5&Je||GngygtrXbK1kpaIgADG%9x%)=yR*(z zP<7h$+fursZ^m({)sHcMVw%~qgV;(^r(EXL?v0m@_fziF30OHVO6mJJ<7Jhl_*A5D zC?(48wZlMM*iDO(xG+YttakmV;#Jsvp`pp8aOYZ7({2!4<1!cF@3Uph8XC!(2p!H5^Fh2`L9huis%p5#}7x`JI zGQ~NbWDGKN;I&+fU8r`A-xrz0r1c*gn;h%4#6z)yAqI%n#CpM$i!@dDpDthY^nR)SA`w5=VrDG2wo(_bH<|)H}XPrM}Uj4H+rY41RrF`hi$>UXsaJYCdaYG1_ z>UZiO3#jf=^XJ?hL&f7dCU_tO4iHhB$xkbfZKuY0jwpwz5~% zbENiFvmKYRS|y+sZ}>q`&avL({F&}tU?WV@zDw2{CR zEYIdTO^M(cE-%Dw8Z0NDS!bG$H3c3r z@w~U_gNW|eMM?45wgcd19w3A9la_qU|v*{DU` zz?gF0;F8LdT;;dP?xmjc&JMet-_q ztQf1VxzU?(dyqb8ExPB)@GSaNV#fq-`5q!jC~=*foQja<%pAu=e7*uoJKOwFXZ^G9 ze6DwVuNKc#KCkzx-v8OFT#l=eiR{OQzO9J-&YqV%c|>nVPT7$Ir7eb%T)aUGgwh0r zjW?IsjAwESWwlt_CN(z5%XNpkzu2ye)>vZie$RXdZ<~?fcNzU(Pjanvd_U&t$1}VA z&vryZMh>aEY0)>m4cjg~_0Gk_3BXor?t(L+3jJ5ME2*Ty7OZEm0ah^Z_@iPK~KQIoxY`B`P2jPB!+0~HYm}6}KY33i{XqUUb z>`62K7q`ag(s%dGeL0p5x`pd}p%zLtR||V~llMU2t}|X;Q&DglWq}RhMkfTeed}Ovzfi&rg!KL=_o-d^LG{(hf_z z4S!^gmU@2Zw1m99bz%PC)UhqIpN~0k)U~Zyzh6G1;fLA>rzZx{3EQ3r>%_d4u4`MF z*7L}NcPIC!S7g2W^&QFkd`N$IVp|xSZ_yl%`ugnel2v|{ zp7~P~HYr8`c;;zA+{)Pe-kxObmZdh#r^{>w^##*95RsSjVE9txyGI#SQq@U11lSgH{8lTRul z+jeNa*_dSn&ekfxDSLG6E2sKqm#GbPP^tHvJxubd&~N_5725X`1=>a``0Dl~wM^r| zHQW+bPa@5*DEiW=0k4fV00q8xt%7poxUF`g`nfs1sVvP^hHU8>~8TNR^m2Is>t zHB>F-+Cv5~R}UkYn5j~u?V#%9;5(4-zDUXia-Ul7VQ?}H0l+3 zu2+Jlm?_}u z7fIKC_7e=0RQSs6oKam@R>jp`QbNY?d2Fi^Hq<0mSC!T@y%O0621&ITW*O-v=f4*mpk) zW$Epo!~|^Ezbc`DWDN`Sm%guGHIA==HSW{m467 z*@Um(drJNIX?jN=<R}(%@?<>J=Mam8kd-(@@DCV)Mpj@ zY8om&cxC>`2;BLX+GOWtS8N35I1`e8CNsxcJ6bzsDS9_bl~|z&^pUsy2I;nvhoPq z?e}fEQa6!{&fP3BXsiD7wW7*_mMH6YRQ@vF0p1ynoXqokXZeHEiE~|1_sMZ1p7ICNFT!$n zu0Zhb`H5rM-z3*Yo_}rn+jQ{XMyZXr0PPmcP^b^#3@OyA_ z=ehH@vT*sz-{=#I``wN1NABB!DCWedr-o^Rc%lL5B)}`A2}_v;o*4vrhaOo{anlht zb7FVYMo+i{2Z7%v+4+(B=_$nfdpXDT`v(s7RC~+B&-uccwOD>cI?WsBCH;i-%o6LJ zB8{*uTq*;zw)nY%N0&0W>9v1$-R{lM>vqjZ9bxtuyN#Uv>ao!0WaOPkKil3je4UWl zqYhI$>9ta=@o@A!ufa)4!&R;PbZ-mOi__{24#U)(I+q`XJrmQ{cBARhd&QhAJRTr( zs|6El4To=S8g7GglCyLy%;ByFHr4y6Dgo-pCt?wrN-v7nkn@2j;``gq{rf%7 z?|#nnJm=i!{^xtnw^L($miKjCukAWRVb#_DogzSrp*QKp`c8tP#k!R8ag9DZj}cj4 zBVSGamu;>-o3ziVeI;f(4=BHCLbZhGbWa^Tn{rx=#!=+GS{W%3Z-$LqluS z!llC6l82$V=?bjFr83C9`=uY_i3Om*sgr;-=S^`^5-0jLduzX$E2vln)iTOOWgOi8 zJN!BsIq)cy-o9lk6p#!Xrr;>%COOB5+Oi+>j-;X6UV_r=Y(b)10-KRlK14XOyf}jz zsv()x7REWJV|Kug=wAbj#1`BM2gA5L#}ZEebiN00iRV#oGWJV;vow*`0xb(B@-{F{ zKG<8d^``C$;%d3>)RCmuahJzw*;}xcz8$)CSCYhCed7WP4R^9@bcFIy7^7*|5r*O zE12PGf^`nK(o2c|kP0k2qVJs(#R-O|m0bp9i@y%<<^gcMe`mb^^4jCl$-Iq%HO=uC z7(sxvR2Q(D91Ch)Li6~E%au1Cz+y7y=};?!FoUoyvRVrffcX;^(PrjuXxA+A>y9Pz{aOO zh^%MybKUsT0!vnONf{`^Y$elndP;J*#{EXng5tSk1B8xI=HbQ&tZxR6{y9+BuGR{OBcaBiJwXO#b* zXa@?bJV3NT7&bhxuWrO}|Ms}c&ZoYb!6{S-@A2`jQQLvVs4v&%b==F9ZLB(QM8VeH z0%wzqzWuKNCySdupYSg+xfQZc{mTSMu}# z=_CPaC2CcBF5RR&FM{u79iO3Y2f|4gt$jR$C+U@YCNufC#S?mS!q9>SHCYvWAJBYS z{o?Kxw$KQLE}3VJf)1@6^JsMqlmfQEDHI|^If&~`F;k#Ef|xHi1Q1-(gV(bYK|qP% z1_o`URKssy3h9#9kpB*Z5{05HzH}Hiiw;nk7ypSz`h}DiGHi9!hZELN{Q)tvxU6Lk z(Qg0?goiv8Ti`BS?wEt2Cj$&-k#H`1O;;>sfRhM`jZo10t8$PP2ign_9}(cq8HBH7 z4sAdl*sc1e0Ma{DJPbPWVyJYzpwl3%hQJ00unM^p>t-(h_*z8?bF~_5TfgjBgVRp8 zyzDemaB}c$j%X8mFFCWSwq#&{FjaU=NXlGy9s#{wOMRPrI|^8 zh|N9EGw71-Y@0?TI6I9r*qSuQl8VfrNX)K2D|Z?HB^}-g0T`(*KOLLe(~xC-*&ZYx2b@JUL4Wnh(GJd zgPa6*e7OqRR1&kwX^HwE{bqcNZpu(E1Rt8e8LrC~7ITJjN_3$mi(#8^L(#{%27W;s zTc`{}m0(wBF~%&Sy%rMUirky+IOePGrGpCG0BKIx1e@b7e^v}TbibLIYoA#xREF_^ z&##+D^=9{VZ;xB*kEbE8_h^u1!^1?q)-NbWh`SD5UUHeR6Jk32la_d!AzcFcF4`4P zYUl$k@N3~P|Jg3{x6pF#T=Ipn2_Wc;x&m48(g23}n}Nx?7&J>gXes^~*ai0>aC9B8 zZGQ$gM953|xC9%W01R*^Y7Ys(MC06y1(*d90Fi6=tJPYTZ@%L*gBmoUCrD`@;ur#( z6ZdEf9Py5q1m0M7`K;BkR>;qC_kxe!+qF2E-;4(f`{Y~mxZpj~AzkN@3BiE)EKH67 zC>@Y?Ls`XkE4}&zjM`D}gJ@It@zy$mahb=RXVbZTRF>^E<{KZcZQq@T)I2|Z8n>JY z5uyU!}BXlpWM{?{~cRdZu?U8>LU<_{VAlQ05CWZxxxb!_n4Rj<%v7 zK{ZmcXhB(^%z{cIHaLI^fJm9n5v%mg^V=`vA%n#UOWeoM42#ypDRGEe%wiW$8J@T_ zi6BIxU@~(zs0d9;(m6755j+)E-N)rRD@Ng{k<{_SITU(kifcr&opy;RAD~d;4h@<@ zXPMOh;pymVD0^Rz#%J5!;*d;@-zj?|VSAW~sPDj$zK@O%ImL1YZ&H1UPX~P^`Z&Dc zc|OR!n$dArq4N)HfY`ZUiUh6D66dE)I=_Qy$?@16w0|5Q7Oe@n7&5M_Yg#p;9i&7> zfTn)2rfl9B2cMnsFWwI~V`UmJ48d11SXQ}fDL$=|7?quxeR(oT#a zV?|%=!@02h?gJK)v4n$*)?qils=Vz#DgIz%W05?$Zt`4^TkVi}GmGbUKxOv*CR3K! zku?WS5#MtcO_+_bSd=EV%JXX77onzk<{OBA^R>1!JJ@pMwK&Q|aU?5U!u6GaT zM+Vio`;WR0%oC+Yy1PS~$+g!Ul%Gy_XXf&*N)6#%7DyZKJ2SMJokr`ep`mh?imHmc z+`Fh7X3Y_Tgm(bYJtpT;Se1%Sj~xCH+gB*wx(WiH>D3Dx-9?Ai=XE(=AtL_OTj?8WviikwTG6? zkM9x8=p4sfnAosBbB?bm2~+w9)f#&YS*P5QmY>`j!uQ^Cu~B(g-{FOG4Hg@D_YX z$@nX$z&m-ssb68O?0BAud-bU;enxA^+_t;1naQaF>pc9!bkWZCz%8AUZ@u*^DIi5) z=9WHi%$rLrKHHX1CSsgzl&G-FT?)zpo2M7fMkjrrQQd-G17x%(GF6DY9RefiwAA>; zW-u=z3NUg5f3|{$fLb5uiw4e5`KfC3`Q~Ny{84k zrO0oLoSQ_D5vb(G5J;!`pg+t2cX$`*?LzWYCkDNt4rDq~qjeJQvNpaunVxABA@_*z#dZrQ)1G z%U#bRthq^heGn~Rs9{--53CSn3o>Z7p zkfZ<-$WEQEkO6bL54=txh$2cUo#mHLMaMgUdVMMv8N(ueBP=wWc=*pOGLxr-ce}N# zUMg)~2iDQue*xMOTktm(-EU%t6*7<0aaEQk#O;`7^h2Zf7)F zcceQ07)@tYFl@rg$aD?E$e-U-{)4O;_9G>hau?$RYMKn|<{kYf1z}<7eL5`W4BpHY z7vbDVm|am>cu?E1dKYB5Q~UO#Rfzj^b&Ve+OCeI;t#df=8vQ>;`~MVA@dQ3v9iX7* zUXCpobf*a4zaTRI0eV5P5ddt4V5fUJut{8c)6wat@a*4o(JFE~hY}lK@rF|Ykcx!8 zvH$jOeY*N#s1$;gyyrHrNAM8=vF9h!HgW`!$Y$VsU&6awPx!E2i%i*G0Iy`iJ%|LM zHi!!g5jX{>lWBznMInV`Zx`%|@KKJFECqSt$%Z^AjoXdkoqxMG5|rFE;QliQHy7w% z!oFBiSU4puvYC9k*t^oKy4f1dyAS>_L)({p)fcVod{nvi5h(Ku4bIGygZASeYcbPH-P|C0Ya0 z#tEDfLbmb415Sw@2B;cNd2^t7tlvo+Y+f=$G2{`_`gVB>QbG6&A0S6%&rtyy?1Nwe zJ0y}g0Y|GIgp7uT5O2`;?y|ED=zjg{_}^6m8)pN%d!WuSq=P(S$aUmLEXPkb;GOAY ztc@7rpf*~<%PKW0%750b^L?c)e)*MIQzU^8U0UCkqHONFp4$zJ2zqrgJ$l7mChwoQ zXvQEpI;zSsIZz(cH#O)}$e;cFj#;B_!Rs@c__D%un{SQHeUp0F-sRwS0eKkxM|Y3) zHKwS%yA7aQE(c$~1%MI7uj*LJo5z6saH!5>d5&x-1r8t|(C?<8#o~Ua5XP{@c?6V# zKP9)wvwRtMPi$6e-T+FtzYLK(OE?Vq!AmrV7|}rBQ3zMo06N}8N!{@FX{Wj{PI!<+ z5af!20a*D5688bGxa%F!&C4pt41y+0hEuF4$X`9*fP}`C&S)LGe=Z2e@OS=YlZ7o1 zXLB;3hzZJ~za6>k2NI#;bT?+^BgT1|qW`|Ehc_I=UH_*rXFS*WLD>9|%r0ac%Tb%N8 zi}?r`ac{d&rIj+9Pyx_41wP?2wr?+(yLEOr+13+je>&%Y#YkaUZ~B^c>5(JSAw?S1 zj)r6T8pQpWuV9S}o>|W-o8aqiNP-8J4D^5bgbP}4ht}z=NIq%rgM|u6j94LrWMjJi zfq&Upq@bfa^IcN+>J*fNNyMVSS_(os2h5|+kXxd5{fBf5CRkTp^Tvd-yHQmuXRDwZcHN2pfsrLB z-CiS=m*9?mkr=r?*JxYa?QC0{qiZu7M#`;xlug!YNpF~e>7Z`BLZP>7Wu8!2b9d1` zECVJWL3L@sRVe+(+s)sAf$tEV^Xflcpds@5hgj~DTtTLCsQX!12Nm!K9r|e=loI`3 zowdb^E`iSyB&!TM%Ne`o6qX1;Vvs?8S6XyxTGXk6HV+P`EXFXILOzrQ|7$$ z{4Nm@0`5tjl`A^w=%vV=M11M*$$S17FIU&7p0&kp?9}R98bK*x9Jr`p#odn+Eu@L05@OKEO)F*|8{K<%acxb331PC z{gfj1)}Z={!yoBdfS3gXcE&A$M??OwtLy{L1!CwI?LFv#R zdCoedpGq>f{PX{VtpDlg)JuBMcjp4s?hFHb+RJ$LHC>ZIJ6tYgyudWsxnj}}(@ z=?uB5w%^~J*NB9%@bcGYfs&@L&|m;8fyvr%o|%Jx+Y3^i9}<3%h{vIiIkx|5SEimB zTyaNRVZ~4nI3{|U30QtzE&$J(x-!f-%9E>YXFWrAAHIkOeYOLelJm%uMLwHkp^g>V%Efp)pEsD6P=zF1FfK=rvb#(BvYjUGLv1qBRkHhB{ zcgaRdw7H3WmpOKQw7aEs_C5E<< zwPE7C{m0p_5708rAfgr6a8rseeB=(6U;{DVIFJKRZ|GYVl%!`4*T*#9$A~r=d)mcw zLYhp2_(l(<93y$haA=gf%ETP2hPA2-=M0&b zqpc!<%cKDVD4mAMd8@MAVh*36J-37$3CU5f_7h>CbaLCziF03oL=435Blt~6f&~*7 z-7^dBqi5abTZBg-@N~7ZG;^?d&B%2cRz(|ru?6V|l%zX41s1K&+gn-*%#~_11@>Bd zxkFgCl!FEFhC=5&s(j78y)`t<0+c=OGt9)jRubic9zelZKk z#&EK|bU=luAPYVDU#9E8ndSCPg2Ca;bvH!}?Vc{!@kPvn6N;7iyCdKy;u0=K&^<^_ z`gpqUaZBGvqK(=_ppLM8?QMxO40RjkO#}88KX11YuzF%*tA>J%JH}V1Z;q+TJ`wg5 z9uZYhYJE1^d{aAQW;N!BV2#<}9>Jb#G)jo%_coR@u4cGgzGO1LSNDu5V5tZUqh4zG6pIcR; z4dU*k{yPxqm>(T+qYqkGn~o=UbgJCU_|YfA(Ir`_ixZyVwkr>`3i|Th7H@E@P0E(% z-{DL*EDAL8Lf=f+PAPGnYQZC2_jI#<^LTIm2g>fn6;SEok)3g%EzE)Eux_<~OFo(b zP5M;+cL>Nff-IvhAmD(C@p@gi0rj8!iH-%dZzs81%^OcvQL(NKOLP7_{D&(7xs&W>PTEG~YUIR|xujdq`w}w65z^2l76~e9(whQUedI?rQVbeBhXZEFP@F4p9bPqc? zTwdEE47Onp9_skNdIQO0@=~_baRkwkhxIz8R<6a9fW`?L#z{Fp3O#cEdmN|ai3V|B zLScKwV+7s!bux=da=P>i-6S4{2sEj1}t zFwh-K)fZcqeEmZ@i5M zj1`W=R`U4L@6YBM68q^XaGJYhigoC0NsZaAy-fGh>6`#dpd~Uu3LNMdkws^K-NBIt zZHPFAkN}hJK{5ao#W1-yF7DA`L>1bCCQ<8S*W;3hgxCB%EET7K< z@G@E>C8xNvPfXnEOM)-YYH`xRy|^i_;-hmz@SZ%q* zJoz9G?v#wfkZe5nJlx?ux3In&xckUT0LB-t>IYEDuNJOXYORQw)KtVcD`~_=TLP z-_|m5DSbxo&H%8-v^061t9;LNEY^U}DfcOKbo|V8sbpxisLw9H;|kTev*^mJP7Da> zvLZeMx-3VIn;E1pwtCW>_f4CI1j$`f@4{KuFYW(dh33}>)2Kn~*$6TpJNAEx%wNAi zvjE5OXclFW$hH%UQ8Q81Piau1zsHc!jCM`K-BLAVi$&ZxL)8xn_gs(|vkK@{3w~Xv zU6{U}fID@*Luohl2s6kIhXh;ES2oI=Ka>L!;XYKUhgEaF;}ApU%rxz$BuQQ)@&7V~ z*sdvk7&MjlM=h@0m_d$yH%`a20G-a#uH;v8R<2w4Gm5%#$y|?#~ zD@GJ4ltU&W%C<MKOKCFXG^7j434#Pc;l*{Lu1aR zlbSEHHtZPPtw@P}`5+ydn(HwQ?}d}h6yg8Vx9RvgsA~5@Y9$2!)q|aC(Y~^%W3j8m zM>SnN_o>;Bmz`bNMz)B35Mb~={_OAqXvw#6`I^sniQ0YdL%ewzI>UK7S#no(2^GJ} z?LmiY+67Cwx|OA*_xoDO-3&cj@-M7_+~OUzD8X2yF=*Af4C`Hb3P~;Isg*wff^ zvZH5!2row=DmQ;W37CsP>?vFh-bhlyV=kwS9Z-k6?SuGE_; z6nH1oj0g-naost_ERr?`KKdM~iOd6CZ>0Cf_GnOIc;h^3FbA<1@>}cNggwSfY1cem zh_4V*88OBnI&dV9!7<`DbVm*wx(6;;>&-Yr7@-YPeWVDi9z@qpolzi?T>28o606j; zZeL~3%7G69^N_Sl*eDK+lv;^6$oymR(!|}%aiXg)1ljs#&#g#0{4{#-;8$jQ|-XY7y&w8BKJV-8jpc}WzwWgMEt8&Shz(l33EQJCBwVWht8l+k0;X8c$*Mxl zGWE8g`cZ#I`{|$gJi2>~HXag$g^9X+EAJ`c}1B&YE1yL%2G zxbhGF^XoU@w1uFvDf7wUQGAvI2@A%40$UTEpRx&DgU*^nKxL=MfY2nKHGw^k>3#r7 z7b2;!KZvnVennuYrLKCbhz<87z^<2TiIzl*}!%A&y0;= z=35i&rmXliY*%rA0hAM>a5)RM$0_MSCo_fcAG@bYp-ro#Z z?6?m(>P~j$p`VT-{1IV}M*1mrW4Y%EGVWv*>%L!Z{3dxh<@TUr){w4(npx6}WRKe2 zRa?{xU)zc?c`ghxM*h0r@h_zSyxd33^P{0`UhzcID@YpGO8NlwU$$(LiLK(m)VM-Wq8E-cL6^DSY zzueSl-mtxk3`IHE`UXDe!u>xnU8Kr#5~MxqWGdPsb&0X(p%n1)Q_jyJF4_PwUv9&~ zlT`q-wsDT_=7R^mo~v^9sC|k+SgedSA>k^5s4U2bd)=G_LM^vCPWM~&FZaq5{xV3J zSns)D)i}w^dp^M$Usprf%;%-h2KZAJoguQusL=Tj3cryt8mHU^_}iE60hd>Mrg|zn zql;}!RilfU>!;^FFV*3nwEoT{Hx+N6;*Y*$9asmOx4t8FC&0@dI9P;hVH<#=C-yG_ zU}ZX1P;4MFU5n4=XFw_PdPeWS321D42ywur1e!@J052d!w?u{i7P%62(jj-)dNdaX zMhaZ_W`4iQH`*GUN2ITff;Q;s9_dC9rcL1A+TaW7VS(D=UzwXb0c6FH8A~64U{j;4 zr^E2rl;B1{+gFC|bo&jHvfT;{z&w)1)bLJ4d2}`8ExA_2Vm@V{RzVyqSkQ%WC%8sg zcjr0{-{`JP0yyB{%?Xhb(z|eE!}x1T2R{Xup7mEBjySWSm1Ar@xP{>KBaaS0aM{lq z0sJ(uh7>bW&tAuumA4^w=h?nWD>-SjtS{8CtkB0b_ZK5{g7RrCW@#3n?HN9GHbXBz z#mG(1gX#LWS4;iXxu?vZg`GO_8gz^%Go;O*@F|(+<36FG;~|!9GH9Xnjr^KXKr?#a zK{W?7Bnob316r2SFgm2PGs*gM!Z5>I(BYXu@pp)*1|F0_`zr4}3KRWcE?^ntU3*XV z%Wc6UfoyF&N7)e}9wr(D2-fggI8x*Dp;7}>?%oI{yCc6xM?T*RYP6Ei1v}7AntVGX zJO~)JJ$Wc`;Hjrp0ikUbgxK8o)Z8UsOzP&7>8B1lnZyLE(H?yjSmPr7wHe86LbQ{> zB*m(}Id`mOqj&z!JYp$*p2<}YCx1xy^f5D>_S9;}czdg0bF@IY*Y|a*Z(pU%#*;kn z)_`n7kxQG$%$^(DT4aS`2lUUZ(WV%4koubjngK6ZfZ*$2n^mcC$UKI(|M>3^bLbRE zT=S4$^v-?J8Soy$Vl&63+FLY6>tAn7k*$TK*b)d|)nt6^)uh(1e>B;@`Lmt~K_c!W zz7YgBfouN9&iX<%fIp?T%K*3Ts`A~qC3A_J&w5`a9KeVe1fUI0IH|@zZNrT2m+el| z9b7cwy5#mA0nWHEIUg3M`x-}@-m-eAU@NIctP8a;t#w7SlGX(|Grn*Z-vUKrhTIoE z$LM=o`toHVkcJ}o_oKqsQC5Gl5Mg~sLhhfim(GWaw_mGSV_B<*kkj1W>cHb2T?SLxY=t+xz?dM>v2pU&aGe}}eO zqyGvI|6jfN|L3FsKNRF|-K775*7)D@HeT1HSpcRo141GrA*mH1RS@Pu)MMth1aJfo zGVK9hM`X~5zzs1nf#={J*r^Jjy#N7m{*m3&9`7Fh&?aIi8F(u*(Hn zv4lrWj9}>v& z^PyPw0QPVXB;2{k!K4qHvLVoN*EnVCr}#M_KJw8xuOCKB#BK!0|TBvd;K@8!PFUK{<6Lx?B2-R5+R_jXY9kEAog9yJyynd{SbE&!`Kj*tYN(I z?>nD)QJT1UBBK0i`GQV5;ci93uvWKxsIcid)V8k=FuBOznFlh8!9H`==L`fU^{4<_b61SXFPNUiVlF6?a|`V%$+burs`wE_ey7`nL!7#QoO+ zMj_KscHHOjL7H$~_CJKGR_eQxA>oCvS^aa-giACpCSirD5i&$n%fx&i&pWYjDln=s z`}&xG?D4w6dx4a}KMwe7$M%smWM9_0Ki;^*%^EkzS;IhS2)X&z6wY?}RrX+MzaQXO zDTl@2*Pn%hW-bQ;?#`RO>mKcX3**p6fwVo$i;g~L{Q^g?3C#Giq4wAT=ZdOfMKYIe zESxL^P{XAgvWK3{p7(j{Yh#1B^h+w>x!w5q@2>a1@r~2};~Ss6-2DMJf}~-34EXB( z6RdPXC+cL-PfmnvE`}X7d(2k;QsoeHla>kb$&UKwpj|gL#NzEiKk}Ji2hiPrcn~!DIOPortnp(ikMb%Mr}1e{?Z@KR+u?1+OD8oeF>C^&y! zh8G|QWOR6FYZEhsAUTE%(w{QD6AF6nFKCr?+A=s$iXeGQ z7Q3ZdLtV$cL~;#OsAtC=lGXJZUs>DV8zu%XAI{63uFNkN` zodRIUt_s9NB%ATqr|hTfCpQX=&mer=`!Mz^jWr>!{jb2kKfGkyd&rj11j2_e9v~Jg zxGvdXD1m;15%pZbqUs{xAqYZ3CqpM|2zr0nJxpfz#V!K&amG7KJs|+`Z0Teh+X7AG zC1<}6b#zy&qb8zDK*C=^LDpv$Wpp1jf<b8b3(<2TtSiCcGm{(8oruLW6JDYzg)=A*z))TmKn=jdY7$bI#8eIlgl@sp zhtQEoY#x$4hD3u+IllX{0~I*oH?ZRej;no_hEZLI?nOA35eaS|5WhAcVx3|~`KgEQ zSuC(;V2+1>M9cy%`vDS)C6U#3{`FJO0nCyS5+*C%!FQh8!U z=*D@bV_M7ouw?)VkU|g-q`D!xZ}Wx6aNZ*cYOJjwfG!=Irh_(-ZglAB@KBf91Th2i z1AsC_Wq{=UB5XO7$wq-CN)ZH$b&;aLer;YwCCBuEr{5ugtRa4L4s0M3Q|$^Ri5g<~ z9aM;oBSR!{FQC<#FmSp-!yYPpd@E8)dA_939dJd09zF5$797V zZ@A`JM@IptcBoTB1O?F5qW?4}3~ZV;oCXM#gh2h>(R8wHij2&t z;JdeO&6En$HzM)aKl7OTVZ-~hrv$N4Yv%VtvL#@KNUpCnEb3vz78;tAA)ph47rr(F zgCAgIla=0AeK_Im`VDv@8BsmMF+{r*3Yoi5$w^aM&EG+#^@)^^NEKKlXZ$h^OWR_% zQ9-I;xO^5dBOp{5u%ujLxk}5sQBPD2!%Pnx0nY;=s18Dna*$9&QTu^O@>O`r7$vhH zZ^1vXLM0|qZy1g+w5+E= zyW&p`M}kV|TdY1L@QopWumhr-7xWW^#gATV0H3gWgpHgnnJOdp+e)crCZSMIrVcf5 zQ{E93h5>i&_qmba2>pcN-mq0k1hzoSBf(exbyn|FCkf@-6+2y>Z3um@!NgV*#VX&Npy=KJScfM zH^i3*S5*(x7KRb|8Tg#TUFi9_OIy7X@=k6PAL!$UKWsbvx%9H_U3dBJ=3Bivd#sf5L3W~ z0i-;Bq+`c)amNoOs*tN})-D?!AvPJy_?v;N)bPQoR818k*TwtKBwpa2jv-Oyg+J9> zGN|ovM+nXp2(FE!3Iyvgg>Og6QeF?4vKM07l++e)cY%Lu0Wj*!zR_)LUErSV1WDC2 z`7)T$?K3*trEh2HX2+Fx!G;Bs!)vz|eY+~vC_;sEsOJSR%3hzLTOAVOYo%{xCnxje zK-$19I2(<;m^kB`p~*nfS0D}UDdW>YuW6}9*i*(4nPGK)WB9uK+jMmL2;R|R=Nn{~ zHk)fz;PcY7XeL zx8P13X!0@nmHvTeerZLz1t|f66zaO#ouaVf4mxp(cGZIh-84_{VCp-%X)pP5H=1DIguRM(@~58XMxg^>3n0c-@&3}kWVo-c1xn_?@4>p^?F=|R#iIuMjjC_bL}gLpOXBJ{q*fQB^_>*PmZSF zyo|%Ik#sb85F(}Kzh4x*PBys<9w$>!gSiziN)`^b6v|Mt6B?s4^-LnQ>ofTBzxBn_ z`JiLZD70MpI3^;pO*PKrSMaAC%f<6rUMxYoJU`;SmV0AqP{HyuJ|{#WiO22pvrZ1J z&bvUvxXDL|tSY`b2ST{hGq=#u@E$*;f?T?k;;laP@LsG1+G6stjJ5A2X!@4kAYNeA zk@u)iSamc!%>~xJDX&tiJKLl8+@MrK$MJIF0_ytvwLd;qOCW%)b`Jq((tO>E4&bAR0>uw}t30+2vXJV4n6t5A@x%&Y%DT z(!)e+lLQJPa#Q>{b}vnFFctT`nQ;cxf~9o!s(4`#eSMnuo>mQF7SA1D^0&Hp5+Aif zeJ`V)4a7#d&(9z>Lx;>)1!Y|`_tDp^rSNPkqFixB@+5KL=P&G4vO$*jI<`#U1YP5-2;Zw=@rAO>sUzWj|u)aU4ajA2U>Cj z(XMA8*uUVBxV1$OH1+%La2z;6%!O8Ir;6|)!(*8jrx;Sf3O^l5Y<%}9dF5CO?yJu9 z4o$SP?4(kOQKqV{6O zDF0g5*gGKfN~yQ*>9g%vM(y!&@lKR`3z0}=A&S-8!audv-=tE1KsIsU==hNHjQF}{IWK)TtP)YXZG^vulQW?3qMk6pd2nmRrzw}SXVpQ+ys`6| z-j2G7-V8c(%r4DoGp5eA!c_N^jhf>)r!IBVjjJmAnfYpUPUbUzW0mEX?fRpVXVZJZ zom0|9|BgS`hok!PiDzcr zvKkZsMs;`IT=sk^+49pXI#{}|?iKYyc){#<(N`HcAwIw55j zBZk5(B_u9hqKk6OQnEk%f)4fb3%A78xTLS?M&(6}@vg_U>TWYQVnhna7opW*DR7R4 zVDe6cVYC*UbmM>0^i|U0uhF>tA+?>4 zS=Qn38BE@v2--}XL@@5fhv?t3uS$ut3l7_7OrC#2DLtV*6N zLriHaO+}^TnLnLZoE&n}hs8s(%_h_iO74BheQ=UHy}K)ljhb=i&I|t1g<(~bc$eqX zeNM}_^*?tn(5rOPf>mwzwich#N{8`~cYM7aU*!Xu7x?!qy@FN0g0T-hcTL3{St~Z8 zzJr;4{M0ja2rqr}gmqf*+X8HhkBUHS2fQb9I9aXb44j~m%UvxJR`|+G<}nN^^C5K9xLC{4&_84?f1D;0=i>^z2I9CtqGFv>5G&2Z;_ zgX%E6(G$9gDM%Y5IcQ%_L1Tu_O<- zDi_+K5^L3)#Po;?I0$#>-$fJ%i<&x&yIXU^?I0cMl^Y~+lozRs@5TB2V0bCQ5*GI% z@fhiIfqk2g0)?X@OmSTyg&Gq{EEV6Cjxq?6tft(QnNUn98XR?~*)MyVI-Skycj79w zjzM()MHK4rO(lgZ+6td5j{D}e-D3l|z9c49?Q^zpdEO~SO$SNXFt%ifc`^Gr3d#P? zLK|gFB zX7M-O(&Kez@XljeSo|7gQ*uUd}-J;Jr6AE;~o}RbI$Qy~bkhw(vh4PC?!5$0n98@l9%+ z%%is;zfy|wip;bNV$VK?7QxE55+ zP<9vY>)5G9D(^TbPko23&Ul3@Q>&9{CCPU@xxnM$Swg2)FC^y=ulpHA{8;_<{!Y#M}@lSfM#H7|{K9k|cm))>n(#ry9 z>9$yI<#S7qH4is5wds}$6p^|i_?JogJ&+7x`KlqL69?!crwSo$SpBwGWoY9ki*NIcwK}e*t ziT&0=j!Rr-}?%Tk+R?dgM0bWME4Q?DY}x6fT}o=I zPqTjWnW!XjwZJJz4HE~$cj{_YTwV;i(b zL1G^89iJ|bv{ndJe8hLV%Yq{`-~mLT^*e@yIlDYdA|Hz5#w5Lnt0X)cG+Egz;`8fD z(3iwTi?=d%bFZrS#dMyKmeioLrmz$~(Hy*wzLoyu8jn(EoK@|Uy@Ib(zHIk%02KM? zF0IZ;i?K-NMD#GSU#*O?b}oNN=)$UIvQPB7a!lo+6$UdjeX8qMNNgMRG{w~wJPJLN z+Ld!YXnXW`rB3T#RTe7~7Y1p!L8I3)=?sy%b+}u2S95Tq+t{m#vmj*E@TrWSu1K`LlSRifGH;@l@S=^*(4;(FXn$*Z37IxC+l@Q_ngy=$fWgnU-cZBOv1Y$ippJ=J^ z*miu-`yy_oCk7J4;^|h$H$_TU@+|f(e_L1;r@=9z44;CI>V!(?4{=m?b(x^JM|YGU zeVzEwV3BPMhu{creM=M%T7Z(!SQ#K|^7VisxR!`n`l=N%P&kJ9qA$WSj!K&}Y|5zv z9c6!yuzD*g5?cU;Jso$8mxp?#l&?g)FjZE@mN|p#mS9sHf!VxeaH(hHU<)<+v=QY4FrDR?r~2k&C(`8ZF5ZBp#U&gONxzS@ z`Y7_)9Jon5Vdbz`V4Woxz+zG<@lDT7K~f z?Y~%t(46rOvTW$IGd4PYOsJBDaeTjOs&$9XnQmmhB@YY=v$qP{A%I@9O#y+aJ79LZ_H-#e4^C=4Vu;n8P!E3k8X zVQo)!(i5J|An)bN+cniZX`))){zrA^ z77yjV#_?BEWQ1y>Qj)Z(v6Vy)E9Vl*F{1{B!e~&ILupFJA+jh|Dk5h&%rJyOWEdSS zsf3|Xp|l9&Y#hdDKd;Ze*mwJ4Uu`#T4Db7n`TzgV^ZPyDpNuDV^CsD$mXyUSTaDs3~%?dfqVe_~M&H#WCOdXlrPe_`VzNEsEWLu4%#psnJ{@ zYeN}YGfB|DT9HwbDuL#=Pv%vIiY^n$)ByPWTGt?u7@JW)Tz7tWIeSj}T|g(v-y zu{i~nRdi-`<8;rk4LWSVIU~TOVe4gFMnWt{I-5J+^sGT*H-!D9?zX;vPR<`eJkj^MHb&sS_Rv{Y|tUl!7;>pR`S#WqJT-|$^!!piwkC4vOdPLMu z1HjGIt?y91WrcNDD;6}F77D0C`z69KKN)dELZqOhG*EpwVCF42N3PD$DJZ@0W;)dxUMJC4z;d%M{eFtp3EC8;u1%2RwoL@ zC6d*gq4lm6D0uEh{y77xU0p0T5-90YX%}mootR{q{u{H``Fpq`iT6OJ_rLG87?7X+ z7Mm41o0_>*4)ziIZh9m|em-Nw2z(dy95}J2=?3Q0y*L9;^lTk(t~6Cm(`7{WI7ns8 zUji$&9lqvzDQS%e$m@*Xy1UHf{A?!92PZNRuJmnAK>Cowwgf1->Mq-2?{^+^s?7>Y;kN+=BaAh zA5%FZ&Fjq6Igf5C}7Rt<}1g zbg-4e7d-Y17d)*Eg_(*-)IsRYIszH}3a2;h$wvGOC~+hog*MU=PA@0Xldv8Jz;|=q z_-Kk^e+Acj&w6N+Btwo08c3IhbAx9Ky%3C-q$wnD2xLScUtkJj ztqBj;D14?5FM@zv4wEP~v#Lh6agx1y_(SVRqSWJYT-6`VNw04gL+-+r&o8KO6Zt+3 z`B)NRyUin=wwtf8IWNUt$w}shc7vAY2ODqrQ}WE|+m9xvv4wNxl*2x*Id%uxK{=h{ zksT4H2?U8E&}on9SbtijI6&^*)2%Rr(3^#a(B!(2?7r+gE(KTC`NJ{~!Y1G<{1s;)$ zO$N-@wp&J@5B4>lc7H;<+j&PRWQ+01V*!d>wNOeiGhg)80$nSh>fhHD#kJ< zbiWv`@4PtWWu`eHj6`suZ`Oi4TA5TyPV9^9d!u<31{N?FeU)J6q-Hb~0?bn1BhC;K zR@5nzx9uYJ_?nLKU#LNN4(dYpD#I;ZtCLp!wwtHZZeP`rCi()+?73Qajw-u+LO7=!rw!vGl zvISNX+O}VH%|Yrwj~}bz9=+;Rz&C{rq3m>cfI6>wQ-IG{8XV2P@}%|Cmpb;yuW73c zT1EX8jhbq9iU883`?e7zJA@TO`4xWISMBbMj-(w5U`4GZjM6Eb&EbA!`;Q_EKzeeC zll+pQ?rsv*2T)T~kzme8^7^uf;gwR~phY_Tz^5n3&6VMzJ9pTXJ-Xn>`tiVdm7hVx z_*};zv1dpn<&QgGwZYS~`z#8J>HnEX!r1yT%tU2Ko$^!7s zq|w*;e5qwh1YVmR_Jmxt0I`0SYsGkTvF%e>((EYsUhs*a7=ip97O_uGB8079g zrn`MhA-qdZsk&3Q6nJ%--^_5si@0pIB9s3o_w`-}KH=kyr+*uL*pr3EMnW_mG_@{N z1rA!Tq+xFc(nY&iLdgy+4JalLCQ?v}Gm1ppveRM4|d_8=t?~wB7rzDT18))?>ba05?rm!AO9sVk?m< zZ9e8zqh?2euA#Z_~~BY(}AW<+v*mCgzO#==k%JrelC@qtDyh z9G`u*=_Z?eRT1`|1b<{)^?}SGKopijPIKQFIzfj9<1qemVpps}Lt5(a2WFc)2COLh zk|!U4JF#}8Pq9U>J7K#lf~-FU(rJ2KhW@pxx;vj!Epq)jYAuz1zX1~Mp7*8Tu>;Qc zTSzC~#=DMz*6~W>NMd79VzKNN`2Vj4$oL`u2I>2uS))B?=Of9$aoJo+(WN_ar}ZC( zBD_eFf~@LkjxkH>Yv(VGn&Cx2ojw6;%3G_Cd9#I=D!2j9(4tCv23SODU^U)1Bz3kR zOkw83YUI)WbcarhPw@A<;p#pc{A2z_zFEI_?`G!~6)1ZY(P6{27+Qdo)NQTUyH38h zN~fm!8PfM4mB$0f%D*RR0S8B^i~-_u?b+niD2t?RqA^XZ%zWvh(EAU=;!Or%imnH^ zIlgEcn5REe(R6{}h?g9H?uzrq=11#E-E9;TFk=mhM zbDn&WN<2FvlC&S-7>tdu965!|N6vhH8d%r7R_2_~RMQ)rt_2wOAX-{I9t}Udc!t7? z)`0O74)dEWAqWUHqcFi%+c>@hp;pLgdiODe=4o&IbimHNBXt+7s!`zp4m^!6%N{p? zb9*I893}`q%Tra4EW^zBmjJ^}MAHPgJU?{u*HD$}XjEeGmUfPDvfz$H1)9d7lOk=9 zcnMqy_h=e6`dMH;8eOy-nFPCN@EmIFvNAsnOHSMb#D3o0#S@(48bTA@$^rdwM~N^S z(FkZDfPN&%3!3jSz{}WWc^-O6;Sd@-v4BQqf`PpUvX(X!dh>y z*{X)dn?cQbjb@h{y^`yGY6ec9JbX540o}>Co9<2n0fbtXwfr9Syo0!2^R-0-#`s6R z)K&jzP3uBavB9SllwD3afHw0`Hs?t9lUzvKao)grywjP`j!k<18oYjI2wJGIbqQ23 z_~!l%wsF*>-$f#p=xjph(t$7(>kDm?u>)`j2~I|dqi0(I=u-j6?ibxkCz+yb04&2x z8($tlg9X-`F<6V_9w2;_z&tDivI`r9Xp8{r_FU7RV&nm_?(#P5gW$H+kZv=MYr#=L2RlqTRvYjtB zS_H2mA8b_0Lscu+@pQ$ByWY(E@ygRW0N7u6Xz}yn7()MwGdWrBKXJ7G9cYVQ373TRp71bAM3p?&3(*2e!Gu+D}G-!HFxLbO_{I1jLUF~bNKD3;miW9e&#_x@f+cJZL|z)p;&wt0$so^0SR zCjyM26$yqQA$()DWcXIK*KQ}7lJJWY!V5KpHH1f#`G-9UhO!PQmn66se>ZIkb?R1; za-+l~oFx>;n7y5=+MeSvtnrx|)->=8k8}I|rf_#{AZ=}SAWhVBE7|GW?Tg3HUP&Ix zf)!L~NH<+ch!G!Uj&mv0i z&$aIk>UY#ma#SpQ!aZXOkrvxoxDi&u%FN*JFY}39Q}@Md9h*{~)o#_auguYG?1k;u zamitdvei3FO_zGtgZYP~%V&#qqP+JuRURv-g_w+|=SZsy2SP?R6y*$^vfOV%Z1wM1fH?Al5H`}~Z>nL>CyfuvJeia3^wV`iO9P-l& z{g79CLWoVQBr@jW(6VdAh1Xh)bZ%oWaNG0MN=c`U-Rq|X)_*um%$ru!TjrLUmd#gf zQ|qOr{%{_!-sFupd40R{uQ2(Sy-X}mjo&Az6rD^uJsLDu*rm8B)KTg%;jzCc64drd z|G{iHz5C?b7tGjzc}Lfo{07OW=JcNqo4*1%I}cFmJ|B2w*86L3w4A?z)*(5kWJs|3 zWbkGEXeF^-o_+6?4yn);t)fxFkU2|%OzeSpn3$d00jq)QY~OglJ;0?zC>~<)ct>;v zQ>aUhNvQiU^S+5OxY0XQcu6{}sdBkhsAoI`)}Lg=*(-OYO<-;_IePmHN^AyMI4c&e zCzWu%F5UPQ$Xe;XYOy4`w-iyW=CwUrB00~*A+}IM#5xPp=q&Qft+(Ga+Inf6HTQ$j zc)w*{j=b>V=inimsf@JTioLDrU{U8#dXc3oQuJR9_0DAb+wd0t;fn7AEqnu3n*{^yx8&Hv z^9|q!>I4-3?%+PDu9}c<-);x5f(bm!E(UZl3pIu&G@ArcyP*-OP1kPo8&t z&}BHMnWsx$VQ6nMDb!|d_k`HqhVkAWZwb9a3$NyQQmANVQ1JflymxpAoe@8j`JYN`r7GfN%LwB9?53RMOzyLB$rKbI6LUk4Y_97mBW$)maGo3>@Pz9oig znAiu?euSvTbA7O+><298-pMP`R_|r1M4d<0VOT4ExD0!0qLpEr|xXZ4PK z@&M&J!yV-(=h6NfhdrLYGTocEG?kU7@eO|6)D=oyyN{Q89rEiavUFh|hPR1svkf;4 z*jDS6j=hv)K|6YUsfm-}r*mEFRx@{7Ol>s;*0=WiyX%@&o4=zQC*r;J1$I}uvfhq# z%5e_Pgm{pe&c1Kr5O}`1w>`hSwLfd?ot3Ne#3X+w#-a7ijZoKM*rr*HDG~bGZ{SBR zUuohfw@L~nyY_Kyz&8GIBZ;c>R4~3hjzDcEKlyu#OA<+=yKPj@J zYyvmk7TTj+QB^lxT2F1M22sr4qkMhnL7qLtcXbq7@}gmLN*=iZ<1pi)x4kyIJl*?> z&bjYACqMj1P2b}*e4m_Ppfq{S^leJVmmW~Blh3civ>JT3Z(`4yrktOicVl4m(7hMy zv^25z^<}-p|Qy5 z>c&`@5(_Lf7n^&d_b~|Su`@MI*$M@*jy?D>!_g#uY;|a;7$~Z_D83$R8 zyTn*#`6UWodOctqDk51{Rh&7VnU!l%SZ4e4DP?P~)B(*Kq*p0Q3}9GUAsfkswUytA zRMir>&QopZVTMN_1~;HP1WP(enurY5SS;=ESO`qC96FD&7|(B@9xK z&?7!XI?1EqufDCz#FcTJJxaTn@AI=$i6k+JkFcLw^2tByk;+P(>UrfmFC}ccNXuAw zR@7rdx}+4B8lBlR1B-}BmojhrCYi#kyzTYs{lQ6N$q}-jrfdZhaSuZ5^qyFe{-|>z z73E#qI8WH3m1^0W%XGlOcu6!!*292F-iF00JLfOUsq&)VQI>qSc_~q1x7g@aOSwb* zuuFkoPk#>5{n@06QxESgZsb+2^-~QxJJuF=s8rm$f67pvb*3wMk(aGiE&JYR$d3WF>5`QBQ``oC}X2rRcSo`Y=h(vsi0xXwPEjl;UQyH>M90G z1CRTwO9ichSzGQs0lj4#Z1q~+wSFh_^kL+wEwf*=@vpwA8VwP;wZFH!y!p#P^TS%d zWfz3v*al;vi5LfJ&z)1o%XL?shdq`R%J%oR8@_M1D|`G7Z2LxZQkbiAb+?HeLKr*p0)4~4G!>6gq!sI^xMOXLvq{8NM{CHHEO+3u{~F9ug&pg!MZ1uN}`*0<0eh4dVIr*eojq7M7y|To~wD&Q*z z#2T`vH#fAl;~I!*gDWQ%e+)xz_Yh^SGL!X~3v{xctUW0vAMddwBU;_alEt%47m5+A zfQd@ty`yxVx$yF%?%Co24zb@GOAvlj^_;(75HqEXe7641l`fEvMSuP`^X}Q@4r z-&^0|7pwx%O-h*Gk+J9Bo}{c3OIENY8Ln=YDQNB(l)04dy7Qamaa4I|WSxWjYZ_tK z1?k?pZPN3~6<$fV)ZX!uJBQ>#n!kAHHJ0f9lZuib_HwzWcSQp1QX=2F$)DlQ-}`ye z+swI?vH38mp~Q%))`Qt12a_Hc{Kzbfr5OKWDQ6~GV8Xj^%c04oNa179cc9WFOI|##R1#O~2gKiVC zo18<#ByO5ocb?K_Q-0i-CsqG)F~;tC82Oi^Z5S#IeYf$Qv%zb8ELuqc4lg7qzJ|X? zh$?PT=mZ5uTrP$16h_;Bl7dR_!v0!GbxH&M12#V%m7ioi0@W}|^FuNrTp3se^)_C$ zav}!xD^G{XO(wa)_{H`2n|rs0t2`Z^lV3C2VTU;3 zw&^>V9DPJp>H=Zguib$5JAkZYsx=qmf?Kx9+wMEs=sfVeAN}bwnNkd2zgd#U9$IB2 zdQbAo=gMd;q$K(v5eRhdhP2$P%4pl3tT@@h&iko)a;XC1>TjE>$_ML;5FtEQMZ`8n z9@BWXjSA@usDBVp&=bq89#s&4LSy+kk%S#hcp$))Yjef458WI?L)*6c0@hbIMnl_% zvq&hCqL*}RE5oam7Ttn+v1O%{QQLd+B>Y3J#V%Kz^6sY8% zeXA7IMfFqK(IjY{aj$;W@|2!o(iPU5tHbp)wz@f6JC*a_`?4)bXL@{ff-@?s^54Bn zp9`D|m!MqgpQ~Ox8+?D3nniROJ2y1xH3axA1bZpw2DQL4dq%=OVHrc`CcYo14 z=-Syk>wV7_sc-yf0N@$L2fJL?gzAvsuVvD<<#n%I%lM{AGmg7Kxx0txT@Ck+eU5z7 zalivGd)rTgCt0K+k|~Cjk7Y4tK~U#wXY}WPvnVI8SM(yj^;X~5w666*Z?4|*W_`T_ zYv8X^3C(Z)b2&vuKZ;0>))!62xy_YM{5Bl@remuB1p##uc8GfFd%-=?;E}1vCq=1} z1}Hw>QnTevbN*3X|0#V^@wTqM{K3=lKhra93RMNL!mYL_>Yhij2`^Jt7y zjwVScUl9fBMA0p#n54}p(RcEo5Z`%8HJ%(OLpFCVr^VdLDWi#9XB`?&e8f`!+0shB zo}?U+AyZ2V>K^3kUW=vvb1hg7-Dmj}(bUFV*J~$z&xX=)Q-xa47E|TfTInbG_P7Ke za9yOHc~^U)<>0M4wx-2hE4G?IpUjOD{LDksFn|2<)d2sq@>Eof^y(cchFycBj)obp z;S!HJG{@+&mM!?Iw7k-(cwMJ_;J5_;Z2$KR8t$@Cb=oujw!`Gy#VFAzIJrlbICfr57oU3&=<9>XJ`m*9@M?h%@e_lZP8(mA$X7wBK-eM<(=L zg%|Db+kx0`i53?!Uql_HssDCbj!mU;YIUEETXoa4 zVSYEyQRLkx)*ub~>=3lTSWgsGbG}}lCZ5r*49Y0`bZ4k+Q&s`RWE196g|V}A|0>3o z&z$$8eOb;^Ka;QWol@QU{%vfA%$>JSDH_Snoq4F%{CW$4r~wH(d#LDd>?w3SBnvysYUk;Wlk3G^!TfkAL| zD*hg6m3W!7u-;h0j)K5!r*5=keC9omDW&s;wNbFW;+;h-*(T$_bKq~rdc*Oy^L*GB# zMM0h>;wkI_o{UlH_4ZAR6o)!=%3S2#tYCg~_hlp0){*g2I}B}EZ#z75lL}_WBMkJ- zA&u&r0`jt5D(Hym)qjw4Wx%R{ciXFOx}&}Q?G0847H$P4f!m%Fu`ZIOCjc{kBPof# z<9n6`?SV_SF7UiY>KoH26|W&7$4O;eZ{Dz0^~q3?H7yhPa_gl%ps)6KZktaP9kH$6 zb@tL%7?k?q1x#Y=gr=TTkL3I)K&;G_sUFM1s?qJ40wojqGt*+3y^BzcpT!m5@I96R zRlRWVQ0m$$9gmr^iT{PyW=nezIQJTXWzgIe>yNJ2Z`g?AU8Vl!zVXv9TDKVJm~cds ze4Te++%%ix6eR=j)I?R;is5>jjv@BZU@Y5}aQZ5|+#Gf&z8WApR1bq{bU&){?`I?) znJVhlZzAe$_;3f?WvM135#GcY$o8^4RC2XxgIbC$^Kqh(x!T%IUL|=R(jINBvzD5B z4&Vs4puE(2XUpS>pI9=BG;W)O?`$aoljO!2SPlG2DK&d0@x@o^%z=$LuAxcP?E1r4 zI&Us!9)FG&mG2bXQIeER8*D9HwZ}^PSxg0OvZpPZpHqj&bhPl7oo}FWx?jGL&0e|E zi5StwuqVwz^A$4HK);4YWJ_E^anEaA`cU8`i{$LB3c39BCs zDpGXnu?IdlYs_kQTo9~R_dv*fg1t(0aWG(lc2GHKz*6mqqz>tDEjAx0iiz!z##Yzi zz;*8agvbTv{#WVU1CTZ)%dCYX(xR;8=gH1MnE!(z`2;!xq`XQg!>3YdNtNI`z7~bp%7r0(b(#iON7-}UJ&<+xw`peb> zg+9Z2TFh0XKkd8p6)l5rMaMQrC zL&B}OF7i%^?QoSFGmgrbtNzw8pi{lWxVs23LAfDGm49I{T&==DqHN|}MjJ;E$f1f~ zn3{PcMED%GN($zat@SReN#bK{Y)h`YaXi0pPmGmdVfCdqYK3Y zHqp5seg&9Dpy)t<4+%V&#>5tN3@VH>?zP1Q>-Pe!c(_Uwy}~70REn|lqjUb!L7bYAc(t&u91{TCx+y3i?E}+rhAi~@|j~aa<$ibQe14)`y zs>}+=YWWC#cXMs1NvGc{qL;a<3bWK$!n+T~IQN^oy&hKA)Z=XtoX$3182N~Uts^Hz z=cMMog-h!~JjEPrufk4Q*5IWo;N^$LoD4h`C8qlFje$g;zUqbaD$FY&J3Yc2GOj&7 z7+lg|#!X|6clpr*~yc2^24*`{|*)X?F;2PW%lCMpYJ%{tvt{3b;JA>4omO z=IRBE{AE2Txi;$}?Hez5K`MNjIrJCr!iMa}UDGg>k zXDAD=sgdbMgSv#lnCT?;_CjEOtwc~?0{T?S2|B^CuGi9N<%?gw*|!dLbn-)@L#YsN zZ*LruR|eKx4)llHQHUB9KEr>=>)#$1zL3!>JN!t3GL7bI+{^)8H!I0S9{xpChA4ml;Lj4 zor8^oNjRhDT?-_3Seetb#=JUVe}ZDH;P~T{SPH`@-W?nYz+$og62IA$egod@Gu*Me z4%stDW1pZ~HDbT_ghct7S=GO%<>5(PvQ+lq~r3ALPSt`9rMe z1+vC|Ki%Fb{TU&}zPmq$C+J)t2n${Ye$3Lm#)n|@J*3>fa$uDkJ9!th@ab%=bSbIM z4^=L2e)`ugcKmJH~a@Pefe-&l{P zfdY-qhdXhxj^XMWGTg0KjV++KiGsX4RcTS<%7}&q-iN9g2q@7v$sz>#r76MSAa(jdba~? zcWlF}!0~$^Id*5Lrr~wY9t|CKJrF8_MI~n)7+{*|BW$1;Qw)>8A`=C4Zr8O#bgz7m zoqZWG3v=J^y}#Q9TE>gKQ0&RkBB&}2qCHL^cD@o{^XX{~szeRxt#1gh@p?bQrV zrQEWqOkFyD*hXo%w0KeA%S?M}N^WH&BkqD7G`#t?&dQ1wDcI&)~(&)Z#R!KDL4E6ZRVb(2h6p-9_&mq#I_W^xyHL z>ueR8kI@Het?=_{x#VEJ{!8p-lZgV^{^G3lJlU)9`#Tru9GbBi2Gw5X9}iGAf7E$$ z75f$BRI8V3pan5AOoQGiixX5AF4OhZz8qaS0dVjD_d57h{V1EwUU@#R1&8ptq<|#TOJ} zfOKW`evNbj$;Y)2$i}hE^Q9AUJP^7j_V+g$#D3e&jXXZtx<1&B>SlwsXq&Rjha&S8 zu>uMe)ujD(yCE5_>KjBJ<;^w`~YJ&Q&b8 z>fWaae@L25KH63ZqNF*fo2@;jq@N_dXnvSX478WH4KjiC8QjoiJsF(8H4Bk(& zw0lPYvcla@~Mr!kq#MlwT7^p2&J%ZH7G0cDC$H-Yg=|YwLU&M?`?hheUO$ z4C4n1f?be7Xu293e4aeCE}0*yn?4ftr=$l(!K_gF9((-@;Joy<3%ubC0FQqL!8dK?1K>wTkR<&nwf z3FAnJjXwoygA^{MUsWnYW29RR5r~=XOlU{=m&FAxFI-wT7AhV9HTUZSS ztcLVv=No3_ubv=Xljs>`FZt0qWBjcuq;A92m|MtJHP2FBw6kw3SXOvAUBhZh)EnwK z$>FT@FKn4N8zvs8pClBA8!~(qFqL)nk|DtXrbJ8biQa9~>A^Xv9_(d5*E(FVLUGD7BZWD+l)8f-Ph;rHg=j zq}Ry`$`vnRef6F#5~Fb19i!_PP>#3;NAO0}$&)Lm9sJUG>J=7&f1xB}*ujEW;4Nz1{b7MF=4bhL8dd{?o)gNblHwE!g=uy+PuGB~XoDD*{e01S`_S;;8%D%$xW& zWUb1gWU>C>%wJ34es6mGu)2VyV4^GFX@g&1tr$lE~xiI`}LzK zDX+Cz+WKk9AFCWVYs^!>+L2Oz0r4vh5f&q*k0?g#NKIk$bTMTi&xPRRFdVn(9^{)t79&#mR ztGh{0C+H{*DdT}VEWc%|nx`bDiQ0=uhS!T?00+;`KFU)UbJF#kulp;rF?0AImH`nl za;|cH=trzzT$?Ow&MV|t6231Zw@v%`LqJw;nMH1LP_GDKO;U&;%OG$u3a#&G9$FEb zck~D{+xWLBJ7xv1o5N&$xn?FG(0N)S{cL!o%6JLD)89SiNTHh4NSzPg8$F@30Fvc1qkq@06>#Pb1#U&%~mcqz$Mw0r}l-z{TY2k}?R6g*b|1|eu ze#jr2W2CZ|s8_wlx{;FeWJCu0HuxAiK|zT_Nxp4gKU8$fOdA|PT>-4Qk!?405v7qu1iJR6d&WN=`fA~5!=$#aMinRtIhx` zYT$T0WPgC2V9|+=Dq8gz-+SAg>VV>wQ48VI1HK()BL^Sn#`M0V)7F`#L8;(9p^r5iB=&*1Xs?d}Yhvkj8z|W_^qnEy zM}r` zQXg#U$Tev%?;Hh$q$CJHdwDZ`%(4+CT11=N)7T=I_ZtkFzP5RHr?wL~Jib;2 zN^KhQKR@zj??Am3xXf^X&YIQTR2DTYM`|9HMlXOL?j<0?@)f`{V@;6Ink;w?jT2wm zL26MNm2MTDA{3}H#h-oS^j)Sq1}%0&<(3*f$EQ0UDd# zg=`NJ-jY2;#bN1^3|`<(Ln1WS!PM2L_r?!oG(;l`f0b6dd;)ljye4ofw$Hd<(@Df{ ztNzHL$_HscFuEo;i2>&J=m+#qnz?`QBz?@igCPkyZxgd-pE!dEH)c1_u)iU7f?220 z{hD0z-=B^YC@F11UfwB_-L2K8H1NYxK?O3B3)^bAh$B|esWA;9cB;YL(p743FG6t| z?bYjk4ag?^Y8OB`F9h8p(+8bx#v=S52{1L%uy%K@7$55?R+RxcK4C%?ulkuhptVJ5S(Nq1I&g5M` zgH$DO_O+M02r!Wg-$cg*Ifc5v(p|K=@{!)M@7*1=51S_*K(1$wGD|$X)|aDD?mWAc zaS5ZX7|nD3{a?oTX!{>s>81q01N!iN>ho^r%fM%t|1=A_Q}#MowlOtNI$gRHv_fuxc#@Arf{0nHpB+h%cB7SR8O*N=@t~x>4>^8C}SDf%4clh?tJZ`?WS|;QZYL7$U_@QP{EC|KfK# zgScBGBXHo%)dnow1;S6)*lr~-spg0~2bo4M%CS?Xi|{Dhb47viS1SPjgVzMF$ua%m zxJ*G_SAbOS5M}lLYbSz_reG*^1UF_fs9mCi;DAi>6uaSfLcuaSU$P$(Nf(M*ufhLQ zIOIM(-qZ#}cHzROqiHB@B@cLl|Ir)-Cu#WcN>zF>p$Qy>H2~fj>0mJ?t!_xs-oLrE zJ-i=h3WOJ)gayb|>OKfLC0+we?~b5=q(J4#o7amdt^Lr6vmfu=K#nLp@s1?9+U5@m zPyF&lqpdo(n4dp%FbhZxW$r zzj{ZFl_InpXn{1K?Nb0I3Tf^-O}3vmSO*S14T%Ckj=2~wu-s^R`*&1K-^)6_|B2ub z?ep5gz1{6Jl$u_xlRo?oKO?Ly9^{0c1x|UuEI3GQ&2M#q0&tMf`w1RzX$U(;x!Hks zxls9cJiS`R#6J!{;mgN6PrRqI3l!av^_*m%(NK6QJ!xqZK(OUO+X( z9^G&%V_H7blyHs4G*rH?zW*g)3m`+FhIX*`m^RVh>G};=ZlfzJl;TB`DngAOg{IE2 z-Xzg1x7vUiTgCx0w)kV7RP=zm1Jr3Rkx^Wo?xA_WoOYh7sl9PIt>l|#R zHb4Kxgyk?)lfG_=-xnB3rS%;D0InueQr?7jRCuCbKgMWa^_-y$&KbOG)6|xvE+kxOe=xs zxJ)N2!BW}@{L|G>SFpYYkuaBCXvg)h>t!1+QuC~ghKgnGCG}hZwp9rP0_)@n!i0ilk&BOf0x|3!NT*pXa)WW=86LT`4fAHF&l~m=i z68+#8L_bPr9{97kwt1*sd-8X$6Puyj1YeuC6zL8|p-g>B?sbm$Ul`pILlE7Hm|CV5 z4i5+^JllPiIYk;}0D15PGV0)Q1Y%nwFYvH%i15;vAHk?6MG3b1m~;t}?mHo)#Kgf| zPnqP0s2tb;+?8_UgaFtku2#TK!hIMGp&7R%WdECste=6~Hb(*gAS#a&m5d|qqrv>Q z0KIeqj)`h$qoG5qH17DuTR(bPs5lLBR{}AMz+->CrtMf;2_W`Xnqn-pw=`C!j|47m zGEN3^d_+N6pO5*d8!$(m$l`xj^T&1JZ@`O2mJ>^z5&+l$`TfxNw2|+K>rR-GsDM}l z44T;L%y;fXzNE5fPD#RjSLZ#3cl7vruD0wCV3q5wr<1z`j`&ddeC z!ro(Cwm{R<;G6+Mefjtk-Wge2N^;B=to;@A9pwBR40MRIJ~2_|JR5aFbYsM^?~b2( zo)F-@PE{q5@M5THcZPP8&UVR?6QzW-zQTdlXkf`;bnp1ocnF!~F^&4PNt-PoG=49ujFbN) z6R$i*Q_E+5{jLoDxBf4-sI_OpPR@0&x|PxGq=#YQomIBpR#PGrhb^OB1j1bM+U&mV zx-$r^tNL^a!8AMj{cp`853N2Lba06PMSVo;y)DZqFH4mF5`JQBm8+dza z_Lf`*W=3nC9Ny_OoCO{Um~MQ9%( zI0qle(O2ttgyPH-mbBZ1j;3Kk9FpU5W{0A{2+a`ETGwEBZJFYQP#{R<#{EU?X8<(Cbv3{dX}ae5S!F-~~+*{wSM*u#G3r zkt}BlrgGFh_U(ZadeN}L!hv7}8J=JfJ$3}+e=3JwS@%CLsuGMK&tfYA2(Zf2024t2 z9%7*|qk7SYF^7OH+1p#MPO}*-OM!B?^6wYPZZ>pF&VUM@0rEF$IFNxGa;nrs+A=T? ze54q}-ryb`<&XwB!(1CEImfb(lzOW*f_j<6@%Fk)w^grap8$zz7jE) zJDS%15{BwnN4enxAH4VX%pra%A;%N_{z-K4{+FGa7)0P?KwiNC7?K10G0)uh)70| z1bI|&P$J}XMIq9&=7Z)IhHD3sJ}$q3-g0RR!rfo{tCJnZk8&x|Z3Nk-Fc2*?FnG&{ z%cgWxgF45|=>Ylq_dXL|dGxt**%P00FxGGIWXr!<7MPg{%o`z(#O*ir1KNI`FQ2eJ+X#d;y|H5zmKWyH=nG)7yzlzFNQ(veZM-iiF58~_Sd zK%z|n&tL|=0T}|QJOofBk=1TgNRE%>eFLQ?9=r{eD*!)Jq59(l1h}6yAowTXd^0V> za29pnVtVR*2M9C;h%FwX5JcB|@3f$0Ww>Jw9wG-p=bh^7UjQ+D7|KE)#Ehy%pTrzG zM)f>Ia4LPr2)cCzN7s7rwi-c8Zet8ZV)Q(O^)QI6`mw}+U9UmxS}ZiItm=!P&r_xK z>tZ&b2`~+HHauxUa^4D(+?{ZpCWUiJ57y@t^w%L=rJ@|*wY#-D2R5IPb)2A^5a;=B ze@{lA4l)Ug*@4hu#WF}mKLGM94 zZVvtv1w5?>7w&@ZjX>SrrnUoVO7SWhhxqtL?JW5=w&$lRkF=YH=ZKh3Mn4R9&m$}} z@23*qzT)gf|6{=K(A3@cFK3yG7}|QgH?j5GOofC9du`P+>^&C+PGALEY5tW>V@NT3 zVzELlt!XewEgv4B=5S0xE3wbz6lgW(K;D_n18;!#ncj^w>>oxRIo*~*@1|nt|NPgvjvRjGjXo$^yYS-0)CLUw5*xm) zG|t8i5TYDd4D>2B5#qtrXF#fL1+@+b#g3*sco{p9gbr;h;XGD-?}VoZk@m~iXZW}f zaD2+2WrMgDv}{N)0F|~JfqVM)fEPVy_{fs%Z~8~)0^%Ovlz}WcOdk+wsM9{wz7((s z)@i1X+5IxY2;y#pHQ>+Mmvp&}KA!MI&j+~Ti9h^2{)!1b7Wo7t$oUlqf*DY}+~1G{ z0`KjKG9;R5!8Xt`?RW$JPX)|h*kJxV(lB~x|iY)Q((Y{ST2|$rAD+i)x(NfdKG-V5qLR^ zSO`o)5wQ=xm46sq#+}gmCHBb#eN>$F1jZ6L6O&a}E%bjg2kjZQ8uTx5)*Q?+WG)0F zjg9-l$3v$$jPL-y~Z^00y*Rm6(gcShHgf? zvW(aV(Mc!PJ|QjvP-OqudqQ#anwbf~2(T#{p?>%%#FxfFFakP+>VNzHAGsGp=KD7^ z%I47Xf#$O6(6#r>1ZW_AG_D@Ca|k&5h&4Rm&wCoL99>xWS=$9mHVXAVplz9o2pV*v z0-47y`r-sO?J>3*|R`@Q&_#s!(;bMLCsxlwD^XcbMvWqHU>&~FF8q(J@(F@{P zJw`+|&?%k@pWqPzpuDP&A91rHZ7U2FqQ?=p@1BSEbrx!~^I<%|Al?a8~3#> z+;e)AOM~9>N~XbGcoMX_m|XDMtbqQ;jZajJ6?$ig5%QP->U(6QKN3MjSowbNY}EGC>Ibjx_Fs zpVm2c1$$VUZ5bv)vE2#nexEBDFzl#V%nosVmiBKx%t6>vL=6YTdi>}jm%?CNkRH1= z%CH}a)-`j)8)Rg>ew6(ck|_@V+i-8M;KJa`LR2^?o6Zyvp|8N?{Ds*8mI}f}lGmgb zk|fFQ#-bMzf?z>Q_71~9+qZ3%8H~SXqVC;tT zX~64FnF5Kmggy%i>6jGdh$A)c5ZMc(&71?k$^89S=BBp>J9u{m9FL-a*iDdxbEb;i z(I=FTIAvhi*kB#vqa<=bBM)_z6#zE6oU@Pcp^JhZL&3NZ4d;GLS5o6YB`7NAj^e5(S}u1dd8U#%823Kg#>FYOm=L));tgUO-k+901LheNC2d<_K~O z3Jl_`M={9ZE*C2us#lG3kY=qVo!YE1oK z=Rh&Ud+0^z>CvxbxVo=PAr%-a*L;jxBZnPWx+3w}f| z7k;_z1JcoSA!y(!$V?;x8K2Cm5?vs{N_*_DbLhk&wBK_N3WItz4XqjQr`@9H3x-P` zG+?|6!~etX2AG%8E7~~}G#FnrI}il^24#2%We9EgNH+>gjk3u6zvUZy0OJfTn(8cyBv}DCg;ij|ISVm3$pwk1y zK6sc^i=Vt`)d(z7rJ;ib-w^X}2Wwf95saYv<;F>oSIFs$cO@g;RRNH#x$jw1X{62o z^;AAp830$nD96c&GZ&)f?_D^%sC9yA1=$>KO|-^#A@4ZKq&p?hrjUXTGD2kNP}WiO zIj9iRF;;*j`|WCvcfqv8A#ThOuth3doCEI$0ZvK**_=@D+=6;h=igfOqrB;8Lz)%*4e8ZgeAWvSkG0;Qr zPA`B46ECP&aB#^sPRSrL{LV4yWHu$RX@tg)sA6b3>#p|RF9a`2YAh_eLYrRj5irBC zP?Er%zTR4$j%d6)QrOk=fIOjvMdv}Ma_`rcsuQcp!32ez#`nG?7i=$x#~!=nPjoh#i_iZvjx$kQ=1Usu=~8JQla#O;^-yW6a_~DWLZ~Dxpj7Y z7(}{T|Gpg1TiP}LPd!c!ZvOu;I_ZC#llcB?=(78VBTy0ccl{`@Y@JGI_wWFPG-|UfgG#Ln_{A{P zJmQ{r8htFt1-%2h9;trr!{LVVYlq=UaT?|WcqIQH&VB4}rt%rR1!{J5#T-bg@(6Pv zv}JO|t{?Xg^_4z7HjDm~YHo~~ZR$IF1veuqtbFl?Y;h=$$Hu~R$k=tw-D?WP4l$s>he@X)IENnpV<(y^Gl3gem%0F{YwYd=h0vIWP8vZD z;H-#YL>5A)W_Hs*1Sw$+@Ku+=lH(BEI;kU+H4p7K7Z>o!zGkBfCI#7q0PO!W()E}> zB-a1$M`gm#|F78g|Dgi%i3rheKOy0q=Cqcr5z#YoH+{xvOcoto=4Dl0q$ zfqi#BZvG-Q&ZrYEWh}FO-7+EAys{8m=(_eon|<~uyTX_>s9_zhoY1QuufpW`TZ5WHII8%yv74G74%!&+Y7^V9 z_-Ya`9bkeoK*BKqv>y;{*9BZ+)mIr&eP~bg02U?iU-Z;shNlir^c`|OEu-ch>0V?| zE+w=BMMDuC|lN-b1{|&BpIwGl0Ae_zdBTytyw37 zip}^@53wV-5;XP?Nh*LHA?RG$DDLHmRz7_qu=xmOwlbkr==4i z891gbkBF*gO%_?YfIEA=?+sPbvbiqBAu7gWLCReHuZ_wNZio?zn>~eIhf+Mhir64n(S_77XOJU35(w}&V1n;| z0@*GN^}f$1!~+#%_#_7!|*KYj(_`;*_*5SpXn5mnUstuGuC zGU%2V7Xpc1-sd{67(ErsHR#xt@{~gR(s4RL9<GV|>Mf~QFK6D$3bCK0W6Vi(>P?J)@cs5m~jyA`nQ zjm5hA_zs*^snnc#)le*RteHV*5G&b>BJ!tJmk#C@P!{n8Hx&NvTM*zX(rcuxWHEk4c6GBMz4|8vy6gr{hb69^qrgtyEo`SulWLWB5AR*Z4=5$Oy)iCY*aN-{jt6V54pt|#(;(h@D5q~^JVy0gu}t=HW26v#GSTj7Rl{*O-xUScqt?9K?Iq@cPM zR6<^Px)rw23Yc)moSFLr%v=(8{W;uFO3{W8Z|~HK4)$Xf)bBX|53WPr=QH=wNt>klA3=fUx9w#E5g(oL z(WLkIEqm7is;TrZ!%2ypWkW!0n7dsp2U8Ocy5PTtdn0q<-U}hrvEr%=jE_x~g4)^> z)VHEJ^Vw4bj`-fZ@{^&kB^O)Nu)^3U1U6v%50pZ~=wJj<#v!|k_C7n*lkFh%9#}}= zA#y{H&!YomP`5nZq<{|M8^YLbTooTUJ#YzGPLTyzt#Sr7f-I~aw<@;!9|817JBBI{ z$%%~SsBco#X8~5^*aD~H{E!IBt!@+}MnN0{oXTEF1;N^-nl^rDj|tR)<6{ZFrfmq48}@DwNC zlcdaf)ItgVb0dy>p>tg500n(h)JZNh1}XjliiY%EjGx&w>&$6;5K&{_W9V4$1`bc1 zUSWH97`v?ZcFjZWv=VegZhFipHz>NQosl9}oO#zk;EQN91}zXO1Tw?<>P`{WeSG>0 z8z5Uvb#ohez>z^e&c_V6u|?(^{tb&X>YDpzI=_6x9cN?Vl+a}p)KGi^lhkO72>v^p zBpB4X|9>@i=8sVC@B5#qP>3uuHI|}~eNvXlzO*397D6ME3{51mhZ+AJ=-uHE1*W>=u-BM}(PtSr@g7Bb~SHw|9g*`{8)hD^UGL1~=n3{6HjcU9qLro*{W`8-(Bd zame4EH1y=O?G^SyB8bZ<-(UVDT5SA!YG5sH-DY)<#g%d-#43b@ zQS~*>tfCMi@B3%=iErS~27h?%revWR-QN<9u`|QnAkvGs(FlWva7E!~r z*_A#|G~SzWWI?}-e;~*25BceY~^g5)#b3g(6dX49$XY<2`JVKJTf@$q|w_CueL#rQ{ zo7xbHL*;vtdM%8mqKhVze->9f88SBcqRys{3rUnGpN%z}sj%3Q#J|=ABmJ!MtQ<8l zRFkgLB!(FlNo$t2dWX=w5XttNEEQvCbhANsr=!Endl1hrL8aA!eNRtwkjhd{wF5Vw zyfe=&I7mv33eMQ-`P3Os88P|Ydn__CGe^|f5A#csB83nUL2}16u%eRVv<1RDzR_Dr(dZooAHUMSd|<61Mt-rS&m8WuhWV$#D7}WhJlIz|~NWvsn!2CE|aUDKofrreZ7{MW=EBW19{OLQG>xXXaZy7e% zmxw!0nRgBtOUVCm7Vs{!s>5_lo-!qO-f^jmti8XfLMfV5woiF|Xuwpp9Qrr5VntRCTe#Ud?9g^_ccSoO#N@g?JoMeJ1WdGGVoSXgdk$lIPU zS_+*9p@V6ECd3?1k}YR_Py6eJRcG!z_dc4?4qgmh)AVzq0~S417>z?MafxuV>jyaq zF|WOxMt^!#+yVs#>D&xFos}0gd2!< z*g6e!oJ!8Qc0Af_vw6f7s0~N<6#H$*2Ydt~y~2|;j>Td>zVM?6-)=mkhUJxdqAvgQ zr3zu%Cz*78(aqw+NepjGoyS*NM6DdE*Xb@t;Mv|I(q^?dguit($0wh_`?r`_sJTuh z#!aPwfFF7_?vGAy#*w0lS8(P}ZA;8(Z)Ermn$hw+H-{7N$eC7CS3F7_5wM1bFVvT9 z-m*;LNTQ~kurdWi*Gs%1AcZ8sxWMvvUt^;aY7M{C@cC8TC!X+M7~J+pG>DL_arwe z5xSe&*8q}r@P(VB6>gPt1S_(SAtb|Ep-ggp2xXfJlzx}zLlJw+2t>Qv7qKx21HlK3 zEQJWTe0c+?AFmI3HN?*RJh92!+d6to1F%vxTpVZv&gxJ>wIUu5jSZNpsX&={eLb}p zBoFz3HUqF2E+eu;5r_@t4jT0UVV@m=-AJ%p=O1*sW)z_MOg z1MO(n1pcTnd`GQNlDx=@oB4(ue+W}P9gLiE8~tCjPn^Nm`$0kb!WvrnE^Z=-$zz{j z)=rxR5RhXA469GQAT&Yil&t}vvT^=J^okmB4O~Hon??}JecdWFf5l}Qw2m{JsWlkZ zrz8FQ?+y6Bvx3Ap2kEo974Jc)VgArF>*YsXnuE0A%hh2KYzwknVxCWJ9kB+q!l~xo z3sQ{=LOZgo`ieAdHAJa6uR&-5K9sBR)GtlIG|UHVfJKjCfSnFMT=6Wt!zouhA#i=t zP))l9DP7aVe8oRZ)g%UH9b=m*fJX9$k`a2L-gls-#I>wCH3)I@X z&8zU&uJ2q);{97xgt1uf6p-8!N8%SwUYLMRqMdsx04qVBN4Wa4*NB>0HyPkyGASc&Omu-BYPb1TB80OP`_;E8AOT3xK(WOQ5X44#C`wgI z<`H7$XTe;?_%R<>W{NuYNW#ckbdsKCA$6E6G@yG>Iz474{wk6_;;qU5Hz}q67Iw&j zZv9x&s+3AqEVzj0iv;NF^Di*u&{GWv`}~5ntN3L>(vMo|PNhR%#?u=1q~7USKgOeG zM%yH9)tgrp2oBE>tjY6?|R@6_>fX`(P6CR2%)6YGCaPnB8{Cl31k)79mOJ=RhFe@GJIHM zhmxqUD*`u52a}?nr)P;gqU+TK!tH|jZis|o+(i$;?|;;10pcKEq9yRvPg#Tg-f2f? zeTV}&+9(S5KK|S=%0kt-+I9xbaT48p3!16CgP??2KDqZp3KJt2ki2#efsp-UnlR*L z0t`UQsu~u9w3bJ`ea5Uac~8L-aPy5{iEP=~8UF*>jgz-1jrhMK>p|kOnSTxCba7Rh zB-dRkj6}#_sgp(3vy!{cmiD7g|bIAL>Ks*NDOTl;_5 z7~qbd7yZNIRA+2yLoJ0tfh8m zGYZq8a0oN32V~pW^fVH9Q(R5jr47B$iotppVib)GZ*RcHPoc%OOK2@K7LHLwSK%-l zOFDv&?7%mrz=^1jz&^q$h4`a*N+D}&(hMr^C5fBP5>{rD5fS)FcUcndnm`RqOvnf z6dAw2B2P`IF|A+Rdp!%z-|&dRJ7&VWzSns_lAqFNaFZW~A=HGLfXU&OI(wYSdCIhN zwT2|VeHaW)YURft5z5X6Mo?|Y$Hnr#l5i~V)dtuh!OA8tC^Jmd6*iR2SNIQ|LLECm z--kgOdKrH0Xg}msT5T3LQcYy(rb>Tjp;A&fnKoKY_4v`}6^5Y=!?zE9VD4}ZsOT0p zz{YbJY44$#esr;#EL(|LI^tVRM*Rd8vo+%-+{(*_$V!|V+@E8?YX(}sQ;2D)u45_L z^&uDEJ(bxa@5{;pkf>!vhY0o) zed|vc&COk+HG8(pELn0*VEhm4H_Hq1#2kPsb<%J5UfYxnti;vXW1 z`VCFZ2T4r{OMO=PXng+xh4myK?-&(fJR{mcN|%&$R1Qv#G~FeUl?!mN^EBL=kl>u$ z%YMl&QxsNesXHr`KsrvY130t6-LtB)d}eVp@3tqWlVoebaHaESFu~M_4x=_b&HSn! zos$Y>@Q)p?t0elZp!Q>EBV~HpMcNsu4Ej9njg(P{1SPAA#{ zNexDpsQUhwEbl-&FKtAk$!|b^g&4oS^(G7-3tz=Z8j)}(H8t1ijKKDG>Hhcm2?vy2 zx0U940^}&V#Wp;^*y^pMepP9sD-_fJ4I|@NjoVieb@S~uC(*r`TLV@RRArijkQm`? zfH^bu2GNe{cO6L{)`-ejO?rYm-5=%RE{pw2!^w6T6TRd<%Mj9xUuO#K$h@LIlIwS7 zw^L(F`+cMw_>=i9KQFfw0nzCHkW!?6xO3|{$S;Ir6+>I{ZYnm_=aZA_b(LPhQTvTI zjb`StIhK|s*1yb>B;V1zA!W`H=R;HF@|_^+x^Rl%CO($;KDoCcFDhAP$9XWJ=0*0= ztsmJWiZ)c(M~J+tj8C^Xl?Mu%5<<#EY<=Mo`_t)F)cY2v%=}0ZM>=b3gehS52){Cv@EnjN8?7&-(9w)Q<6T5aD@Wr zh8ix>Y{b)cakNUeNHt8)Z3KlItL)UDu-8Ky*_Gml^-N3oM-Xv7@|Uvqjp6)jQhI55 z{fM3Z+A^y$Rkl#X$Uq4C2qX6y-d#o~azo>5kq+1Di1t-^bIQmz-x{( zw@wKp`)--9ys}{faqep3dwDbm=^ug<=6U&q3j;H|$!+p2X(^wUvYB^h5yyPc7Y4&G zA&3^YTmStaWXxN;B0GIsghM;e?LYKoPufon2kr@)*4H-|JN(f@{IhaW0hskfewjS!?er>%XQG=CTEQi#1V1<$ z**K!Qm9yXc+`-7w4YQN7QxZ?y3~PiOr#eNrmPFL6Zm0#^xPM{426jMhp#<5RbVs91 zpL%ByDYCk*+kdeWG!o%jOwP4%shlgn3dyFo`4w#j6ary$=CUKYOep9`C`lehkCF4&H!zq-BW_{WuTZTDGi;uX+%x52bd}I#u+Z zS0O@k&sjxmzZa7P)Lz{|k;fIP4HqWV4 zd+Rzev8(3#T46s2L<8%=6c>W=H=8r}&AM4FQDZfcYnQ!>5gA@@KBlu_MO9td)#xw%a5{FACL=paM1OZIghQgM8)sZ@RIkS*%$DpRAmNsXtUT&_#Y{fq$Lj zf*xKniK?F0RxBV=@zVUjv-S^d5>Wi6c~C~a?Ajk8JYANzNEd!hfuzPJ4DQ6EFSITY!ek^EF7myptY_dp@(qyPWDRj!%DNs?G}DL ztVO+rV!2<*glr5$siiT_uN?0_&(cm>%l5lXRZCh>yJbLbGi&E@C&?vcN~u-tOY^_F z$7@v#by^3v-4`cm&i#fJWh8>~nD^HpNLg1sI?S4kfa zvP3xIqU)duBa?}1svYp0;w*=W!#NkY6;1Ri@p}<11pi_U_HBp{(QZU>KSY%r#u7^G zW$K}^pK6_u=6H4XwcKF1#IZ4%nnNp61hZhr7TlR2smbJbQF5!u=ESR-=wYHI-P76} zGw30{wKRwN`mobLdW>=%xad#5zC3%&F73ND$>G`j}eXQsK3 zo_|YjEc4E*fLA22A$MU0f7>Ya0zKz9JV5 zy7vpKnma#cGq)&wL9L`XE?I<0*78uR11C{dXm)0#JUfLweJNntoFyqySK6hyR~RME zwILmEBzp*oq8M>ZmP`^aPj2XgY0Qy5${BgH|K_(pKiA_@oO!^Y>X)S0jYHzi|CnQT ziET}>71>0=($|kgdgl4T&P^WA zc2g?2otI%^PwaUmSCKEC`br(1ziV{ulelm8o3Fz&&OI5hVxE3p5O)RUETd~w-Pm2t z`Q|%bD)o`*R=wz!!7z&mtfJwhH<*?L!u53OO^p-d z*g!$%t%%iEmO2eC<*uDw>gzB)Yz62ENn4(squU@d6l11A<$m&R%G058< zNJQl>&`bpn?Bv=?Rgn6G6XCB#{5@psL*WfW6o@kUvN(K}+}rKYer+7!-xKF~_e57@2 zg?Y6{CAgOg-uP@HSak|pJd&Vhk#|kt8T~+5bmsi+r$ycsjaZ=~ANgJP=M*81FAf3@ zN=JtK9qe6N_k%#aN{5)p(^*hjySU+|Zjx`@Q~ej-fi7I|8dfBKnf4}C==HJGJ854ZuA!!Paue~ynvgVLZEaVry!`3zD3eyPd^i8 zvuDG>0#xwkN_WvP5~|5C0!`Y=rGF^0{e(=<2BiJ+%k+(-=!?M)Z!R(yOxg?GN(2!3 zH#@hQb-fZ`G6fvWASi|(delIUO$UpblSXltofkF3~g;^PY2*rm&7;TwPgzm3a)-u;Ou1tyDcUPF#@q;06`7I6AXTLBK0&Upx0QuRCZ zPv~Qdw~`tXbdJ=|!JP%xbml3nE$C`X<`#n2Nl`X7 zUq}i|o`0RiJ8zV4hk>=(!oAdUuz*rR8PT}sOqjY#Vc?k@A&~vNkzVBofoeyEY*sq5 zU7>3zX^7a2yp6@v07z5!^=k+H`?2gCLUr6gPq>v35IVdM=+wpSFnb9%I?&BU0QA46 z`!}xFY8$uhLtUWIq5bniAP0t~;zlwv@uKPK_9kena(^8M=ruZOtXDs_q|O{^Hh&~h9svYU%Zi#X3f!EAp0bYba`Sf^j_1VBjv%Mz zx{GRdN`_dlh{mcVypw9(JH`+EEb zwhh^%lp^z$oM%s5gQDbab1hr;HM_NWNSnAD%bhWk`WpQA5#_BS)QOl96=)iZ18b}} zJI{HM0}sxixgMh1z?-q*aGqC<1s+THaA?ee>F(T3@A6sO=^p~6q*UvMVeWKteu2Jl zLJ)iUm#UvCie#tRRL2r7c)6se%q#Cw__a=;^8?64yy=k3)|D-!5MHnY--;fQiTc%< zvNe@nb+~(u2U9*2(e3aP-QTg4ZPjVgd21E+vgW&A>vELea=Y`>fueelORTD%fZ$Pf z74{G~-rfc+szyN%;Z@_-WeCqx4cRkozzNf{kddo;5V$AzD3ycd`#8xbmQ-(1(;`=;k)nH`X$%^`Wn( zFfRJO&AhJJtZdT51KzPKm{*3-DhCHyOG)-mWSZ!Rb2M=8oZCZpZJ_Uke)f@` zvxNhIq#$PhlNV~d!BR*PhzW$*&6iuqE4e(2G{}F}WPxaDg9t|1L<0pOa@$Ze!tF2^ zkG2kxd=rhvKj?>PSAOr9-;glI2;cut=ALJvM)GwVxVTf+HDdSX5|G344`U0#mS;y* z!w#>?uSpAv3uXwm_S*`Aa=XeASy4y0>bqghT=v67#F%zZhEeBCS}UkHXHCT{qJeps zskzU$FJa&wo6Id#1K3aApyb&6F|cEq*8Q?T`H2@R@NpaV0tH)Fb-~<}II{*sm~7 zQV*Fq=K(%WWn@B@eP5A*O${t1hC9r63+H=sQl?mVKrWd3Brzc6q)L1$2=g10hihxDZgP4M`7tW7ao!ja(L z2m)2E-Abhvhw?OHsvN(*;NpTp0z{K*U!u>;j}MV zTZ)?r9ttRj?WXwCqTa&1c~qWPDJrR6f7G%6kin`AGDa{wjrhi&A%R?|YYa88R*yYE zIgNY-{?*zWE4rqVRxC4Xs~GU%jOMZ7L!u4+C3=csh1v4k&*0GgOJ&=3dS_|=tE9YX zVu(sAh1pr`I>RC+7);5hlIT$_qG7Q_AB53KIOXvBP(WU+fMS^%b=xy<<_5<)g?oP} z^4C5RHzE6Igtlwx&Z80tRigxYl2LLE#O)o6C!m&L1HvSk0{ih_{6ce#_+z&?1kNpR^xDOI(MUe3UB$_CLXV?VxG#zxksn z+T&n#?5KSXlfk;kC{=8FJ;dOTBi>jf3i=804{&bg|9&a4;%I^0S^Em#%PJN=DCpAoDv&p1p9_K-+@y{L>Xi-%k*t zN8f4V9~ZK5>iA{dgA!u55BUIPJmBi90IP;gIRvsm7+&%#=A>?!Bb7(-#%}R9uHEFB za=e?I!mLukxc`fJd|6wCp>h~`!Ym-7WrU{o-{;MK^uoH7B;~ry1G*2A-rcmk$l0$P zNg3u-J2}Y_Kbnhdb1{$)Blw(i5JziGAiwf1&S$4@h%wkxwynEQUEK}vG>@WIGoy4E zn#BYx`C+TKJN~E`EcLs@8@@_L>dd@PI427!8*<*UnZ*qS^1I^ypjbi7&vVNd`4>*a zeFmv_5-aN2d3MBK{t!KFv+7PJs*_hzBc9XI}l0sUM>6KzeTO9yiPjzzBym|wGoAJHWXcqrf}E%2lMRmAFM!5uqy>DMvR zk9umW!VOjP{Wg@vUBh9A-Fq?{Ee5I#q_*cs$=R#S_J)@KYp1d*|PKt*;8Vd(AKKdM&eB5*!{Rq1GiMh@& zyKTE-R>)}&CjMqDL!9Ge+~zo$yb7uFqUa~d+A6{a#qVvgC||e3eq%0=v)xLxEIy*| zVX{HVU0xt=bUGqlK3jyZI<-^kUi>7y@|EY3j_;5UQ5BX%mIGIwB$k-4cl_UP>aO(g z_m<8A{Nu;}YPSx<1f+&_tPb39bGk8!OE|Lp6wO}7>XoGXl}3YzAnqDH?L1E3lbhy4 zj9S&?LA^Y+`oI$^{O|=fKQETdO<4MxHiEB~H?76Q=nC(a)UO0bueB8&i|Rbkbj;f%eRwyG&x>G^ygv05xGMYY7(Zr7zCRY z7L*cmn-Va=O91#i1}DfxF#3L_^qm32qSO*e^J62AS0wVhi2jC6K}JVtjRplN-K?c^ zV5Yyi-!o)aG_b2@N5JQKe7%u;a2X(3qp}rL{VRJ@1L<4CL-e+yTIGdzGHipbr53=3 z1#7Au#^|X*)@7m~lDt+>kWSWAQ&=Vsn&1_W@RW$$yDgF12i_?IQ)A2nSO4b*_Gc&E z(anFgq}~V-O(j4d7p{O{O63!-(#)jI?zV)UAc-6l*Po_)GfY0PJ{%C|Qyt#pS2zYY zJm=elNN!GG?Z+w$opsFB0cdkNHhDG=k^Ca4cwL2k%EJ3T-%99YSYRbqoGsG5h*5&& z;#gJ+(k6Y->aL2gSZS96-twYG-T(~e+b8gOQ*WFldf^z8 z$-KAiadR7`*Nuv`M~PAmPiUtl-)({o0$;E+-_TnWc9m+_oQ> zRFd5H^pWa?XD&(y&g<10%|{&xZqK^po7x!cS$yQ{XrI=YhwqWa@V@?zw17|6C#OGu zc8EGkxFWzSwzI6D>WhKZs8>qI=eE-0fp4|YDp!yGcR|}Z@I$cDLz~CLxnH2!EI6~b zM{9?S!FYV*v9L@lUqZ^(*u2?VH~L;p_uhr0uA&C6`|d7M9A-+xsl$BY&kqF#qu%I6 z*5JcBsTPj=Gb5gTxt)TxK}Blt7#ED}pBQ=5ZJ5@%>o9kCax&ohkI&De&sw#1pL(oo z-&_fxk|gGe)YEc#~cU*)| zUS*hMpt0ABJt5ce-o$79Yk=^Fj=zWs*QdTb|B+R)zq}_PCe^?hE7R`z{Fv?2zP}G& zE;x7n*HO8vPwp$%Noij=WIP(rvpVU3I&I={>+(er0^9X;Z-yuSQk>qE0pqNMG4nO} zD)Y&&)|qEt^a_QsInDv^C?eEUxALXK7a?46vo?g{_gA&{Q zuj{KzTFX=Y_5Gd^=G6inDd9~`6peSQg`EY>u*|7@lW!4W_Q*p{MsvG++Xd|bPI*vu zx{C9{_K%i9lXJlW8(z_EYc|?5qd9|c_^&a3Dmb=;Pyc>#cN5C!QBY3?E(oxLMP;@= zE(969P5m|D!SDY)#+lYdGf@rs?Io>tD+!^n)Zw{pT*;E0s#H}tw%zemap`%IxwF&BYx(%(LEx$&IxC46L_ zZD!KTEu%)24-7j_-aVsn^JHJ}PE^^6^VqsL&n(oPKEDCQRbJHka|sovkhTYEFMM|Q4m|Randm7l z#*Qb1P50Lq7neI#m3;6q?1#j|k0)X%h1HX2Q{~0e7MUo`x!+f{`bw}fA?hU~dxl@H zdIM&qbX>lpqrK>Dd%f!B-r9)rjfRHDvweqqR<<`3zZEnFaD84;^ep&)ewp7boRmZ$?k4xqpVE3)|9SNA;qx)ob5$iWL47sP&qkd&bEcFMP_)a6f)Z2kZS z_zKTrRc~)^;HQ6W*2ds7_ZXh zI-IM6Zq^#S9+02={U7_qVKi!B!y6$giIoEV`S;RDTgLCqA9V>aceVQyVgxq7;r2YR zLB&p=+Oh{e=4m>7{1~!<20kU-^Pi?2s94!Q{|SW>%-jaw8488b+_VSzSMdKB$6Kxc z$4=(`ztH5S{{{FPV*i&$_G;K13m=Wr^x`pwfBrxF->U|HOAG#WvN+f5)ivb1J8ofX KUS#TZ^Zx-q(?OE} literal 0 HcmV?d00001 diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/charts/locomo_comparison.png b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/charts/locomo_comparison.png new file mode 100644 index 0000000000000000000000000000000000000000..727eba31be6f7517eb5f4b4eb3d8d1cef495f8a5 GIT binary patch literal 43814 zcmc$`1ys~)*e^O@fB}k%AfN~+Dvi<-DkUHwQc8zKcc;n*B~>IO6&PY@q+5wWQgUbn zq@}yh^RmD5-E(8z^R2VaUDvht8r%*u|9Ic$`PK7(_fSEa>^RMF3jG0Xc;Y%mz=Nc10(jNO}07!2;CjKp0P$Ed#p zN5Vs%ZHXuQXBh3H5|N3;a zvZ4qtuW`rqCb9h;JGbjx;!|DentOviZtDXxTMsdw&JzEZgEe_XAc2E_aIr;XJxcBv z#$Wo#Wpp+NXBW;85)y(fnJZ>BE1pU;Y8IS$&=kQdXVb_nJv~_FQao2Y(-zKc$Xpzt zU*(NG!BKdbUt#jsSMdm^@z3r3HZ9R=x^Ansy+%pcz1?laRX1tF;FoQu0~CJjt`Eg_ zX+V>)LTo&ux>P@@37aYcRg)xbWMy&#V9o2P4W3N44qpb{Ee(4M*iX z^6nb5qg$$bQ$O4!qc>!Hbp+$Y&(Cok3x)>rcgpv+W^Ah|q!XTnK4GT!&{|fA7Q&8& zYI4k$%$KKF43(c14hmKfy5SK}yNMU9&Ymrv$(ULt=*H@)8cYefuXd^%RllW}T4`5Q zl;bS3HSWZzx>t-8rk3ksDpgdDVji|cS=ZnAu=A;jf5^3ygZ}2z`#taxk%W|~d^nvj zxBcC9-nr497qRS-hJk|2g#mV5PhQ~WZ#UoQdS@dHSH!r#R&@JsFu?<^c|E!+#ss6j ze6t>p{p}%|mc=$XPWDKn#Lve4(thXJ>*p%=_i}EumN?Akz^zwYaQ>U$XFggs2WP{o zY2Wkg+D0)MU-uVNSVo<51-Z>|D!Tcbi(ce#mWoj6hB& zz*SLxj)l*tE#Yscx^8sZ`qXz4dN#44?!jka+G=Y*uf=}a-I~dKcIfEIIy^n$WyS7V z>PoRvyi|zYRNK{x46D)FXFl}qj+?6fT8r^I%+ zEJnw|bU8HbI#rjKNbkhDF8$mQ#k(vmRGkXj-53qpbSq!@abnrTu&jH+YwcSl@rgg% zZ^(tNR}d<8mgIU&GRxGu;-!O@?2LPBLp9Bx*f(0xT0P;ijz<$OL1lZZjqqTiIl7^$tlNQ3Tx*_mjrlw#@Z#-!(?O?+rUoI~$%po{ zd4$l_`600i^+mhY5kJoH`8S19s&0Fm6Jxp-Ru^4Xr$dH@1cLT=Cd8hxrsqD|9Pg0CC&U6YJ#^F&Rx-}VZoOuYtwRzYRQ+bjSBf3y| zXiWbHg*2Vg>0K2}AQrYG%Na|Gdu z{y&uU%$;v3n5^F`_u{>9?5R5VJkuP40wm4WkBHFW5=!aFvMM)lfRa+606mmFB}bQnBg($O6xVQ^4%6(?NCk?>GD-|qR>;orDhc4c4kD%MX$l` zSZS43wAAE1;z^+qnose4Hlf=%HN8Gl>F}Uxjh(jBSje(aQzSqA{7jw!hWf67-H;8% z18GU)w6pp*FkE)fCp|B)VAxdaBsX}h|I``l(lgY5l;ikJ@Y7GE6xrJt4e_>9@VznM zP{VcYbs4u|tx8FN$Id8yhY|P=i;CS)(Vc~=9f1sG zt<34%m#vJt6TdQbD`*M|7HeqEL}w9*>wL3bk$|EsGSz0H+=>m(k=h5;y&Y;8*D{?KAzR$ z7a9nCU1CwbnfT~h_rq!5+al}QO13)bGG4mDIfHc{nD8?z8pjLb8cgKl-k98Q^_@!# zjc{_&kfw433Z;cQy2}!E3yRP}p?F zD5`EHb2`>PuTG^G4Ay_Ol-GXabMnHYQPJ~Smfv}bMof00<~9Za^62be<~6p=eelN=<0*3$pxblmDsM7YkN!mY$YU9rR!%j9 zkk<+C%G@>nmc&Ki_L*OlJ^G?%7n>)DKWJP6CO4V1@x2EJSUkg%J%&2ZimR6y$wk|T zG$PKn8qm$!EB011SD?8i(tGT1>sAS5`VNl+VlwW>CminT+{BCReL#y|E1l}-N$U6J zaG?>V7@ICjZr+A+nzXSS@r%&*$5rGIMmgiBO7~GrrL7XTXEIH$iYhw9nqNIXkCi6Y zmM4X^iC(jhq9lCo7&Rp&WG5Pa{Y%a-a{JG~vT^=(u7uvdN- z#JX?YpjSMZ=?=T!W??;cf9IOm`dXhwm^!VRTZMtZYR8@iF(y%j5|uutdr9G9?4QOz zr2ZtKcYQx^7?|e-4Zl=y2%7&l85&~&-Zhc3>R*Fxf_u=jc{8ZQd8PfSm8N*}REq=i zqUAEKWJntgR`lxNd!9$83tOhKsXyUDtrrI#Z{to{;J&n&(!*ylQd`6TWs(l9`Boc?nv##N7XUkZwy>ipObl zvSb(}8@??fAQZOr4C_tFePy}%_Iy;vc8pG!hDo|BJ)Rn`5iIJq<-Z=$%c6~+FW=56 zWw>u9@wON?_XK`@z(F@)rpDD3Ysjg+Uc3U$Z6fpYG{7Owsm+PlmLS4o!vNU^S>0>A zO`BBaCA0Y|gg6ETqnIj1(anUNO<1C3uHEKSGsQv{rHfLVuCIc(@(yDjxE!E|9N1+O z+p!*dg3*p6?>aMVdY*-asIwwG7ZPd*fK=F12mpdjh?YFA8PA7>qvq=~{nn9fh+ncv z%Q$P*xNi?g{12WrE$g0!`{vj+BYF{+HMaSCuD@TAj0?^(uL|w0SM19JfO(deDG24a z{rC4bGb=)BD}cMhbTe^rB^D`d)`0laa?^M|tu5l|Gx>Pjx2BTX<@pCbCFYSDU*qu? z#OcdvzkdZre_N^%=*v8aak(+kZa#0yxAmqRSXA z@poK|5tLphI7_)4xC5GTtvC3*FJH2%rVxC83=q~uPGMi!@|UisZ9Yp@o+G1R?0)x) zW8A>+%b$1GV($CUS>Nw(mlUK!9^>;0=?2@Ax^Z-BASm z?FicK*f5=lFsN4JMsUvZcI@K|x5qvyFBMh0T*Pldbx#DOUS9f|6E~@*VT6^<^cx#) zR}^dd{ufa6C)T6e>+`V+V!JD@?tajQc*ORu#k$_Ko9h?!$y^`u@YsUuv%F0|udJex zQ84H{WvEI&>=J{x_a{VNoT4Mlp+dX%!1w5mipBu4%mH-n_PX!!i)g#V zt@6<*gJ=PKM89pj{8(Sa`*&tD{|Ui`o6oard>ZiB-+iV$0;rkAq$g+Wa3&s>FyG+J zQF6M+`G0d76c+5D47>LLkxL!0Fp4fz3*NRrHr!zjj;+4wM4T}jE^K5I04$%nHo!Zb zqT+OBkL|u8Yf2ScqkzqUI7U_)0|^_imnsw??g)VM^sKK~ckBC%)n_Chbshiw!g^JA zv^Jm}E@2p7zh#wHI<>04GGj)o&`eq|0&e}1Yj80e#qUi-*)(hMQc;>?Sqgo}V_fF6 z?oz3GmAR}p2XJ|lUyPu2m*CYu$rL~cq4Dm>OUC&x`%M=(4#me=^cPwgE!I#e!v1wZ z46%NqKJ|Kr!%(@~^6I<#hHzcZa=sB0wg^j!Rl^#Fg7$j*KC>{N<=Xe7lPk0#8Lh_z z0xjS!{#1yx<#-&G|qr)a3Y!1e_e2V$FIIFIq?W)I!WpSao6fy zP$=9~j$by6DJR{+V4h2i|JUS)Ean5fHgCL1lHGaVJZ_ zB@rRJ$zR{ckX~~AJ!&YtX1zQC)Z7LUv*H+mLvI%>{&r{ExlyApc<`*9X_*4kjFAoT zI7C98HY1MY6+M>e{vw-*u12E&B^BjyaIq-*aJ{V?tgLc|_`s(1%8piO1U&5zZjEhk z-9OMfj!@|jAu#2AiarMF;i+Nz2Zt#3@=SYf=9v!&!K%#A`R5h!9r&EAe6N zUKTQ1J~pxoH?MfK)U4~ko>q3djP8x+>r=aAh#gGBuF|(96Us##1xPWmhGmTok=f96x-WiYJzr<@kr!fy@&oD_CLpt%>ebUU3*;|Cgk z^QZ~@3BYc01YHj|)IF9r3t({$g#yyH{c-GC0zh-OxW0_*<9&V;hTFh^@0a||k&Cct z7HkKo7b>j*ZVSl1tC~metLc{3t9psnw%9o*5v({oF~6_q&@kj zp`8l2BjKV$KR1aF5@g8R%6HfLasd-00ll{L%~*$W7s>&Un(n%Je$VqMKhE;|En@qw zS+Cntk z_L!CF)XYTGBR&SM4mb?S`&_htY)r_|)_{hwB&zE?g|!7uXKdy)ArFT{yo)Y>vyD($ zYn7yu*EBDBMk8qb6t%SWf2efALqU3@+mplyxo#}jBK0B58kDW*Mpg}^j(}cz*H1PV zi0>auW6+LT7&PGCE$6^xxC!7!1hK)X+;)YA@14~rX?U)qvM#t&0*YTdBGu)8r;=iu zzYcsV1SHlDJvin-iI{L+pNl?(X9^O&xjSp&(eTv}aIKwcF+ke(S0tAN26JAB%(CJ5&;p{3+T~5Z(NvRZwM+roDeqfS+I`BTCRcQQU| zndNoX6N28a0upUj^(g4Kmg|Ln$YM4hA0*0>Ip@d4`?`@7(^p`jtJ74sagqor-!!?& z{(cMz(L#3h!x}esPQX>u%Mdydrt9vswcH|ht+7-iLiiFR}_DpyGBoUA3wr>(58IM8u9h2E~6h;Ud!8-`id0H~#cis>jTA zSjqskK!sNO(GzyT@_avKY(RmtzF>BxPw6#M^gBGTn=|&%E|(axDJ8Sci$7-S@MFl% z8_?-|pK)b8Kk^K3S^0`<=jMj2@hu5H@7u8sgHBUl-jK_}0+DW3V&w7v3QKdx=F)lL z8*#na#qdUkL5CqX^%U!y-Y2C=FHnAZAVT(>)H0z{$zF)J#&P?4dI0C2yedqj21LF>k`mucS=Q^)^_4egF-t5`XHQfQO zp@WUcQte07c}>V_}b2A3C2Hb5xm#dJKycpLW=B zA4Fm8Ej?=gxe9C`0qGA#UaPIr^i5~YYQ4g_7oUG>l&aBhq=f^)aEU;BRNZ#<*GiLV zOrjdrZN#sG@|b{V9Oz@ZPMTJJ56$oOWyo|g)N}-(FVd}3UOuq^Fyg1~iV`_3dl_qB zKv4D`xAYwqB6pAesaNSKJELIg@`1z^WZF)S%&E;@U&E<3 ztXx1}z~&Bf4G2GNhCw2s-#!+`g6fLbyi{eAl=mp_H&lxEx4BY@qnxX=f*H;%ByXJn z20&-AiEvIAkdvLlV*4)mwmZA6>6!&g3dNngn{S6^4L2-6o3K{)dj|#AZpqF6UYde! zNA#J30sFqg1)4&!y}nuOLg(>W{Ww@}fmOj8oBV>~a7uKhfwcS_NPo<8mT#1&j@vxO zdlsYM4SF3LE_VLTLZQ+QMAYEEu&;crR@0+k{Ea`23_O& z{EnW7!&1mBU8;Ay=oEh!k0_iZp7zxrC{Nl`MK1GY{6P}_iAh#b!Hmu{)urwL;a%KoNz-`xcS?%N8f}VW z!hYxH7covO=_lauR(rZ5$NJw-Xm7H)bHcnX3SVZTi2?r*8j$)6^Zh;y18_MwlV^;5{`9)|-9=YYzL zI^G}m!ysU4=-7i0l?WBx{$oDXE*Yvv2Vy>Gy1YnMb29$+vYtZw)u^zJj}(4Vx&3CJY#>ZQF|`b`#|*Bx^0>b)ZgQ`Qv^mP~!379r^tO}Y)jTY0ML%Hb z_tcntTMJlZd!$AAXT-TzKzEp6DIq<|q~eQV!)+M^milh{&(In;nsyq#TVV0dl{)n* zInM9BIkbnS;a=eX%9k(-&4P5czwR1^sH(}7Ij_nh3O49*ShsZtcnJSRi%9Hn2{zMb zn$C6Md(rWm#MQ}J+rX>aL468krC#r|sG#Es4J;VpGw!uNIU4;-RF1&Wm0G+5RapU4 zWBYP7its@oE=3sDoL~X`W3@%zY#kM@+U@gqe^0i?FE3+d8?w)K6LL>)znwX=9e}If z*o%BGkYO~yhE?iWp{~|GH49C03K*melJgZpuGPT({EXU*aLNn$^gO1iAt+tT{0_E< zV`zwe1AF9>!GlM5r|LYB#K{1EjKQlT=j%Ej@osL*jGih;o8OAbme-~y_P<74M@>O9 zoaozKPaWX_&ONbxeUy!i zBx4#bS9&f7+)GTy5u)Eu-*PWtk6kr%_niI|Gv5n4u~ZEG8<*r^&tQgLIebC{b`z88 zn<3D8HL0TvH89MyFUFEM1K?J_xsZmQy4~Mw{P<#jtMiuF{-)yQvd7M%^xVmFXGP2b zux_rR-*~7_(`fn`8kxNi(D8=-i?an)(7YorIsN&tG!fslu4U5{LXVe7Y5mSGl&?Z5 z?t=7jThL_^<@tK9q=QPC0>Vy_h@I_hfc_7xefxiW>H~~`?c}c&pggL5ZUqJ;g9wK? zM@rLKNoYL({LK3F1{fC;8I>U7SO@`+UP|v^M}$Jp_-*T3tim_4PH+sOkf)LhcHtQQ zf!3R2m=e1gzMn&MpyeBv%hv%Y46``+I(puJZClR^x;$E77pf|QR|VGSb86x{=nc=Q zDRz$SGY4=~qQC7ggG87BIE^R0XgneX^a~@@QJ0(ebq@e2te_`Gxa{ztd4O4RndMu? zy|CQd|9)$U*(Q>R|Ml{W#uC!i5c09ew43QNf;!UDBx?e|B{Ly_e^8(acDmCprMeb3 zs;lx09YXMPQK>%??{0 zbyJyBq8oZxJHx?>!Zk!?^R@#;$pF}C59*qxB0+A~1)5l@wpjkzKEGLVYOc%3PAg(C zrL&2$h{7kV_$LSb3x*Ru9Y9*BsaWLOB9`T&N-+)AmV@9g7r=zdEiodj+Nsj?ZagI3 z6t);jU<)rUPlXZP%P*=YPXkxG9-rSINt!h^2uiJ`fMM6Q}SU?Sblfj zP`zpeUo*O@f^<4Ahy6$DN74$$N!)atABQaHN+!La!8XKhBN-WW9<66o3|(gxg+GXA zoz_=VDTXWOHdVUZ-0(_O=I~SCUFtg5bUeUK{>16`p6y5R4a&ZNg?ZpW53WU7-orj) zk`8=;6ie&``$9-oJpC?ox&~@mPFc<%;fb}bZC4YwY_Z)h`R;o&l5pIvJap<$eIx0$7x$laxY)FaIFy4zP?>;SfqK;EUH$MK ze*y1b_X~fTQ8dm%%V*jx!P$w#9Kw8SJXl~yPhE-%tC>xX52Uf!gn&Uv7<*PFaY&5V za^tP#W$?U=^R|VNQ(JJmh_B~q@Xi)cr4qFnC~Ip@yC7O-^^O6zch)KYah&y&m;<Ph%gNaQ)0YSl1yjL(raAE>RqMA3nK5>*D79i z-KQTUmjF9?=uT~hD{`r%zh>&oQbM2-E!Ig$uZ9aH;%slORsH zeK~jpuCpv?0Xi`ioa49spVLxw*JO?1sys;GHu#+I0T^}7$57QGfRQEvxBwX(i+TQJ(DV?@edr0%dx6 z$CKM8HLTfgkAJ z2M!lYn^0X$@vwFmvFJlTNY5qZ)vo?6i-m4b?0cJ;YDF!130LWVKD!AW~zgD-wnrs7_`T6#B^;M`yB;}tRJd@unH1lU0YGr)y7a&G<_F+Q0qtuQD;;yx9nR|7|hX@smy&O-olz4rE_FPTU2wm1P@Zz+15eabX;f z=%2E9d)9~ox;`}H9)7=BaK9NR{TRNtzGW=ll(KmT>I)}o#L9Nm=BhyIWs#C}iJ4Zv z3;B1#pLo~kqfLNxE!aX0c+e_}0w=<0d^FFh^W(qQ00oXJpci)|0IH|+4wbM8ByErT zX5JsijYlT&vnn*Y_3jsvYxsS_%IKet5J8B{uJ0oO#yce zlIMVvYL^E=yyX^&f2-$Jp}E?uQBniBi?0k{TGQdH*+3C<+WRqSSqiu4j|pYp4%DP& zVIPRJO@Rg?PivZW*ZAdOPdHu7e(Fs>mcrzZC2)#1R|mn24vAE$Ug(bzaZy;F$*m&4 zod959;vw4_y-lwq$w2a6AM}hQ0gNyrAfSLmR23E;2wAqNX*sgJKu6nv=LifgbSJ;{ zA1+Y}p#~NthzCVe86SAD9r|+=x#-4kVXwj~uYZs(GFEN_g15fh4eB2qPU`K235xH= zGG6vT9BpH0xw--vx~-ZFH>}c1Ak);fwPUc$pF&yd#H1~d{9?$VooSt$f`>iHf)7+yy`vpKQWFYT^Yjnu z_5bGJ-xtquIQkAys#oB~W4GeK2ign=;8&!gI{<5YlNxTsy2?4?JLqg^yZ~IX__$C6 z@?DXi2E8Btyrf2{i4}xU0R|LgoWkDy_l&r-N%y9`87kpDfFP>wjh;vGs$9rsB_hO! zl>7ATTAC@4Q&Sgd0xsNQ-ks6iuV3zi|@MhSEw#1N-17I*Gw*`|H>z z2t1@AXcU$PF842#sGlv0=AK;z7DG!mNTn6+ifB$5`JZdF?TmhGA2XHv{|h4Ra-J4c z{sU|)6+ojpx(R2kX#|=@1EI`md3@7t5+oC$>E!q}*tW279$y+MYRk*m7^Dh0xRJj- z6u?UBfajw6) zH+X=B*V4<@dYvLcwNdNU_qm93YdS)u^bk(!1*6u>{v{Md5t_dtxHK6c0oe{p#k;z!b(Hh?)^JV6;h zq!8yI@M{MCYDm87fC=++_^Bb&2$p*fh`KsPJpfIkV=W+%(gPViU85dUeceSeKoK%7 z*oC)*qHFvN{G9Za9nit*?FZ}#JH3D>ElUX@`MW@p)STU64X_SP9J?0vaLT!65v^-Z zm(8&-*0MQZ0F5pD7K5c}4)cSfv-8mF8+G@uqV(GgJ}>1b^0^7`3?nZ6_mW{}usrHq zvb0)PHDRht@PTXekTVKE3KBF@Rv=z=F^r>ZrELXJ+&DuSph-pdJ-8wY4g+@T(;X?w z#lTLNfA#v3Lzh_j6RbG?H6QI7n^o<%#>I}>Y%@LIUbxSJ>nw{vHtimh4vb$r}S76x_TDYBH23@nFH zsCF@*sx$BnRLVvY(|nbo)!3JFGXF^3-Xhw5+J- zf%k|Eo*trpz70DK3)x2|Sd}JDd*1=9ilO+Q(}pEVT#?2-a;n0yg zn_{(_@~^E}`Sd#3k-*A12r`5x5TwGXY{n!Dsq~+Cy~BEPdececdna{hqiuCTDx^!3 zRPX=E4HB~Pp?cIBhSd4PhP?c#xfxHH1ZHu}L8a(g488>OqTQ{Np0bwGtk!4q@( zzKHpDh12z97dLySgp5-F;*zo9+w)#$749R=t{8xp^743-iQ8=4cd5fKDV9RUSN%5k zWf@QUAFl*X#!q5wlAfmqLE_G*W&E%MH4&++b=iYb$R3+y`ADLh&;D;W-Hs~R{tvW( ze};Gj8^TR!{5}pBBFwyWu@-5d4_0^~OWv^Fs)uDzQb$>Qt_z4P3!l=lJvs07b4 zWnz5&Uk&2{5WM0WB>D$?_*PKzeZO%UB_gUO-VoarO%V4>^l>r40RiJ3X+L zR?t9N#69I^6fyu)vCTr-V+xM4xk(zhqhG(ifW+7c$R)RB`cM|S4dS>_taGTQ*}|qB z=lI$BiK&=)9;mug_Y&-_=)}B={oVN~i1x`|i*YoV)1YH8E+s*w13?~gj0NQJmpFcK ze{u%=T}Sk?ae!JpT@$RMFx1LrB~hRa#H_lMSJPGJgv@$#WfX-MC8i;}acPUDNf_mf z`}6pU+vmWF6SPFks@BEK%ei&7V5Kk)x-4W|L1g*ahZtwlb&M8VyY{PLS|5E_r3EgC z`@*H0?00->P3OKbEI26)kie#H4aAr61@<46fOIQro2%M?K&~h@vlIE5Z}3p%#VRh2w3#^&XCzYaC>a+lKtsf+w0riRaFQ$q2jczta^jf#Pog% zCm-21321)4Os@lyUmpjaZt3LHHBE}q-|`re&+Nzli}C);5r(t>BEQQhuWdorgY?79 z?*Ni?2tGHfP9OZ^omadmGswyYq`9>{v>$UYTmUtJDn0bDC;I=5T#OrM_vC5BJz*1J z_9lYF_U?m>W29lz#H(ClKih`PW$@F>b0I#N1Wx(49U+ha*;~XB-NLHty73;2Rlw3w zM0W@fw}LYyNg4>dEgpNp#SZf^1%zU|nOx}ANl2!sHQD?BPBuh9tOKW-S{jQc43JSD z4dHI1mC4qSE)U@NZIBF%MJ{CouSqD#vZ1*UTTe90?_m}J(Ic3L56!6I)gY{$Sm(9BPo^|PKm-!x^8dp6x(p^tlBjeX8N}S7b)r~W zFuBOeFX5+lFhOx2QCsKflqBo>)D7psT9NIWX-`t@g)ZI>Or;$HQPBmo_`Lsg-G`Du z{k$AYfXLHVY4YU~m%20bd5!p7o(0mT^K9Qz5GUOZ3q4MZLYn;?h zUT2aB!0@i|;^}PuEZn+R2~7+Od8jL8fC8r=gWE{-!VyFdo8K3{>VnTN-d`h>u zmj`%i&xMA)&a$x6LPEe9@B)hn)z=qZu1XA4csfGzmDhR6X@0oxq`iz+PQt~uAm3-~ z@S84!{-;*lOpz#6sd0u#)62uE$pKVA+3lb&i-4&zI*V$uFFF2xm6DdmR&mX$|0=4& z?I5&<9I6DaBtnv_ipaBWaJ#~7cAq$gW~Lu~B$pg7t3Q_#5M=sSz)ET12;!McT>y_M z2_%MFAU1M2?-+8CU7LqWfu|Pnw;UjoYwxFJ}UfKV(}&sh$7{f-k)o z%YP`oWelcLgSZa}Z(6fUlINdXfzZqDdPq#r=3((pmcox9LYes@$kZN)h^2o)p=u#8uuzRgJeUlRTe{hN`$sG3 zP6W^Am598y)l?(%O2B`tB%VsIs8vI;EEOLGntg|Lf1Z(J`!F9LVIze-hB95he%Dxb zown(h_eC_dcS4TAz^)-)>3(wWW5)5}6RlD4LcAhkH{uwrP^4vfy~fq2)s%_2ggRf# zZo}emIJcGnR8nCRRB-}2!qpNTB4u2bA}8z8L6AVNmY4(W{830i-KR_-eTUFnlh zV`RXqY0@8sxNk_%jq%g|SnjFd- zAra&*RuX0GbneuR@ln6p%ZTrhm*ae!^V{z=7Elps{^fsN2?&gz#FJZEho`|2qQPwt za|lj7lq7LDFgpO~m+-vdJ;nsT<3Ah~4Gsi_vt$@1VSfzQ>#BbTIe<~|Q*t}}X3Z@9STe0Fn&p+){ju@!>Sz%3N56l z3jTLVvHQO&DaKpRA%!r`O$%sbQ8Cy5gPZhBAZH5Aa!vi; zBt^-B5$cz8BnOLAA5T$T+y9DNSkRjp zZ}p%NChCGGIQ_}-C>mF;mfeIpPm9fCHMLL$pyIjTkJ2W1OC2oQg$kB$$*=ub4q_VW zBPb42BqMpUWwz(r1W$zmxLV>U(n(%pu+N>qD>Ny!It1P1b`8;QNS%aSf(B1lTVq!D zo+sW|@1QA;KjgjFC=f)dCdZiT_qBzaUoY*pLG@eL|GB0}(`WjYCPZ>artJLC z24&wXs_zgB$(RxU@_JluaNN1$?6|Li)R;}#-oj^{iVZE2X^oRbXv%bXl+-wa&rE~h zbNC;A=Q9=mhoSR72!w(MVa(@uTCXCgm%h3MLnx-^zIUts9Wf3kA_G@(0J?sjj#z{Z zDTbDr=rXLO%9F{U0?JsZ8&~}=0owoB$g*-L5V$Y|dI46B*LNKvz60O-KOFagUiRN7 znt2?#b+!8uhl@c%QwRJG4}hD4@S+=dD^3tqcv3v#%i4xyDvdnjkX|UmXdn>+T4J4! zWKf&ib0O@8{pY;Kbx8gQqrB{akR{gn&S`n%vB9OX{|XB3%`*uLa)5QFh_rpjz8w%e zmiSRv}!9idwk&Rc;5>Vt*6|5Fe&a}0ds&|)xr13_v;Nu8jAJ&-+O z1BI9Ba;wkCC7^+2lfpJx*0FW23~_OW0|vGcI~sVK0}Yi5Uhf5i-{z$)wEQHLcLF2c z!U&?6b^7f{Zb0$$1IP&m$W;;xG==F%$X1LM75swctgS8ce=)KxZ*|6+;ydbE=6;ofo!DRX#mPYC>#Y{ zUH@98WmiC`myzVX)$B+f+1>xhbbD@%K$%m7DUIjkvp-;j%%-plhTrJYc7a(IJwnq` z4GkV}qPZYJU=c_Zl;{JErM_Sl?Dq*Uu;&>&P$0M=OfpYH<<|gYGm1tmz^A!5|CrI% zFUtfFE{De){r-k@G0=1y%ME~ISXAZJG2Sq5Gj7SbW(O0dV>T%)YX@~D5V>NTGl2pl zZ*Ad=CyepHLnd&LkSd+c@3WqKqU7}tty!)Txdh78(cS5i^0KpHk2lVaLjZMkvXx0) zPU;~|=ew&&Y{|IikEJ*kNF zJCBzpd!^9;UKKqBU_wiB{r8(wiM_tZe?UxdYLgVh^d(iJ$pLZ=pLE;de?5lbyk)MT zk0>`|aV>Bjj_3NR{d*^q0`(v*pXhYm^5hJM(7#g$`)5O5sIuqOG44~v@fdwfB{I>t zwA>T|#yQ6>u3^rEUDYZ2l5?=v12#6abpcO)?0D?Y7i zNkObexqQT%bdlxgbMOb6f+iqj_)cf&amDoM-swQBt#G!xU4tC{Oo&hCXVCDM(}h|e zGo1wWv2B_AlYloV{?~BCr{+995PNgAe)50aTxT)Q?C`vr;n7I>m=+Bqmj?-+yEYD$ z?)-r!!E=t%$5){~2Ny?dd5_XTRW-^cGk-hs)97dThiWOpK)4vE_c46wYi>EN-C%!L%S=%#N$7w1LrK+$ z)Y42cY3MYiCrl4mr01*SZ2isaxU~xTR6UjKF7ye35xVqn##`YmV*H*dY%JqwI9B$n zx*Nqsw^Z2a73gsc%cGOBT1)Vk@onD#MOlM7n@(?rv$wv^LNigE0^y8qeiL#6xA>pG z! zlEi8??ox%=0L_=*%z5-;KwuB3p6c;^q^%4RF*=!h+t9BZ6QkO;P*%i zbkZqvoP+cYO7C}HncW9s+w#31bcs+j_gb0`PdvD30iytea$q68$jkKGt<4%Hy)?>8 zab+&6ihMIqjy+GM9O?CAGkC6X6mNthBMX*p;CI5(P{cm7E}#U<3JMqSH@#ipikTr~4qcfQyH`yL z`K86`6%bDoA#HW#)MCKpn-Oo20LfdFy4QI?)g)G3PaTHr_6wcl(nVpXK>DICJtaP}5zP9JL%;5)^TSeG`6YfzK6n(& zYZy6h{o;+m3oJth3JVs{pPXMXRh$G>npx3y6Nwh63l&Ue1fcT&s4e{I&HQgDHkH$T z-!VRo$K2Cr zW(Q!h$>-(IvVhDvYGOj4hX){7&vGgC2rC&EC%g)FE$+D4~|^m4<{M~BKaMI(U`FzwEA9RlGK zetG<|K0J3MulKnrs}Ko9{THX8_}Aan`-uXb;BS=i0GuamQQvl$(;Ce|rY(#ySe43F zU3lWt6)?SZI{8b_gNs^0&T;pLCV1FH2{Wf+r&j%1?7n0KT)fSPoc1Qbc$?))@ep)4 zKOw;_0t{YI^9iu*b`&ano}1Z_lasTvyAGP?#C*lE3*a~z_dE8Ix0#(5_e}bSbS$oZ z9tg1_>gQYVLp#i2+NkQ;9u=LO)|z5{P=QuZUc#c83{)!q}ghi z*&yFMK(ZM?p>2nfbck35AXY5eX+ECl5gf?>v7r{=h5@~(mv(xc=2f?HV!gv)Qt;v3 zz16?11!OYSY_!m@xDXUT1b2vXBMD#b`>fZJuw=1XGe*~C6-5F zWW%a-Y_S#%SZ^*O#NLA^gLC_JT5_QP;F@Aq?4dluW#UTsYSE!nmxK#znduQ&EJqn= ziGp=dKR$_Nq!?s%>{OLd zalWD-tcjQssFurG=9D4h9xx&};Mjnssg~SCA!{5GiYEO?^x&}=ieSq`{Y_~=dCuI) zIKT1P>a{o`{zMWjGP=+lDJ$gkqNMO8B44xLUw5_TF8`7U47X;rq@R}Aw*JlWv z>X=%pM&?Mz5fS@}>%lF$>06e+;+-cCZJU&FCiDafS+&VPTDvXGoG^7cC}o&?RA(e0$eXj2M^ zr@^k|Z;q15qRA1I53{~sfbzni1Tnriht&JIXAe7}8NCnHah^h{s|3Vi(!ruDTK`U^ z>-r&6M$3=-k=9DuZ7>B;aswSi z5G%}uZ8Y{J&3H`9>j^MUk7BqPHm4t-i8f^td;6Y~IO*et13c-(`i?p;hHdw@&TT=2 z?_3`s7*-6S9~L{0Ku=cU0^BQoua^`mIzTK?ikvkzjKew@SPO^9V)fZ6q?w|qxD&VP z)2rCu=^Y+luMoE+r?D2H**s{bHXDvGOsS&5|D(B3Dh~ZUpw?)9)aUT~sXC_6s%z_bjeS?M~i+ZG8 zDI`d#f1><%6q@jeGLD6Tw-B9MPwU=yRCw%1*{iP~X@{qJ2{SzVRJ(`9|Au}qGGqbY z`BRM^I@PQTkBY-o`XA_Wc|c$tQMT;x52SV`UH6;*My)_s zBym_%nH9rk#>%n}GzeRMgj*#)4iOd-H*U5yU@XZUpA3+x`}Jih2kwt7Wgyl~VobMl zfm0$+w={p6Ta|z5CT;`e?=_ zubstySFLD;Q}o;d%(|)jT{)`-s+H!_>~;=^qru@>#1Dc4v@M ztKIQdx=i^>qfziNB!@j0foZo;x&||iA3X_WSTnCduF7LnApUp`4zel}n`EiJBN>lI z35>>E;C{!M@|x#S3{_gC%0LXn#*vyIj#{r)-O4v}dVD+RecDLBy=n~?!8fjrEy7FU z$Zr^K4fpeA{OkAnE9t(JKlKNK--N+A{d3oAcAUA}vJ9>cCcORdF1rE(tV}4(B$%v; zxTTdu4*il99yJLgff8ZR|$p$aylsxJj1rbw8z#t$OiSy8QI3 ztYVZtQ&awUtVg52c^y+kGW1&k^+ju#Gqm5ATtczatBg(mNdH+|t@! zuB~iL$NgmW)l!HpWqtO&Ny+2I?luNdb-HmU>Gz}QPS-w&_}L7-5tw^8{g7^0FI6p_ zxp3CE%Gt2n^xP9LGQ|b^{raE+MO!3AS<8|OPe3HC)LdntW)n|Zk0Py3PoMW243>w} zb8X#{xH_EM<#9v3tF95Y-_B*I zh_0f{!{Z^wH1}vI9eCd<6e$U=5AE;q55QrKE$@eX`caN%W=X6l>T0<9Qf_$D;ddM; zFy2sv3tXS@$dVx?_1ydm=v^&$4LaRal4OFJ1y}s^Tl1cX*@CS;q)v8TPIlO|@yT_} z&i-Lf_(sfGCF3)Tz#Eg0ywa8fV;>y+hoT>DoL`=kq|`ZS6Hv6sVqX@Z$I z{NLL95^$>be&1zCWr{>9QJD&bGS8Y==DCs(DxxTqnG&H;BqAxpVi6kbL^6czQdZ`a zh(*RSXIi-5@7nMCo_o%@_uPA*JDqc%`#$gU?7eHP|61!m{C?l>XZrSj1?;pLl!Zw` zI-=HcDG|AUta;&3{uk4YgqK*29^BG;1=?8~&L`j+jU1$HIdC(B z;jmjV5M4vCpVUKv%hdq_v(kV+&kq9ru2JaBKM6vDeW@VK^G!{vkZwag1U>X-a{TNlyi&G`c(2eC@_0=D`HD-j_dweFK{s z8&J3`wN5$pDB!u z0$P)l$CYtGS8jw~{;b6?>_~#Y1*+_ruZcG5ioB^>Gw$&DqL*$Ai-c@|@{i>kPo%Hb z#=TT{`t7Ln`H3%|FI|hepoX3y!Syy<8c!)TjG!-u_a3s+8fk+LWw87A8-28B%XwV- zTdqYi%RR~2?!<-^p(CCya?N&AqVAp@Bas@fXm(_^56=I2^?Re|9s;UC0$W|dT>EeH z0%F_nuAFHB0xoWgXGb9Q^{* z?x*zb(L3|MqxAnGcT`JT{yc3Z_@@x`<*9;favK~a0}3r^f{*oQs5{~KA>=)8u8SEi z7p@F|B+TYy5+^-e;)ori)(O~Q_;J^Gr%>IMw7J}Q*Z=}~n}Pb=xSb@BPsirHZ{hHdc48;-JTkHUSq z0JN2-cN4T%xAPK0$db8BXnD_bL&ch(`O-$_5P0z?t#HP=-ES)RS*4h-kr&k!K#b9v z_o;9P!#4O^xozAgcQ%I-I1Vg1AHoKbV8Fwuh5!6_9L@^o{y%QA?EnArbe!TJk-(d6 z2%lkTMuz|@SW!6DlBJ(W=c2#|WBgkI+W<6ZXChcB@7g8>K{?y8`@t;MwA?yQyj|mS zwO55;>?TG|03Y)D4l->;*a|@-E4CGqc3nUuXowwV=1*i=ef^QFL2Xl$nf`9Nxu^&P zK-oy2oWR7nQ5!t)Lxto_mR{&bbkU5BW_u9a1`NY<9kdK}@Vg$+o-Z{K#Z&JfC1dUG z3HpS0ykXn~TOgjuAgJueM9W0{FPDpSqp`_x4ie)y03i*tNM`y%tVPi)IxYk~+zzma zWLkaohaICCQu&hematRS>#=i&7!iy#${wbJLzS6lxfs5~mYxU!s*q9u>T>FCJJ8ED zL(3fxy2AvNr&ynxg8Eom$O=vF2hx}Oi0`uIqEf7F@ zuvig7ekbby$~j^UXr<@WEs$EMU0%V&lPQy4#Ajav=Z6u-_XZXI8H6N@bUt~fO3`_X z_Ul@!KAs-_4g_?!-7`PB0J8fhcYDU&zwJ4RJC2fi;ZG=;a7)*=WAOKUO;A1cPzhb`de?k13qUE z7;={$1(M#kt0L-jvI~jM+9EZID*QiG-a=(;RHP#FLq;kpEqrB#E1|SKVAR+d@_@7zsjOa1(Nv;0EE_Q#l- zLXVB;d4-o44CV(3{3m?%J!e~6`lx{3e3*seCUx+)L(UlH1PHCbZP8j_?q@4*k8c<@ z3ta;x!cRTP{5wKe-comufzz>oaPv_as`K(+bb#q2z^<9A8#>#)-IzCF?$L*Dl{tl< zmA=nCf+Wy*uUI6P5z=5PX#g0kGlJOX-FE0`>Fg*5T%3At3~^fs^{xLF1fdRJAg2^g zl^0<4cM1{9C3t65F^AAORBC!h9VDwJE5Pkt1g$j*0&|pJ11dsyVxzDttDPNUfl#F$ z2$d}LTp>JbWbi1Ep)hj6yqpuk84@z#T$tTjVm=N}sKrRse*BaqK7eNV{UFBnz$*mS zP*+&uCmp_8$MQSWmGRa^&a%X>$Ki=>Mx8bA;KkE`B4#7fk958Av_C*`&1ggY*nlbS zgtHmMv-YUi49JUyRztm8%n1{JKrT0Bp`t?n%+k#kbheQoC&#FTfAZhG!J>MWU&YJ0 z6z=(EG~8f4(E@>LULf!sI0enE3&75WO@KG0LY_7#Z!waFuW4}wzqb0eFp`u($dmcuF)fTl z_4Y!1fgt0U_nfs7oiz{#m8C|@BSIo^Gtllb-(oOVgb?=PJ4Z-5`8nYU+*w0NH4geAxd$a|wU@FOzIA7^3L60=!^zmHPs5$x^0EO9kOM2I zkRgpc#f3-9wB;D#B_eTR+pv<|3`?93o&PQ~@?wec#{=23uo94YMz<5951H5(Bl$l+ zj}G&5Piy&Hs#a|@07Tty7%VAxW=4i(@bLEGR%aOOUqV;VjBXbpO?dR@mw`Fek5!x0 zMS6~&x9$bEQhWb){85$c`NR{y0w7gM4~Zs=f9msS5cpC zA$S0*YVh-Gu`$4PK^Q=V-2BL}34C_)9CftBro%72e?qebQt7m+w=f0(p_a|S#M09G zi@xq^=R7BPfRVrekk{P2Z+KL#P?IwZ-{HHZsV>I|s+SgU;wuTPg>WShFh-2_C;SBT zS9*JcfE-pCXYV%VK9Kk8Ax}@2!h$*~nU3zGoDLEaZx)|t=hYa>j34e(qMw<=ZJB!{ z3HqJ{PC+%LnP1=h%e);x{qk4gOE_Ib7>91UjhPvbA7%4bklltNy7-Y`@}0GkE=t2d zG$G`bUBJ4YJ92=2TWqpTJwoWh(0S}@A9YLQJ&tovRlRJ4XmKe3iz5r;cvQGJ*$m$? zXZ<%ksdN|8`3hW;ihVi(BP&xokFTm^=Og?cY8F4Tf^OC!K-3HX=sQlw=6?coNY`|5 zGG(xiL*tR3SB+mdWRrF3st~|E&z%J~mLCAh%@9;@58W^2Za`HMF^9tz>+gPVJQ*RI zb`k;rb!$S`J(#tvsVeA{vmHS8v!IS1E^^2et_rxe%s_}HR%-j0X9RyNVV?;>2~-R} zBq}2#y+PTfbRvfSC`OUsa#cXES4lJGXdI`iz^jaGvtjXVp81ULe7G~Yr8NeC?$0fd z98V8*13-|j$3}m@ZwF#~XyH_wwFC1?GdRr6|LHkrBW#3*wFT-ma672}nr2Zp4pCy24obWQdeP!dxR*VZ7;9J`l8=2nvFN#^_tAw?3>G#`&AW@a2RV z83H@^IL5fn(|`Ad%)eiA0w8pn21t9Nh48G6cydUM3Z|udsbd5WVP_OP@P)vu3H8mq1^C>vRDcKnW`iMIaM|fuh#yHB^m}Rq z*fFVI7O$;VBd#!@HX>p5smMfr2XgCF!~@FcHJ}XBst~*#P)el&7}XRNaFv}%ZFTov zP}^AVkP0%PX9b+}k2&pZIDEDat%9ZC)1s^VP#7JMmjkkHd6y=U6IfirZD06LH=-pW z9GuA{NUjydrHbwYn7(lwOodILBD!Y_IYmW)9kl#o{b`UO=wo8Q!tb9cSg>OLLs|-~ zL9VUa>5qHP0E0b|IS5Bq1fhH{8)d4nzy_44nDC=InGm1d@O~7vL^exMJXGjWuyQXM zahF^E48F&v!&}bX*%FE`#QPV+_DEQ3<8X(5%4@idhG4_j0~(0pNkBFLuZZi{j+KjzL%0(n_H~Xo6-q4XbPc7OaT9y_4{RWDev#C&+9L z^7_?T(&PzKEcPeE_=hG`z^|yovq8dm1WdO7&$cFl2U<0L{N4iKh#y7*!scpA9@3Fx z!Q_eI!JYJ#>@BYGaKW2U{IWa-DAFT)<`7;Eh*1M^-V4qcB>f4e1K{4f1ehAuDXD<_ zl}wJ^F*kZQwQ{JvJJ)lS)m9055M;{~VH~Am*k53R2=*6#AQ*0fbv@DQ=*h52Zw-)b zprE1%T7|aOlmFN@eu9Z@qY>aB7XewjF)0uXTHcFj;+4psLck8|S0KEK{&6$GHww=J zl^@3B$nVjFpw&qv0|`l0ebAiS7e>Gz8-|AZ@UI;-0hV@ne z(A_hn_Y?FhcE{O-;X5XsVaWyUrgRqY49e%AQ5^y`2N6`E2G+1#HiJe_-2W@?IbWbq zx;RHKk7o`+9+MFGIa(lx5mlrZ^^^s+bv4+^0*A1|sF!+fFbq#0Xcgkc{1-6>dI6XR zOwrM_jHM^?kq*L9fEFDM2!^aG5;Wa+H9?xN=!N3=x+69h4sG7+^rwjXE9Bk4G^mSN zJq1$qEp8!Qn>hrRakQvRQelaSDGtCm3S3qJ@3MGM=Wd4a-X+8Cu1Vd3RYaZucys-1zV#ou7gC)$*Mk$)I`16j z4q^I-%cC$pi8MP8JzNalp2D#KsJB@FEWq5<%Q^q}#lR*v5+}VE8{3^*Xhu3uJNA(B z&E?~O-%`)M*Fh5{w!w1-s~?u5dlj_=ZG+hPT7QO<-wg2dUDdV4ZXM4G*3mE4=JSNl zK+Xqff{J=``zDP5pYk*uJyZG`b*btwjKFuf;{3N#!vseGC6Dh1^OzhttJCLPe2&pu zRiVZB5mVz2D$jm5bIJVtL+aDOuc9CSP}LM_29TIc>THxoa|LX`!FB5|M#( z&)$jgeCq;p`K|Zw-{-zYov3dr@LP$RM^bY4;{l-gf_^auGH(Ndp%NgW9gjAj!wO9! z(-al(dnH^xn@qZ~=fCcYZ;=r`3zCg$T(r@dhMK7?$b*r;?gv*|4+Ag+3tpjqc3*>1 zMw4qe`=+>s*)jJuC{!TqdyJ8i*z95QDP~Mf!|j<3$cxmvbuqb>C=>#{b9%u#y4-`5 z3ok$V3=04LtwZu^F_1;x15(PZ?pWVg^zlOusS5q+^A4(4nBK+i3DPK5h2a&T*S> zp#cg#i&*#OgF1r@M&Kb>d+1xI-(1i2W{hn^2&eJqO^}Jf3@Lfdm~)R|y;5d|h1T8= zj%bFMEyAma^2woH`1(HVfOEbQLPHy*gPmrD3(USzR=sfT$-bjVq_VqZ- z6QbL)po~rw*wF=Z5QbpA`jBu15ZEJr054g5IeXeDF^!;)OUz{B&+%7P0j=M=I|tRM!*6ckClso^&h0D$cLfh)Kh=9vFY1K5NK z*+R3U9iCXY9}06oaF^l#~%1zS7DddvcF;SNH#Rra;s;zz)lGSBrL^K8ntzro|lsLMy} zsr*+Pt;(TA6%fu{6#xFY3tP{Tb{OLDw{7;>X z@Y}Ri3!SNL|HK5&T&H;1M``NN9@`df81f7R2!wor>T)~-)h^kIP)^CGf+m07@D;T6 z7=L3179#j&T1@x?nA{XUhGFwm739xmA>*mu>1PJ`KU@Ga_z(+#%|hpPA2X1Td!)Bs zIh1TXp51c_c72OafW#K)sLPK0xho|+gq(NkiUt@=(njF>OfS^Zi-4_3X2EmEUfb?! z8t|jK*w96He_^u0fUy9CZEaEG>So06O!a z^B#~Sb8!xK)J4NqD)!MKyIQv~R(b$hIj}hRVG$6yL`@+}ATxaIgP=ECjZyMX7 z2nRq^cJb3?ctn3xo`*IJYc@S(NmP0xgv}u3|Lh;B7?+R0Rhj=7lDpJW95Y7mSVB~q zA2L>QdH$}=S3vhvK%RukDu5;oLo?`4H#x8BZ^IEd(4-0f`*$>Q60X+&>K=tT1b^ed z(<?_zQ6U7a8NbdK_w>z;J8@X3xz?&}_~rgp-FKQEllC5b>h4D} z1YOFVP6#oWF#nr#DgO0A{x=U%<&gNdH+5Hkbp7F6PHeI;W7pdL%BrUA*9GEW?an3# zjp9$2QAU+K?AqBVpTqzL)QK7)pp3JFnlRuJrNlT4kNBkvyFN@03D5*)B@2l_`SYDh zIB}4U?E(KVFQ-ZeAan9zxy2cSWN_NiYXnGG)(2X{rxZl7O|To^rRQ`2PUQvcZL6O@ zEMS3Oa9RZ{qD>gbj9?Oiq=*rdu+za@CNdDMkfz|4{DA&31_sbZ{a1Tw(-K9HURs|U zgFfF7wS?AiPH@7+@nsqY>7srO^+9Np%mkZ)Lv9+K|1gMx6c_BWIjQ?C`Yr#!yg5t& zMGzj4TSw5O40fWpj7**D4N7OcC%y7}XDHf-=xa_0@``+aG;QrfHL zWXjA!^%=jV-oM&d|G!l4^xZW}H*Eygb;nCrfP8pdBaOgx&e{+hv!W2+c_QF+8qEw*)n`;%*46IXl~;udp)o*T zzHV;cwOcY@bvta9+Mqtb5i46! zL9F5q5Rf6dMZN&(Ic3rXOo!%wVN(a#)}idD_i)QD%vUVYuZLSYrMLt$Fb{1!AqTa? zIWh#!m8i&+<FWbppf+_Jgf?N03IehB+Zk$YY{HK}}65ml3A~w4~Q&{$T~#!#aq2 zPReuxqPh17`{7mJwO4Pw11!tyNnb{+gF$W}At@=D*DQ>;CxLVi8g@GT{6D;>)8ij= z>A~bso@(Gyf&s&1a8~Zw0Eae!vIJ7Ps@>+I$g963{Wez1kq?T$(!7Gw_))20! zpq?@dhta^9&ZCv^S{>bvM4>VbXKqe;Vy#l00R^fa;OYYomFzG%x`m91JJ`R_9StCL zln4$2hk|Zi{tN2`Ab{u{3J^cT(@G3UYNoH%!Jny0JE=>wHt+3Mm!G z3tD-_H$cpK*>@S__LIN0a0=3JCm1Q>fHru6QxviRU3)>BhgVCkyy35A zw}om|4sa_Kb)%5*0=do+z(A#f_b`?X?=OSQ`z*M-)OJJRPY?X{EL{fl6GOv^KpFdg zfrCy?bB5nyXi1%;#YmK+gmT3ni%poN)VgdczU<)%>gOzI^>%%P4ak=@ZSH>TTbSQG z7TkGqMw+c|iQ+GH)QF;Lb4C(t562NLR^oVrywNCyd z$SmOwSrP@;xuHI|)8$K%%W`LoW@RvZj&G`uKrzM&;?w5>w1v^%L+G0E0ixTK0MN^Z zX%_uaW5723d3^26Edp24>E6QYS@UUUbH{hC3Z*;6fU_yu=ra7p(9dgrFqO<;-$b<*b1?F9g4SFKf*8NQ04+&<`!36= z+A@;~rbDzHbI_jOk#>i6>w%pU9I&A}M3R%6#eRX_mLTtqOAPJCNm?(zWv48&)Q_;J zlNH+~w0jdPlg8Y^HzO4sNYmWDoaBiEt}x{42-{n=LPAS4Oi%tn8Jc7I2zPIC%6Kh> zilW{xAV;pxE2R#MgJD=BxZpB80JIs?pRaL3%Fmkn-(-U?lZk6gD&`uPNs()Q_g=rA zWGoWC#sbGvWCM~Z4BfPmvIx8Ehs&Vnd{@nib|sX?O$rHO15XPj(oDFgZeXI1@1g_) z1F6uti@UnEcjKG6&<2#WU1!qCy2L&bt=V=hhrn(+v$1!bCT7Dd-;6C71E#5As^YD5 zGCX4Uin;AKOGN)k{&i*m>I-4a)97D8(@!%Hjd7Y4s3@Hr&@YVEt3+kUx|wLr-!(P` z7zXhLvL~A{OjENxlE#IxZ1qN5h~Wny|wp^6~B3z z19kDb2@0ds(DI|=;Wul?L+AQs#3Eva)bxT+z%7@~D)VBaCo>K%@C_u8ehv7scjOmb z6xJBa@ROEM-%;rbsm6kWmq0Y#N14mjn__$`eSPA_sSm4pS7I~=Yot?vWUfyLHSOo9 zELuADx|HnGJEd{AR_Hr*pZfAALCrID)2gPmN4^~H?44=6FTK3NrI}Zk4+cBm&+c9; z(WJnMeStuM*B1-%_GD?YGR%a+(;_laz2C(MP5m>$G-oZQuHMy752i|or|1K6_eY;L*qTXQ_!L2d<5M9;QPec$3>2|O^$XbRF%(Y$i zw9QHF2@A&34=z$}Oq0q}wNll?2S{^jG0G1dKD*Zr3Fz_efQ-9$)m08J$AGD(kA|b& zaOlL~BfUi-IcJmQ;0PHm`{76mwTUu3G39ylNhGCX?wG2?G_I#*Zs_z$?SLhsi?(kL z@U;7{=HI$XT5&W0Cp5oqfqTO;taOytZp<#u>Zufk8#w5}3vtpMx!@=vXA6u+6eA85 zMoj42(RT1B1jw$lARlm#u;X`hm_@HGW$pSNJKEKa3BvH8+xP*;H0g&Z+5=&U7n>^7 z8emB!gCL<~hm0+zpC-6_0*FS@^r{KMFh$z^?UgX0*CKxw7V$pKhJ*b&7Q?kdr_!~} zX}w2TS*TO$8rd0Fr8T~0RO!&-jx4C{S#<@(Z6B-b9`jV#9A0(#!xl?<+6~I0GuQfH zc1HWj6)ihU+Grq%b^2T`rSBgDg-Jv3%}C1cH8eeQ`8yij!u&V2q?bLWK)GVMz+@Ux zQtG(DC?ck^@e}#%oFbIFYTpjr>>Ps9WwU}?Qt^giZia0pl37B89RW9bN+$H_ZnXnt zb3%c;S|pdkWW+dP>}bfI2(Vy`rf)+9AZpU9{bkFx{b2V6Fh+9e`}*n-$2l!G=^&3? zkwUt?*Ap>-ZspXo=>u$pKX`g9!=H|r z(oDrj-Vt`|Gnz6oH!#0FLxb(-!Advx31Ll;R_GPRo;Ge&azf1m6ez~gsl5zIbCdzz zywTq08b3aCK?~FbYjm<^eTKO|?U5*{Bl5QL1);QA-r%;F5I7$zKqy%*l$49Scn*kC z7K9o0r`5(nUI)sd`qJKIczVztiIc7=fqua=G$&`rW2NicOxE3DFsOk*lgTB}CfU9K z-)Evc$lM|GML`nEossu=k)OdnL7#HpKav2W-AhLyKSjzFK;I55(q91bn(!L1J>O%{ z=1g5u_-r3aD{XSsi4i#V0(zkzr*MhSf(b8!8K+P!*n!hAy2=ST^HZ;Kp*T&aO`!5d z(=!nkoM#{gE*b@5X{X&K41vS3tJ1htzfW5M>)m@an_|ZD0A+rztgw^isrF`>KDCFf>&zNFRPzm zVAFbj$h6}!lo11zA341-@5^#}Z)`s)n>Yd}wI!KBm;EPBZfQoDb6#N~Tt~J0#YG>n z=Ec^NBI*q_`;U@i8E>BSDmbJ8R44($!A)_JhtkbNb4wjrBcUqE|1zIJNP0Q=UNFH& zR@%-7K5dHf&9rKx`N*AQ-x|s%SdnaP1)E1}&frkzdpZL)(1G`BHbcy?Ol)st$xFvO z#$K+4Ml{VHKr$&?yBr%nCl&ew+D=Ig$xvQnqUplO$Z>R+i6qAselXw!*_?Q?QB$tQ zq>#UjS9IyAQCWF!UnOs?w4dx_NpP}Na4*+NhjS~ubsCe+wiuOMUPA9b;mP!D1b;GD zg}~eKD^V4?ci&XxR@gJjl+U@v_%1n^%??iTGuVBY*!aj!&PpQi5;iaW--4;nRl_gV z!VaYGK)XWf14iOqb#-sz`zD$1FKbt}^d?e3VY_8wC7j=iZ@Z(xTJB((FF+eibv~V_ zKNV1}b9pu{m*Zf_N_*G05P$XVSfQ2A(Vs<1&a^k~VKc4&c+KR-J;Q6HQ(xp2iCa94 zAMxw$kOcD~$#Orb?W}&1=*EkRilQYO<0D|E-0vkjO8eN#^1^-hZci27k-4C>*b$X> z^4NTrfQ@Y$WpO6xwVgCsZGqJE>YefX5>l^ioo%*Z z`aww(+A`Tm%|5t&%ibdQIS?jpI-spfFk_av{HxPPNa37wYc+v8X5s2VhoA-b1FUQ! z`I`zJW=XsayLUrQah2Tgi87xHBE#Tc*=o6UKJumkXU0?MrGqaAr%D${eyHZZw0dYZ zk?)2SyLoTS7S8nnYjp85<8Wt&R5lI(8OT5-%&-Eb8k^-UpJsA8D=RQK@N(P zQ=eC_J@jjK-t)^EhtLOBY8#k6?j$+B6M`CkFn+{OEfqY;9mYYE@oo?w6}`3At0;t3t;A2jc8i3f`JS40dg>!N`y1(gZ5?cqS^rEM;w)=H@~uN<9B|$!t7>`eIhPu2R*u#pn%S)5P=H2Taav@7X?{;5Q`Ff? z>4h}oxLodcO20aYgA)U#U#L$at33xxKC-NHCG=`OP4S<+GfmwolP>_vPlW@o9n2%> zeJS0}lFQT_9+N4pC`rg-JrE}y8$Ke^lf!&cQCj`5rC^{IY%UuE4;&RS%c`1?VsU>G zk+ScM(dh3h8TL}sgA5m;HmY;%6v>QRx#n0S!dbOH?$GwgN>I-_0V)yH8G@1d7X-HN zHt`J{8rw~5jX0{iKRTD~R*1f3r=LLUq^=UFd*!pcZq3meA=FKl#LDxlM%C+~&;sJ_tqRWi$Gs_{6^NCI7oO7hf)qFK%iV9eVyU z$@oG3`IOb`eNb~MGNnYUHtlYSm0nq{@n$7pfWznU)mv292OMR1)Qic3V&&$oCyvx_ z`~1~7mtBzIt_1tXprMhvbj)^-U@eEXM_d*7sPydBN~BeKOG7{FJYvyun`_iW**Tj% zT>;R`Q>{#6AxnRe$w7ug#yb}xJf=f(wKlJbw-C|(lliII`GaQbs%TC!u~&#qXCnTn zvEg$oahgdAxD$ARf}l%DdmJ|U*GoYB*|cQS{cSXFT@njPwy+uiC1O8*-v zggp!5sa|IJ15YZ2=Zk>}t~sJ|_HffJLDPT>j+N)_hYUZUVi}c>DaxKdE~b4fs)KxG z(Mvqd2knSv-`W$`-H*%n&hsjieB!;0mZLg7fAy-&TKCO)W> z(_PSL6x?xg+n>OHZVg~}4YuJ&|jlytU$Eg`&<&l-lMnZ`ya-Aa$%ZMpQL% zdSda1VnxZD?&vM-v9o$jQI4t;aZ$Grc#?e_ntMxsPl<@a5$A5Liq}8^KYUs8Orkw} zdUN<8--*ZaAZeWE02`yn<}^NmBXQ|1-8ztbgR zd6YY2o8x1(w@PlI&X+;_(kTon>?~|f7O&{B`!#D=z3A4y%~G6lQ8)U-z|$>t4!6sN z3}_A$5l!z-2&}FKu7%G$aEjwwqSUeVH;g5i$cJ4(4*k?LprkuGEh}xnKUOzRBi?~} zV4eB~j?c=Y%VxF;@i;py%8wA|x+y7ui#J^fQIaA0mDu{=PNq}IESjc6_NNq88v z*#%S1?eE!{womAE?;@Hl$-XY|_h1p*3PU7hO#>nK;zpGZkGRpzaf_Kd) z7Mk42JbOTLRot2W^{%@#5`SxatUD(tPTHrPD#7I^-#EML+=n|iCTa#ivsjl#<}*lp zDj}=9WIN4o@J>_mkO*19!BvCQ?(J~B#tz_hhtO2&D}jNf#@L_@?<%JKSVCkxVvoME zuexe|L9w>)($6ooY2a;20jZInyu~^GN*r6F zn|z@risN2W&GGWY^}K>2bA#2>fH3db=zfgQWOYR_UtsCu5so(Al%bzBGTCSjnOMwp zt$P2q@8a46{(tD1ovX3xm7V=5Fm`GuPr_HENYx%VjG@{RIN9^uX`6a$E_BfY89oG6%aP4Ikrc2noN9Y^sy?Up7l_EFO;4qdZ8~$Gdnqjr00s?fb(w_JHg+2} zs|}9Dper7Qz?le2&!3wx04Jx#Xp5?Mt(Tzjgj%3cDd({zabg-#_Cn8_utrL(V31qX z)R~28u#ITs(v-HxdvqrHGz2QN-)y=r>=~yuxMM-vwCisJ>S7+`~e54em<1RKm%YY z6)nZ4kw<>b{d#ZTqj&Xo$~w}`lZV)h^7e~Zs(#tQwGVu(0{PrIqxzQqz9?M6QCmuF z^_3!iO0rdnyG8*i^t}YMN$c7ATQNL5BU`NVF94QQH;d&+T%68UZwmnq4Jb<-wC}|Z zNjXdrHG7pgejIZ$`=|sg^xVA*wr27>h*o<{?@CMp30ao2K(+V)eo~>7DOzeq888#k zn&O?=JnduxdUsvVJsv;fAd?^X^3sz>njYUPN5SS2@tG1h4{>6*mtdob%%}i}RZ`W5 z+Ye3t!l(3|j>oxKU!;f5BrA21wR+B0A@5OOyJdCYK+K<{jA8+vKZRCN*Hi-3 zz9}KNzsnV{k7>z8m%o=M3+B~$88w)k`&Q;VWj$8MY-cpc)l6pX zgt|nSb00Z%(bIa4D;OJi>q6@5dwd3#%4s^9S_R@Xqnb(dEH3t}3qI;NmiOKKH{qfT2spzP<6)uI;{c%--RbLZ~O@<>hPBRwffG zZaP*=u_&tq0H@)Eys6E)A0O`V1TOigt7k(*j%mQw3L$gxc&}^G-S>vx2u!?&Vp@1e zC!MS4Eq%P`qe_Qbc~o6Z)2N z#GJJCGJF)S!rEs2ruP>>{d;Wp={W`rwb(1f1 zZPTZN4l&HufImC(To>sCz0&jfsR5mvM*JjgcPohs^7tY!B0?8Zll1$$A%T2L;lM$| z+_@iTP&wR2HqVae@1zJpLHLwxXZ>+?!KUf;eR*EO+0z=SS4`c0c$h_$hr0D;@*IdO zQi0v#R3_VHjmWnb7|wGZwKr|8Gh{rqB*0_;m&$dy93y7u_g#Olr4XSjV0K~;G-GSA9hlg zmKr_hRgtF9+u>0GpgmP1jX_XkDQVou?W3I{fA4LHxttfI^jMBe^jxeD;pp%wbT?0n zezeP+wBf6kZqop%;a5mpdZZs@7PH~VRk^SIu1FV54>f07RsV*Dm`~x0bB6N|RmZnC ztvLcP>(S>Mup`v5A#b^DB%+$(6_-KG;d)C~=$r)mZKE3mw}M!KIswAm<=8qu4J6jeHG;*9;c_hz&xo|?ktmb zNo4*6L_F_Y*`lAj*+MJP%45Lvo#FG}VQd>ugC0EM5}O4rL9j&hG>;CsDTDF89#^kI z;lNHl^#!FJZzsM1P$kta_u#0J_x7}^fh7o3lm2~I&r;lC)%uaG1@QZo`}XfaqrW6KXz4{af+`^@VJmbHQf1(!txZE4RoGbIn12|E0iSt*DUDBvzGq?p zV_D3Jnxc`zT~ZG`)i=_zX9b{{o{$IM zsy2>(may?^JQN1R7jLqhiy6&E@Ho`K8<)p)<2J`P$C_Iz@_ri{ z7*ZZugu9jvGczn)DXFT(Z4{qi2l=t5ASKheY5UUSWDn4^k-6jGx;s2G2ZhD)mtd2A z%z8U>WxGRQ?jTFHYeaiz;((*CUqf&04K}QA)UrWL#a$dZ6bvKj*sy*AzDMIiS~<+k z;X6%vaN_qq{jmvA^3$h`CK8UI--$i^Q_jISX#B_XH zc>9jy#B5>(P4pFvi`X$y_a-)lG2G|Vc;cZQb#HVTFS3V3#u;1_(XNwaJimXq^q8(r zRnm0rMrT&anZ}zB4vj+gNh0PPtjucuY%k{{YVzHc+gVU}vP{I1&3gMZxOg4J+2qS1 z=qA;9uwf%n+j<9=`=;Kiak2FwNVBThp1-KFThN(L6Dru3oV2+*&6rFFYm4<$_n0YH zZ&OY&MO^z1!Z+^tkaOSn5MBA6rruqLA*LAddusJoFB7!~X)P%&EiE-ms!ZxHNo5ck zD*Gvtj|=h^j~!THj+>r`_Ax=bpXKPErfcV3buaB%P;1PN&DA1BHHu%e-8MZY%%JPQ z2?%wd)EZUq;SH)0Qu|BZJrvrYmpylJ>b9SW;e91pzFQb$BxPWprp=c)pTa9kl9vN5s7p{vMFi#0?)!Xj@=~~f_;=2C|GwKgJIav< zX4KlmHZAX+WZdaA0<)tQUZ*ojJp z7{Bd-wi;F_w>kz}K*rT;QCtqeCo1&ymM1iZbf3P{BwFu_pw`d;gR6Nh7plSkMMo;z za9#C%=l<%(u9jSJqO2MR>y-n^B7q&p7fH3LJ z_!-T`V;3eJZI@@|hs24|OuRuwqdg0KOBKlL2?$$jfX=2vY?*fz~h)Qja7 z8Da@#)$S^so)=pdu?)TQsa|{x{J5LfswtR7)LD37xN_s>?X`!kML*>wL~mM9y`sR5 z;Ta;cT5rZca~W%!>mtLG&?Ybec4S>CmdKIrg|_0ey9v!{LrXOj7%?M|CQMwoXrCcy zBc4NGZfCTneVS6G_@5^xGyvMQ6^zd_P!k-c+JCBxO+5guIswZ6eYuQ7Y=?ODl*w1! zPhDR3?c5y;!=paj?!>uAkmS_?%7ygeE+E^trbrc8e9z$vz1?&nAFW_4I~!kI>B_4y z;`2qRuYqgk5lP;w0<$}ACklON1RD(hr1q;-3Kz|I9pcu<8Js-xNOw|NQ{F7D{-mT`C#0o0pAH!1kv@tGb$rju>vLTU9+t==2Hb@@>3B}DMgNv~b z^4aG;gSIN2VppDf>yY3Ug`#5*^-?N4)snidl!d!gGM5AsiMK)DVa4Nd-dakCp6>n$=WTlRLK|}{&Svqlk6@O zs*01X2CLsY>6pwE!_Ig3`DKSoL1xl*I0TWe?sd|f?&F5yqYo(~a}!jrbZqjV51mlQgaXgBE?3GsY96+n<^~2c@YKRE{wZuIS&DilmkUO%&Z_ z4A=Om!NO~(XN0Sc9&t+)A;w80@P@>RkQ?p!S0{DqyO+~ix-cKa6$;xIUPDc~%cw7| zJt-(Mj)S6bD3WYKVAa)C6a0}?YKwp9!FxBudP9uc9E~H-M<7kYV0J6iHtddiy34X_ zt}O>U17+MX{2Dv7pu2IwB5R1Ws-S%QF^9znqC~Tdix3c0gCZ<7RoK|56bR`*iKePr zZo8PNOG8bfU!jgo^gmZ#<*%Y)aP?D78hr8G4(qU{A1ZJdSFTn+fuFS)`hf$hNQxJW zW+Cd}9ggZn1ik|_t$o4baQes$qx7%Jn>%9!Mlm-KN*)6n*fY2ebz19k7jY1{(hezn zCE~#%#OMZ0J3a@um+S^7Xpn0ewnbK}!KY`2@1q|n3t0*jUq8pFraUtF9nP`JVU*gs zupJK*#K~|Q4%37!#9}5fJ}UMeEgs%)^?-ibO|474T-X^pz`Xt(S0uv~{6S5LcMoWZ zz=^4u?grM!30Y_Z!uc;nL&?ht_~Fu`lJpZd0~?&*=n;5}wc%f<@@c<{I=e6&kH zJZkV__`gJwc?kXyyp#1bqY3^CoP2wH$N%$x_+FxYg=vb_-O%Rx>nEtVEpHPLd=PU?3?$MI$+M<^6IeB8%+a6kNU4Q(dCA7Q&YkL?t#4DB3so*AHIb?mIot?bNAbXgq?p4pmM zS@Lo6aC2U}!fI@1XKgFY#bxm?dvIDkGvfNLZr=oFp|HO9#1@63k3jyB+_!)A5rrzh zd;j(grRR~e{S=Y1t2?!e_|PoBtRM@jCp5n~AFN&Jq0Xi~?>l?`^Y$bC-*S&Wo!L5g z+~9J_z5GKxIsU)O`SLd;$_L%>+lCEghB^%mVVLE`!a&=JNZV%OfNjmxT68m>uu1{j zakv3wlafAxrQG|7qOv`RBH8y-$_j|F_%mo}~Et(7uiJ z^@~PLky-sGSlh}sr!Ny+%$TkBXIw&|j?s~@?mhg_Q!npNtrXW*ER~s8}m~3IwEVjFkCaiVR$cx5#c%TSjr}&Lyud3(2cIuz* zOr6`Ttdzudxo#{aYgf1xyAhpVSFlwO*CMpsiS~q3?OMy?8?e3XM7O08F}rzHk(XJe(0jp(jSLGcG+;_=4i3w`O7~%51+bb z9b5P5YU&XN-isz}U+TgIQcnq*e>JH8zUfrijn&as^DjFYHiX7a#JFqqWExN81?UJlV_7q63kPTH@aJxoILu}5a5B5h@?gDMcC^lhll_5{pOSDZ&~_sIe|_w8JZ=|Vt3!4xpR|Kv-r;qe3*5=BX<1MiH4m6_bu`j0Q@SYHYlF z*O{v7t^e!Ya5QFZ6{CHu$bLp86tg?!MieQ?|;G(h>0h3MEk+PrXE0u8OT}${x9BQ4<#SmCKsN zx&6_iPD}5go8Nw3-mt;L7GqKcL-ORKBsnHW#&Y7CdP&8`><$DOa}OJ6t#{A?N?r-k zoOaR;Fd>!Ckstoyc?+KH`6iml4A~#rlAE~`S3@vQInBD#*zO0h zQzmN^J^S^B_VVn6g!PaI(Q!x^Zmfc(E%wGf7cq&AKSA$^1Uz=olf`|D=bU!AD-O>y zQ(6{yJ>>p}b6IQ{=n)p`vLy{pM-LwSM)>j|q{$6e(m!witY2G(5#Bh%()ws1)?Itt zu~T%sA&e&3Y;(Y6zNnyVV?2VbVtdo<*WzgH+=O67;CLP~Mt9gpM#PSwJPLgvgnN+v z(IOez+F0)2(Zi;z3k7Df2%3kw#PI0VJSWB&El&727J zsn^n2Jr*q!>X^AK@U+sy8vE+it6R^;>c%XM)@FMwSVu1v+AtYGa2?KGILGDjMsA6ScMY!{G6r9mVKO{j{3+@*l+j$J3okKGn?z-<**kg~OtxS( z^p?3_{OOlj|Fena!w|IQ6~8`Y3+yx*#}C;19c2=JEjl^#r??{2o8k%#^7+-r_~JzN zNWPgE_w9{~d0d=*RVb8CDuNd7N#bZx#VL(4LJIOXyi-N=zrQ3e+b@=}<0N<^I4QP@ zTUjJy=euA(?=8N#*H5f&(kVub`(Dqu?xs&FX+Mwc=}JPj(`cJ$C$>C{Xw?8!~E+xqZ<%v$5_Z*l##N|XIW z5p)ISk{#-r?RbL=^Mn?rqfcF<(_P^>9Zm;@`l&2K3OQglcEhj@RdwW>)&3Iv&m#eY zg*JKwSBN9R?%Soc#1z#`PHk=NKx_l+X559`gN>bDM$t|+a*{*PE)0DVynrCaR&7Il zDg1a1>Mq((cQE7YA@*wy6xy8ayM0W8xDrIj(JFWCnW>W5KN9XSg7cL+V_OaZYR*k7 z{w`%8;fr)|o!q0RrzPC_^h6qMvh{zD%w3Ll(a8o)=UzwK|hx+)O22ndu(FoTD1&aal7j(bi|@^L+zDspr0c293e(>fSuF zIbPKa4TncZth8Jg@09RQRLYzj@!df41^v8@KjpoRE+w4uHe`=2!tgZwWc3O=JrO9e z?el&{e{Ffqii@}5%axiCZMWp9_9U#k`{EdqKQ+eab|Gt4bAz!HuJ1abkM^;Xo?*K^ z@fiO7S*h78366ZZefZOvnT6phg-xO3n~v)`TB9uuX+H*D zrj|&&HAxNoy?#Uc0$)xxMM8Z2mP@#VdwCgGo_WusIcz~e4poFyFo*K2-;iT7^J&$+ zuA+osOm)gX)C%`bIJ-<@JRLGoXZV*ThT_7WHr{)Grdmv!u66Be?pa|Af6IIJ#LZQL z=`H=W5<=wNr1a?CjkX6|JqgLWcAjr)07>LR!X|7%U_Xi%R-^@Jb&HWBY&usQy=e^S zK=3)w3;zu!-eZae#(l;14~~7H@X|QL=rKSf5V?0`Z%>@^J{jyT5{KPng*5ANMOdaf zVIzvS!gl=eIGT=2%N)D1(k_pkgW&5Mgy%+~MmRWN>`R93usL{I54;{=?vy-#hCPz| zMetO>sUE?fUM*~OFrGV;6)#jPXXJ-Vb6IJ=I~#xF)&8}|<9(1WFsHbmdT3fjmkdm7 zOvD_+3)7sL>&~hYkKr>4Fzrg4T3kP_xjcDD)7S6^vz6v8F&m;=?RbUn*Eg2xdIVvo z>62pi(@L_D*IFI!3u&eTbh0eTji{4pk0LaL@l6kuxfC{2$7Ij}z&}h2u)fmkTm}!` z0oUi$%r|m~LWZvN^z~<=Ejjqf_`tiSrwb2=$!REs@$73)l4r?yYohVz)J%(#LzvCM z_o*+*3B8Z6F2S4D_Jo<#EUsJPFAY+HZ{hdZm zZaB}R^Xl}NP0VnWH`eC2Zba(LaBI9&XTpP!gQ+t&10(RLZZw-fURi*J^amp`@Yev+Io*5L*8J3x-%JAG!jS#e$wN*@? zJX9%Kn68ZukWw#knYI&w`E34jYF3P}7+UQv9J!BiIvIM;hI8oGY#R{4CJVinhq6=#7m zB3&Hnnom(rI+eJ>V;hxzQ!FP zv^cs~OPg;%4e+J8khxWVIL&$KFDI|66{Zeo1S=Uv^^E&Y$xRN6UD)jQpe^ zY(fU<#s?aYPY`r!F7sFLC2r1C(`o14_{5;tCgR*qmvujxx$nup=G?>IGu zSLd)oO_~?z_}*IOnRfD9_FW^)4=Qbh&FH6aH;PTZh)=h2nYB0Jl9g3Fxx0;jFaR+{HHJ2r)@byzRKDHe{9!{Yl6WRsQ z?p|{TgQASFKIuW4#8AoKqNO|&SMx(#nw9JYEc?2>gxL%2JVT=UW_rWxve<5Uy`K6! zmrlskxp|)ET{t&Ji7p?oMS}y)>44MM+x83a2;|3`yxNEoeDR)P1_woYPT;TSP(+#g z7?Pg1MQdtS+~OO(;fC*b%o5*@(W$BBs|d6zUGx$;a=4K!_%I!3?y#Xoi5w^U7k(PI z(AqM$Bw0y%x_JZ;+(Dwe2lueY6ua?FAsXh;ca>Z6*%a4gR)Z)Gx)E1Z4IlyCEiunf z%d7LEOq8xAGf&h^9BHx}e8EGW^!^AK(J zaOm}z#8cdYVbjtyeW7N?8R7yFNc#PLUjxt~nFE8kcPwWl(0t8WDtd8`n7p5IFhP^=3m%H&87j z$N88eDE2k7Zo6+SN|_5C@Ub`Cl4m> zp;0rlKd`6i>IEvmhm1{3Twq~gS)Cu;DGwmD%S$A~fUkSU_6kTN5*6a_inZtk(DU4ccu;Ud7vd`rQS9lOB`QEuWQ{jPr6L8;Q%G3I z=5v)Ik@VZcxSP&|xdH+7rrlg`-UDEp+R#i^*nA75VEr>Qlgi?@ej`;ivrA~uH6CgR zgdENvS@0Jcc>Z+mA9rx}yArA#7zQ0TxsQ|E+08HZ9pE^3?sjJB(i@}_)QS;xdL|pr ze?L%cEi|39ugs-j?&OiZyZGrvrbx2R+7EP=3X!2PLg-il5r{`+K$)}40B^)r*7gQC zij&Nv^_%0omS&lAr#P3w=hwApjTu>0$Wh3~nootgTlM|bfjM5dl5`kvLM=KQTGmke z@9%==LU~XuK*ezO7)VNbV9-D=>V!aL;Mc*EV^BYCk3}e#BUQj=_&-zMyC#)mKM-T( z6Dd9i5eSZK9Dm61t;Rgew^@zUaAD^jbTfAFagqh02#Y|)$B3T55ukHcKu7)VFRmUaHS3~4qLVfK?P z--*lc4riAD3SuBX&TC0n4TwUbuNId;Z{`B^6Av|tw@CCFrTPtR zFpu6{q{Gz)vJ~{chX3w76lX-8Q?eh7qQ%Nu=jxMJ1kABt>zxhR#uJeX``o8sN0mNb z{48kMH)*rh-@>=B|G)vM&Cvd0dqo%y7F6pqnH9ZHUfBNm73VmJ#)xa=kpJV3t1p~` zt!sek{RasCB}bT)NKs!=ekMw&nkz_khg$gdEC%`)WI3u34dVJBo770qpR&MO2O79E z73D71fjs*)dJ17RycsVbX|pOXQkX`jSKjl0tmyx#g?33AIQuoOa^(&Z z8wwIu*ctV6^xX?f7nFxj3pqXV|8P9_?ubGC{JO59Z~*1e{ty{9$o5}$v;W~JBJ)75 z@t=JB{~yozX^6BEs_4{53DVLLg6X8Bq)MPxQ8Mt}7P1=fTgn2>=`+O86p$i7;0va{ zAeRO)GgU51Snb=>8-BSgdynmNNa8#zh&P>I72^L)OeAqAr(C@A=7d_7t|ua5dLE`T z1x0ZQj}Pp~LYNugS?@x~PZE=ZuY-X>pBt>u1{&oR+78#(Q~vj?RO-|+3z6!KEIh-1du95xB47&C~j&O1c>S#;?!iLDkdcc|Slz-!Dc8a9R}0CUp4(EQpj zUMjE)Bp>Sdm_2lkDhwf`LAh-$);jd0s&auC*-$2a%j?J~AhZqhFU(YG#ItONwl zJSd*;G4Vag)_>pIOy;+DjHLN}Dw3gm3+!y0(lng0epKA>OkfM)?ibLO9j-rLcp4?< zBG@2Ea|Rl|P43{MOsz74x!B&pRA>ra>UXaW9P2fJdbt#*$AP*v&v?9j6_3$&&(qrx z+nkICVKmZ(nu_me?1mE%FVa!MtNS)j{JnU`c^}jQNTv=9R8&LuEPU5}Y!*kV?=iKX zu5UQq2U0COUc>$uoInPv0&#GHo}PY6(6Ihf<4pc&ZQy07Lx(viM|1oxx$2fs1bq4_ zq-QSly|szpl5s8|HA7@_z(xpnp69L0{f0MIcI$xDoUhJKs)`ApD^9snxLSAJn=IOIm$?8ze696tN^=^3eSlRvHd zcxi44swyo!&v4Xx=w|nra+Fh59CY6Uz3>$f32qhOK?h`et6PmVdd#Oz5Vj# z%QI<5OI`AZ*JuBXz3aAo8GpV6&6^)FWNeag$MUO3;hCaWI3~wYE$W=Yke_Mm4qQBi zF|5UfG010$VXc>wBYQtZhh}W&%*T@-;k!F`;+%_-u&i$=7ajMW`-k!W_8UhC|3jeN!x^XTWB>&^g^S|Do#Z3%w?AHu( zzh=@WzW2~SB`D6I_%)4XF&cU`ehkC;nd${iDk$&*C_$eLSa=?H0pDF{6JLvV+ZdxO zX$ZdTM)~V+H#dT=7OP`8Be0+H)E9u|#h8ecN|2PgC!jvIQ^hd=zo7Vnlw1j3HFs^@ z`?KC64`%`Q!7e8Z3ArZi{AOKO2&I5tdDfHnZpXvr0acI(J>@C>hZ9Xvh@>BUNwLPu znS_Kyt=zS!DMrG|XoUz=?vK)O7@xg4Mu8xtG2@;GU64lx5Zp7T*!KG9S||gr{x^yF zzM^(qd$z#?7%L-j=(~FtVg0_}qXy!Lc?%wtJX7E~ik+4dK~zcCDsxU&FR+vbhsT#d zWf01MslIjgu3rlXm2-ZT-@kuHVi@oSajM#AWdP8=t<8T^+U;{U$UVF$#?CM{&FQI3s`4N6pn zr%z*D2+?9L9pJJcPDt!1fKGrT5JA~Y0dC`WbF(~D+6v$>%hjzu1kN5sgt(qt1Z=dj zXxUe&^XdT&M>s}|xY7>EE=Vr&S{$6*bY-%2vtox(L9i6si%nGXq?LfUra+x3U{eE8 zTRBVjPTBg%Tb(r?+83Y*70^218Bq|E2ff_cSJca#^S}vWh=~FD!s`Y2nA*V#k{Nf) z`|f5ac#-mkV<;|NkO_?iOIcw75C&3^jAzh!40~7eDbVzg*k0Ge<~c%sJ0;?f1}%Hq zlBGRrEjcN=4OFsi$${p{&eRZ*Ej}4zqheOZL?${;4WEw1Ph8p=U~x(Tj%IT_!s^Fs z_q(5O`zj^Mg#GKYaP6xMzO352hzA$j^JXHvphQ3%gBTY`RWcwcWjD))jFtp*X=?#k zRkgPM26D@ZWW4|ezMB5@YhcR3cmB3#_n*Bu5b;MQ^(?@_Ku~lTdz>N3>Vfln}qhoaNtg z=iDz8H<$qOdAZIYKAR|IyxVEf=PjI2(+}ITD3Z`S5Bl`lZ7dlOwt%Kk@IK1aCs6|L zt;*}Aq3eykc4Yld?qPVk9GoPRknpu$6~O~`FYYwxv*Abo6`85#4FHrqD>?vv#=d53 zu1P$!Nbm1khfZ9KMc6CH`GNe)HqeE^R--Zo$n%F@ct+DeC})eI!!Y#ex=Xx?3%Hr?J&YoWgS zjT+Per}r)7VYG6BJVJ5#iHGO<8)i!lW59-4;`cx%LW6YQzC63BM_AA9*FJ_CRbXlC z{(VD6c?kG^-MsX@$Gz9Bn@;)-y1m2rAiTNDV2eW#s^$(*GO3Wx)$&Y(DO=QXjpJZ| z_F-HSWWw4w=LJ5+rXIEeyV)7EXgrX84t*6Zu4r^;hNcL#|9$XXD1Pas2ijc6K9-@X z<&vD%wzkik16)0GL6KebAIH^x-Xi~b{`ddKhy97LTgR$D10glp?>Ly(ta+mcyn`!5 z{NN0jRuBeprBhYAA5Zwx5Z2Dwc#dUY7w}C!7Z=}g&_T~;)r#Uq)!Zxi=>LL^#ca|R zj2CEf4ONi2_T5%Did8U-EwpxB+$C@W+skjEm7v%apoZ>m)FkBYJS>58z zaj>J7IL_y528-FG8?is#Ys8<4k5f^utDUN=O(OH%6DIA6%3#E_pfc_Xvms$oTA!G{ zsh+KW-x2IKUFqsZmr=5+17Hl1nhs4QWDlpmod?HRhe4tU z3ccg}U+VN+A&MJ}I5?XXW-maS-P}zwYYN~GL-bm?+aE5-Gg}On_siW}0#4M^sS(P- zTp%@>u35A8E%NIBihYt}cnJz6=TU#5*|ds8jutyML0Uh<#38`(5^c1ed+p#6Pg)7~ zS}cP*gy@Zy0b6hz3Ff+NP9^1m$5y?}sQ3UNT&U-M4Gv1w#SFv2s;MgJcH3J{Xo%** z&;IyO5|Dr+ZAG`+#3cR&k$~lIzvQfgyNZp#EoSGPU-PSqn2cM>=tfSiA9itY%lC}u! zj}7ywvTH1h=&`J6JB!-o%0RGsf?9Xq!Mh)`N3>UDV`Y2O`4937bu{W9l!nKv%cyw= z5Ru8r$$KWiBv|+1`~!0#ioJt;@VaHB)==tkJdE|ABua;HTChM(-43d81|)%6hvmKf zJf2@w*;fgb=QP+#G2qLXGp~T2ly_{|6^w5VB~E+0p?r);hQK~-v8o-;Z^Cak`4Gu$ zN%ArIfy)!kX5HU(1ds45d%(-~O*aJJ46-e%rICApA+2a@Yik9rn*uo&3eP+jbr`}O zVC^eu#9%PEzwOP+Ozr{1ydu!7%4v^zG6~?v?ZD;0F!$~i+&f|ACEcs2(r06Dcnurq zhusWW{=qVPJU@Xl1Wi+gWpLrW{*hZ)o8ez#-kA1ibyR zJd3?uJQUZfyBP7)oev=FKY}#K4{UH5yhtb~L;BaChBkqO>o8jZUZiB;&$BvZ_6}s# zdYgyh25viC%q1U*RZ#Dnz@XL1_kR7lyT=1s#}bbyi?C4zl1h#iO+>dt3$KE}kC=Xh z9cDu{S?KA%0)vtasrG&N#lL+(^7IiDCCx$^7&ZQ(4^*-RdF-AExGZZ_q3whmdVSa% zNnjXs!#P&N7*ItdI`aQwU%BIEBtwNZqvnMnY`6Aj5<)E?DYUOIjP3GgfO&5&c45~!si0X5R{J}FDm@B zvRW&Gviz0_LOjDZ#GiE%6`oN68TyU2M2@@c_6Xo%JMn^wS1>2yIbZEe~#W8Z#0>`A8t z5krLM@2*H^ZVKa*I&k##Logro&j?x#q}hztAd%4FNF~f6xT;;Q!DrOu_ismy-|<{7q9C38fVogBJnA07*OlKsdjcHNVpAiQP59Y(_!@IXO1x16AH-xvPK z%?hG)^y@?4@KN6pvg%hTR{H(%f~`l_oOn{yjgaf!aI^w+T$L21`_L+_09ce#Rjz_- z1{1JjWJoM?NvkyP$*xx6BSCrWdws|J8!M-Sfexawl#)vsFxNcZ? zHfeKaeWTm*<`y;eXKaDx#h%s6OY+XLV~6k2W%hrD*zR$Sql>f>;(-}VBbjbb3Pitt z{rW4IusF0z3teKMmrsMiQIHj4g3r%jajJj+b}97hSyaTQg4&U>1@kxoIm_$x=N1SJN(;yXE-wpP5BJ+R~!}jlF4G-fd zfDog|^MRjMa7%a6R5CJvF_VODgu>;7YOcA4{6=;XriOCngescQcm!J$-9O+rmk6hN zd=s&SpnfXekP7U!&RlAjuXa#Gp(IcLLu?pHWjk9t)ti@EcPjL1egs6NAzBhvID|@; z%pJ0^cdi8h&5@bw#W1VpV{@AqB_q;*67C1x zz4YkIL#QtJVaCpZXZaAWk|4-lhF5_|xlG>Ht)K5O;{~t{-7gFqegGSm0^T*}Kkua= zxSm0xhC>XzER!z3Wapf@`V_6WFK_Hm+xJ8%Ps7Hao>x-n<*PEcawhk!M8{(Lhr!u*lrPGw1Yh~6ZXbib`-*_? z%!-Z2;kmJtIZdV7lK|j6c8!^+mc&7a$)b~1)3Zf~>yC%sSv6DY@_Z1I=UioI(Y;kN z&ZQOS`9PHe^xOBs>PX!TS=8M6F|cs7oEndTdF?dIEK3BmFo0oT5p&MH>qoD%uUgoC zdb0AxJ~gO5(V-Sm!uCtOiJ(xYO1>aXQArPPiWW=#azCioqGYEp8SmntSiHjUaVAUD z`Lp$L%~ESEk^G&EMLj9wMf~7Y1o?wtL5t#xdG?qXk&c~L0}u=Rr9W||9;RzDnMu@M z?Fk+3EpRNTk=+vRGh+{6iI3YS(E&FEC+tg6EBo{^QybI)caydWycELo&f#jI(i5=g9wwt7~RL`I7S?zQ$R}RajH8+TT%P^Q}HD4g$-gL$P zl<-5ABkSj+Euwg*Z1d(jG@}Nm&x3uv#Bv^|Y5J(b%T2BL9yZ_nQR~S0egcr_k8kaF zjfKWFS@4lpF0p*YU5l-7EC{>VBW~3;dp`K8cI$o*llFuh0_6GUH|d^2Pav~w!b?nB zx7ZjoZuxlOXNHgN_|-BuCbI2d`o&p*q(|7?;YFaI5eGuDOc<}u4~cjP{HB| zU<3GK6mX9cL?M85SJlZU?}Ou7)XV+u_>~ueGKZPA8Me27cg~N-A$j`)%gNhC>7Iw^ z6fn$X{lhWuGb=|^2ES#Y%iZRtxe0;H;@zAjW$hO<){XxBR+V&G8V`IX0)zO+uAVAM zTHs7HEHQsnk@$g~ah6Pa}wq2inejQH_?`&d>M>j9KCY_ znfQD+h{ZU3g*tHbXGGV!8Y@hfGZ|O#S*)5B4zd6$9MB^_@7t|4m}p*e*Gg{x;%ldV zSvggwmZel3SGqB~6yqLHf{7B|2y~e1v%K}IMppgqCL+_E*^N5yDw#xV+Bj_uj;$2- zlRY+!pg?q@lf#Q3Lu`v_WoR%LxTv!{InIPx-FB>pc!tr7zK)h=(zZ&!8sSzn;S%xt z=4WaYs924onP=w4Hdnt6s5#5PcgX;J8gAgJ@bIC6(2uHA&`@@9T!smEo8jVO z#k~v7ezlk)7kS!?kMAWvO0seE=F~2Ym5bi0oyjV@|M8;hDyVCfRWc4foUK(4J;S-WIJUPFOLIORx4YQ!mc{} z`1phaFJq9p^vL=@W+npI|xw~%3JLm>bZv?C@XM&I zYj^5CN(T$u7tS3nk763DtC-~0beA0)hVjjB^l8G@QAqWk3A+RUj|DHp1I_xvOev}O z8JS`0;03v_fm4VrsnnO&WB-8>|G4-z&QBFnzIza91w#^3QV`-OqGbI3;w= zJY9HtO^(HX>yMtjDgF&}eUK_SGuM})8I0Zf!i~AX5~){S{tD5h8~wB%%Bh$R7}a0h z8sBheW&3>~I;K?xO89Qe)I+a(@)hX}9@65<_tZZ%KJxD3>XkI#{y?i%U=tDTe5OL2 zsYB{;VgNFQ_Y67_=7X4~5)nD4%KO+%$$WV`nAFWzV<>PqZwge!#yF$;T*1@tq=*Ux zP0#|0QzsNE#!NY@W>&_1KQoonZeL>BGOJM8&bbo5B4HX5h8IAKJ*vER2!ij!g z_)vM_{_LZNmc+RoT@~^D-}=oKo}Rxv>12LRlwL8w(pNQ!7hEyVI6(2Nr@(*sQ%+(U z8Sj~$(3IgJ^E>z#hA=2j^Njqd{)XYDGfBO4c7pj4R8~N_s;f3fJ#6{?Hhg>8Mw6m^ z?ib;{MvY@o;j4M}T~&2NYhn!O8$n@!6kle zbQd#)uMT!_sc-vON0!f*6XX!myV!1Owix8v9yE-=1gPLr5mz9T-{1;_EWr_cQQ;

pU+nd zHGw(ff(IKuU1d}SgnjNgnFl=nhr8vPvvJYhkwO_Yzn2Hr66k_iENIMK+p&M)3ym#eyjQQJm8u6r+HtU31QJqJY} z6a(o+zJs@{WU*Z)&_RW(2|(CF-hJLUj6xzj*2 zB*UCSt3h$Govr2GBHJVknp3Nkek}FfS*fFeGmT_ty$XKcfk&o3#ZCfkF8ReNLKD+4 zJL8{xlz&(o5N57ny5i-Hv zAq@JF^>B58*!z+<)eEiT049I$0Tk||V~4oXJ=C!wS`SxiGE-frbPqLb`8eRN$Ig|k^-x&> zgHZ<*Bb>$S2t!Afo)o+xX&6k(;E**Jhz(nH1X0I%z6{?{;qIOR_bqwkGbutXT2>KO zTe@GA*-y0@pbPYaj~-xB^{*StUBcdX;0>JQR(1nU*nhEsNWG*$A!=I5EC^ z!o*BLvPgcD55PsQq0|s=_qJS1>si6R)U6f8NM2~+;23I;|Iw_(0CZLdbmESL2Xr9# z;Yf0k0k-)2-TP56ZYvv5Hk{nO^1W`7AxQSKysWI1GvbENs78VNc`rN;HP#Afh#WxB z;TFl~rSi=OD)^Zt+%zDBwt>Jjc7j2XTm6}g^hgBH-N2ZZC_G-{{%KWB$Ck2D%f1J? zpTLRq+m$#r5%gGm>3t0KWrfqzTh<~9)YU?41(u#08{jP4$v8&QnqHEBN5xLdNQLO2 z-yWCf;JJu%JseHRYa``x)@P=X^I(QfQOip?$)z=G zqn&dVG*P7J2Kgch6q$>#SOwT-d_i>8#TU75eAwngrwkY`XLE#HPJUUGgOzIJ7+d`v zyInkK@VPLNt3xp(ZECfC^P3*3ynj`;#H}ffLD>1vH}KotI{#KQW9xd(*OB&W0lNj1 z$aby}c4uAI%6=;LP{Lad_lz{>l&6-RDQkY>+ut%@z6>ZHtTGo0AZN;(r`v>R@-CJG z1+<$<$iMZTA7^fMRnUNl$h5;H{7TD(dQacmcsX<80+;tsm?U-;^gXZ|OFgi?xr z`M(H!W3OHAKO>)1`6cUl@pge7pPVvkbD$`$P`}+uWJTkxko{U8`PjjR&zk}qGd~jQ z)yi~o1kHOB$=|wl{7ATLVLU!EL$JCY78Y97%r%RtL;R4SNC{rMqV)CMAv(@9a1f?| z)ZPn@f0v39Z8`>67ogqe_}N+dCibVIfUTjX*JFXbRA9LOn4J$d|AV!v=!6`IOm%<| z-ME&)?fy%^qPrZO+=X{pDWK)>nMC|48ur z0UiUJ4-T{4ac(;pbx`IMezzIe5C6Ds(Z7-^&Lp7W;xOM;=&S8baHXjYY;AeL(rfCW z5jdvOIh}XHl~{a~W%~h5O`tLquASfR+tOZDxC{}M361*Uv%2Obj=SLB_;AiB}5Tog)^D)NlJeQ)ZyKXB1>7jhv!l%C&MU-Y3 zq;{pkoJW|9Ecv>YZ_pd!n_g7~lrO1Z_`+F0j4O!iC(>Vk_66LQiO6J23{0G_-yZ;S zh$=YCEG8!}UE6}W(VajHT4S>fnlm!Dm(%M$rc;eSDEyz$C}T%u@VJ2l_EodVq=dJ* z7WM43SGC&8c9sm@3g}lJkPhb$+aI{Dls6-I(6*H5;<$qyRCyF+$dzoq@kS6O6TunP zkJQvbh08_@^h&vt6`E%OT94X)2n67aHOSJfK7-a+KGhz4Zbc6f+prLw`)7bl*_ITC z)GsL$x=aidK5eJ*giqr0FwwcNN@6riD#zs4j(tIMtsetT^1J|~=xjai5SR%g1A=dW zSqf@i3b>~#fO#;V^DA(Wy;&r(w6O^9y9oF$EQhR@dk$AoNg!NMtqAn)<@kPH;)>5R z^~1EBpEJaqOR6N!YQLyeuF!!c5s1B=I%_OquHQ5rT$1Dq7n*+sWqK_AX=E9507&xs zDx`tL!xq0k>}(s@@7fR}vYyJ2KZRJN4AAtR|8>3nC2K3z{$_gReZ^gO5>t=@JU>lq zXP)I|L`P=wm-|)na!amrDbf+_!IY~qT;4uukdZj&k4|ZT1;Ac=ZgtHW;|zsSI(P++ zGrP=;Yp+%pDs0BeA6uOsP=Jh-e9SrGA-``^^!4<^6U|b|>A@+T){ZxhUUeLBb=YW@ zsO_JTi`tSZaj-04(;rQ|8oz=VLZl}(J^X=sp8=N)<}c6hm=KeE$c}ivP_H%sC`9|v z45U8E06T+)BP|?rLD*qraIxpvKgb(w(K6lcLQR1dk(Wazp0p#Rh)=+bh548ZQDLoN zN^yGK+03G6rCw|hD|7OLux$9f0w~g(bI6kJEzZ+-8k$RlR0I;SI4NGkM|Jo79#BF` zegYFl_;bn-<;#JBr(Z$FQw!$vP_D^1_QN#msrW$gL(Add4C>}Avm1sw$45fu*b#YT z$(`~<`M?apExyr3d)$B``Jt1KQaVif`s3>9xV{Em*jaQox1v6ONSg>fYvCLCZYw!~ zvS+I$5KF(*zl;xh~qNaOR8^p5O`F(X{Cl{-3S5h zh?2_-32*82wusQZf4q++m5hh=XdHBpD)u|BPAh{OKvT?RP3gnA;o1_D{^C61Wh8W z`>dAt5%A95}o_qma_|Ef9h)nT7$r(dQI&AnyfX2>`MPj?0^94ha6T~{5 zu11jbz%7xNvl{Mj_YNf7kHYq+fXlwIJkfM0XRaVUg5UgMw~nsvquLCy*=IlBk*F14 z+)Aq=;Bj5sXqP^Tqz5rKe1p!lPAOin-|j2&?vjJI{^BO}h znCct0M9h9%aL%ZPRj~~6PjYD|wN*!VBwSX_U7C>1=UC{GHs~y{8obBE1X74eYvFo=W?eEsw;+pm*$-8cSo9UPS~9|ndsq?R z_eG{=3`W(;VJCUa_G_*HL(W8i*D#&}Vu<&w*&RF%wR2u}}*;k3%7 z`V8j@^WOwn+<7@_uz)N_4(!h9i2Wc>+xE#Irz39ltsqrIg}!H(MPFi4^sh|$1FNks zljFsXhKu;NDET7rV)=cS`4knivHfpB{uWGjh z*N3c*ewfcQXurF$ab5+x7NwNf$EUNlV?AUoVA0dt)McKsf1)AM5UGPW#o3PONz(Mu zVI;e5+!#189$+(#|+CH8r=h8*I|x<0J+p-)KC-!}JQ zen$`OyGupX8_qjeqnjBhzbk`R?)*_Aylkp*pczRQakQM7E}}Uk?4;1{E}n7udjXS^ zgm3-2CAH&1|2ab&F5f484Pt2!8_A4YFEaiTj@Prk_C#QHOuy4&D>) z$NE!ztsDqjOe+%u6(9Q~!s8;Xwr0^>$irdMd$TeNkkyFB1rQ(4-GI}{z$0Rx zT0x>#%Q5svmxD_aOdGO**;Sy*Seeafo(Ap*0u@pJ*3F!&^3kF?0Yb1k4^)pbFxFJ! zZhFqz1Tu@IV6$!;mOtDgJNw!uMZeov99ixuoi*>5JOkm+0m^_o<(hqC*B>*Ctfr7g;ORqN9yoXnv z#5nZnJm@VQuws}1OdkPe_&c!}YKyPHY&)KNXKCd zNd3D5>*Nvmew-fO(Wo?pr`!8TSm6@6cAd^HC&LB5CY&JoeHMLJ)bk}UXgQ-K&e&)n zE3J|(u%1?>35`;m0=i*gT?<~|$Uhx!cWrWzJ3}%$QbCyU`m;X^g#$7J!xklB7k?f+ zdh}?H_{EFAIBjA1uEo%SE=U!73l5;DEa3GQXU@U(eVkm+!Vexm|7RKL;J0sOIS4|2 zpId5SYqnU(XNFz&Rt-RPepTv<_IxsUwJSIkqdVrOl=irl6G~{DO-YyqGN(*IYF9?? zOjo#n=P+#xA|_O02D`#?CaNNXXE3kwo-dBjiz!0C%+fnx31$;01C9#a_yyp<2-Y@5 zwGPom(Lu{pK52N2wq-x2I%mj~t6$w*r<|Qw7tE2eVxlqxK@kf#v>AZ2t^cAxbvT-zk}~H6I%yd1I(|vyp@kL@Gq40c3t3hJhlC9pFAV^Q;E7cBfF!1Np2|JLUogttuuXaH%^)IBfx7 z)yZeN{_JN)Gf+~w8C}5jD3!Z9fsIodjBmLXnz_abuSZubdJAvNX_;iS-C4Ti9&p6L zRdjS|>_NDIUdn(uAv;C&V)B`dY^+q2>6n14wL9PtwjX-lhPxJm#X#okX$2*vaoN?( z7@DG-uck#Kr+D-|EXfYI5M4#K*JbDJW@0%E-ljZ^PUCoTv0z%9;I|(?pQ?95s()T5 zpV2VNsiFnxh8koZ&sa>mW`=kI^$u*4E4 zDJo+888R}g%EuQ4puSd!FYZJH`SWs-k3sBuwD`dQmY9{5-E>V8%-9hsy{eVs z2Q{@>4A^mECm01Nm!WBz?jZU)s?a}bizQwCnsUNhT8GRTp%p$f|CTM-(;fgv=o*@{ zTEl2%3|>9K#}PoSQQ|d_bOaH63W@Nr+zxQZW~u97oJiT(nvf9eFdHZ>>c_mcMcn*g z)X@QJiLfJXOOjNwJpicv$v^7Px z(4r^X2&1~Qp<+;{^6>NR9YRF$UcqcZFJw9?iRK;(0@Fp^m*&oLMMrzL)DSW4nt}e3 zjxTPM#9X`g`4&B2K36{|#%q_GXL%i2mjtBWK6E@L#i4({PC$YCyyfrO+Fj29qduo* z?!&*H1CQ4)cEd*@dcnb)dpM>wRgp14F8C&xM=9GAFl6@Zzv>|p!h*EN*|+$R*%RPt zHXAHQtCc>_aw)s~ldxFJ(m#W6x4_NF0^)lFlNl|(8W*2r1OXqJdMYC7m$JD!<4v)1 zgho0%MRJ)r@n%a9?-$kQLebXhXs7iaVX>Bc=atD-jkh*iQ_{XD%eN(099|HIr{21NCD z?ZQKelo%jL`v*lpKqaI@1wrW!2}Qa@=^Dg91;GFmBqWpu5g3pzK{})d7!akq1{h$@ z+ITw{_wIM##os=;b_UfNEB2vrgHLCd^~<3UDdJgo zQI4kfs(iYiUab2es6zu8-%8tg8F4Bgwd65&+FWCh>KNG(deU-K)#2Qwb9ez{Cq2ha`Rty}9(7aj&_@c^pr zu|FgM9udT;JGBbHPXG`D@oEO$r7{T#(er9}Ca7kICUyW8EN$cqqdtKdsSGgX)A>xx zw14Wqt9+BFG^bi)IWqc7_!-PjWFx2e;lU~H(t%J?v0O{zX=5pplADz*Bzj_0JyC8% zc499Vf zUAT_>6mT{E$FjHcy04Lx=fdrG+)rscyOqXm*++qqw7Gl*mGYpPb9^awSax@}WL@`46jt#8K5qs>E^gfVY@-sfU$?$ha#*pk+O*mtCT zwhUT2_e8d0wK`fi9WyDf8WO+Neg_sWR;W5RASDY%J(dtiCDD1h39zKBy%;!JM2RpSv5$0|ZJ)hl-C zq6svu+I$M6IFs);BD-sy|lNC$~8^rGjPdEQ**>P*a z4}G;y+1)*!LqsBMh|fo;sqeSO zWD{Hl7B6{!dBeCa(|3#`jPH7JeU@%aIo?f@5u`;2I6V#}*4}kxP{-C#;q&hszvuQ` z6ZKHat(RLWYbGE%GY5)o@Id@c4k`UGC8+{DU$2Tt+83aujglZe-~k+MsGoVDpQ$YJso6|c@D|j=U^K? z#vxq@PkoCZLRTRJNE$V( zvF?a9zG<+VeWRl)6U5v;vbty89Xmfh1Q;k4><=@+%x!gJqX>&AW8k@IULs zqSbM=oz-ySw-52oESq5b#<$E!;Xme=-G>cld(o+gha=>{8lbzP#g~FQpw$4?2Zjv8 z7(n*iEiq1m z$?k}5gZE#(Z)XeN(`s9`aFOCiaPzC`ju>tfd974My-JjL75QD`j zV77*@>I1jzY6$vi#@TErm%z9!2M+YUHGI7T27!W|@a5dOmB)ondWKLz{{$)^sJ8B5 z4@eLv>`5ip{xc4Qknz`Rkl}`|UWE6H1AG>a_FhXAcjv_5k8RwBqM~9#@l@-~#`#5T zu_ok&<=~b&m>>pGWx4>#tnsEZFiasI8ELnuWRX$$) zWdTIcqjd)`$Jpj}P<|JKOP-c=kd(qD>UiDxfGXLpZ(oXFJv z7H1GB-J~&jjQw-7E7WEDCWYz95`x5;4>;Vhk`F%1?M!pGLoY#rHsv76E8W1mE*5kG zA9(ZFDbRSnxCcIGoE1B>S)0eAVxCm66Bcvpy$GNvn@37W;@@=$KmC6>-s1~76AHN2 zrht#@K$6LL^yo*(q!2|kcq`aMh8H+v#p&22v=BN80$Iaw2)|Z99i{>1dbzp+M-Pvn z_pd&ZRzn$*2^szl6sSs}9M(+0Rq|I-adNW}$p8ewWdG^9kIk_7Ngtq)lK$KOM*G;iCHHPaxvc?$l)%&o!*Xx8fVH;E zQegLm(5H~9$)wCPCx^E6&6!ZTLvYt}q$C2d3H9v7$CT?%C5GEGX%cQgtKx$8%r3aw zFMI{#F^q4=ng7usrq*hEdmB-KISiNNBOdTT@;#yc?v52S0o*ANCJz=!4Tau+4#{r! z5Gy0#7OF2KKZfG?4?#1CD>+=sW^zA3c|bu)=?MHA{9Yg|yrP>wKMm*|hCe_l_fDQcM5n1OMN^q8NJK*XqUj^J=aClmo7^BAi2(#RdnIl)CU1 z{O`A#Po=|Wlx|~U8bQ`~%I3qV;=$|#7(xyQ+5bFefbORopTL!G2;)nrbDehE5qBQ` z=$R*KIP=eTr2O?m(sBuI+pA~VVlV2uVgihN1tRrc`wuRpXcVoo$dyzH`OfJ7a;8-f z8BH*D4M(s4xLq;whb~HRvn1h1ypnzkWB;?$WRRi^%uQie8eM(UOeyvGO!c0u-qmlq z@=YH}R;RF>4oDH;;i(^?C;_(N zc`MaU!81DqWge01SA@JbioDiFdZnydqD>ZHSzaDe(*~H z?qNd&r>mdx<9)KMQ$lCJ2Vf}QDg+EZPfZ#j4)BdrN$6Z~?}?QE2~oTk48O71I)8Zq zqc0x~^2~FPsWk96D?6p%ap<&nu{}S#xI^?X{z_3H<5H$}S#AJ~ya-Bg z?GbH-vm)^>@i=Pjd(Yv+hnGtuV%X|S0MrbqFsusPn^&uz^oC&*FV^QkgZ0;55NeOJ z@v^KN!mr)*8yMscC>g-FLA-SSN1Zv6@m;b&Z*TKNNBI9vtG~ezvcKtr@NJZ=csqJx zIc?JQ^8Z=%;0(BAuov>Oz>2Rwhu|9Ydv#|n&k+Ga)Ce$r#i?ZK&RI)8FnHX6-yEPa zMiQ2}EWM07jW@*Keg)*cHCJRjE+%r}Id4KI>`(_XRNyx)TE!&31cE8{MP2#2<3GG6wk)l<5j>tym&Yf}CBd&gCu6J^<>=Z@;04amJ*=5APA?Dwss zKLIe3FB*R_5?5^MoijJ~m&(tZWuGRK)9Sw4BP)oV1rqDlTmmk!djiJJ- zJxLIl7Lh$~kbBGi(fNMIlv+KWQQ0Hzz*JZV_iZfx!igFm9gIIk1n4@>IDTx_PiJ0i zDMhdTo|V6s9xv*$f~QFy?yJ>2*=4%4?iqqs-Y1Qi+g5(Lu;$@t7&3_d-C5(g?b#tW zUMtV-T1ndc_A)vWi{wmaAf(5-nTH7&=6a7zK_v0CZ+P-N9C$lOZ=|RLY%DBo4~M^i zVM2$q$l^s4#+q7AA1qT(X{UGmGDh~fBRDM*dei%qBhrfv|LVBiA}M9OuX~!hOK>Di zmocc)%jKLAWn$jHc2c4J_s)k0umSw9-q zHPUHv8QsV2ydnmw`(O+zF3iaemTnt5?|VGjmphp{*PlDMcAG~d?iv&-eKR6cQNS{| z*I4Os{dRajV`P_Vrcu#Tmk}xp#diW;a$X%r`{&g9$K<^bGDO{3L2jq4W4@fO1RTQg zQJQ>jBfkc&>CqY%R9#VHN3V}+{<>88p`IO<6SNEOoW#EwIPtR=4IvO%plN2KvWGqF za-E@muJv~lMeB>7`GBasr4N%s+;oK=%Swrybu&6d2eiSZej~7ctluGYl?XpcXgU!` zfw+s+t<^iU2()Toz7+QcP15HByALz+5VWs${V}NJzy_j7xjN~T$nF2+n}X zbv_UOiXx2D&Yy%O=d(0a>?ma(U{%!BrUI*iv+&W?c=QP=mWaEpsgYf~+#nrSVLpBh zWp^l19Fhqu-tFb}Go8CpgBQ#+lJ5Xb4rzZNnLJWx90>vn#;c?Iq|Wlb+yL#NCqKs}Me z%z4YO&|CH2m>BasY|7ieC>9=bjfO9%CtnMD3k*z-atQsgLWPJo84#F6tr}>-0hhPc zQ)H|#19%|4h_^*|BRJ0`vCB8O>s}nU0AkIZj~fuy+;pJ`Tj(bD9Bt@YYk2H4Nq9^3 zU%{wJX=5<*b}qJDqrCM~sh`}@hBS2g0vff+fZ$N)YMTaRzWHvvFv#2Odin|BYh?Ch z)H%z0@K2VN+t9U6E>io?{8GvUYC({pn=Mxyb065im5GY1K^om#G3uY;-gcoMxW=E3 zY*4rWCjfC70Q{L{(Hm^cbRY+F+PWOL>s>1k=X30RJ~0gF`HyE<*iPA8qCfF5=2b`Q zAt${&^AVWH^l;KIdVA5P!A(f_a~LV<{=!o>eVgFpt|v-P{iSWF4cPdiQ&nOvf!gc* zmZxW=05Q4FeNQYZ#Z*~YjYPSFPQ!XPw(Uv z8TyhAFnfbV)F}+8XiK@MY?F%5(Ys$1AP3Teu1jPq4(CGHdaHeL)Eo*=Y4C~!!DN9e zoEJ=K2f5GZojh=H;HZ<=t(n?cWr>a98_14m<(4X-MiN0LXrlE%N;5CfayC^OEDCZ; zOrd1}-yUHX4kWg5nk}9DKLXln!l#_}53{l7SomF?I2)J)aeijZf>5-YcBb?gTXyAM zh~1B8Zocn_yk>Ka*O;<2_L$uNtXl7t{PuGmEYuDReBO%*;Em$LtG(ZoXOp<4Zi>N7 z4d~Ie$4It8-5K;?`>{+I$&Gs!x5#uqI9yFj|9=-`dpq{mQ5BBl@#6!$81|(9+gi4s z6=C2*><;>IpsabX{t3DJ?&(2m&S0a!{`~CF+D%t+zNBhHscW(&1x`S^t`2vPSff$(D zw^yn!y}4lkm0w-?Bv9--LEWnlRVV8_A{zplRepydl$t;p$$**^Fs!hIfK6lY0dfVS znnWiRJIo)*$68lxIudd~;5h?~bA*W{y9jcqjUmT8d&uJGx5MBtB*;j(;2(DoT8Rbb zl+h704gF1Y8;^8y&;0z>(F#IJlZrX_5)m;luSR@mKoRJk=LdF0qW)X=5I_Pk9sw4C zjR2U8%p=xdhkI!i%u2!q|LCRY82g>H2`Dx) z%ry^<648N$R~82$qjA9oa~oc8js5TIij~hW3hF5!q(UYOw5e@-0{sJnTnqX>fH|$! z)sRaN&R%l6?$ApPS*c<2(;)cz%g1K(N|ENm0Dnb1qd4CMiWBjvyKrIN5H4d``~ z0TG1maeq(#Ux-amN#Xv6_ZlOiYa%AldVs%}8Sa*bVIU(m@Lc{DxG}xxpX+xH1~hKrn#W zaKIFGkX>sP0U2Oy;~v28Jb)|$H|j>!?rJDPb3g{RLfk^si$$RJ#aDu?LmSFu@I^5B zr1JU-v>W?D@lnUV2Gc+{-%CE21pB38(Yrs6%C6L(0Q=?Q#H5=9Lk~-$dBnnY0QMx^ zspW?Vz+fFJo6F15lARWMQmf1`c5bRDD|r`dWxh5}kk!vqR*Leyl!st5j5ki|yxVkM zZ*(2Uatwvb2mX{EkxIWYn9Lu^n|0%4a}MD6DK&e9S)dn^CXxe*g-{*BxFTX|r*bE1v zNDf(FM8X0FicbMK7PD?rSUOzHoudW7unuAHN`rzCcxHH4cdVCRAg`0D; zfPs35HPgX#T~RwrvL*z=QTE8R$40A|yIpvkHTQSal->AVn>zX~gtiK9p1@2~Eu|p={iOLy3e&Y&U4LuN=aV@?V z-7|H*C1a+P@WT~d>A6tqAnur#h#s+dR0Qdw0|?7}*SoGSY$AOEh)sP652>ya5m!VY z;5_SUOmi75%}i)HEjojBlL4fg8!(Px=tUJ697^6Y1=Yc4X!s(U)kP22ij8QbvHGYE zp{hZJj3MnT2_Rsga}Hcr8KV+}UZNnjOv0iOwBL z=TUuIYOyvu=Q{iLbt0eD=hoZd*0r_b!eV`|NCNazJk40Cm6bd2I~jU0_ix0XRe!%B zFSK1_@J}UfPEbGlmRZy8-Uv2Y*wiWOD3qX|Yq;g~7fR`B%RNDo|0V(y`MSKst`B*| z;7`D`UK6C*Q)unTD!~FVw`i%rrR_$*{)~#p{1CSeatc@eDj4q(K3@0p+Opk(OrY3P z_#0UTglf=F)VU`Dk-Z`SM4hV_Z0}bGkx(BwVnE#A19au7WCK`l%s+UbrU?;_0T?Vm zR=0ZRw>h7`G)GOO)qObtg;s$zFbx#)WErC z_m3fBtRe0!1={y^gJT*5hvbDmsDI-QhBu)~dNWPrh6jJV6=iG<{Q^NvKwLUB~I5tqywz)G^!d$wUdG8>ofmTXJbWImw z-fz~uIkoQ=3#A=%EmRYQEAJBZ@!QI`F&{F&>9@SI$0dxE-WxA>*(;u#AHRwo{8Lj^ z?j^ZdAJ$gv@I7M3kthejMZ1_N&4H8CZL$sbvaI6Q&sfZz#1=g$qg!FQOvn1nx4;*i zH|*u!@_sg1>ADqR4|Rf+`*P5IAerAl=DDoU9K_yX04J!t#C6Dbv5dW81ludsWid^X zv?qRjdmpq2;dj#qN^5SJIvcdeQwuxE`@5_%PaQR1FnFRhv1ZN}qe2qkA~wj$G;QK0 z;w3o5Wke@7=;rYWx#ktWiuT?Ko9WzrdFtY|iALJ0sV%>E!o($SG$+m177%)TwW3gN z0H}B2>o!e?QpwVwh%FLJ?n#g;Kr|_NvdGU8fUYRTy41@0;`M>8772<9=mB8!>1;I;?-XChur73erxCN+vXp+bO5mjH`)Mt3 z4muAvqFZqst^*$<=1ctXSiydeeMdRN4$OvmcdELd#S-L6zvK=+U*?{TlLCMm2y+Um zw<7sQim8soVtP&epvG{HqL$9LKIiwxrCA$_oJI_sv0g{;v2~Qh1yVNSP^Q2sj+Q+8 z(B+!+teb9jXYaXtP*l3F+)2$u72KsOH1kvcK*Pkx@<|+WxvAR+-i9*wKHtgp-9R^f zApl9?SI;SMmd&fnDRY~6dQvdXHt}6xX`J8QleMXZOBYPHTv|aQfPGP!UUF|g&di36 z7F{lqra1#4sW#`b8JdRS{N1dbNrcIQ_xZe9^)}`F{ywf6nxQ;Srxfa146eUzg_Y42i8z1bWWP>q;7?7?|NP!#=Bnf666A z>cU!VYpdvY*0nm(jYFZ?K%4s3$S9hW5}GmZmxle|IivdDgd28>B@6#mxKXvKd>yDo zLmCd3X4W+8$_=O8NUI3qE>8+zSGM^(qT*sQxVq&a5ruDz3VU|DiA%WyY6^UsT!j5}xM z!)mKsZV40b?mMjCp^4BB?C8t1Dx-_|Uz>bLe^~k#m!%91Mm{@EzeGKp`b{1Yh>3{P zQu#IwI3%kfl<*_-*evc4`b?FsU*P1^L?%@R-#+)i@9Toq=YaiIV816!rN1ms%{#B{ z-w7jmqs|-aKwrNPhzIEaEE*#udZY;n1WM3dX#=y**KXnB3Fr4hP0)qQH+$F*oGaen z>Cna~E$_YQv&wk5peiHz1EKEzOAc`rsPTGY^;=-7gh`a&=QmOw0oQ?ccKj2*BVj_j z&7%AUxBMx~tM#Dan0bBuQl>`5)7hSiG#mfDPyPKEpu%Xr%leq{me=lQ!v;cyp+2mV z)M`xjw?=2$^eVX<^VOtq*LDx>hbLjzFWPhSC*#*Ysu$~SuNlk5MVic@?H#PA^ab^g z_%%Jf>f8ZuBZb$whhMBJXzBTGZXxK2!9aS?AX4e~5y+ z*^H}pUD^1OoF6F546(frdAcjC5A2;pZiDob!(u@Mvnc z7;}X{vLe|=juK=)geqXjyff1a-l{o(YZ^e)lqiR>o-0mko_lc!7`g>}(&B?Vf@KR8 z#;ne`Br9#ngjDNR_3k5+XZb?&d8{u+U-KFcVL@&_cqlM%ACL**$7>f!gOI0?CsUBQ zf2d@0AnAv7_rsS+`G*WCw&C&Y4mk>{0(bkjdaHmCnxkJynr+s(Zj1cN;67(UOqpuyaC<_toL0k&1^SDQA6skjH>U?$W5^a#5H zoPmgycObAxLuN;I$)=iHh;?g@_3 zC&K0Zr(eBfUM($h8mf-S(aQYTo4ZMm)~K3F_!=+mSuYW|k;&LcsPW5DT*!oZYgato zzTNS(&f)hY0oXH*=#vZb`I_H)EF%f-0jAp3p{gT2S#Q6vVTAM zloMFjz`-_BxT5)ixaUMKhp9}u8oVE1p-HGM#dS3uVV9boa`0~}0xrA7zUzBw$ER(( zo~5!lwB<1RgZyQq(6bw|AB}T-)#))pam0!WX@|#x#)Z{#@7H~{aBx+vRiXR_7_UZ6%&l+j;co4T(dWNj zc_Mt6xVyQ0S$4~dcr{=?cagq9Vsq>vS)l(Fr#;8ejrqV3N}_wN^ilxV?$S?5DK9&# z6Acx%`kDI0tV6i_wQCzjS(wX1BEscPw$^R4c<0>*kIc{`z5BZviRzoxXW;%6I=HhL z$iH#hXr22MB~dP~zCXfQvKB4z2#xd@B$f=h@<1KV!3BfVcd1CKk-?z|5p)i)CS39T zKQ|EV*a$dvW6;DvbURcRIbbl@bR5<5?OF`_b&b5X=LZbQb>D@eekY*ndFnO__3cNu zU*u@{a5-_qZ$sVDC@t2uY2<;0ruI!WRXL8NI`|1S_HD-o4JGl~!{!gt zBQARu-VHV#yU{2v5EMN-*eDM>1}ro^a%mnQKpiv>ZR;=6OXbkdpS=sI$~sHXY9lAQ zUcwyvxag%J_A%ppDDASDN3Z@km&Lur*+UMYxfps!R@Xb^F;Bs%Xf?YYY(x{*w~*$R z>j!biQ&SVvv5dmT&5lzU;fZ?bMuWx6nVyNHk4(i~rK}Ate7Kj_P7uB>)U5uFofgtE<&%P zZ2T#!6BOJl90foFWR}m;eir-q9N9e(^(CB1w@myx&9DoeWr>_=1Q6p`LYax*@y#SM zIn~LUffBQTSP9j22hs<9SQ;u+`orQn7zvj@SLLzhxk2b7%KZ5T&ipoKZbo4MQ#V=S zCHNN5i*f5+mH;XaXLEqf`! zy(NX1y#k+>dqK;3!6>t?# zRl3G`5V#Jv0B0%={c?{$@4YAQoTuN1GxHj;MtYYJ4*Y_gLCyp1Z%LXbS9oO%AIeeu zUhjp3hJF=cMwS0DR96b7txk0zGEyAD1>nta_p|5a#2CB!fu*u{U@>9e&4V5}?tZRv zGO=xj-*Ix#2*VrgfJJbtrNpEiZ(__XpWqY8gcQS9}4` z+##|Ugei{RE5`NiJst{3yFa?UUD0Q*f}P|& ze(gETGWeA=j);4%n{otY>l0Sr?#o%P=iI7n?aMJ^pPI;|PoJ+EGlKo_HWd=p(+aH` zg{lOBt`!#{Fd-xdl(eD;9kTr}u|Z z;rn(;>F$l^ZX zVpfN|p`}10pIaRl?gPg4Z_W@j2Ll{i=$NUG^%9W*1CL)o=xA~~vf{gQbc1ibF zKnve`Je6x&)*zR#wY1#vz$U;J$?{;F*YOF7J=Us(%_nGd4TVx2z6#&2V*2e@{rRRU zjMJubwN8o3@W%E(X1M*6dHwhn&6`fkb9wax_xI*HeSxUrx!kgLIj|b)m@We!G7y6LC#a^iS7jE1%C_S>!JlFSp*JDQg+rWB0Hbqc zttrj@SH4%uN=S(F^=uoOh5`@^T24YZVt6LRMY|?$__-Zep_2WBODkzwq#N) zEj4sAAo0twtLDX*{xqBJ?^4LvmE0NBF)g!+($DaHZq`5Ri*Dzv+H>fT-%F>Q;MdFk zRkVMELuO2|gl?{Xa1i|O@Cg;QRaV0TU7PDx^ro-xe9v(jL>uK6Jn*^tdV6hhXU(Da z+c1my22QPB*I6>qXH07@(AL(k2={;^vON8YH5RiVOt`)1gfkf4+1VTyY=SF#O{+|u3#2K~6A_2$6Q#g-?1YxlgJzWFE!_3dG?e8R{mXnD{>zH4=P!Yx7xCvp z1{XYv%A`yetj^qf2lhqG9U0nL81)$OmQi-GS`os3%g~qNu;u!uWu>YG%PVv&X3mh- zVBn|+AlYzUsombkpEKGbo%8Jo@cfxmY+Q4N>2X9Aas2uM*4|>^mlBQBO7iypt$0(# z?*{!iyMFWjp<;60YkSIu*i=Z?mM>HKp8eIs_g2s^oXw!ZMzAdD>h&dl=v5r$(><4K z;H;us$S(*;W--(PUc1=GuiOWPSn3ilQdATy);n9<8;8rW3YuJuPR+JszIp#{lf84O z$CrP6UYN9#N}hiZU8uX4T0g!QJ$jcNdH^X ze~1e%L3cKw&?dkTJ>l;U?VoJm7Vz!FBLBg9@KTimFw}EgDqqtGjxhAPbMV~7V@M*e z5Q~7hDYoqJW4G4-zFvU$`i~-l$NZT*2MT4+j^tQJi^89T@8ra)JPtqc38i!8ocr(< zqMskTYy*K76;${P3CPb(Lc%#Q0oU<^H7S$}KlX`-$VhndQbIPdHFejWmy)bJx2E0p z%TcIQMcdR(#OX>!^0aB`lwr#Uu{`UM54{Dbj@H1Z)^gkh#-Drk$~aX+VssiK*z)Z> zcDM4OD4Pi(=COXAXbd0ds-s+3t_i^S5l<7G>&!jDV*i!$!57tD=r@zc`WUZaS32EV zJbpX1lnQ@{f4v>n6HXj#*4k-_Y%a9@rX*nfWqh4Y)^nKX;9R=iWyL;}mv>++qP_@)-!D|TgSf3 z0e9K~1!|Y+sH9R<1gHFHZu#mB$iVVIu=^8S>F+=-P{95Cyj=yDdOBh;IgIDDs-b*> z3?RDDZ>gdmnAvaU^I$C?50x<56R7#Fln^rzsex<7c?hVV%^tLgVW2bO2cFv5ijNL? zj#bZCld6Fz9)jPWVgP*)HbfZepkT!T|+G+01*k!S^({#=JNLb?wV%VMxW^dFh9k?5Dzf;1jNM+q!bR9 zN8H~!kCqP(6}!47X`NFdn*IHZ>WsV3P-9TzEEOXjlW<&5mgHgW&$nI1tuIA=D|R%j zbyz8fIn+op3r*G6VCpL7kA~)%)ACqNG@v4c@es~=aQ#u)L?g=)9`~e@ z;v_Yv{Qtg8;f7RQa@?g&L{ziFk>XtT6|zI}#HG7aGVyB^(5$)N?c6$h0ldC$hn_6k zWWTZS?r85NYsvRR!lyIMym!+-@y3ikOUUOI2jJqn(VMT`SfhW!cvaSk(yi(`dcLfh z%r{?66TO&k>M&TT&T5MoZ$zo6YICB(4uL_#D)ATtdDuyxj`jq2zveSUoEO-21)8R7 zCEb<3{K_co4ByLq@>7%(syD1;6dS6I-gGnHJQ@$}w>wZZ+Pv5gPp3yg%YHDh32Vxw-8M@gT zK62I@Q9@=?4Qp@i%l&;Z*BR^f!=_~5ggcG{AL-4pu+d zKeSJ1;!WJbBQ3*US^xT^H)m(bg1XZ*IjaIzzB4(4J>fVZiqRXKQ3!bG=Y(o^Ai@W5 zru2d7F5^^Fb8t)j0hI|9>H@5C{@z>r(9YT3MnouzjR0z-Emt88a(GC5qdkUl0$mzR z&K@a%*qmt%4gP#_=b`0_O@Vs7Rc`rsg4MfVq$KK(YBwH6?JX%U4ol z@XBC8qPg@6V$cZVewMKQ&|R82^{5$D4xA7t7^^WZ_3-b}DmXMVP(Qhnu>*2xTD};Uxtcqq<|N_IFxbLI1OhAY0u@ zKnuVk)|v=h50a^^0Hg|mN;hruCfTbaOvix5;)JkIZVFfQfWpQV7&6qnQulHmg@vQI zUe$!bwgl}u8(?P^5pTh4CPJh9{3tFAd0HqK0mYY*Pp2N)kc&q= zKfp-ex`^A!bYpib_cl#gFWr3x7r6;6B<3enK*qFyToizL9?NqCkt1a|V&5WnSrXoB zYab{$LuB+amhcpPjeK?Mr6&Vxl-DYtA6s*Xmf4asVEwgg79X%*#Oyk&tH4D5KmoV# z?l4#U7L4Bt_$No{F$4e;5Cl2{;RXY~nike*=%mko{@?=fj!OeEhJ$+(3wtsGBYF1f z8xBSpn%hHaey~?V;Ayfrq7aM#bfsQPE&vu41CPNNm_ylcmx|x1KTH5em)#XWO!|i~ z&MjNPv=OulTrVh2N$jB5z>G65cw=z{3kmNuHn2l#F8f3{MbK8m3H4;4p#x$9gt48+ zgyc}6uYfBNs0k1-mW+d-+x!I@VXGnJ_k!2jr8^L#N832Jrz7A<)Ccz>L_OCq!5TP! z6Ia+3g8gB{563QCy%@@!cDP}xh_Q~j`L>AxePFw!+Hs^O_$maS?W}*U0*ZR{doVc; z8sb|E0PW;}q7{*{0aC(4PWCLK;jnXl*EK>gjnJ2V3-rh%^c&dJg@`GO<`XB(X64mJa7x^(=EcQ6=9KY%#l`jN zA;n0w<|L37H_j_CPW6jdSpJjarmR?vRF>$3OV&5MuN*<;3l9K=>wcc%E(E}>XiJ1` zewnOGsnmfHGR6-s)e47PkE{bRYI`HJUISddf5Y{51}N3MQ3eF)fmM^dihLv^X8gX&IkxwAmyy zRFs^1(Hg>^xeix}{QZ1IC5%4yW?KkUwTAmwNuMb){bAwGE%`;=G30|#C7gnUep`5t zQ?P=R-;2y3ry9KC>zVjrhS4#ra+0F~6SW{Yo!=)c6~j){S+Jugphpv4YH_EZpMH{^ z-PVxNDJ$tCu(@Ytrx%3U$q}aOC&|v_kuD^cI&c3&pCuPWJ?x$$xV-iOD;1fMVNxv zLnm?niEN34>%f;JGnlGmF1!q#qmP(#K@xmqUw~3)UVM{j(d3!lsfQ4P5yaz9TZ>-p zHTH(yRy7(XRPs+SQQ9-PwszEmqJNS6%-MNhd%&MDj%pwMMkk?iKOF?GVM?uI`#&Zk zJ+i-&!RcOEB~?&XLLF5}VJP`YX|Sxwlx=gyhDL2Uj<8tPQM@xlrh{XGY_(k$e+vm^ zX@ga;Xe#ND!EAR)NVU%~Yew1^_-;7R?;;LOquvvA(%^56;74*@;Y&Us4p^IVz>dNf zVYYfAYsZu7E6p$PF?0i%Dm*P$4h0n|hCZe4c|N-IoT{0^>ggm0*img}%R6`d+z#G; z(Lg77)~+~MEkQ<7(c=b5swR>dAc3H4ga4S~5?b&$1B?jpVn7TsSL6&IoM^i(D%_QS zJnT%(?k8wS8UUm5CwQ`v}@>D|f#ElY%EX3HmMpW}W> z$uQe$TvsLJ0y_AvU%c@93G!m_ z(8|F%+`iJ*jJlQwb-T&ZXhmJW9FPW==Z;&GWH~+0eR9W@?43Iw#wVZR7vo1umpHUwr&cU`5A3F{EPL~KZ0XBdX)~=r=lc7=AvPXuyQ#P2bo82DHKflD!FDT( zObZPJEY+ea011D97avAf^NH=>^(fVNCwDiBBAq3OKP(HE*DH$ZQh$rHe3RW;iD0U% zWMBj)I69Rc7BuT=Uv}>c^D*+RV5d7`C*L4ZdyMl{JH?V~n|$QOY~|Y^*H~Hd$j|xq ze3G^~TQ%&*$J@WfEw@2*<$p|XP_~)+S1|jtz}Hq+pmNF9%h17iuiPQN*ekJnh>UQQ zQ?~0;uvTXvXLxzDS+8&R+S*AE>Rix##!Zo2A|(w{U`eHw(M~dfozbgYIb06Kc(_MA zj*I((WNet-{XDK}sG?+n1S=k_=Zos4Fgtmm_2TySM z_UB{=xf(wOQ&djMp{uj&-=KQtd{nr_7yDLbYwVHgzAQ>7S5gbgIawbrX3gN8kE%%z z?B~DzdLiJSq(~yuCdKzm!{@dY zlwa{(6Ovsn?hZ*7>f@Go*He*)b9pfw6ihh(qB$sPi(#r>lS}2cO(LxTbNUqTX0U~6 zh;8fp@s57V6uEP~U9M1R8ho)-B!VwYh2K~Idt@qdLwz8>XXRunq*KP4yb&oAI?R)K zsuS`T=e&x|*a(W1u98nNUu)_JfLf#l8^$cocnU-4ig}lLhyB3&kufGc^<|ch6coq9C zyhzo-gny+`GIfAT_!s9eP;fR(7#TKsDGiCYbYz7U7+R(Ol%;nm|}ci z2B~KNUh#sk50JvMp$Z99%$ejI@_iTS#6qzPD6bL$7H!3;n9@LSCEqZ=%AhdBK>k~d zvWwYm)Mv_P`>6H$5#|fk1>U_pDm68~KT=$$*aTAf^n}hWbvo|G)4d`Fm{q6=16uOF z5}|A&-#6H?1T)11^aY0xaY7|gVkrc;eVyZfhM4}Q_(uAOzd3K+!|XRDo_Dj(grXcN zW0|IRwDeYpKl;z5ekZdJIoo4$PD`*8nZ@B!v0)$pf_zF`kG)~cr6HELJ!)*)&cb5k z-QO!4UH||xu+)F&F~`r@z|Lb5_v5r0!V{RdOes|edU!)qC;b-P!NANSeT!}Org;uK6eQ>_|Em+BljnSij>QTJjgDoBCwU^rQ|58M zO%(Jxg}z^D_;7YXY-~Lgny9R>*YH$`cAC)v1g8FXD8kW2YhS-uSLzcGbR1 z;SaT_xt%{u&)m1K@Z?HK{=u|-Ij>OQV_JJmX?l2Sjt~UpFG#{T3QLt9b!a3=pQwBa z&Hpt^&+}70@AYb->CH4%aWP_0Pt-{kGKwjk8fOZW$HP1vt2<}ko?GuQmAUQR2d7c? ziNS&VG|99HL6>$#Ewt6;%sS_6%`CG{x-Fac->HUqB{C}M@l>;0@$a`)ZqdYsomQB= zxiG2l`oRkrJwh-_Ijcxb4g&2#&i*C){@i#Ak@$Hw)IR5_plTe~k7$TgFE!nZJT%uI zevUdHkUk@@wX8>70iig{Ol#Ugz$w-#Yw?NBtd%RBOjHSGoIhJVp;?fqoCVih9vwfDQJ$CkUBEX;;L--@p;a!dlExA}dA5fPg_x868BC6$nc9N_1xxD9SmdjnnpGpDTF=(8^B(uIHO1I)st=9Ual4aY@O*{0pNovpcq8mr-MpH`tS!@fpjt;(`K-9Io3Es89S(`T9 z?^#*P%Bx(hPL#UGZzZvcuU{wVln<2IHG-OE@1m?Q6o5%k`V%^!uBYcrnXsyw1ST1<3QZV%p%S|a_M(TO z7XN_`4+un|g)4{vXcOR#@ouZmrZC0cj;2h%j=WY1t$$fbZ}) za|9T*H*oqvO=Ts6fm^l{c;s^t6c*4Irh;bTd6~1{#X`X8$#^>6Ym{o_ev9BZ&JRvo z2-MQi>HYLvKG1Iy?g6s}Edaf*fHDV<2cL$3tMqbPmg$gr?I*B{+P&gr4O0aGfX?iG z(;2ytZ3qf)7mA#4m3xl~@86=HUq>C4cs}rU2A1Iyj=-(k#4%7z-8-VWQ0OO8NIZV4}6RZv=45)q~t3mgu;O$QKVzm)NAXGrPUF_ca}6mEOL#)^CJ>L(y;nR{z@O3WdWa!$BpEVvWA9d|2% z9<{Xq<>T1W6nGmYx+nd^Ct9QNd<_=4FK5Gsc^tUAL>z;pm2Lr7CEtbaJI1ZNRgOr7 zk|2H(0+3cE4FOaWPeC>{Hu3T#2h7c7oA}1^7OCM_F1y5ay#AHK^#>7vmV(Ra6`*v{ zQqtx@m7ADW&p1H-N-Fa4M|@hYYVhNS)rIyyck=;adI}SPIVUbwRBa4cMaZ56(B5Lp zgkSRH(U`ZWrblPFs#VW9n-y&p_8M%Q)pJKf*Km|k)58rAJ@gP(ztCFKbhmIs$)o5_ z|B7yGMfYHouRKiPVBXm8(Y|RxFS8u<2T+T4X9d80Hv3t77=b&*xwjAb6&76W{&+tcL9O=+e@gGyD?T5-J` znh`!eBmtbFF-(+HO@Kc1q@vdb?fweK}Q^#tk&D(jAfkwlA9 zU_}nnV<<^fScCETm{GpEvma?$th>uPtl;f;>y=Noprmg$aC|Pdv-=Uu z{Uf#OG3HzgSdjNlLzcX`;LU$RD^=xK2TPy7;sb^#YOU>8KN(w3f7VRo7#&;&jA=Un z3RG%3<_lbr(52M^zQPrRW5OxoyFM+^__E}yKWx13kjcq=KK!Q1^B;j~D%pjmM4{>1 zxPw>5fp0@A3s_(h@bw>0!JgIS3~UsCf9h%9t1-2uN4ms`L8E|4;?3*;m)d`DrSYie zX;f1Lit8SArrR;6_+BL+9r^cqB580(jxUIU3YZRIDHP8n0M52Q0JS9JEU-op`zb29 zJj}QqO_GBvG~s2Fpue?@JE?M;h3t*vT&%;g{iU2x|CjV2>{(m6j`e>;vxHPH5LLI5 z1Hc_vuEwB3!FeLI!g}iN2`5303+AW{Zll)FxnK!qQ@uG|;GuLO2m^h#eM=su{m_IG z(DJT8`bU?z<_;swMmkcSw1e}P?}#wPDY5-|0IP*pJ^Z9Z-}x+ln|f2k!5sAVX1Jv4 zg{S9lm0W;fYgaj0VS}FZ2#z98qEHn)5{KU3HgD#61A}fo&ru~oy%1=Y0xsEmM4)$T zY{};5ePZzMSVtC#U+iR-Y#K1E)LNr1x8l?Pm?v_~vK)qKBO-gO|11;Ek^ z;{6uCsqXt_T2vd=iWlD>p6H=C@F}9); zQiOvUgpSpYLaRFP|x>(JIkqJ+$=Pp<8-1akE|(?Qwxp6*!YsgRNqHkjcU{|2%4m08nC> zODlvY2dF8|wu^TN35n~%mZ9OuAt#Rr^DA#P1A^I!LXXd1Va++v(dbnUAJwXckW~z! z(*ONe$?;Q4h8>7qWIqIwkg*!ja$Xt47Ce0nqvA7lJxd60q^8In`X$xl1}Gd^1MuVw zgwgqM{p81V zJM46uLHkH99Je4Z{M0v)&cnke|M5!%Y=QG3C6geV3a#P7{aa`$;6He30mRIR8?4RQ zFkk+f|F~-+>S&sLDV=wNTf$%CKZ~)`$>px$?*03E(WJ(SN)w zNtW};hBV$!4I1`TY4^TbMn7*$*_JsC+B1^#Rfleds@UYOC`1!#%e@HFKP{?c%#bPU z#%o5f-YgL4Eo@(e%<0%kCB>cJDDy{PW&YQk-$+*RAf6Zoer;IC-525mQMi|5l5ibj zHmP5+j8DMz=-LrzXEQGX{760OFqqg=H0nMiD$TOg!HJ}tgl9~GWfx&t+i_+x5F6uf zSw2h`V_C!J_bU)t1owj({d#~#6fUsVg}}yNDuXuXi*Ca9OC=NSr>y6`k2GwFU%U$> zJsDJ>oT?viPQWWX;#TMVv*#|qpskR7gbSw;a|QIxGS*XfY-;GUJ#HW5x4a~X?grZ> zYsbue`GfN9!>s|cW8LTB5N3lRR~Iy{#-OZ@ZY#v4d`e=#3NuHO7`7~0H6 zRd!TKjjVZ^gpnJuPmn$Pi>8WjqF!bU?KPtbz+e(2*d zo9|&qw^`~s0)>hr{awA3a~8HBex(^`V;@aq4!c9fRR)8p-=pXw+NOA3x>ql{Rg>t6 zXU#{^HSg~{f<_5lnYXhyTd*k+?W11`3QNk-TyQZe(Q+W>S-YZVZ44$DQn_w9d*?H# zh1gP!GDmm7TUz#xeG>ysN}CMY71i`EpS_KJP%;cmZ_e9~KqA;&vf)?&k+_sCA)Z2+ z??Y$3p)RX4uqG!keT)z(zTn`0;JY|kcKS@w>lMh3I61#x4=l`oOiqX4t-;3UH4rbA zBnBc1h?R%+;ptS}zSb9zUe?!O;tkFfxIM1AKnwopfzp`_$>Jj`psLB^Xm%Wr@@uvNVzpjz@8`tY{3FbRA1Sy}>)z>bLJG=5@^Z@r52Az_noJmC^$A zmB${nkX3~^)+w}f?S@|O4)0gvL~jT6xjM7@;i4|AIrY28h-m7yNoXf_L3U5tU|lT= z+3lB)hXBf)aRa|1Iz-$b0y%dhP^XXB3PM;BDFp=f3`NSdg{_NEEnJ6)`}+#C98kl1+Sop(C0sDXU`+e&1N z`$o>y>E2VCG=)$|aFhLyPu%MaszcQ3BXu<3oiAKU3i}QkcvJ| zB8mpRrBTjg^JI_cF>-u_#_qte`fZq$sGAt^J8L5S_$JkmmHU%R>@HhQrSVoECd|h% zJad=JUMW9bWHY7CrA{dX#oFnJM<46se!Pf zYqqe%9%YzRlz5+sJUeJU7A}g)2q%Q#~YVMXbpGy_!M%LwV5UY zk0d2bk_70DXUoj$miXX7e8t5gY#B28rBJ*9ebKyudH*H1q38_qHjXXQr!$NtJ<*G| zbPDfgdksui+vW_SS)B~6_>)(;<%CSUGVqsYoH`Z^TyDCVYi>0J2Lo-cF_JfVZ|%@w z1<`)lzD#Y8>z(heD=h+F^)@YHurci~dEvJGp@D_2-9XCyW(W~!u)p{znl*(*HsWm^ zZ=#57T=>MHiq?EH6OYa61VXERT16+@>r^!?mn6(vusv(arPU!~%CTKcgUsgKP;l0B zL1_cZMeQHVCS=U@H>)zsD1v%o`rl{sJU_KAP?@iEj^z8)`abUA)g4N*sCyzC?r-CE zqjK!8l&vO%GbqU?#N$zcrJOYmsxGp<~c_ zv(#DZ?m1gF4LZktA0vRZGxI8t>br0%iR{784}N%k#ryU}7$tg^bK2!YA#lZ`s`5LV zW3wtJz;Zf*Pn%c(qp&drGvwse4D9YUd%*yV&-u8&g*2d0hjyKc0MY3mKm$3$C~e&d z?YalSa-KyeHfC2M-kg(tnqbYdoUVOE&Zj;7QnUV`(3nY2qYoN82mOU@+}7^!cQdiQ zQA0oHN~QKIhTjoQA(x45GT&Ku^$Qrwy#6fPfIwMfdNoTKH6SF3m+mwy5Ii_;AlBCI zW76%P)|BV!(yHMGKKM~arIckzo5nG<6hE8dz0n5e09=b&MrAl=H6qucJupMe`RUEV z)bIwkJjMvV7m9Ccz4{FBfxEm28pcCG1&1Co*VI|X_RVfs#Gb@8F%Xs+N)IvL?mO1G zk?b*-zdyt|iNSn|! zFg#X9Ogzmd-}PQOeqg=(bPwS0DCNAx;Pl39<~2-tU`(pKW3}$qCWg*IH79E+B&~S1 zUdAwlS5iL1ZXnnP5Pd%kB`!UnZQH|*ek4P-A{h8 zpZ6r#SIlmI@}aX>8xxdWlJE^xC++JsOERz>u>M1*an0!o>u$dwqtdE~#u*!_pFBpq z{(3kPCDw%5{^L27yGxum9CT-oP~MT92~gRhr?e=s3spu@@L5`flxMbp#7OoPI$q5P z73*GPI(hW!fEQuxvcXQz%50GpdTn-=E3mPOpas-RGIi7q^Lc(nqW9BL$6CmEO64>h zG%HpfAXW?>1-ZX&RTShNQjndi!9`(ZYUH%t=O}r{&QsGEFqa#38V0JiKou_fSDE(qjBxFQmK&8Q-$9lgd!p*giltHCKcbXV1c-nXA;7&Tm!etQ3s^HE& zq^nsS)vV$W$Oo$O$^4>6qbS_~$)e8pj$I8!{M3gC2UZ zjDKnhmyafq*iDb2y6=6l|0n;TX;DIC`!{a6ix6v8H4pGPK(2}Mb60n}fp6M|&>Lf6W@TDp?8^KHQc&%s literal 0 HcmV?d00001 diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/charts/membench_comparison.png b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/charts/membench_comparison.png new file mode 100644 index 0000000000000000000000000000000000000000..8bec145c818b33f8026ce642a92fe0022d439ebe GIT binary patch literal 33854 zcmc$`2UJt(+b_;L4%-}l0UUvvu%9%S8b7`W>=Tf2MRbG1Tg+;e~E;Oy>TYjMKU%GJ%* z*-2bjOjP*Ng%c0l-5UhkFGR#5!Kz*~6wK?$1*+ z;%*i?4UTr`Xoa>WE8~;IvGny{{e{k%pX&u$;n@ehpRF=U+nb9ERR^`GTW2+5*`){F zbsUPvqjDC$?o%GC4YmBT=V&8^(Sq9+KdTvUko;YyHsm;)O(MoR^V3(B)-ufUuZ}bw zhr&V6#Xfr?T9z`En$+Ac=QYo*5i9ZBD$4L|v+PXfF=;20(8Lxmvj5R@Nq(*uN@qW} zW*X)WbTvu!U*Prm^^P+ksjr+m-czS zYo*<-zg<#5bNx<)%H=jH7S8C|jiaU#xT9@uTGR+EW6EoFn?}*^k8)3%Do#q%oPav4*&*JkdI`s{ayuDUv zr9@x2*8A<%>4q*d-#LNsG<~93FI<0vIo|_3vU!>CS9GGWPgcd|v=w8Ef=iX9em|FV zLCN~Z+bdRTBIWLplhU?r9~_1&A8o`hwQKsdK!+gy^)(eAHVki z=NnVcUMqBka|yZ4#kqV{Z<{;4TJi#n0a(&f$4r4Ruy@Gcj|E4pK3zo%+kG z>9Qp3)~1y|Z7pkw)45_}!aYf=Axfx$*MFr|poO^ZuSp6{uV^7m%6Tsg+bS-f4wrW= zaoiZBV5K`TQyTxuOq%kW5{+{TL= z(Rx`%0B(oCIH&2gH}0AEmeQ>=ZZ8rBaT$q&EcR{=Yv?gb)c$58KrSm2BL*!5Q zsyP4xdih=5yzmmkcyKm*B3=n%uUm z`tAq(sySBOhZQZey*?(mr)X<)RgmlA{aY2auuk+n=kmLw9P?UlT1gK)ycS7*-kcye zNvZak$-?hWacA&Xm;ZdNR(~y8vfCur!Fmq%+4$T{d#YA1d=hghzx4&WA>r;k1;_C1 zwadtoLLZBK{C-7UmE6yscYsSOb2fumAy5iMpq>;w6rqi&$S>4;k)hQ8*;d%Z>-M3oq zRUMFSboAbwN^MZw{FO}iQ(PZ@^4ZwAJl`P_t;c_ml)lM}rYm|)wVmO^XGdb)ro1Au zW3boS_o1M&EUJ!dz|wk6QX}IS{@aww0H)pbtg^q}(iIZg_8c|2%|EO)nL^R_A$3Hp zkVD#Va%~g`Rks=1@2F)irW2^!mIRzhYWGB(vze63XmwgxuUm^kLsjOADu=g@>WEQV zTW0!Yfxx@AWT;3cYa5eE`n^%?*>AQ_COwXyejLKy`~h`emH#q7PedQ4=UxzcL(x7f zUoZdOl^UuFRR?`WH~nuD?8bP5AYS{7@eTg{{Af=F`a%`D;9keaJL8WYq-Vxz*VLyN zI~7}{Z}FcH+VhHEH%6O?eS6N!BDYB@&JOo_*Fc-tRBqB8 zetNorWK?>!U0+3_f^Sri8UOWIZ5AW7FGiac8W;-4lVWxOf2hVfrl--u}rHde&be73&t5aTk`rR0-*daFnRh#l0s`qB_FGKVLui z@%D>uzgNFwVVU~aGQp)6shDYDQ$yvxE3>`gMAAx|TI5)H``*0dBJl-e#af#lx^-3-AU$JVLsBjnHyo(D`+3iu$A$e`ve+_QvGo>fzHjt9Kuizde`VZV&?R%|G|K zL3rHJ=>Dr4qX%*8Rl0Q_C!5&}141H(bov729t77SSHe6w? zM_Q5;r}4Z$sf}W9L?-YL`Ykt&TV6_8{Ik6Wr(@*5+BsIa`(Wb?-~?wnmBdJx zAuFSz`2Z*d9FN+&q{Vh$*D8$=C16f=Z`a5uuWeHkG36~$ms!WLFh#I`+*%(C@2qI6 z-rG)7rXyQme2u;51MOCqfn_N7XN`;m+g+iy8j81BywdHj#s2+Q;HZi^nvRC>f&tP0cMo5_xqjg&tws!|JEZ*rlJhiJ{n zDV-)0Bo5zFsiA1!*u8iEH)+V+!NKl zIjF1avu68eyoBv-k2Xfm%MX8`)8p8!GtgwR{{Ivl!(snUHSIicHwHd zRVfeW0o!G)Y&51(+VU!t+iAjtnbc3RO?e7d#_hQpBO_4L{%qHANtsW7^K~)imGxlx z?B|B13nR8SM5Juhl3l}%s)PD!DID0&sGW1u zqU5G|H)j6*Hh;rZHPQiTduB>gA0n1~So*aI4>%`9{kz*TN*265)g0xh18!ptBBYQ) zGFEPTqXjo1Jdoxiww2vp6pR2(-AUat(zNb?l8B!fl(IuWm6qSUFg>UQXd~LZjImAY zoLA$xjlfwm;-$kkjI`L?J0{9gPy~sbnmFf*xtC!B5)zj7CQNcbRdA2T1m1S%qMPh#Vg<$qEWf{krjJ4;OY_p zr6Z{u+nHBD@fZu07A9WhmbBAtRMWm3bJkYHiL|x0B*tccDpW?_wcFGS?Q0SnBNG^) zlZc$(Jk+11)fZ&Ki$U9rnW4DHKP23o9974}5&?9f!!`_k)}AmQY#w9QO9kwRiFrJd z-(_SUR&W&c;wrGrMmJMMo&4Lb-_PKyBeWD{n#OcGb!?Xc`rbT_k(f6`ziR*_9N(>@ z-{=SwB;BR|HQvmJ-zzvis$+~GViDu>RZCVmB3K}>_PCgV;Qi_WBdx z+1#9Q)NPZU1sQQQOoGjh1trycKVluA>(LfPnnPli)ShFu7Eg4)zbOzY`SpIB<4V45 zPR)_vtimB#APJ=L%NI4u763-cbpec#FLiJQ%BU_$n+lVgKL%*m#!QRt^aq!I?K49F z1YPx8-dzi-g`zpuDBjZ8@`3$Wv!?CSt~L*yRm1djRSvhb18Jv3_sH%L(UPf@Vy?-7 z0_bSt`fY{Iron9*IXc9MvN3e{kU?0v=r|NTI9)C5`Bg2o|X*YLl`39IC4j_u<4wW^QwR zl5q(I$n9C2G!c$4)%;0LjvMG{aX*EzMP}5rYbhmAPWo%6^--X?gjTW2W%{KIt#CkdE@*RoHIZ$ZS`M1Ut z(c5z+GubkiZ;9ihtVOa^fpFVzolN;;4Nd0z6(jUX(>~81SD4WlCl>3z0>`-BAK4uU z`W_Ry0TI>@G_(U^(lb-y-@dzVG}h3hB2D&l+m1(Yl=n)j@^Z#m?Mi$JYii5cJ+Zc) z1N?*lQFPq=rz%!4k)q{hIk$y`bg|SowIS+`LLU9u=L6sJ|E2zpoV2}?N>4w(m6%>S zKO0rMCIA~OCMqfKgN5tGnN!#FJe<;9CARrc9q}wkMSGL}RvX_3 zYx+cx<`f@IBTlM`!zfMr{2m_Lf_Bh56ulfX9n<`62R^f}c+7MOF>~>Es4>+(Ij9)D zJ9PSkuWF|T(t5J()H`+ff7AG)8z<%HD&2LBRpu{!h32%JT{@rM^sR%=F#gsRph}o^ zQ|x$*CEn0FX23%x@F799tXN2)Mc045noaBHi|@KFF$ILlHv*>-T(KV@91H zY)~TY5Dc`eGlJohOgHD>oK1*{a4K7@9ng^WKP#A(r#+uyyeXXO)IgZ7MrcR9$~A2( z^G(y#mSX$v?2YaA$#^U?zb^e(a=1V!?st^BdHH9F6ChgMws!(`qX+2KatQ;&ifVNF zpM)LM|3b5Byif^GgQoTW@b~{iBJ288z*V^-={zJ4X!{V4touR{avYYQP}-m04CSnW z)-~DV`#WlKH$Od2!dBZ+dN({D(CgrIUvXFE#^tmn=blgZK(s<$W}Pf-T2dWvz5g=u zxBu6=9A}s3&cXY1^z@E^G*GX0ccLRhUxZWQ!Atz|DW(%}&<-*FSsLgXV|9GE5KwMZ z)|qp!urOjJ?SU2NJu7hRZ`Qf@c1H-DehDnTZegU#ZJoJs=&t{4PtG_I?w#`1=5p;X z-(8pCa38Nh11McuAi_Fq{cP1>aQD%ki8Q-2(R@{;#j(1&XfaERm0ztJ(otA@Ak*QP zNB!32chlQCqDt~=PI<0g#xgIAywQE28%I&~=RZnZ~DD!X( zmUsk%Hh6ueE6Wl%cUVSti2(dvqN-&!@Tl1{kd5T~@@(0V+knvQ2RIti)(Fb?pt4;FxP>&%#$GN@nX*rgA-9kc4_Nle2imb zb>YOdZJ=HGJ)kpMK0g!nE$IfJ@%M;E`1Ox-)kn?!?rihg;$9a-)}k||*~SEqgzjAN z0RA%IJ2}19aW(Ktd`2H4wj2X3g|#3Nx4pq#v;~Ol1t7-mU%jgovJ%4AxVf{7MqkL> z;*xcL!0rCy$|4ZV=C)-DsH-M^i#0O02S8I8g?>UBqFr(ynhRTdyeAxpqiK9H6#T_c zY+6&blDwf%jv`(1OXrFoXh1H*kC-PGD9$*VF}K%%%Q$T>D|hs@QbQL66$up%{E2k` zjRFwP#w!Nybz9qjj0PzcHy?+p31v#LeWRwmO|OTL{7{`oMAefx}>aPj?YS+(tL zDygo>Wh{lZl3l*mH$adIG~uWs{T28e@W!tT{=3pnWm~M{XQ5jc)8=7Qy9F3V9*`dR zRKFlH)kaWrGOk$=vF*wZXq)*k$$HCt_?S ztXPav_cLCPhJ3vn>sK5kq0fc|Za!S}q-{uEUL|5c~BFxk77^It1@ zEm2^a9uj9bMV~H6wJYMU8xu@ZIojJXC=<$d>HfEuCR_S?dQpX`*)50|yDWaI<$a2F zz9o8WG4g8JNL7%2I-JE*o!?^w&c|SOU557yU{O7(YkT!MKLI5;r#7NrQ)y=u@mzhz zr_+&`Hf374vW@mw&T0l=!6r9fWVs*eaV%vE(0Ai)l)>PIFSbjsKUmzbV&RE^I&ev= z@@u2`K2bRCmc2 z2H=J#UkhR#w>+7$mF7Diua|y=Tg>8m;-i?DbHE*-%D)9;Wn6qpPx5;mNM){6a+Hwq zgo;p#4G2Xo4T$Vkzh!%K(9mVMNm^^Ugg{G6V=o{a57sFhkaFx(&sS=?B%d6K6Hf+3 z6fBhRXnBmQiPxZqr0ls+jpz^GwbuEVPQV*o-UA!fEnATW%X$o!I+6V>!LAfh?=f2+ zlfzy>xm2h268^Ygl=tAU2!ME9@uieMWV50082W7UzbpfXLBJ7F8<&1NVOucN(R>_f zG4k1D(Bo5Bpa?A1TpE9)W8uW#w95~~@Nrs98?5r96cPiwLEa~ncbk<-CvX(%mvncZ z3^7AIBEWBbB&fK>CpO_+c^m#Tq1SeLN@;sBT-<+9LSYrr_q{j9@os<2f({kM{9Rq? z>B<=3(twTDuGE#_jqh8N6xsglNb7A4ratR9$0LmOIwe=>9%TC=7Lb{R zs?w$m5xoakMHp-z!7!PL=pxvg7Bw9Djcm_89QQ;7o+4;fHYMd#v7j*fHlG87<_KcI zlz}paA%lUz4#jb4zbWy`lfB^j*i^gJ@f-zFSqsqeYzNF9IdGSYQEAW|)@Z#$U_POgNg6 z`B~MEHdmNSn1efBIgk&cc`Z_tNwh(V>xroJS#5*vT$`47G5Lkc-Q$BA_7=eX8alE3 zm(;)Xf_{MCU8B2&e_U$#YF8yJ7jb!7M|Asm=>(fTIK8L&Wx3_wPA2*;{dR_bR&KGH%@UDB-05h2Qw}Iq4-&Iv_B`q~ z^L4~_TCLVk+)SOrTvo-jAC=m=?YlZ1d96gZRyLO-z`1Pk=KN&R-fDHF$4LKnW&jj$ zZIgdETxs%0bj5?*bO{3+k9Gu>UDi;K7Cou2aRza8Ha@VT(;7GeL<_MtZi3V_anpeI zmCjtZzA41tIp|#sx&b~2ry?Mzk>)B8s^QCDko##mHKy@5p8dhJ2YT>F+{ws7)^g*$XmDGIGW{tVdrH}WySXn&%UvaSBaH&Hk%Ve+`I-p?E1T#-4_n58$^OqdA%-- zXAG8K+(-f7v&-KgPRZJaV26`E_*I>6qShbOxuXn~{zJK7Dt8zPP;1b_Ta`lvdGO(R!*ef`qp>xu~5^&-I98q4c4rdxQjWfW3aa8t6do&MV1xBedRudivI&Tor8pZ*99U8a1)wWr*9}jpJ=$@ zhg^)iQNN>Xt|InrgHU1P4B~?7(nzpt5Ycvd6MPjV`=lMuym1%PumplVTFZT@JvK@H zDSci5Atsl4z*hDL(lSsSB>@+w}AxRpC^ad-R8b^;D*^uVRr9QzAG5ioYd*dYhm zF@hX>R_6!hCV$gXSk*0q_bq<-05Ou6f8Vj2K?RNCi*x-OMDCw6`43u8A*P@=l;jM@ zs(m75hhS4-c(sIiNc934)nK$pH&8uyKyB*Pl=}#xdaeO$Y*UfhVo9r^PW$5USgs-XYvyG*KRw8HS{Dp-4m<}Z-7}g8)h>lvf@{* zH%}s--#88jz6=J_iPo<{H0=t1@K+Ecd~#JnbQ*BUF#w&h(>`hT6m!OA_a!i;EQ5s# zz9~(x^2q+=TLBl#ReA)pT%z|WxF(^Oz$3B@mF#+^A=UfW5a{9F;5EBt zzg!anGcPi`n}lQZKxJ_YC;&_~F_YrS^PgCBi$Lvzb}$BTyq+x~`uA|n7dA+`Nc>QXZu?XKXqE^RGfPB^1QY*htoY?$Z0OA89bsF7V zVeGPZ2>4O=dMpTXCfg%~6Qch7?coGq5ia%osXz5?3+^HdaQh-==_j`lwEjIFg#)f? z2-Y8`)rE55cbD@MLWWb~oL)e$$nfijN=q}@@~mWH{<*Nws?68K5un1dw&TD9Lvvg0 z@6sE+}>jXiy`A7sd-W%Wl> z7aK0!14>?4`Sw$Ds)BDLAXXNn-EbP14s`T@r4Hc@+y4AypW zMIcG8sOa>+VlcHkpe6dMZ5gAZ#;bn8*(jZ550n&qDXV@V$+*x71Etekug{G0i8Da| z%%yT|>fpwWX)}4DG-a!BUWJst#FAa-(p+Ev3;)2OEJBVa42bVBMDqteL(hpEI4L#$ z=|16PL)%Yobea>b>}*ra%i7!VF_T?T$By%U4o&+fcw!v^eo;%?_QoAfva?ZJNJYC& z_2fm7MRz1hlb&{qHvHtE-CVEx{D6#z67Wv}s(zurr#<**W+{umr=o?vm#OV=O3^?h z$;IOGbEiRLIO0?5q} zdXJtrcg*ixGVp`_XEp%4Cvpw=8!o*-jB*%-lvVZP(H%rS@L8T&;Z3EO{nOMIU}47E z?9q(GAOsWM$S+K>5V7a8A>{xr-xY4RMzI{Ai+B>_DVNhRph-Or3Gcj!UY3|v)@Pf=?0b$B+8<-KKo_+ z>Su??>$rVqb6h|WzX@8wDC9Ms0mPAz^_-2yGegkD!Is%gb0=OUcu7gS@^J*fFZ??<_33H;Ao33=>p`ephmM*&yb>C& z72)Mh85&F`?*1(Rd2Y`F?A!6@vUYZGH#npie4o`0IgyotEte>o=-! z3~|WvoWH$1iJ>*_eE>`HJSi0+D+PL9S{!v#zQDmP!6FU|JYsZyx|^6(SEq)+a43r& zvmolWW*HaK{}*6|wqiy;ch1Rq(0)<6Qh9xRN(Ed_Qcd^y&F@aVXK=BOVb7rHcsn zA&5vobOs+lX21t7wPi+DLH)_TH`<2^7HhfczA*E}U;D^D>j)mcOi>AJmt=4<;2GyP zd`Y`LL%Xbo9nD7eTh<`hD?lGbXY12_lS8FqM_RIBFUx#XV=^iUM;LMVOe_{57ZDoihL%ylvb>WbYAgj#Jm&YU?+-9BDz7@Tz7V)Ha0DE?c9TW}Mr~R8? z_trXNm*cqQvq9I-8SS=-uwD@5ky-? zkycnoa6}lenbtikmi>YiEQAn@*<^FV@;_*l(gli%i8!LAj?k^33NF@P*0UM516bB0hv}%V?7D}NS143xd`!x z%GMx((!dMB5hQYj?o;%f?fDFH&bcICh{>sYm`(M;j=xWsgJS8t;hP9CtH?xsJM-m< zrsWFglND_kI4Sd?AwbHXLC4Wp#X=2o+TO z;EG(9Y=)qL0T|$dj&i_S--F1y2(2AG-G)fGb#sMe-rx(LZw)|3q4+cLy=Az9l>T8X zKpP~9h<6r2F9Kn-9PdYN(K?6#mZ=GzFA&OF9PBV{L}vsSgAtMVZBUoCG9_M>WSai0 zAA&4`j{N`@-OzV^!gWs?Ao7f8*rm~D0#px3(-fZZ*q8m$FxPr997jLRDdi9?_`8j6 z3DvDx<{ao6w<{bGwrct~BRxc&BTgm#&;YcAl#(HULwo>}OgbdbG9j(+#%XbLuuA?! zLTx*Ym;MF7t6T-RR0IA+5O))&_oBPYB^|ZKcr;~r%~e=Vr=AVrD@8L(1*zl6es9RY*n{Tj* z@|7m8jEz#CE)+^Utwcqt?dS3DAavCi5^@$_10t0orJM=8$3Z?FDikfS8<1X=0k8o# zRP@kkG+#>2W-~PRGNC;Ixfi$1wPV-cm&_qi5myEZao9fLq7%$^_L(_1 zp*i@a^kG0@(l6eE?jpdK`tat5PmXOLREmw~-;9_;bt`OqI8JUvqO`11I^I4IKkfV# z52a|*%Jy;O`+p3A#=Q!@YmH;WgqTO|Ati(tB`>tSy`UvbsK^Vbeld()lnQ-?e_W&+ z*bP)}X^UW2sckiq>CWI0y?c28)Rf2qIIJmV*UOIabfS&VCqT|dq@~N`^N*|h#t(_2&tst$+|eyTAzn@6Swvr9{mP47ACZqyPQRrV zVgG@pi2e5pU=bKSGW%5xnJ3OHt8EUXG@nlvtLFSBAY$)vJ38E!BR&&t8TK4Hj>3+A z{9PGIILcQ&bOknF1HmKtYu)yMw2<(x3!Hcj)DkwlITn+6MnShlW(<4DY8 zA`fbbVzUBu0Pz5#ZG<#W+5rw8{kIG>B`(l(D!xS98EHi*4NK{i?5?I*$p#t{#5&Ky zbOJt%Xk>!(kX?n3z!nvIVq%YwaX|Y(iHcuwuD!Py$p&HY$k$n~iXm&31+3{b$BLr4cez979@mu31C~Iy% zFwpWRUe3W4Xc{qQ*?mxM_LH3rZz7wf!fFRSb8~Gj?nxa~zH>NX+mKrKa9i^m`lcRe zR(+iZagQtiU^~k$jLOuPl9s(EuBQ6LSx@=^mHuExXS36j4K*D?Ya!2F^{=(5D`X;=F1sC&(f<_W!d%l|ExZbD~uy0bwf`pC`|yUbGn?QQbVA|NQ@T#9JW zKal;uMC=3X=qv47J9fbTok}nUeO6E@8zkCdt~msxApyIc@cHoLeK$eiv4l1m4w^&( z9h^}b8Sxi_kkkw1|7AbD=BW8k65hb#`I6Wk zHLCVv5d=<%^)nx`t88)8swRVL&=+fv&ho1_*QO$jJM=`Wf8pI zo}#h5+ne-1?m4&pgH00STsh@Z?GPDMX&))gdC3}FBT zfrMqCHWQT(0iD%<8A6lz7A~7^1GCm4=TnJ)_1pRGGMj~ueK3g-sA0RBE!pW}t3NsHo5}~^=XzoKqqDk{WEslXGg$Oh<@q$xONg7!}$&&f6RG$p+ z;V|e(ewl!rHlkr)$?OHE4uJt!DV@C`9q~Kh_%2JtIj!b*XcmU{Z7--#)mQfk=~m!? z-pTxkYeVFZB0$O(KzjtpYcw)LQJ=&EA$l2kKgb@q?YaBgjn>gBE!6Vhu}RMFo2s4_rY6SuxefuBK_*STBLc(Fkj zqt3Fe?J*2Sni21RCGsX7o#@xcPtmFZDAt95B1bl~H;zWfH$AmuP_!|`pW!IE)6xL8A5U)%i^P{MNSsj7os zlX#SKc$ziq{03=giooWZ4FRh zq*n8t$mxO>ds+8iFYEq;*$X5c5Wo2kACK!Gn&YC@TEf9>YkzIMl8|^x5s!JEieTB> z8aMiY)7`djNpYKO#%%nD0h4PCh;;d4sPd&OJCzC3U6*&x#G#9uN(5Ea*Lph;-*BdH|>||HjJ! zaXew9m&j}mu!4Lp?>UTYY;f)p>>j^FB(;JvExfSl(h}^0ELR59*(flSf!B`$mMClxtndpHTtvf{j!swON(mSLp<$X?V@^jf4&7ne` zN~f6pO^?p~X;17DKjP3Y`xLiIUT^W`yBVh^G}eyC9bjs}ZitDT3#I>cHuQnqYR%cL zrJ2zE6LBSaf;X^OqqXoXaAwfyU*vD$O zWXI-t<(yvauKXGOg!;nM(Ws70EEXE_qgHX#w+iZ(7Q1sbpAql|E8VD<7`+|Iq0i|z zNx>Q+4TkA&#z}2N=u@|CBn&HDr4OJcLb!q|CR#6=>!Mh<7#sm=1@?zRbx{|SV)25g za}ZKV@>{aE8iR_+1Kc)!b)fE80Ib4)%MzlNJN%UWa_pP2uMNbE#l>^m^8vqx9G}bd zot-$a>oWXuz^?Ua61i5xYsh?TjM{6(#dMeFIq80kr`B4~x@ng2k)z%$A`-Yav~b_ejti zII$*A|9R}9!}TGsdynAFy_D&m$)@i+we7Fharo3R)g|(h{b;87Ei1`dq-RxH^5>!q z-7+3jRNJ*W^?JOgt}*|lrm1pDi{zgPO{O9%B8#ecQ8x1vHgBpx&*4+mqs3V@BA$;% zIhPb1>Lv46pGcvT6D6gTmy^hV*^%sxjUOW$v1(Zb81?u|V%x48t2wR|p7%AcGbVj| z=6m?@h#$z2!7zCr9b(veaK}8b^)K95KXH^>`*Fk1g7m#g9<5WhbNCg}=3k zmV4#a=!e!|OI2pEzn(AF$5GUm2mV8qfYyEHJv8Z7sE5D6f{lOp^$$p-o2z@!p>`gJ zM(K9}1ZHHMY-gOUWIEjihE~QWZ|m;547V)6F5btCd}*ObJlQIEBB%GMy>f1f#(T0q zBW$&>U#Ws`_1dgUkBUnh!<0KgR+v3*#Y~V>;v=8*AO`19(0^~SL4*xxpea>}vs`NE zOm@Zi{DXmycN{84WEcB+8|t{6NqjV3*!%7SF`d{W2Nkv*SXoG$oBcfx1|=<~YL155 znEMHlx|fM%$MsEc$_}~IJ7P`)>pkA(lF4qVtYQnjc-MgS@53iM&Ag~pnlHqK%jA#D z7u|L!o!Z-4f(|a4*{*Y~q<+nC*6^V7wU90dS4>YFQ`*M=%3pHLnX~4)jt`WpeG;w| z(ceamHMaRhTPclb3C(aWmjj)M&pvB2uE#Zb^}R>KkyggGvgM%TTCgv#Us?S!h#sVB zsH8`btfhq0z@-9^$PH`~JS@Kc{f7$z#v$5jIJJ%hBK1sMY7ht45)eQRBJqG_$AkE{ z+#Vfwa$sg|!Y%mYzeKdVW*&1{)M~YZWKIP=11ZR~wTxNh4jv4jG{c@|wpZGc7asDr zzI>s}>Qxz|{}s7Twttq5fuST9%{)@J4Cg7hKdzLmRhrEpY^9NrEFE#QRQ9#0e^ zGY_?o6Cn>HT!(qlw>;d z8ddn8j^%5jdNO$`$xhI)deLR=wU)+}8Oj?OGk@Q-7PRf9+%M>SrK0{<0wCWx* zvoiJ^uuE12@;TkgJU$+jgyjvI&vk=iN<8>+LQ`?N(!BrTM^|G3)3Kf>^;M)c3{G*Y z=vUdpAnrq&hL7-Hi} z)FLg|%~n($iChb(JL$?xEcAL$KGCx1_{C@OmBFV(8Ee7L+0k25lj=g&mol7}Ru$KO zE`!j|XF2-8oJbu~PKq>l4py9*SS40$NFjWYdz8m}n8-A_BELRN91}$FIeMNlQ0(Ro z!B>4&MC{6GyvW#hJ^1p!M3U@;qltFL!4n$Gt|P2-Zogk5LK1fVy zVzov;O%|1To=3^*Xy1*D2Wn{wQ|sw{v9-eL+T-`oeR)pD9gVb1{qYxa9!!>TOO4K` z_7kCj4BYUZ)l$;$-{z|g35}Jd5%p^~{E6QqpGX!Dy&NHFRw9CpGUWU0S9n2NwSQSO zUqX8NtbGX*Ec*YPp1$s^fb41oBUDUUA>tf^XgsA@s&48cP1wUJoj5{`d$rd1?1h4b zIjzMPqh8UWu}*Ifd!8j_sKgv@>-t7wVyLrlAd-r|(NxmP>iys@D@teFlI*4xcA(2l z=*U#j?RmR|Y_Glz|FISv$tkLITr*K#M#Kxewg*G>?%R2?X=)E!Kkjtu=&HU#dp32H|~P=gCpl%$oNGD6Bm~n(KqxIte<{ zyG(Sp7^@G=65ZNfxh$B;Y2=gAdxO;?!%p7}c?R93`ynVBkS6S@>Tn&z@91 z+ykh_=aJ@FG}VlY^is<@uaPd6e6lZ7miihuK^0P&lie6lTk63D3%Bum_E;n@`45%~ zy~0bAF}RQo7YD)u<(p1(@Rw^(TYmV8P#Hw8)hw^cA366|4q93;EH%ShmFTigqi5qJ}x;SYx9b%ST!izW0vqxO6BuhS$8$75dL&HNh-Y*rJCOI#2b-7;1S zPIv(vNyHIo| z9l`H$nJ=EK<%9T=e6~5{xZTjhK!3RbO{%b>kd(gx$mmOp$iO_P;3A$tP1mdVwhzJ7 z>%C0Kcw$by>Il6Q7>G2xOfUg+0GZrq-O)P&(n3hs+>Pn93!fMyOYEoiuDEIelHtj1 z?A)B#Ld?;z4gH}Im56i7b!&4LX=jdvo?ashc@9kDIK149%6$%dnRwizkH6p=?JDIB zu`%~iV8+Qg6yKpw{at0bVHK6`?#C&(fHbFE;Z0@v&6RzN2qxRpUl11BZ7!~VNs+q5 zCBCw)=03+wRP<41y~bR=a#MbNUA)!y-5$%rIjL}QEY83LW4UVT2}z8Oo`+QjlEGeC z1P7-YiZL4~U5#T~3Erd6G3GZwN;E5jOMt+q8pz}yL&R#Jm5M+l4?*U+>5Dnb@4+9< zdNVbXzjEWXr|&}mJP_0OKLPpei9V-YmlGb2{phwU{X^80jjJ&1UHLZqYM)o|nSRM< zHxHyIN$Q~t9Fh+5Bnqy#_4p=L&>uEBb6yxYJ%NTU@(um$`~L#rGo65)2q;|TGV(Yb zU2}Af`(KdVlNj5`Oh&S8dx~U97mRFLg3A+(=&R;7S@~Trz%QzX zFGl$DpyK*nEDv=2jV(YrC%Fuhv%h(`sSf<&rW5q4}s30>nBd6THmK z68)CGw?cAgX%30GgT8c%XbuK|7+f1pQxP=lS&(nx=0~nMg4vhzWiQvyRpgQt3?eG9 zsMm-2&r^{D59`Wv*1BldORc)U_bc+rLY~Z`2k$as)m?W}j*nX{^RuCZv0tabCz@=| zPyww*$+<%CtX87QRBZB1#-Y&Cl^1IbS#q5f+YFx%bOk=nbq9j~SRLa}A-u`ySdknf zI(@n5cu9E2xU}hV%u%xjoO53LmvKsjneOb4n`dMf>Nj9r%5y}dY_n_s;8F1u&EEp5f)0l3n{DKW)Beezfmwg03JJei zWCnWB-U%i*7XfkBCV>n63+CYo0+sBi6hW83Tzv_`jJ4FzA8>0MdG^gRoWo^LIaSO& z@}8*RvIr$-%XcpKe4xf6tfTtHAU-<}z73!P_@nH_z48*G^Vs2>9@`m>dg`j7*Q)nw zlb74KLZg<*`odJwQTSL;OHL+6pF(l3&jycAHh-KSjCwXBEOSnO&3}vh%4PHwO_`Ff zmypo{1v29F=+vZnXklWJT$G_*P8s5sxb*xnS-t8tz=hjv;8{J%$S+JzGry& z%ai5L_TAGubGl3hi#<~+ra0g~R31Vb;YNC&TQK6GU z+&p4wNB4&*IV1@HpzS1Q0(`|^XiK{mFp-ZVWZuV-tXb*0WoX#YM3+qfu=I`+_sE1}4y)$?^`TO{Yc^kylFHQbDy}T(dQgH+dTlCmm zW-xN9Fj%tm8M8dm$s2K&Sp^iYAWl` z zyqT}yn(WW@V;B9dLvmAeqD~bj^6B!JfqlN2*uvDLCBTfDvs{@FY+k;~E9hLc+SSN( z_jLgQwUBm2WIW)(R=$4R*vZy-KfUi?dHR);Z$?&;Uh4@N{X{(+C^Db8Ju_V(dl(h9 zt0;$|=S6W0uPJ&iifBykda1CMIJKock@`cJxEnV%5*Zw2AZ6 zTH9wh)y1yt*0ImIy!Gt6PLMXt@4C&FkzW;O%wtQPxPl-;ccI+7`$6YtO!oTbuXYiO z4UfXm^Ksk8v+W$4kpUh(ohI{cz9| zjQ;u=xOVI6g-C=lT3TVbZAOMZli%j*(o|zJzKGJ1?$Y*Z31`OLnJXr^ppv}TYP`qh z4qE)ACRwtKM{JZ zC-+@BwpDtnGo#y`C=H&&((1IpUbP4uyX}A?y#25~8K)#Ga&&GWtzOx+fAAoh_R%N1 zLjlC%v)jwtbuP4kpydJyrdukupTlRI_zv;mr2aZ$`ay^AT-^E$B7kd0(&R+wGexaQ z2aX>Bs<3*U8ziFH3*}1r8PJE)_Bl?+#vwTfojE>8A!MUWkhBn_N~7mj1vKXu;cFvO zCxkJJHRd+Uh8BP9W`eeRxz(LgaNkT(?AJKx!dilJrJGTuqs_Q2=g&X*3G?r43&;U~rIYd@>ls6Sh?zdM$M(F4suCQZ#1}j3&n4I0_kNHk@j8{GN@pq@snT8NoT< z=?(o+t)r%54UTWuI-a9MRdQF)A_1>5NGgo?>!61ZD|5f<4wqm0wp3q!AM&wCZg{7Q z*n5DvA%9zbBHD5kaRu`jFx>JSyM^6)HJ?+*>$Z>$7c`*nW%@c$buWaaWPbkcl(R7( zRF@c(#4VUz=+EuaR_wCW%B;Ce*ZwFCBAJ1GC;4Y@fZ7pinPdACCIfk9-}Jn%o4#ru5O3`kZW{Ev)ifyDEjGjYP6H z8;<9UepaiCXesAL$P9yGK_}&~0GrYp@X{f3-@>l+9$j3hywdvVPF2wy-_hzM&Gmfm z7aKa|pWo-fZ3AmJ_`ovA^04)3k5wEI_Qlv zFj3}#%&s9BoXM9D>FbNFFjXlZHVj{cbO;g^4Gy+$wnv6-o7m+^y_;{$1^;4-UXHzh z%dOildUW8@ieyU+W~jP0%klwH=|=h5pv8C31Mlzb&P9pi>^?mVkJi_KIRXS{lqIsX zq)6{u82IKtZGyV=Kid29a4hq_{fpMoR8x_PQd3hzAxTQ8h8Afflr<)mCD}5TxLZtR zofg??if|(}D2bbCLb6Skh!8@EvP8=AJ3p76nfX1>`#i_*)uzl=dR4Nc_zWLsam~M^VnA_gk!~g zMQ)yy84Xfu7}ZTKUJM<8Deg1P8he=AVD;=ts>|puV37dT`K1<*(TS!^}7blFUt?e-mQHi z%WJ;0=PJKykbk*;+Ll7g_KJ96@E^RvEmTL3-xA!O^+;M2*WCV{?#tGm_ynWjpcl9I zU#YO%T~l;=o~27+O>cP_7ceY2x-(bCW``QB zd8TlT?fK87tpCXlegCj1{(px)|BKuS|N8ExEBYJ`KDTLx%H1Bvc1064_wP za1a62{9_1_BdtIzdK#w{a|DwSv{{Q_6Oy@4y1_h40)qL}(1h{ngtzQ4n$oGgbf|`_$E0+aMkMdP>?YK zEAuJPfS!{ec=%z2=Ir-^g#^mA!{Znwh!c8xAQxT2+k^Dt%=-h_?5b{RKm znosDlbP3&0wV2xnJj{v$DHQ3`vhbNJT@+{TzGeK`(3<99WPho00AD+axBOqXQq39)V6Umngbj$4I=+q{gATA_tcO%nfy1 zcW{}6@86`jfkoZjJ z!}iQH_Advu%eby0d@`SWC3mZnT1F}GQJD6!Tqdj*Y2isFd1#`}+S;`#Y||s+WD^9X zq_~wUkO6v>MJk()%B>x*^WQ#HaMf86=oK#4vB8>pj-%jDnM9Fj>hI3M4Pf+OzIVxkB-J$5|g3^Dxg?7k!m`K+aqv)IMCYZE8d5|+(zx;Iw z={}5fDsu)ZH&WI}Pag#;^XyKEN~?5wf(PnGEr7#0lj_{L3q6?~Ei z!WHlR;I&7fUqt5s%%r#}Z{baIp47iXYw68mSA4Dr?dAk7{RAGCyeZh4{^2L8CH`ea z*jE;Js}g(ZHK4ZDM@yGX1CdXu!-gxc!|*($R|85~@2~@nnqV1(pP*UoJD4Eau8Juv zn-%1mk_y*2*W+9PH(-U=fes=BK(MhE{PaZD#Q?5(ws@l)$eQe44?JG%|LFPQ$PK@T z0HgiO_nAdp-SmW@*gb;(6p3XV)swZhe7naXc*^}<;arBb$4wvO1N~z%mWM$N(Rppb z`+BwC8O*y;^+@HD?D+B%xfXsvuwoHRY{>ExT5Oo(X~pxs1dnYVvE0eS;NsH7SSLn= zV7QGENl|RdhUX1mRc%{veu7OzTHq4RqT~yFKa}{Qiwz3HoW%@ zrJ?wpAQ72DC1amxNbk3MN$PCo<`*hg;UUwuKmneJAkn=sI68~|QBX5+0tdPZKLoU0 z18HzWk^&Siqgze4k5SXIdVzpk*8`a0cZ?&kQ-?=7kv4T;0eeF!o zl7&ERUCA{;tCO^Uy|PEmU#75c)N+G4PW*O~RDiDLsb>)vZF*UAO@2)zvPBiN*QVf_ z`yf;+f<5YA1Aig=7mo-7B6kD{E z3bSmt#Dms6XHu#`H|?u3)G)mp!M^1S6`--A#AEMkE6aN&#=d#hL=N;qswp=0$^>1| z{voAFY6WZgBA#%dgWrvqfdJnuJ3842uroH5xedsikkM)vHYLQ)!Yv?ub+1nraIo0K zrS#^i4HXBN4>2mk61o0jVnh$~nJahmZV0^#()355o_2oI-Fi1qhhHSD#VGFWk9W)3!Bi;_MnukgoMv-H@*7I_mGqajc4z- zmjOFOX9OKsLLGdeg*aMBCrPH*M9!Juy^-d#^2B9&XA_=ohY$xNA8V+aHk@;ZQp?4i z^aSMGfGKH*#RF&d3XJ0`n0H`#`jJ}^Z4#v#8{Dy8$wUtoXR5Gj4TdXW zzs#yj(1H0wx3MM*oOU4(2*M%v<}z>5XjWUWx2mubH!uTQ^XnKbkhfG0%sa%eELYZ7 zLbA6O@{b-K=!>t+=o2`D;*1-#U#3KzHOh3^2ah8t=wxZoWnfK^3L~h7Wev%JFJw47 z2GSgMteZ{$JIJ3uduRB%=H=OK+wvUVRN3B2VYi&u&1d}Cpk zOv&$W_-nOG6WO4FF)fUxeBjGDG7FJvBZ<7z!=sHuE=fxL#sJkDqb~2{CYpa388mrX7Sq)@@CRN!t{s>y%JZ;%MO zf7-Me(9_zbQE<2L-}Z?-tkZC5f8N%F*P4u#=I&nnzyBFA$`BKGlp%xHgG5EP9?0c+S-JjE-5(nomssImi}6q{X1-z|S{5aFq3obVW{W7b1(VW0MHzwY?=ayy zPfoy?5QGxZdD?JkJTlfDM4w#XJz|7@Sg)Bu_qfF}RfkWn6jP-+&>rl-BwyW6H25Cb zBguy>V%z$lym8r=1V5~REZBJ%NzxA{BXXo{GMF#^B|Sk-u$JYa6J&9CYeWRdSaj!> z1Ktmv^$;(fFLVM*vIbW2JTO`y~= zgAxbOhg13AZh?*^-%HYdzd-T!+$k6b8d}S<=>Xq~ z%@yemZF`T_$dxF~N<5J2g|EMIv=;Th9wz#<3F4x4*tI>Bh_wx}zV^Ht8}#^;lycA- z>^oZOzgqS&zOI%=BKt)L^|2pG4n@P)W{;R)4=wQ^{%kLr4NxhwB862-ws_#<`=P2= zMsCKPoMP7uT1`Z7-l#_M$WBF_msbou>2`3Ol^Al0x}oMh+ZOx@HGHW?hyc8N*Hnws zbnvQJFSJ-W2-~DAkNtfs9<9>H>F4h@!?EmbhEq>MWR{kpHYhAX?rb;|$kw)12sTx67|!HHUQgkGDS zalLpv=l|+0|JCTZr0>1|y7t%);D0xvJ_t=Ddqak^f6rn5Y(S&Z(kQ=Py@pC=u%Dn) zvQ4d&Mmqq9t-ddfU4bFnVg!|N_F$GWiY^B@tQLc$V$sas5k@{%P^O%2Z6shom))@4}LWjLqiMdfe2H0Cfegq+prl=aC_Dz$uX zbJLV!FBsx!Kd?ciNj^RwFnYZ%zAi;Ze^vI2(zK0}ri+-YoKum~UZ>6!x;F`2xky2> zy)rUc-UZW%uvw&NC8b~Qj)$m#<2zgE<5g>?GWu{*3q<_|3LsnWp)k40ugRHM? zQFFI;kGc$WZP2f#w9}?D>89+L-i@JR1t4Rf1wfBiP7ZqT5c*yV4)v7k-r*s6!$9zkdQf!qlB)tk!e#Xady;MGxoG^<)>u&Ehkm^1|-b zOJ^NzKJaed~=FQ+`6p$up3;3M+%vcmLrvQv(?Zz9wAmiG|g zT62d{kvrB4G#^c2_Qi1y#po2804tPx3gc=ygF3)Pc93~KwRs6pjdi2_$s-S{2a{sB*<{rb zvc?)(aWCH=|Md7-E17+xs)lNTeNfj z47I0+R=!ep?Fh-Qir@YNxBK679L!&t@xSB2-Cifm^ureb zp>IAp5w#SB0;b*2h%*d)P=FQw=Ez#Hx{eTFZyzhTsIupkpas0I2B3&`_OPP}lGavzC#?MvqH+E>NYA&Dq%R zIV4*`H1i^ubb<4$i$}_C78#_;(Fepf39(sq+UQ=B0aRMe$Geah{Lrh^1X@w0?D zM2sxJLeX(8#N!DHM5P}RH}D!|7#$VlJdAW-EH}R7Jf67by>+*@HhtmVYqsB7f$v)J zkyUM>?8k$zv0r58WKM{s&NZK?5z$*C1#f3KWDjt_s;1np!|0s8r(-%`=o)m6dTle#dg zdToDrckOV@1hOnnXY+`T7{ZMEp$c6?%Qi>TnLmCR zAo)mZnElRZdx#w8qlh}_gQ0Y3KaP-DO7>hIu*~fo8_-xr8t=Q0>A`ihsP=tz^Z^Uo zT|VQeugL}7-b4Xb9fX=S0?YFI+yxul=pga2ERMYDdF~3oyIc$Z;j+k)B@Er*YtPCh z@e`kUFJ|f9fP)fNQeKI>+H`9)nOHfDwC(9h?o6|NZ8?V7KaX2{T|B+e^F>Ut&lq^* zc###H{*|L}Gyj9nTl*`)(uVncqWM2l8Qoqd^e^*s`Fqy28OiAeIvc`lD3vslZz{rp z`Q3YLw_&c1L`O7h3SN!(TO+(N4q5?k&*V8c5-QB0bQ|%mbAevKKZ2Z8O@VGaF2Ec~ zigazU##!#F`r8nNiwyQ}_qw22qhL7n24KD{h6QP8VHznRV+@)D*i8^k zgf1NkhjEqbxBK@$7tNHW2M`p2jwJ$&9A{_Pe{Dh0@Z=s&Gf*ZE>5Pcw1Bii!H#!V~ z6L!NgI!9f^TJW5x-PXZ=lBJVrE8##aHcFJGezwEs%-!>WIgT?l6n!xLs=Gp7RDaIa z6|Y0!A{E>368Gdd9Z%B%SFsYQy9?-g%rzUv(TP=yrO91xSz9cO787t(!qK^~tQ-8` z9^Gr+>_WdNvrI^t5moyBxDndv)M~T66HIcq!>Y5gfgcb}PLTBxKpT+65kaE(l46e2) zKjO#Rf-gq4Sds5aMaEasEplZD3P?umZR!l~H zbo^;bKGlVlG|!LgpFVH3xbB-3fEv-^(rgpe-O1ESjam=N$Qe8wCQ)eG!Qvm!1eKW` zWCoH!quxe14M^8VaQXc)ug-9X3AfI?5}QA$cO}}D>oFL*LEc6f@aCaZFyx&i=+_0b zoF_juEe*9=b?z-&U)@YJ+K-=dKJpRo>#R{I;a=L5iGn!~Mn(2|_h2HI2g8clnxP{$ zS09mOT0f?9;8DX+MY-ZV!Spo8nl`M1dQz#b9%KHQWX6SgfvoQAtLXycnBDedloXVS zGRuTcX;8;B&$LLs(udxH&UzeDSqUB?YHA$m6@P)9dSNE|qDaBhJ| zCJRQ%?245W;?V8iEnE>Sph|y>^&ZcmKmVU#MVU&-0-USy7=w|zIf@HD)?&(f2aYbe zD5&&#QyI?v&Ct4Ki?Z?K7lb=GYC+r z33PiOsv_x3B(j5pP0_kLIMRu&a$wry14D2ZlrHvEG;p3b!pXvmOxVc&2_C~jri87- z!B;*Dob*nKaZLG4L--g3t)SBl=!h62V?ed+j4!|=`khVBt9)6En|F!eN@g+qj7*Ic zbWYBL^z_rn+bQOqbS7(Rr8#FZL>lB~-kg9y6QRUp8!SYrWOBD1miE*YIhXp=_9Y?X*52Y&tLLlWKc`h^xK>l}XEJKU zXvYl&wEd6xj@SG&8VvR#%sscur0EY$(KBdFLnmCAVGvwYaSG%NHf$2ogXrdsdnO4U z?7inbJebUB(v#3W^)lgyQ)B{-%`LtXptl~*5ZU!PQ z{Jx{Ir1mr@>++#*Co>!`TAz-30{vuctA{~E6O3d0G3JzEjpnyYCJNH5hz72A{if>) z%zWU`cIiwO;YFCc94<)ZV`2~dYH=64X;RekjMY=fNYLKt1$MZ0#A2L z;zZvWuK^Qg*MQuQ58#r`R`qUHHFrC$c#;gmlJx`)1nT!T8%nsvug%Yc&onz}8^Tn- z*h2v-sFiRW79Aqr;p9G^(1YQt&oC?ONQ21R-l@rN{`y>Kj^+Y|49!`On|y%2+Ju1I zN{wIibDlg%No@3a5clyItzfdjAy*}X326wO&F_-R4Hd#w}SY1Q+F7$f#}%IL!ksTYYq7*q#aGb`hxu8|zraPYTarqjUqE!&5A9 z#KFZhb0{#hKm5&*q^B<^yT%z<-1k>V?6yL4PGpzClUKpGdhQSdn@^3*pUWqWXE+bZ zK(z^6-SY^(8vvf~K3Wgb#*fQB>0wtC+#~N!DiB(aWsAAJ+Xy<>PkWsj=I&W1?fV|5 zF+R-2d45}fMRJvJMmXz9C&w*lf) z$l*cFQ^wq4jS6>EEt=#K?(D-mGi}`w?#f@~N`DZ1LoO(~!a5o1Aw!EZD-*PZ`4LV> zbcGq!U&w3vzKD%LH5ri-64_<e;wbZS3fcYivc)_*XN+^V~g1#_`aLI*Oj z0Pe4{L3GbwbTFUJ03h?Bi0mdhNmmLk{&FwsN+-*u_wzG_B4h8jcXvOLjR7Oz+`?pO z$E@)EB38FILR6mqiLekh*pkMa6>v}i59YiSYX{T5rz~ga@D_BiG&(l!;~VZ* z889cJ!$S(>_+l=o?ENYY9JN;**`mJn@Ni|THuAGRWhgK~tGS+xA>Brhs^`(!1Q{p> z2g6qXlt&iA?!%>3r-V*|T1TL7Ygf+EQP8Gkw5%r~-0VgxC=*DW4yrs`R^BB|>0tlr zFSZC&&>WT$W8KxOlxias7523i?GQ2)=m7O7O=-p!Eb%6A2=W_@FL9YNy;J!FC|IW3qHEs&+{ z8Mu68JDlULt(E`oKvL`o~))@_n4s1C2(IG;( zs+X5l$}d`=hDd)E6O-B%kG<>TQ&Z)2`KF1ky?txKVry$_XLS>bMepj(*$n%20%c&X z_VU_IS(D%NoJ)v2>ycxIxLc5?*R@Bg@yL-Qk6RBH23<7Dau2Vrsj>fYAilv^oImr! z@QJP$!PCT)!}RT2i$Vi`%|mg~%Gg1~T3hYup{70b#VEs!h_T^h(Zh_`7v5h6VUE&iT&!nwpvwbKTYC>q8Z6?B)pX^B^?wSj*I>ArC-BD1Y%H%uh@uLf^sLi)Xyg z)ljPS_1vmq!+@_DrnzC37;1yBAA^&l|8Zp4HR6tN9q(jlNzUE+!QR#wP$-*^(`ii= zmJcD(v-@PFJ!+WczTup`CuaLC-^`o7Kxymz`SS-2HZ&NQba=f>sv0kP)`MY&&rrdz z^UnJFMdE0783nUsSJo8Lbi6qWbPhfaIW}C!pyl zMf;Yh>0-*Sramy{xy5}NmN?Bj&>eDGh2e~-QFe(rQiA7p%)>ZqU4gTyEuIl>Tg~4)VfDg-1#+HGL!TqJ7 zAW*stCsc4dGgW4_dgqNl$>2ly7?(Cw|AT4h-cn3+yizbdTa6mZBHCDcRBa#Bb9E1# zl3Z$`aOnBkH@4JUkyFn*+3{?ms0gQN5CrL6%XUTVz+f_ep#FiA%`Jfh>$kqyNwNLj zz1>1dqsRshErr40-z|PC$Ed2@#Gvb%@UvTn`r&tk?T0tr$yD&0F>$3aPpM?k`*^wS?!zI6C;Evfbj2hgW`BCI-3?Y~ zFZ>Kr1WQ8XB~$O-y}3R55>U~XC-Gr1ZZpR-ug|P`8Cz&+Y1uKGnZ3^$H+&~#w!6sa z*E(|sqwZ*O)^5*)IeaE(&)r##RJW+2zYY#Z%}JXKW+i6Y8m91VdF=$w%24_YLG^%o z(5B14L$Yf&-n{wD0N6e&vKb~*D0%217H;pIyVWZnX6wDVWAe-CA^Vh1?ju9@=k??y zWPR27i|(K>Ys|;rzq$)GUS{ppB59l` zla!?I)a19OvyPviT^}nTC3w)}tVefPON+UOJ;S(*jGE%6dQacyF?139=8K2lt4%Pu z)2tvEzyVE92o2)98ZtZkxHWojeqPn%8vD6h7x8amm=ED8uqo~PskpefysGN8jR2GF z@dGyMOR=CybM#$ZkAXzb@u01XpY4TTs?wk0vV05a&nur{vhAPggWu3WqzuEENxc@s ys9qTJSG)FTJO}^D>^9|Br9b~OUmlk+GXA*WT%OmtA2!leZd2R&?X68Fr~eP)-u7$& literal 0 HcmV?d00001 diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/comparison_table.csv b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/comparison_table.csv new file mode 100644 index 0000000..619c3c8 --- /dev/null +++ b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/comparison_table.csv @@ -0,0 +1,65 @@ +benchmark,metric,series_name,system_name,mode,reference_kind,status,value,example_count,dataset_path,source_path +longmemeval,ndcg_at_10,old_brainctl,brainctl,brain,historical,full_same_machine,0.9253,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +longmemeval,ndcg_at_5,old_brainctl,brainctl,brain,historical,full_same_machine,0.9204,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +longmemeval,r_at_10,old_brainctl,brainctl,brain,historical,full_same_machine,0.9894,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +longmemeval,r_at_5,old_brainctl,brainctl,brain,historical,full_same_machine,0.9681,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +longmemeval,ndcg_at_10,old_brainctl,brainctl,cmd,historical,full_same_machine,0.9247,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +longmemeval,ndcg_at_5,old_brainctl,brainctl,cmd,historical,full_same_machine,0.9206,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +longmemeval,r_at_10,old_brainctl,brainctl,cmd,historical,full_same_machine,0.9894,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +longmemeval,r_at_5,old_brainctl,brainctl,cmd,historical,full_same_machine,0.9702,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +longmemeval,ndcg_at_10,mempalace,mempalace,raw_session,historical,full_same_machine,0.8948,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +longmemeval,ndcg_at_5,mempalace,mempalace,raw_session,historical,full_same_machine,0.893,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +longmemeval,r_at_10,mempalace,mempalace,raw_session,historical,full_same_machine,0.983,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +longmemeval,r_at_5,mempalace,mempalace,raw_session,historical,full_same_machine,0.966,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +locomo,avg_recall,old_brainctl,brainctl,cmd_session,historical,full_same_machine,0.9217,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +locomo,cat_1_recall,old_brainctl,brainctl,cmd_session,historical,full_same_machine,0.7614,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +locomo,cat_2_recall,old_brainctl,brainctl,cmd_session,historical,full_same_machine,0.92,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +locomo,cat_3_recall,old_brainctl,brainctl,cmd_session,historical,full_same_machine,0.6616,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +locomo,cat_4_recall,old_brainctl,brainctl,cmd_session,historical,full_same_machine,0.9738,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +locomo,cat_5_recall,old_brainctl,brainctl,cmd_session,historical,full_same_machine,0.9821,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +locomo,perfect_rate,old_brainctl,brainctl,cmd_session,historical,full_same_machine,0.8817,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +locomo,top_k,old_brainctl,brainctl,cmd_session,historical,full_same_machine,10,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +locomo,zero_rate,old_brainctl,brainctl,cmd_session,historical,full_same_machine,0.0438,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +locomo,avg_recall,mempalace,mempalace,raw_session,historical,full_same_machine,0.6028,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +locomo,cat_1_recall,mempalace,mempalace,raw_session,historical,full_same_machine,0.5899,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +locomo,cat_2_recall,mempalace,mempalace,raw_session,historical,full_same_machine,0.6921,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +locomo,cat_3_recall,mempalace,mempalace,raw_session,historical,full_same_machine,0.4604,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +locomo,cat_4_recall,mempalace,mempalace,raw_session,historical,full_same_machine,0.5809,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +locomo,cat_5_recall,mempalace,mempalace,raw_session,historical,full_same_machine,0.6188,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +locomo,perfect_rate,mempalace,mempalace,raw_session,historical,full_same_machine,0.5534,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +locomo,top_k,mempalace,mempalace,raw_session,historical,full_same_machine,10,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +locomo,zero_rate,mempalace,mempalace,raw_session,historical,full_same_machine,0.3499,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +membench,hit_at_5,old_brainctl,brainctl,cmd_turn,historical,partial,0.93,200,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +membench,simple_hit_at_5,old_brainctl,brainctl,cmd_turn,historical,partial,0.93,200,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +membench,top_k,old_brainctl,brainctl,cmd_turn,historical,partial,5,200,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +membench,hit_at_5,mempalace,mempalace,raw_turn,historical,partial,0.885,200,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +membench,simple_hit_at_5,mempalace,mempalace,raw_turn,historical,partial,0.885,200,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +membench,top_k,mempalace,mempalace,raw_turn,historical,partial,5,200,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json +longmemeval,ndcg_at_10,new_brainctl,brainctl,brain,measured,full_same_machine,0.872,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json, +longmemeval,ndcg_at_5,new_brainctl,brainctl,brain,measured,full_same_machine,0.8678,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json, +longmemeval,r_at_10,new_brainctl,brainctl,brain,measured,full_same_machine,0.9894,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json, +longmemeval,r_at_5,new_brainctl,brainctl,brain,measured,full_same_machine,0.9574,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json, +longmemeval,ndcg_at_10,new_brainctl,brainctl,cmd,measured,full_same_machine,0.8753,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json, +longmemeval,ndcg_at_5,new_brainctl,brainctl,cmd,measured,full_same_machine,0.8715,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json, +longmemeval,r_at_10,new_brainctl,brainctl,cmd,measured,full_same_machine,0.9894,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json, +longmemeval,r_at_5,new_brainctl,brainctl,cmd,measured,full_same_machine,0.9596,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json, +locomo,avg_recall,new_brainctl,brainctl,cmd_session,measured,full_same_machine,0.9145,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json, +locomo,cat_1_recall,new_brainctl,brainctl,cmd_session,measured,full_same_machine,0.7533,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json, +locomo,cat_2_recall,new_brainctl,brainctl,cmd_session,measured,full_same_machine,0.9107,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json, +locomo,cat_3_recall,new_brainctl,brainctl,cmd_session,measured,full_same_machine,0.6607,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json, +locomo,cat_4_recall,new_brainctl,brainctl,cmd_session,measured,full_same_machine,0.9643,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json, +locomo,cat_5_recall,new_brainctl,brainctl,cmd_session,measured,full_same_machine,0.9798,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json, +locomo,perfect_rate,new_brainctl,brainctl,cmd_session,measured,full_same_machine,0.8751,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json, +locomo,top_k,new_brainctl,brainctl,cmd_session,measured,full_same_machine,10,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json, +locomo,zero_rate,new_brainctl,brainctl,cmd_session,measured,full_same_machine,0.0519,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json, +membench,hit_at_5,new_brainctl,brainctl,cmd_turn,measured,partial,0.895,200,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent, +membench,simple_hit_at_5,new_brainctl,brainctl,cmd_turn,measured,partial,0.895,200,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent, +membench,top_k,new_brainctl,brainctl,cmd_turn,measured,partial,5,200,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent, +convomem,abstention_evidence_recall,new_brainctl,brainctl,cmd,measured,partial,0.0,5,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache, +convomem,assistant_facts_evidence_recall,new_brainctl,brainctl,cmd,measured,partial,1.0,5,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache, +convomem,avg_recall,new_brainctl,brainctl,cmd,measured,partial,0.6,5,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache, +convomem,implicit_connection_evidence_recall,new_brainctl,brainctl,cmd,measured,partial,0.0,5,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache, +convomem,perfect_rate,new_brainctl,brainctl,cmd,measured,partial,0.6,5,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache, +convomem,preference_evidence_recall,new_brainctl,brainctl,cmd,measured,partial,1.0,5,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache, +convomem,top_k,new_brainctl,brainctl,cmd,measured,partial,10,5,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache, +convomem,user_evidence_recall,new_brainctl,brainctl,cmd,measured,partial,1.0,5,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache, diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/comparison_table.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/comparison_table.json new file mode 100644 index 0000000..31ff305 --- /dev/null +++ b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/comparison_table.json @@ -0,0 +1,834 @@ +[ + { + "benchmark": "longmemeval", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "metric": "ndcg_at_10", + "mode": "brain", + "reference_kind": "historical", + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl", + "value": 0.9253 + }, + { + "benchmark": "longmemeval", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "metric": "ndcg_at_5", + "mode": "brain", + "reference_kind": "historical", + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl", + "value": 0.9204 + }, + { + "benchmark": "longmemeval", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "metric": "r_at_10", + "mode": "brain", + "reference_kind": "historical", + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl", + "value": 0.9894 + }, + { + "benchmark": "longmemeval", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "metric": "r_at_5", + "mode": "brain", + "reference_kind": "historical", + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl", + "value": 0.9681 + }, + { + "benchmark": "longmemeval", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "metric": "ndcg_at_10", + "mode": "cmd", + "reference_kind": "historical", + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl", + "value": 0.9247 + }, + { + "benchmark": "longmemeval", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "metric": "ndcg_at_5", + "mode": "cmd", + "reference_kind": "historical", + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl", + "value": 0.9206 + }, + { + "benchmark": "longmemeval", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "metric": "r_at_10", + "mode": "cmd", + "reference_kind": "historical", + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl", + "value": 0.9894 + }, + { + "benchmark": "longmemeval", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "metric": "r_at_5", + "mode": "cmd", + "reference_kind": "historical", + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl", + "value": 0.9702 + }, + { + "benchmark": "longmemeval", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "metric": "ndcg_at_10", + "mode": "raw_session", + "reference_kind": "historical", + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "mempalace", + "value": 0.8948 + }, + { + "benchmark": "longmemeval", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "metric": "ndcg_at_5", + "mode": "raw_session", + "reference_kind": "historical", + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "mempalace", + "value": 0.893 + }, + { + "benchmark": "longmemeval", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "metric": "r_at_10", + "mode": "raw_session", + "reference_kind": "historical", + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "mempalace", + "value": 0.983 + }, + { + "benchmark": "longmemeval", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "metric": "r_at_5", + "mode": "raw_session", + "reference_kind": "historical", + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "mempalace", + "value": 0.966 + }, + { + "benchmark": "locomo", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "metric": "avg_recall", + "mode": "cmd_session", + "reference_kind": "historical", + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl", + "value": 0.9217 + }, + { + "benchmark": "locomo", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "metric": "cat_1_recall", + "mode": "cmd_session", + "reference_kind": "historical", + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl", + "value": 0.7614 + }, + { + "benchmark": "locomo", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "metric": "cat_2_recall", + "mode": "cmd_session", + "reference_kind": "historical", + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl", + "value": 0.92 + }, + { + "benchmark": "locomo", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "metric": "cat_3_recall", + "mode": "cmd_session", + "reference_kind": "historical", + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl", + "value": 0.6616 + }, + { + "benchmark": "locomo", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "metric": "cat_4_recall", + "mode": "cmd_session", + "reference_kind": "historical", + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl", + "value": 0.9738 + }, + { + "benchmark": "locomo", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "metric": "cat_5_recall", + "mode": "cmd_session", + "reference_kind": "historical", + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl", + "value": 0.9821 + }, + { + "benchmark": "locomo", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "metric": "perfect_rate", + "mode": "cmd_session", + "reference_kind": "historical", + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl", + "value": 0.8817 + }, + { + "benchmark": "locomo", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "metric": "top_k", + "mode": "cmd_session", + "reference_kind": "historical", + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl", + "value": 10 + }, + { + "benchmark": "locomo", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "metric": "zero_rate", + "mode": "cmd_session", + "reference_kind": "historical", + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl", + "value": 0.0438 + }, + { + "benchmark": "locomo", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "metric": "avg_recall", + "mode": "raw_session", + "reference_kind": "historical", + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "mempalace", + "value": 0.6028 + }, + { + "benchmark": "locomo", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "metric": "cat_1_recall", + "mode": "raw_session", + "reference_kind": "historical", + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "mempalace", + "value": 0.5899 + }, + { + "benchmark": "locomo", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "metric": "cat_2_recall", + "mode": "raw_session", + "reference_kind": "historical", + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "mempalace", + "value": 0.6921 + }, + { + "benchmark": "locomo", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "metric": "cat_3_recall", + "mode": "raw_session", + "reference_kind": "historical", + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "mempalace", + "value": 0.4604 + }, + { + "benchmark": "locomo", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "metric": "cat_4_recall", + "mode": "raw_session", + "reference_kind": "historical", + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "mempalace", + "value": 0.5809 + }, + { + "benchmark": "locomo", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "metric": "cat_5_recall", + "mode": "raw_session", + "reference_kind": "historical", + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "mempalace", + "value": 0.6188 + }, + { + "benchmark": "locomo", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "metric": "perfect_rate", + "mode": "raw_session", + "reference_kind": "historical", + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "mempalace", + "value": 0.5534 + }, + { + "benchmark": "locomo", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "metric": "top_k", + "mode": "raw_session", + "reference_kind": "historical", + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "mempalace", + "value": 10 + }, + { + "benchmark": "locomo", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "metric": "zero_rate", + "mode": "raw_session", + "reference_kind": "historical", + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "mempalace", + "value": 0.3499 + }, + { + "benchmark": "membench", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 200, + "metric": "hit_at_5", + "mode": "cmd_turn", + "reference_kind": "historical", + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "partial", + "system_name": "brainctl", + "value": 0.93 + }, + { + "benchmark": "membench", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 200, + "metric": "simple_hit_at_5", + "mode": "cmd_turn", + "reference_kind": "historical", + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "partial", + "system_name": "brainctl", + "value": 0.93 + }, + { + "benchmark": "membench", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 200, + "metric": "top_k", + "mode": "cmd_turn", + "reference_kind": "historical", + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "partial", + "system_name": "brainctl", + "value": 5 + }, + { + "benchmark": "membench", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 200, + "metric": "hit_at_5", + "mode": "raw_turn", + "reference_kind": "historical", + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "partial", + "system_name": "mempalace", + "value": 0.885 + }, + { + "benchmark": "membench", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 200, + "metric": "simple_hit_at_5", + "mode": "raw_turn", + "reference_kind": "historical", + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "partial", + "system_name": "mempalace", + "value": 0.885 + }, + { + "benchmark": "membench", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 200, + "metric": "top_k", + "mode": "raw_turn", + "reference_kind": "historical", + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "partial", + "system_name": "mempalace", + "value": 5 + }, + { + "benchmark": "longmemeval", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "metric": "ndcg_at_10", + "mode": "brain", + "reference_kind": "measured", + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl", + "value": 0.872 + }, + { + "benchmark": "longmemeval", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "metric": "ndcg_at_5", + "mode": "brain", + "reference_kind": "measured", + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl", + "value": 0.8678 + }, + { + "benchmark": "longmemeval", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "metric": "r_at_10", + "mode": "brain", + "reference_kind": "measured", + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl", + "value": 0.9894 + }, + { + "benchmark": "longmemeval", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "metric": "r_at_5", + "mode": "brain", + "reference_kind": "measured", + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl", + "value": 0.9574 + }, + { + "benchmark": "longmemeval", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "metric": "ndcg_at_10", + "mode": "cmd", + "reference_kind": "measured", + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl", + "value": 0.8753 + }, + { + "benchmark": "longmemeval", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "metric": "ndcg_at_5", + "mode": "cmd", + "reference_kind": "measured", + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl", + "value": 0.8715 + }, + { + "benchmark": "longmemeval", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "metric": "r_at_10", + "mode": "cmd", + "reference_kind": "measured", + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl", + "value": 0.9894 + }, + { + "benchmark": "longmemeval", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "metric": "r_at_5", + "mode": "cmd", + "reference_kind": "measured", + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl", + "value": 0.9596 + }, + { + "benchmark": "locomo", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "metric": "avg_recall", + "mode": "cmd_session", + "reference_kind": "measured", + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl", + "value": 0.9145 + }, + { + "benchmark": "locomo", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "metric": "cat_1_recall", + "mode": "cmd_session", + "reference_kind": "measured", + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl", + "value": 0.7533 + }, + { + "benchmark": "locomo", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "metric": "cat_2_recall", + "mode": "cmd_session", + "reference_kind": "measured", + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl", + "value": 0.9107 + }, + { + "benchmark": "locomo", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "metric": "cat_3_recall", + "mode": "cmd_session", + "reference_kind": "measured", + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl", + "value": 0.6607 + }, + { + "benchmark": "locomo", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "metric": "cat_4_recall", + "mode": "cmd_session", + "reference_kind": "measured", + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl", + "value": 0.9643 + }, + { + "benchmark": "locomo", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "metric": "cat_5_recall", + "mode": "cmd_session", + "reference_kind": "measured", + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl", + "value": 0.9798 + }, + { + "benchmark": "locomo", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "metric": "perfect_rate", + "mode": "cmd_session", + "reference_kind": "measured", + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl", + "value": 0.8751 + }, + { + "benchmark": "locomo", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "metric": "top_k", + "mode": "cmd_session", + "reference_kind": "measured", + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl", + "value": 10 + }, + { + "benchmark": "locomo", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "metric": "zero_rate", + "mode": "cmd_session", + "reference_kind": "measured", + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl", + "value": 0.0519 + }, + { + "benchmark": "membench", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 200, + "metric": "hit_at_5", + "mode": "cmd_turn", + "reference_kind": "measured", + "series_name": "new_brainctl", + "source_path": null, + "status": "partial", + "system_name": "brainctl", + "value": 0.895 + }, + { + "benchmark": "membench", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 200, + "metric": "simple_hit_at_5", + "mode": "cmd_turn", + "reference_kind": "measured", + "series_name": "new_brainctl", + "source_path": null, + "status": "partial", + "system_name": "brainctl", + "value": 0.895 + }, + { + "benchmark": "membench", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 200, + "metric": "top_k", + "mode": "cmd_turn", + "reference_kind": "measured", + "series_name": "new_brainctl", + "source_path": null, + "status": "partial", + "system_name": "brainctl", + "value": 5 + }, + { + "benchmark": "convomem", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "example_count": 5, + "metric": "abstention_evidence_recall", + "mode": "cmd", + "reference_kind": "measured", + "series_name": "new_brainctl", + "source_path": null, + "status": "partial", + "system_name": "brainctl", + "value": 0.0 + }, + { + "benchmark": "convomem", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "example_count": 5, + "metric": "assistant_facts_evidence_recall", + "mode": "cmd", + "reference_kind": "measured", + "series_name": "new_brainctl", + "source_path": null, + "status": "partial", + "system_name": "brainctl", + "value": 1.0 + }, + { + "benchmark": "convomem", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "example_count": 5, + "metric": "avg_recall", + "mode": "cmd", + "reference_kind": "measured", + "series_name": "new_brainctl", + "source_path": null, + "status": "partial", + "system_name": "brainctl", + "value": 0.6 + }, + { + "benchmark": "convomem", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "example_count": 5, + "metric": "implicit_connection_evidence_recall", + "mode": "cmd", + "reference_kind": "measured", + "series_name": "new_brainctl", + "source_path": null, + "status": "partial", + "system_name": "brainctl", + "value": 0.0 + }, + { + "benchmark": "convomem", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "example_count": 5, + "metric": "perfect_rate", + "mode": "cmd", + "reference_kind": "measured", + "series_name": "new_brainctl", + "source_path": null, + "status": "partial", + "system_name": "brainctl", + "value": 0.6 + }, + { + "benchmark": "convomem", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "example_count": 5, + "metric": "preference_evidence_recall", + "mode": "cmd", + "reference_kind": "measured", + "series_name": "new_brainctl", + "source_path": null, + "status": "partial", + "system_name": "brainctl", + "value": 1.0 + }, + { + "benchmark": "convomem", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "example_count": 5, + "metric": "top_k", + "mode": "cmd", + "reference_kind": "measured", + "series_name": "new_brainctl", + "source_path": null, + "status": "partial", + "system_name": "brainctl", + "value": 10 + }, + { + "benchmark": "convomem", + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "example_count": 5, + "metric": "user_evidence_recall", + "mode": "cmd", + "reference_kind": "measured", + "series_name": "new_brainctl", + "source_path": null, + "status": "partial", + "system_name": "brainctl", + "value": 1.0 + } +] \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/metadata.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/metadata.json new file mode 100644 index 0000000..45f5356 --- /dev/null +++ b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/metadata.json @@ -0,0 +1,24 @@ +{ + "argv": [ + "benchmarks\\compare_memory_engines.py", + "--label", + "rlm_srlm_probe_refresh", + "--membench-limit", + "200" + ], + "cwd": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl", + "datasets": { + "convomem_cache": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "locomo_data": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "longmemeval_data": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "membench_data": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent" + }, + "generated_at_utc": "2026-04-23T18:05:25.112789+00:00", + "git_commit": "9368f288279ae1d3395e37e89972038b74d66e3a", + "historical_summary_mode": "recovered", + "historical_summary_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "machine": "AMD64", + "platform": "Windows-11-10.0.26200-SP0", + "processor": "Intel64 Family 6 Model 189 Stepping 1, GenuineIntel", + "python_version": "3.14.4 (tags/v3.14.4:23116f9, Apr 7 2026, 14:10:54) [MSC v.1944 64 bit (AMD64)]" +} \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/convomem_mempalace_raw.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/convomem_mempalace_raw.json new file mode 100644 index 0000000..eec82fd --- /dev/null +++ b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/convomem_mempalace_raw.json @@ -0,0 +1,29 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\convomem_mempalace_raw.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" + }, + "benchmark": "convomem", + "caveats": [ + "Blocked while loading ConvoMem evidence data: " + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "example_count": 0, + "measured": false, + "metadata": {}, + "metrics": {}, + "mode": "raw", + "notes": [ + "limit_per_category=1", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": null, + "reference_kind": "historical", + "runtime_seconds": null, + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "blocked", + "system_name": "mempalace" +} \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/convomem_new_brainctl_cmd.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/convomem_new_brainctl_cmd.json new file mode 100644 index 0000000..0668f29 --- /dev/null +++ b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/convomem_new_brainctl_cmd.json @@ -0,0 +1,106 @@ +{ + "artifacts": { + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\convomem_new_brainctl_cmd.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" + }, + "benchmark": "convomem", + "caveats": [ + "changing_evidence: discover failed: HTTP Error 404: Not Found", + "ConvoMem comparison is partial because it uses a bounded same-machine sample, not the full benchmark." + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "example_count": 5, + "measured": true, + "metadata": {}, + "metrics": { + "abstention_evidence_recall": 0.0, + "assistant_facts_evidence_recall": 1.0, + "avg_recall": 0.6, + "implicit_connection_evidence_recall": 0.0, + "perfect_rate": 0.6, + "preference_evidence_recall": 1.0, + "top_k": 10, + "user_evidence_recall": 1.0 + }, + "mode": "cmd", + "notes": [ + "categories=6", + "limit_per_category=1", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": 0.6, + "reference_kind": "measured", + "rows": [ + { + "category": "user_evidence", + "evidence_count": 1, + "question": "What color do I use for hot leads in my personal spreadsheet?", + "recall": 1.0, + "retrieved_ids": [ + "msg_2", + "msg_3", + "msg_4", + "msg_10", + "msg_7", + "msg_0", + "msg_1" + ] + }, + { + "category": "assistant_facts_evidence", + "evidence_count": 1, + "question": "I'm finally getting around to that productivity hack you mentioned last week. Can you remind me of the specific automation tool and method you suggested for integrating my color-coded spreadsheet with our company's Salesforce CRM?", + "recall": 1.0, + "retrieved_ids": [ + "msg_15", + "msg_2", + "msg_6", + "msg_4", + "msg_23", + "msg_3", + "msg_27", + "msg_12", + "msg_37", + "msg_32" + ] + }, + { + "category": "abstention_evidence", + "evidence_count": 1, + "question": "What is the direct contact phone number for the 'QuantumCorp' lead that Alex mentioned?", + "recall": 0.0, + "retrieved_ids": [] + }, + { + "category": "preference_evidence", + "evidence_count": 1, + "question": "What CRM functionalities should I look into if I want a more visual way to manage my lead pipeline instead of using a spreadsheet?", + "recall": 1.0, + "retrieved_ids": [ + "msg_10", + "msg_60", + "msg_42", + "msg_11", + "msg_43", + "msg_4", + "msg_57", + "msg_16", + "msg_74", + "msg_56" + ] + }, + { + "category": "implicit_connection_evidence", + "evidence_count": 1, + "question": "I need to get the new team-wide kick-off call on the calendar for next Tuesday. What would be a good time to schedule it for everyone?", + "recall": 0.0, + "retrieved_ids": [] + } + ], + "runtime_seconds": 1.514, + "series_name": "new_brainctl", + "source_path": null, + "status": "partial", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/convomem_old_brainctl_cmd.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/convomem_old_brainctl_cmd.json new file mode 100644 index 0000000..1f53f87 --- /dev/null +++ b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/convomem_old_brainctl_cmd.json @@ -0,0 +1,29 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\convomem_old_brainctl_cmd.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" + }, + "benchmark": "convomem", + "caveats": [ + "Blocked while loading ConvoMem evidence data: " + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "example_count": 0, + "measured": false, + "metadata": {}, + "metrics": {}, + "mode": "cmd", + "notes": [ + "limit_per_category=1", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": null, + "reference_kind": "historical", + "runtime_seconds": null, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "blocked", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/locomo_mempalace_raw_session.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/locomo_mempalace_raw_session.json new file mode 100644 index 0000000..388f041 --- /dev/null +++ b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/locomo_mempalace_raw_session.json @@ -0,0 +1,38 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\locomo_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\locomo_mempalace_raw_session.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" + }, + "benchmark": "locomo", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "measured": false, + "metadata": {}, + "metrics": { + "avg_recall": 0.6028, + "cat_1_recall": 0.5899, + "cat_2_recall": 0.6921, + "cat_3_recall": 0.4604, + "cat_4_recall": 0.5809, + "cat_5_recall": 0.6188, + "perfect_rate": 0.5534, + "top_k": 10, + "zero_rate": 0.3499 + }, + "mode": "raw_session", + "notes": [ + "granularity=session", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": 0.6028, + "reference_kind": "historical", + "runtime_seconds": 2106.411, + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "mempalace" +} \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/locomo_new_brainctl_cmd_session.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/locomo_new_brainctl_cmd_session.json new file mode 100644 index 0000000..3d877b6 --- /dev/null +++ b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/locomo_new_brainctl_cmd_session.json @@ -0,0 +1,44071 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\locomo_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\locomo_new_brainctl_cmd_session.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" + }, + "benchmark": "locomo", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "measured": true, + "metadata": {}, + "metrics": { + "avg_recall": 0.9145, + "cat_1_recall": 0.7533, + "cat_2_recall": 0.9107, + "cat_3_recall": 0.6607, + "cat_4_recall": 0.9643, + "cat_5_recall": 0.9798, + "perfect_rate": 0.8751, + "top_k": 10, + "zero_rate": 0.0519 + }, + "mode": "cmd_session", + "notes": [ + "granularity=session", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": 0.9145, + "reference_kind": "measured", + "rows": [ + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_1" + ], + "question": "When did Caroline go to the LGBTQ support group?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_12", + "session_10", + "session_4", + "session_9", + "session_5", + "session_13", + "session_11", + "session_7", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_1" + ], + "question": "When did Melanie paint a sunrise?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_13", + "session_9", + "session_14", + "session_17", + "session_15", + "session_11", + "session_8", + "session_18", + "session_16" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_1" + ], + "question": "What fields would Caroline be likely to pursue in her educaton?", + "recall": 0.0, + "retrieved_ids": [ + "session_7", + "session_13", + "session_18", + "session_10", + "session_4", + "session_14", + "session_17", + "session_15", + "session_9", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_2" + ], + "question": "What did Caroline research?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_1", + "session_11", + "session_2", + "session_5", + "session_13", + "session_18", + "session_8", + "session_15", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1" + ], + "question": "What is Caroline's identity?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_11", + "session_16", + "session_17", + "session_3", + "session_7", + "session_1", + "session_8", + "session_9", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_2" + ], + "question": "When did Melanie run a charity race?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_11", + "session_7", + "session_1", + "session_8", + "session_13", + "session_5", + "session_18", + "session_14", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_2" + ], + "question": "When is Melanie planning on going camping?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_2", + "session_9", + "session_18", + "session_14", + "session_10", + "session_12", + "session_16", + "session_7", + "session_17" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_2", + "session_3" + ], + "question": "What is Caroline's relationship status?", + "recall": 0.0, + "retrieved_ids": [], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_3" + ], + "question": "When did Caroline give a speech at a school?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_2", + "session_13", + "session_5", + "session_18", + "session_7", + "session_19", + "session_10", + "session_14", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_3" + ], + "question": "When did Caroline meet up with her friends, family, and mentors?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_16", + "session_17", + "session_3", + "session_7", + "session_8", + "session_10", + "session_15", + "session_9", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_3" + ], + "question": "How long has Caroline had her current group of friends for?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_4", + "session_16", + "session_3", + "session_10", + "session_19", + "session_13", + "session_12", + "session_1", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_3", + "session_4" + ], + "question": "Where did Caroline move from 4 years ago?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_4", + "session_8", + "session_15", + "session_7", + "session_16", + "session_17", + "session_10", + "session_19", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_4" + ], + "question": "How long ago was Caroline's 18th birthday?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_15", + "session_16", + "session_3", + "session_8", + "session_17", + "session_1", + "session_11", + "session_9", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_4" + ], + "question": "What career path has Caroline decided to persue?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_7", + "session_1", + "session_15", + "session_3", + "session_16", + "session_12", + "session_9", + "session_14", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_3", + "session_4" + ], + "question": "Would Caroline still want to pursue counseling as a career if she hadn't received support growing up?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_7", + "session_5", + "session_17", + "session_1", + "session_10", + "session_3", + "session_6", + "session_2", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_5", + "session_9" + ], + "question": "What activities does Melanie partake in?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_15", + "session_17", + "session_1", + "session_9", + "session_18", + "session_13", + "session_4", + "session_8", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_5" + ], + "question": "When did Melanie sign up for a pottery class?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_16", + "session_14", + "session_12", + "session_10", + "session_17", + "session_8", + "session_7", + "session_11", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_5" + ], + "question": "When is Caroline going to the transgender conference?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_9", + "session_7", + "session_17", + "session_3", + "session_14", + "session_11", + "session_1", + "session_4", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_4", + "session_6", + "session_8" + ], + "question": "Where has Melanie camped?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_13", + "session_18", + "session_9", + "session_1", + "session_6", + "session_2", + "session_4", + "session_16", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_4", + "session_6" + ], + "question": "What do Melanie's kids like?", + "recall": 0.5, + "retrieved_ids": [ + "session_9", + "session_13", + "session_1", + "session_10", + "session_18", + "session_15", + "session_4", + "session_17", + "session_16", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_6" + ], + "question": "When did Melanie go to the museum?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_10", + "session_1", + "session_13", + "session_5", + "session_11", + "session_18", + "session_14", + "session_7", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_6" + ], + "question": "When did Caroline have a picnic?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_10", + "session_1", + "session_8", + "session_5", + "session_13", + "session_18", + "session_11", + "session_14", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_6" + ], + "question": "Would Caroline likely have Dr. Seuss books on her bookshelf?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_6", + "session_17", + "session_13", + "session_18", + "session_10", + "session_14", + "session_5", + "session_9", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_6", + "session_7" + ], + "question": "What books has Melanie read?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_6", + "session_17", + "session_9", + "session_1", + "session_2", + "session_18", + "session_13", + "session_4", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_5", + "session_7" + ], + "question": "What does Melanie do to destress?", + "recall": 0.5, + "retrieved_ids": [ + "session_7", + "session_15", + "session_1", + "session_18", + "session_9", + "session_8", + "session_13", + "session_4", + "session_17", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_7" + ], + "question": "When did Caroline go to the LGBTQ conference?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_7", + "session_3", + "session_1", + "session_9", + "session_11", + "session_13", + "session_4", + "session_14", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_7" + ], + "question": "When did Melanie read the book \"nothing is impossible\"?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_6", + "session_17", + "session_1", + "session_2", + "session_13", + "session_18", + "session_15", + "session_8", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_7" + ], + "question": "Would Caroline pursue writing as a career option?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_1", + "session_4", + "session_18", + "session_13", + "session_5", + "session_9", + "session_12", + "session_8", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_8" + ], + "question": "When did Caroline go to the adoption meeting?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_2", + "session_19", + "session_13", + "session_17", + "session_10", + "session_1", + "session_5", + "session_15", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_8" + ], + "question": "When did Melanie go to the pottery workshop?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_5", + "session_4", + "session_12", + "session_16", + "session_17", + "session_14", + "session_11", + "session_1", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [], + "question": "Would Melanie be considered a member of the LGBTQ community?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_5", + "session_9", + "session_11", + "session_14", + "session_10", + "session_7", + "session_15", + "session_3", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_4" + ], + "question": "When did Melanie go camping in June?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_9", + "session_6", + "session_18", + "session_2", + "session_4", + "session_16", + "session_8", + "session_1", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_3", + "session_5", + "session_8" + ], + "question": "What LGBTQ+ events has Caroline participated in?", + "recall": 0.75, + "retrieved_ids": [ + "session_9", + "session_11", + "session_2", + "session_10", + "session_3", + "session_7", + "session_1", + "session_5", + "session_14", + "session_17" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_5" + ], + "question": "When did Caroline go to a pride parade during the summer?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_11", + "session_10", + "session_5", + "session_12", + "session_9", + "session_18", + "session_2", + "session_15", + "session_16" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_3", + "session_9" + ], + "question": "What events has Caroline participated in to help children?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_9", + "session_11", + "session_2", + "session_7", + "session_3", + "session_17", + "session_13", + "session_10", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_9" + ], + "question": "When did Melanie go camping in July?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_9", + "session_6", + "session_18", + "session_2", + "session_4", + "session_16", + "session_8", + "session_1", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_9" + ], + "question": "When did Caroline join a mentorship program?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_10", + "session_3", + "session_8", + "session_14", + "session_5", + "session_1", + "session_13", + "session_18", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_8" + ], + "question": "What did Melanie paint recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_17", + "session_14", + "session_1", + "session_12", + "session_10", + "session_16", + "session_11", + "session_18", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_3", + "session_6", + "session_8", + "session_9" + ], + "question": "What activities has Melanie done with her family?", + "recall": 0.4, + "retrieved_ids": [ + "session_6", + "session_16", + "session_17", + "session_10", + "session_13", + "session_7", + "session_19", + "session_5", + "session_1", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_5", + "session_9" + ], + "question": "In what ways is Caroline participating in the LGBTQ community?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_9", + "session_11", + "session_15", + "session_10", + "session_7", + "session_3", + "session_16", + "session_14", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_6" + ], + "question": "How many times has Melanie gone to the beach in 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_4", + "session_8", + "session_7", + "session_6", + "session_3", + "session_18", + "session_14", + "session_15", + "session_17" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_10" + ], + "question": "When did Caroline join a new activist group?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_1", + "session_9", + "session_4", + "session_15", + "session_13", + "session_12", + "session_3", + "session_8", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_10" + ], + "question": "Would Melanie be more interested in going to a national park or a theme park?", + "recall": 0.0, + "retrieved_ids": [ + "session_11", + "session_6", + "session_5", + "session_18", + "session_4", + "session_7", + "session_16", + "session_3", + "session_14", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_11", + "session_9" + ], + "question": "What kind of art does Caroline make?", + "recall": 0.5, + "retrieved_ids": [ + "session_5", + "session_4", + "session_14", + "session_2", + "session_16", + "session_6", + "session_11", + "session_3", + "session_10", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_11" + ], + "question": "When is Melanie's daughter's birthday?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_4", + "session_15", + "session_17", + "session_19", + "session_9", + "session_13", + "session_1", + "session_18", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_11" + ], + "question": "When did Caroline attend a pride parade in August?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_11", + "session_10", + "session_8", + "session_13", + "session_9", + "session_12", + "session_17", + "session_14", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [], + "question": "Would Melanie be considered an ally to the transgender community?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_5", + "session_9", + "session_14", + "session_7", + "session_1", + "session_18", + "session_11", + "session_17", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_12", + "session_3" + ], + "question": "Who supports Caroline when she has a negative experience?", + "recall": 0.5, + "retrieved_ids": [ + "session_4", + "session_18", + "session_5", + "session_19", + "session_12", + "session_11", + "session_10", + "session_7", + "session_15", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_12", + "session_5", + "session_8" + ], + "question": "What types of pottery have Melanie and her kids made?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_17", + "session_8", + "session_14", + "session_2", + "session_10", + "session_5", + "session_12", + "session_15", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_12" + ], + "question": "When did Caroline and Melanie go to a pride fesetival together?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_10", + "session_14", + "session_17", + "session_8", + "session_9", + "session_4", + "session_5", + "session_12", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_12" + ], + "question": "What would Caroline's political leaning likely be?", + "recall": 0.0, + "retrieved_ids": [], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_13", + "session_8" + ], + "question": "What has Melanie painted?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_9", + "session_13", + "session_17", + "session_14", + "session_4", + "session_11", + "session_12", + "session_16", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_13", + "session_7" + ], + "question": "What are Melanie's pets' names?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_7", + "session_15", + "session_8", + "session_18", + "session_1", + "session_4", + "session_9", + "session_17", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_13" + ], + "question": "When did Caroline apply to adoption agencies?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_19", + "session_2", + "session_17", + "session_8", + "session_5", + "session_1", + "session_18", + "session_15", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_13" + ], + "question": "When did Caroline draw a self-portrait?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_18", + "session_11", + "session_19", + "session_14", + "session_2", + "session_16", + "session_7", + "session_17", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_14", + "session_8" + ], + "question": "What subject have Caroline and Melanie both painted?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_13", + "session_8", + "session_14", + "session_1", + "session_12", + "session_15", + "session_16", + "session_3", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_14", + "session_4" + ], + "question": "What symbols are important to Caroline?", + "recall": 0.5, + "retrieved_ids": [ + "session_14", + "session_12", + "session_6", + "session_2", + "session_11", + "session_7", + "session_18", + "session_15", + "session_3", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_14" + ], + "question": "When did Caroline encounter people on a hike and have a negative experience?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_4", + "session_7", + "session_5", + "session_11", + "session_18", + "session_15", + "session_8", + "session_16", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_14" + ], + "question": "When did Melanie make a plate in pottery class?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_5", + "session_12", + "session_16", + "session_17", + "session_8", + "session_15", + "session_10", + "session_7", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_12", + "session_14" + ], + "question": "Would Caroline be considered religious?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_18", + "session_13", + "session_1", + "session_9", + "session_5", + "session_4", + "session_7", + "session_8", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_15", + "session_2" + ], + "question": "What instruments does Melanie play?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_3", + "session_9", + "session_6", + "session_1", + "session_2", + "session_18", + "session_13", + "session_4", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_11", + "session_15" + ], + "question": "What musical artists/bands has Melanie seen?", + "recall": 0.0, + "retrieved_ids": [], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_15" + ], + "question": "When did Melanie go to the park?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_16", + "session_7", + "session_1", + "session_5", + "session_13", + "session_11", + "session_18", + "session_14", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_15" + ], + "question": "When is Caroline's youth center putting on a talent show?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_9", + "session_14", + "session_17", + "session_11", + "session_16", + "session_5", + "session_12", + "session_19", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_15" + ], + "question": "Would Melanie likely enjoy the song \"The Four Seasons\" by Vivaldi?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_11", + "session_15", + "session_10", + "session_8", + "session_5", + "session_9", + "session_4", + "session_17", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_11", + "session_16" + ], + "question": "What are some changes Caroline has faced during her transition journey?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_8", + "session_10", + "session_14", + "session_11", + "session_7", + "session_3", + "session_18", + "session_15", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_16" + ], + "question": "What does Melanie do with her family on hikes?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_7", + "session_2", + "session_16", + "session_8", + "session_18", + "session_12", + "session_4", + "session_14", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_16" + ], + "question": "When did Caroline go biking with friends?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_6", + "session_19", + "session_2", + "session_12", + "session_4", + "session_3", + "session_8", + "session_17", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_16" + ], + "question": "How long has Melanie been practicing art?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_4", + "session_8", + "session_17", + "session_9", + "session_1", + "session_6", + "session_15", + "session_3", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_13", + "session_16", + "session_7" + ], + "question": "What personality traits might Melanie say Caroline has?", + "recall": 0.6667, + "retrieved_ids": [ + "session_10", + "session_15", + "session_16", + "session_8", + "session_7", + "session_1", + "session_9", + "session_18", + "session_4", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_15", + "session_17" + ], + "question": "What transgender-specific events has Caroline attended?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_2", + "session_9", + "session_10", + "session_17", + "session_3", + "session_1", + "session_5", + "session_15", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_17", + "session_7" + ], + "question": "What book did Melanie read from Caroline's suggestion?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_6", + "session_17", + "session_1", + "session_2", + "session_13", + "session_18", + "session_15", + "session_8", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_17" + ], + "question": "When did Melanie's friend adopt a child?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_8", + "session_13", + "session_4", + "session_2", + "session_19", + "session_6", + "session_12", + "session_16", + "session_3" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_17" + ], + "question": "When did Melanie get hurt?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_10", + "session_1", + "session_5", + "session_18", + "session_8", + "session_15", + "session_16", + "session_14", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_18" + ], + "question": "When did Melanie's family go on a roadtrip?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_10", + "session_2", + "session_4", + "session_7", + "session_17", + "session_3", + "session_1", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_18" + ], + "question": "How many children does Melanie have?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_3", + "session_10", + "session_17", + "session_1", + "session_9", + "session_18", + "session_13", + "session_4", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_18" + ], + "question": "When did Melanie go on a hike after the roadtrip?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_4", + "session_3", + "session_1", + "session_9", + "session_12", + "session_2", + "session_14", + "session_16", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_18" + ], + "question": "Would Melanie go on another roadtrip soon?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_1", + "session_9", + "session_11", + "session_13", + "session_12", + "session_16", + "session_8", + "session_5", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_19", + "session_7" + ], + "question": "What items has Melanie bought?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_4", + "session_7", + "session_1", + "session_9", + "session_18", + "session_13", + "session_8", + "session_15", + "session_17" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_19" + ], + "question": "When did Caroline pass the adoption interview?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_2", + "session_13", + "session_3", + "session_17", + "session_8", + "session_1", + "session_18", + "session_15", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_19" + ], + "question": "When did Melanie buy the figurines?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_10", + "session_1", + "session_5", + "session_13", + "session_8", + "session_11", + "session_18", + "session_15", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_19" + ], + "question": "Would Caroline want to move back to her home country soon?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_19", + "session_8", + "session_5", + "session_4", + "session_7", + "session_18", + "session_2", + "session_10", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What did the charity race raise awareness for?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_7", + "session_3", + "session_5", + "session_8", + "session_14", + "session_13", + "session_18", + "session_15", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What did Melanie realize after the charity race?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_1", + "session_9", + "session_7", + "session_18", + "session_4", + "session_3", + "session_14", + "session_13", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "How does Melanie prioritize self-care?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_16", + "session_9", + "session_1", + "session_13", + "session_11", + "session_4", + "session_19", + "session_7", + "session_17" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What are Caroline's plans for the summer?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_2", + "session_10", + "session_17", + "session_11", + "session_15", + "session_7", + "session_16", + "session_1", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What type of individuals does the adoption agency Caroline is considering support?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_17", + "session_19", + "session_7", + "session_11", + "session_13", + "session_15", + "session_8", + "session_14", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "Why did Caroline choose the adoption agency?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_2", + "session_13", + "session_17", + "session_8", + "session_1", + "session_18", + "session_16", + "session_15", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What is Caroline excited about in the adoption process?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_17", + "session_19", + "session_13", + "session_8", + "session_5", + "session_16", + "session_4", + "session_1", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What does Melanie think about Caroline's decision to adopt?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_8", + "session_15", + "session_2", + "session_12", + "session_4", + "session_17", + "session_5", + "session_16", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "How long have Mel and her husband been married?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_6", + "session_16", + "session_8", + "session_17", + "session_19", + "session_1", + "session_10", + "session_7", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What does Caroline's necklace symbolize?", + "recall": 0.0, + "retrieved_ids": [], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What country is Caroline's grandma from?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_3", + "session_18", + "session_1", + "session_8", + "session_9", + "session_12", + "session_7", + "session_5", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What was grandma's gift to Caroline?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_19", + "session_18", + "session_1", + "session_9", + "session_5", + "session_12", + "session_7", + "session_8", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What is Melanie's hand-painted bowl a reminder of?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_16", + "session_5", + "session_12", + "session_9", + "session_8", + "session_11", + "session_14", + "session_1", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What did Melanie and her family do while camping?", + "recall": 0.0, + "retrieved_ids": [ + "session_16", + "session_8", + "session_10", + "session_2", + "session_6", + "session_17", + "session_13", + "session_7", + "session_18", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What kind of counseling and mental health services is Caroline interested in pursuing?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_5", + "session_6", + "session_1", + "session_2", + "session_7", + "session_10", + "session_8", + "session_11", + "session_3" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What workshop did Caroline attend recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_4", + "session_12", + "session_10", + "session_17", + "session_8", + "session_14", + "session_15", + "session_11", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What was discussed in the LGBTQ+ counseling workshop?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_1", + "session_5", + "session_7", + "session_10", + "session_9", + "session_11", + "session_14", + "session_6", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What motivated Caroline to pursue counseling?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_5", + "session_7", + "session_1", + "session_6", + "session_3", + "session_9", + "session_18", + "session_8", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What kind of place does Caroline want to create for people?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_10", + "session_3", + "session_5", + "session_17", + "session_14", + "session_15", + "session_6", + "session_11", + "session_2" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_5" + ], + "question": "Did Melanie make the black and white bowl in the photo?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_16", + "session_8", + "session_15", + "session_6", + "session_12", + "session_4", + "session_3", + "session_14", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_6" + ], + "question": "What kind of books does Caroline have in her library?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_11", + "session_2", + "session_7", + "session_10", + "session_17", + "session_3", + "session_15", + "session_5", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_6" + ], + "question": "What was Melanie's favorite book from her childhood?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_7", + "session_17", + "session_8", + "session_14", + "session_13", + "session_10", + "session_4", + "session_18", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What book did Caroline recommend to Melanie?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_17", + "session_6", + "session_5", + "session_1", + "session_13", + "session_18", + "session_15", + "session_8", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What did Caroline take away from the book \"Becoming Nicole\"?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_13", + "session_17", + "session_1", + "session_6", + "session_3", + "session_2", + "session_4", + "session_14", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What are the new shoes that Melanie got used for?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_2", + "session_1", + "session_6", + "session_9", + "session_4", + "session_17", + "session_15", + "session_16", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What is Melanie's reason for getting into running?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_2", + "session_5", + "session_6", + "session_18", + "session_16", + "session_4", + "session_10", + "session_15", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What does Melanie say running has been great for?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_2", + "session_10", + "session_12", + "session_6", + "session_9", + "session_15", + "session_17", + "session_3", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What did Mel and her kids make during the pottery workshop?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_16", + "session_17", + "session_14", + "session_4", + "session_10", + "session_5", + "session_18", + "session_12", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What kind of pot did Mel and her kids make with clay?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_6", + "session_2", + "session_10", + "session_3", + "session_5", + "session_11", + "session_4", + "session_15", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What creative project do Mel and her kids do together besides pottery?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_17", + "session_14", + "session_16", + "session_1", + "session_12", + "session_10", + "session_5", + "session_15", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What did Mel and her kids paint in their latest project in July 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_17", + "session_14", + "session_11", + "session_13", + "session_1", + "session_10", + "session_7", + "session_15", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What did Caroline see at the council meeting for adoption?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_17", + "session_14", + "session_19", + "session_2", + "session_5", + "session_13", + "session_10", + "session_15", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What do sunflowers represent according to Caroline?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_4", + "session_1", + "session_11", + "session_9", + "session_5", + "session_12", + "session_18", + "session_14", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "Why are flowers important to Melanie?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_14", + "session_12", + "session_6", + "session_11", + "session_2", + "session_18", + "session_7", + "session_15", + "session_3" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_9" + ], + "question": "What inspired Caroline's painting for the art show?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_9", + "session_16", + "session_17", + "session_8", + "session_11", + "session_13", + "session_12", + "session_1", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "How often does Melanie go to the beach with her kids?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_6", + "session_11", + "session_14", + "session_4", + "session_2", + "session_1", + "session_9", + "session_18", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "What did Melanie and her family see during their camping trip last year?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_10", + "session_18", + "session_17", + "session_6", + "session_4", + "session_7", + "session_16", + "session_12", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "How did Melanie feel while watching the meteor shower?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_11", + "session_13", + "session_8", + "session_16", + "session_17", + "session_1", + "session_15", + "session_14", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_11" + ], + "question": "Whose birthday did Melanie celebrate recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_17", + "session_14", + "session_4", + "session_13", + "session_12", + "session_10", + "session_8", + "session_15", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_11" + ], + "question": "Who performed at the concert at Melanie's daughter's birthday?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_4", + "session_15", + "session_8", + "session_18", + "session_1", + "session_13", + "session_9", + "session_19", + "session_17" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_12" + ], + "question": "Why did Melanie choose to use colors and patterns in her pottery project?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_8", + "session_14", + "session_16", + "session_7", + "session_15", + "session_4", + "session_19", + "session_2", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "What pet does Caroline have?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_13", + "session_4", + "session_1", + "session_9", + "session_5", + "session_12", + "session_18", + "session_8", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "What pets does Melanie have?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_13", + "session_15", + "session_1", + "session_9", + "session_18", + "session_4", + "session_8", + "session_17", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "Where did Oliver hide his bone once?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_7", + "session_10", + "session_11", + "session_1", + "session_14", + "session_8", + "session_18", + "session_15", + "session_6" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "What activity did Caroline used to do with her dad?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_11", + "session_5", + "session_10", + "session_15", + "session_7", + "session_8", + "session_14", + "session_1", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_14" + ], + "question": "What did Caroline make for a local church?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_12", + "session_5", + "session_15", + "session_8", + "session_7", + "session_10", + "session_16", + "session_17", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_14" + ], + "question": "What did Caroline find in her neighborhood during her walk?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_16", + "session_7", + "session_8", + "session_12", + "session_11", + "session_18", + "session_10", + "session_19", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "Which song motivates Caroline to be courageous?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_7", + "session_5", + "session_11", + "session_3", + "session_4", + "session_1", + "session_18", + "session_14", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "Which classical musicians does Melanie enjoy listening to?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_15", + "session_18", + "session_10", + "session_8", + "session_19", + "session_1", + "session_9", + "session_13", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "Who is Melanie a fan of in terms of modern music?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_5", + "session_11", + "session_19", + "session_1", + "session_9", + "session_18", + "session_13", + "session_4", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "How long has Melanie been creating art?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_4", + "session_12", + "session_6", + "session_15", + "session_8", + "session_17", + "session_1", + "session_3", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "What precautionary sign did Melanie see at the caf\u00e9?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_5", + "session_7", + "session_1", + "session_11", + "session_10", + "session_18", + "session_15", + "session_14", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What advice does Caroline give for getting started with adoption?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_13", + "session_2", + "session_7", + "session_18", + "session_8", + "session_12", + "session_4", + "session_15", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What setback did Melanie face in October 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_9", + "session_15", + "session_1", + "session_10", + "session_13", + "session_18", + "session_3", + "session_8", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What does Melanie do to keep herself busy during her pottery break?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_8", + "session_16", + "session_12", + "session_14", + "session_5", + "session_2", + "session_18", + "session_7", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What painting did Melanie show to Caroline on October 13, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_9", + "session_13", + "session_11", + "session_16", + "session_17", + "session_8", + "session_5", + "session_1", + "session_15" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What kind of painting did Caroline share with Melanie on October 13, 2023?", + "recall": 0.0, + "retrieved_ids": [ + "session_11", + "session_8", + "session_2", + "session_10", + "session_15", + "session_3", + "session_14", + "session_5", + "session_6", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What was the poetry reading that Caroline attended about?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_6", + "session_13", + "session_2", + "session_7", + "session_11", + "session_12", + "session_4", + "session_15", + "session_3" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What did the posters at the poetry reading say?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_6", + "session_10", + "session_2", + "session_7", + "session_15", + "session_8", + "session_14", + "session_13", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What does Caroline's drawing symbolize for her?", + "recall": 0.0, + "retrieved_ids": [ + "session_14", + "session_16", + "session_8", + "session_10", + "session_7", + "session_18", + "session_1", + "session_9", + "session_12", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "How do Melanie and Caroline describe their journey through life together?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_4", + "session_8", + "session_14", + "session_11", + "session_15", + "session_19", + "session_7", + "session_17", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What happened to Melanie's son on their road trip?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_15", + "session_8", + "session_10", + "session_7", + "session_17", + "session_14", + "session_6", + "session_12", + "session_2" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "How did Melanie's son handle the accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_16", + "session_10", + "session_8", + "session_14", + "session_13", + "session_15", + "session_1", + "session_11", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "How did Melanie feel about her family after the accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_3", + "session_8", + "session_14", + "session_2", + "session_10", + "session_7", + "session_4", + "session_1", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "How did Melanie's children handle the accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_16", + "session_10", + "session_14", + "session_13", + "session_15", + "session_1", + "session_11", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "How did Melanie feel after the accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_1", + "session_3", + "session_14", + "session_5", + "session_9", + "session_13", + "session_4", + "session_8", + "session_2" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What was Melanie's reaction to her children enjoying the Grand Canyon?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_13", + "session_8", + "session_3", + "session_10", + "session_7", + "session_1", + "session_4", + "session_9", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What do Melanie's family give her?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_7", + "session_3", + "session_10", + "session_13", + "session_6", + "session_18", + "session_8", + "session_4", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "How did Melanie feel about her family supporting her?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_16", + "session_3", + "session_8", + "session_14", + "session_7", + "session_17", + "session_15", + "session_2", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What did Melanie do after the road trip to relax?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_1", + "session_17", + "session_9", + "session_2", + "session_12", + "session_10", + "session_15", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_2" + ], + "question": "What did Caroline realize after her charity race?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_14", + "session_7", + "session_3", + "session_1", + "session_9", + "session_18", + "session_10", + "session_4", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_2" + ], + "question": "What are Melanie's plans for the summer with respect to adoption?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_19", + "session_16", + "session_12", + "session_10", + "session_17", + "session_11", + "session_13", + "session_15", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_2" + ], + "question": "What type of individuals does the adoption agency Melanie is considering support?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_17", + "session_19", + "session_7", + "session_11", + "session_13", + "session_15", + "session_8", + "session_14", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_2" + ], + "question": "Why did Melanie choose the adoption agency?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_2", + "session_13", + "session_17", + "session_8", + "session_1", + "session_18", + "session_16", + "session_15", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_2" + ], + "question": "What is Melanie excited about in her adoption process?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_17", + "session_13", + "session_8", + "session_19", + "session_16", + "session_5", + "session_10", + "session_7", + "session_1" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What does Melanie's necklace symbolize?", + "recall": 0.0, + "retrieved_ids": [], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What country is Melanie's grandma from?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_3", + "session_15", + "session_8", + "session_18", + "session_1", + "session_13", + "session_9", + "session_17", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What was grandma's gift to Melanie?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_19", + "session_7", + "session_1", + "session_9", + "session_18", + "session_13", + "session_8", + "session_15", + "session_17" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What was grandpa's gift to Caroline?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_4", + "session_18", + "session_1", + "session_9", + "session_5", + "session_12", + "session_7", + "session_8", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What is Caroline's hand-painted bowl a reminder of?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_16", + "session_5", + "session_12", + "session_13", + "session_8", + "session_11", + "session_14", + "session_1", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What did Caroline and her family do while camping?", + "recall": 0.0, + "retrieved_ids": [ + "session_16", + "session_8", + "session_10", + "session_2", + "session_6", + "session_17", + "session_13", + "session_7", + "session_18", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What kind of counseling and mental health services is Melanie interested in pursuing?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_5", + "session_6", + "session_1", + "session_2", + "session_7", + "session_10", + "session_8", + "session_11", + "session_3" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What kind of counseling workshop did Melanie attend recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_17", + "session_5", + "session_6", + "session_13", + "session_10", + "session_1", + "session_11", + "session_7", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What motivated Melanie to pursue counseling?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_5", + "session_7", + "session_1", + "session_6", + "session_3", + "session_13", + "session_18", + "session_8", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What kind of place does Melanie want to create for people?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_10", + "session_3", + "session_5", + "session_15", + "session_17", + "session_14", + "session_6", + "session_11", + "session_2" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_5" + ], + "question": "Did Caroline make the black and white bowl in the photo?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_16", + "session_15", + "session_8", + "session_6", + "session_12", + "session_4", + "session_3", + "session_14", + "session_13" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_7" + ], + "question": "What are the new shoes that Caroline got used for?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_2", + "session_1", + "session_6", + "session_9", + "session_4", + "session_17", + "session_15", + "session_16", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_7" + ], + "question": "What is Caroline's reason for getting into running?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_5", + "session_2", + "session_6", + "session_18", + "session_16", + "session_4", + "session_10", + "session_15", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_7" + ], + "question": "What does Caroline say running has been great for?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_2", + "session_10", + "session_12", + "session_6", + "session_9", + "session_15", + "session_17", + "session_3", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_8" + ], + "question": "What did Melanie see at the council meeting for adoption?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_17", + "session_14", + "session_19", + "session_2", + "session_5", + "session_13", + "session_10", + "session_15", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_9" + ], + "question": "What inspired Melanie's painting for the art show?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_9", + "session_16", + "session_17", + "session_8", + "session_11", + "session_13", + "session_12", + "session_1", + "session_4" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_9" + ], + "question": "What inspired Caroline's sculpture for the art show?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_9", + "session_16", + "session_17", + "session_8", + "session_5", + "session_11", + "session_12", + "session_13", + "session_19" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_10" + ], + "question": "How often does Caroline go to the beach with her kids?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_6", + "session_11", + "session_14", + "session_4", + "session_2", + "session_1", + "session_9", + "session_18", + "session_7" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_10" + ], + "question": "What did Caroline and her family see during their camping trip last year?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_10", + "session_18", + "session_17", + "session_6", + "session_4", + "session_7", + "session_16", + "session_12", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_10" + ], + "question": "How did Caroline feel while watching the meteor shower?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_11", + "session_13", + "session_8", + "session_16", + "session_17", + "session_1", + "session_15", + "session_14", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_12" + ], + "question": "Why did Caroline choose to use colors and patterns in her pottery project?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_8", + "session_14", + "session_16", + "session_7", + "session_15", + "session_4", + "session_19", + "session_2", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_13" + ], + "question": "Is Oscar Melanie's pet?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_7", + "session_15", + "session_8", + "session_18", + "session_1", + "session_4", + "session_9", + "session_19", + "session_17" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_13" + ], + "question": "Where did Oscar hide his bone once?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_5", + "session_10", + "session_11", + "session_14", + "session_8", + "session_18", + "session_15", + "session_1", + "session_6" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_13" + ], + "question": "What activity did Melanie used to do with her dad?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_11", + "session_5", + "session_10", + "session_15", + "session_7", + "session_8", + "session_14", + "session_1", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_14" + ], + "question": "What did Melanie make for a local church?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_12", + "session_5", + "session_15", + "session_8", + "session_7", + "session_10", + "session_16", + "session_17", + "session_18" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_14" + ], + "question": "What did Melanie find in her neighborhood during her walk?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_16", + "session_7", + "session_8", + "session_12", + "session_11", + "session_18", + "session_10", + "session_19", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_15" + ], + "question": "Which song motivates Melanie to be courageous?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_7", + "session_11", + "session_5", + "session_3", + "session_4", + "session_1", + "session_18", + "session_14", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_15" + ], + "question": "What type of instrument does Caroline play?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_14", + "session_3", + "session_7", + "session_6", + "session_1", + "session_2", + "session_9", + "session_12", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_15" + ], + "question": "Which classical musicians does Caroline enjoy listening to?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_5", + "session_15", + "session_18", + "session_10", + "session_8", + "session_7", + "session_1", + "session_9", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_15" + ], + "question": "Who is Caroline a fan of in terms of modern music?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_18", + "session_5", + "session_11", + "session_1", + "session_9", + "session_12", + "session_7", + "session_8", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_16" + ], + "question": "What precautionary sign did Caroline see at the caf\u00e9?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_5", + "session_7", + "session_1", + "session_11", + "session_10", + "session_18", + "session_15", + "session_14", + "session_8" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_17" + ], + "question": "What setback did Caroline face recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_14", + "session_9", + "session_10", + "session_13", + "session_12", + "session_15", + "session_11", + "session_18", + "session_3" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_17" + ], + "question": "What does Caroline do to keep herself busy during her pottery break?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_8", + "session_16", + "session_12", + "session_14", + "session_5", + "session_2", + "session_18", + "session_7", + "session_10" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_17" + ], + "question": "What was the poetry reading that Melanie attended about?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_6", + "session_13", + "session_2", + "session_12", + "session_7", + "session_11", + "session_4", + "session_15", + "session_3" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "What happened to Caroline's son on their road trip?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_15", + "session_8", + "session_10", + "session_7", + "session_17", + "session_14", + "session_6", + "session_12", + "session_2" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "How did Caroline's son handle the accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_16", + "session_10", + "session_8", + "session_14", + "session_13", + "session_15", + "session_1", + "session_11", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "How did Caroline feel about her family after the accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_3", + "session_14", + "session_2", + "session_10", + "session_7", + "session_4", + "session_1", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "How did Caroline's children handle the accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_16", + "session_10", + "session_14", + "session_13", + "session_15", + "session_1", + "session_11", + "session_5" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "How did Caroline feel after the accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_1", + "session_3", + "session_14", + "session_5", + "session_9", + "session_13", + "session_4", + "session_8", + "session_2" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "What was Caroline's reaction to her children enjoying the Grand Canyon?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_14", + "session_8", + "session_3", + "session_10", + "session_7", + "session_5", + "session_1", + "session_9", + "session_12" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "What did Caroline do after the road trip to relax?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_1", + "session_17", + "session_9", + "session_2", + "session_12", + "session_10", + "session_15", + "session_14" + ], + "sample_id": "conv-26" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "What does Caroline love most about camping with her family?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_8", + "session_6", + "session_18", + "session_2", + "session_16", + "session_4", + "session_7", + "session_14", + "session_9" + ], + "sample_id": "conv-26" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_1" + ], + "question": "When Jon has lost his job as a banker?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_16", + "session_6", + "session_14", + "session_4", + "session_11", + "session_10", + "session_5", + "session_15", + "session_3" + ], + "sample_id": "conv-30" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_1" + ], + "question": "When Gina has lost her job at Door Dash?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_1", + "session_14", + "session_16", + "session_17", + "session_9", + "session_18", + "session_4", + "session_10", + "session_11" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_1" + ], + "question": "How do Jon and Gina both like to destress?", + "recall": 0.0, + "retrieved_ids": [ + "session_2", + "session_6", + "session_11", + "session_5", + "session_9", + "session_10", + "session_16", + "session_8", + "session_18", + "session_3" + ], + "sample_id": "conv-30" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_2" + ], + "question": "What do Jon and Gina both have in common?", + "recall": 0.5, + "retrieved_ids": [ + "session_6", + "session_9", + "session_10", + "session_2", + "session_5", + "session_17", + "session_4", + "session_14", + "session_19", + "session_15" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_1" + ], + "question": "Why did Jon decide to start his dance studio?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_11", + "session_10", + "session_1", + "session_13", + "session_15", + "session_3", + "session_14", + "session_6", + "session_5" + ], + "sample_id": "conv-30" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_2" + ], + "question": "What Jon thinks the ideal dance studio should look like?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_2", + "session_3", + "session_5", + "session_9", + "session_10", + "session_8", + "session_11", + "session_19", + "session_12" + ], + "sample_id": "conv-30" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_1" + ], + "question": "When is Jon's group performing at a festival?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_19", + "session_14", + "session_5", + "session_16", + "session_15", + "session_4", + "session_18", + "session_12", + "session_8" + ], + "sample_id": "conv-30" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_2" + ], + "question": "When did Gina launch an ad campaign for her store?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_6", + "session_7", + "session_14", + "session_3", + "session_15", + "session_1", + "session_5", + "session_8", + "session_11" + ], + "sample_id": "conv-30" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_2" + ], + "question": "When was Jon in Paris?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_16", + "session_15", + "session_4", + "session_19", + "session_14", + "session_1", + "session_18", + "session_12", + "session_8" + ], + "sample_id": "conv-30" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_15", + "session_2" + ], + "question": "Which city have both Jean and John visited?", + "recall": 0.5, + "retrieved_ids": [ + "session_2", + "session_10", + "session_6", + "session_9", + "session_5" + ], + "sample_id": "conv-30" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_5" + ], + "question": "When did Gina team up with a local artist for some cool designs?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_1", + "session_16", + "session_8", + "session_3", + "session_18", + "session_6", + "session_11", + "session_14", + "session_2" + ], + "sample_id": "conv-30" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_5" + ], + "question": "When did Gina get her tattoo?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_4", + "session_16", + "session_19", + "session_14", + "session_1", + "session_11", + "session_2", + "session_18", + "session_12" + ], + "sample_id": "conv-30" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_6" + ], + "question": "When did Jon start to go to the gym?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_17", + "session_19", + "session_14", + "session_1", + "session_11", + "session_10", + "session_2", + "session_7", + "session_5" + ], + "sample_id": "conv-30" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_6" + ], + "question": "When did Gina open her online clothing store?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_7", + "session_6", + "session_18", + "session_10", + "session_8", + "session_2", + "session_5", + "session_1", + "session_11" + ], + "sample_id": "conv-30" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_8" + ], + "question": "When did Jon start expanding his studio's social media presence?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_14", + "session_18", + "session_11", + "session_10", + "session_1", + "session_2", + "session_15", + "session_3", + "session_6" + ], + "sample_id": "conv-30" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_8" + ], + "question": "When did Jon host a dance competition?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_1", + "session_8", + "session_13", + "session_14", + "session_9", + "session_11", + "session_2", + "session_18", + "session_5" + ], + "sample_id": "conv-30" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_10" + ], + "question": "When did Jon go to a fair to get more exposure for his dance studio?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_11", + "session_2", + "session_15", + "session_5", + "session_3", + "session_1", + "session_8", + "session_14", + "session_6" + ], + "sample_id": "conv-30" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_6" + ], + "question": "Why did Gina decide to start her own clothing store?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_2", + "session_3", + "session_6", + "session_10", + "session_7", + "session_1", + "session_18", + "session_11", + "session_5" + ], + "sample_id": "conv-30" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_6" + ], + "question": "Do Jon and Gina start businesses out of what they love?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_14", + "session_10", + "session_6", + "session_5", + "session_11", + "session_8", + "session_9", + "session_3", + "session_7" + ], + "sample_id": "conv-30" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_11" + ], + "question": "When did Gina interview for a design internship?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_16", + "session_3", + "session_5", + "session_19", + "session_12", + "session_8", + "session_14", + "session_1", + "session_2" + ], + "sample_id": "conv-30" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_12" + ], + "question": "When did Gina get accepted for the design internship?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_12", + "session_16", + "session_3", + "session_5", + "session_19", + "session_8", + "session_1", + "session_2", + "session_14" + ], + "sample_id": "conv-30" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_12" + ], + "question": "When did Jon start reading \"The Lean Startup\"?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_19", + "session_14", + "session_11", + "session_1", + "session_6", + "session_10", + "session_2", + "session_13", + "session_5" + ], + "sample_id": "conv-30" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_13" + ], + "question": "When did Gina develop a video presentation to teach how to style her fashion pieces? ", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_3", + "session_16", + "session_14", + "session_18", + "session_8", + "session_11", + "session_1", + "session_5", + "session_2" + ], + "sample_id": "conv-30" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_13", + "session_16", + "session_5", + "session_8" + ], + "question": "How did Gina promote her clothes store?", + "recall": 0.5, + "retrieved_ids": [ + "session_6", + "session_14", + "session_7", + "session_3", + "session_8", + "session_10", + "session_2", + "session_5", + "session_1", + "session_11" + ], + "sample_id": "conv-30" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_16", + "session_8" + ], + "question": "Which events has Jon participated in to promote his business venture?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_16", + "session_18", + "session_10", + "session_14", + "session_6", + "session_11", + "session_5", + "session_15", + "session_3" + ], + "sample_id": "conv-30" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_13", + "session_8" + ], + "question": "What does Jon's dance studio offer?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_7", + "session_18", + "session_13", + "session_6", + "session_5", + "session_15", + "session_9", + "session_11", + "session_2" + ], + "sample_id": "conv-30" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_14" + ], + "question": "When did Jon receive mentorship to promote his venture?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_2", + "session_15", + "session_14", + "session_3", + "session_11", + "session_10", + "session_1", + "session_5", + "session_8" + ], + "sample_id": "conv-30" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_14", + "session_9" + ], + "question": "Did Jon and Gina both participate in dance competitions?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_2", + "session_8", + "session_4", + "session_9", + "session_6", + "session_10", + "session_5", + "session_11", + "session_14" + ], + "sample_id": "conv-30" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_15" + ], + "question": "When was Jon in Rome?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_2", + "session_18", + "session_16", + "session_19", + "session_4", + "session_14", + "session_1", + "session_12", + "session_8" + ], + "sample_id": "conv-30" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_15", + "session_2" + ], + "question": "Which cities has Jon visited?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_16", + "session_12", + "session_8", + "session_4", + "session_19", + "session_14", + "session_15", + "session_1", + "session_18" + ], + "sample_id": "conv-30" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_15" + ], + "question": "When Jon is planning to open his dance studio?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_18", + "session_11", + "session_1", + "session_15", + "session_6", + "session_10", + "session_12", + "session_8", + "session_16" + ], + "sample_id": "conv-30" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_15" + ], + "question": "How long did it take for Jon to open his studio?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_18", + "session_2", + "session_15", + "session_10", + "session_11", + "session_3", + "session_1", + "session_14", + "session_6" + ], + "sample_id": "conv-30" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_16" + ], + "question": "When did Gina design a limited collection of hoodies?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_11", + "session_3", + "session_15", + "session_5", + "session_8", + "session_19", + "session_14", + "session_1", + "session_2" + ], + "sample_id": "conv-30" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_16" + ], + "question": "When did Jon visit networking events for his store?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_16", + "session_2", + "session_10", + "session_8", + "session_3", + "session_6", + "session_1", + "session_15", + "session_11" + ], + "sample_id": "conv-30" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_17" + ], + "question": "When did Gina start being recognized by fashion editors?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_7", + "session_14", + "session_6", + "session_5", + "session_11", + "session_1", + "session_12", + "session_8", + "session_2" + ], + "sample_id": "conv-30" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_17" + ], + "question": "When did Jon start learning marketing and analytics tools?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_14", + "session_10", + "session_11", + "session_1", + "session_18", + "session_7", + "session_12", + "session_2", + "session_5" + ], + "sample_id": "conv-30" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_18" + ], + "question": "When did Jon and Gina decide to collaborate to create dance content?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_3", + "session_2", + "session_11", + "session_17", + "session_5", + "session_9", + "session_7", + "session_14", + "session_1" + ], + "sample_id": "conv-30" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_19" + ], + "question": "When did Gina mention Shia Labeouf?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_16", + "session_15", + "session_14", + "session_1", + "session_13", + "session_11", + "session_17", + "session_2", + "session_5" + ], + "sample_id": "conv-30" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_19" + ], + "question": "When did Gina go to a dance class with a group of friends?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_1", + "session_15", + "session_9", + "session_17", + "session_13", + "session_14", + "session_11", + "session_2", + "session_8" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_1" + ], + "question": "What is Gina's favorite style of dance?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_3", + "session_13", + "session_18", + "session_8", + "session_16", + "session_1", + "session_9", + "session_11", + "session_2" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_1" + ], + "question": "What is Jon's favorite style of dance?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_3", + "session_13", + "session_8", + "session_16", + "session_1", + "session_18", + "session_9", + "session_11", + "session_2" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_1" + ], + "question": "What was Gina's favorite dancing memory?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_1", + "session_9", + "session_14", + "session_4", + "session_15", + "session_11", + "session_2", + "session_8", + "session_18" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_1" + ], + "question": "What kind of dance piece did Gina's team perform to win first place?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_3", + "session_5", + "session_7", + "session_4", + "session_18", + "session_8", + "session_11", + "session_2", + "session_14" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_1" + ], + "question": "What do the dancers in the photo represent?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_8", + "session_13", + "session_1", + "session_5", + "session_19", + "session_9", + "session_17" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_1" + ], + "question": "What does Gina say about the dancers in the photo?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_8", + "session_1", + "session_5", + "session_12", + "session_7", + "session_13", + "session_19", + "session_9", + "session_17" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_1" + ], + "question": "What is Jon's attitude towards being part of the dance festival?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_7", + "session_1", + "session_12", + "session_13", + "session_15", + "session_9", + "session_11", + "session_18", + "session_2" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What kind of flooring is Jon looking for in his dance studio?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_2", + "session_7", + "session_4", + "session_15", + "session_11", + "session_3", + "session_10", + "session_5", + "session_6" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "What did Gina find for her clothing store on 1 February, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_2", + "session_14", + "session_7", + "session_8", + "session_6", + "session_1", + "session_18", + "session_5", + "session_11" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "What did Gina design for her store?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_6", + "session_8", + "session_16", + "session_7", + "session_14", + "session_5", + "session_11", + "session_1", + "session_2" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "What did Gina want her customers to feel in her store?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_7", + "session_14", + "session_18", + "session_5", + "session_8", + "session_11", + "session_2", + "session_12", + "session_1" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "What did Jon say about Gina's progress with her store?", + "recall": 0.0, + "retrieved_ids": [ + "session_4", + "session_8", + "session_14", + "session_5", + "session_12", + "session_16", + "session_1", + "session_11", + "session_2", + "session_13" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "What made Gina choose the furniture and decor for her store?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_7", + "session_8", + "session_16", + "session_14", + "session_6", + "session_5", + "session_10", + "session_4", + "session_2" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "What did Jon say about creating a special experience for customers?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_14", + "session_18", + "session_4", + "session_7", + "session_1", + "session_5", + "session_13", + "session_9", + "session_11" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "What did Gina say about creating an experience for her customers?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_14", + "session_7", + "session_18", + "session_4", + "session_5", + "session_13", + "session_2", + "session_1", + "session_11" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "How is Gina's store doing?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_4", + "session_14", + "session_1", + "session_5", + "session_10", + "session_8", + "session_6", + "session_18", + "session_16" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_5" + ], + "question": "What does Gina's tattoo symbolize?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_11", + "session_16", + "session_13", + "session_14", + "session_1", + "session_12", + "session_19", + "session_15", + "session_17" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_6" + ], + "question": "What did Jon and Gina compare their entrepreneurial journeys to?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_2", + "session_17", + "session_13", + "session_6", + "session_1", + "session_10", + "session_5", + "session_14", + "session_11" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What advice does Gina give to Jon about running a successful business?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_7", + "session_14", + "session_10", + "session_5", + "session_4", + "session_12", + "session_13", + "session_6", + "session_11" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "Why did Jon shut down his bank account?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_14", + "session_6", + "session_18", + "session_5", + "session_10", + "session_15", + "session_11", + "session_3", + "session_1" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "Why did Gina combine her clothing business with dance?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_14", + "session_7", + "session_10", + "session_3", + "session_2", + "session_6", + "session_11", + "session_1", + "session_5" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_9" + ], + "question": "What does Jon's dance make him?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_11", + "session_4", + "session_3", + "session_18", + "session_15", + "session_9", + "session_17", + "session_2", + "session_5" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_9" + ], + "question": "What did Gina receive from a dance contest?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_13", + "session_4", + "session_14", + "session_1", + "session_18", + "session_8", + "session_11", + "session_2", + "session_5" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "How does Gina stay confident in her business?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_7", + "session_14", + "session_13", + "session_5", + "session_12", + "session_8", + "session_11", + "session_16", + "session_18" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_12" + ], + "question": "What kind of professional experience did Gina get accepted for on May 23, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_4", + "session_13", + "session_3", + "session_1", + "session_2", + "session_12", + "session_11", + "session_7", + "session_14" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_12" + ], + "question": "Where is Gina's fashion internship?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_7", + "session_11", + "session_13", + "session_6", + "session_17", + "session_5", + "session_8", + "session_15", + "session_19" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_12" + ], + "question": "What book is Jon currently reading?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_17", + "session_4", + "session_16", + "session_14", + "session_1", + "session_18", + "session_8", + "session_19", + "session_15" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "What is Jon offering to the dancers at his dance studio?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_8", + "session_18", + "session_19", + "session_7", + "session_3", + "session_15", + "session_11", + "session_10", + "session_6" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "How does Jon use the clipboard with a notepad attached to it?", + "recall": 0.0, + "retrieved_ids": [ + "session_14", + "session_4", + "session_17", + "session_18", + "session_12", + "session_8", + "session_19", + "session_15", + "session_16", + "session_1" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_14" + ], + "question": "What does Jon tell Gina he won't do?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_16", + "session_14", + "session_7", + "session_13", + "session_12", + "session_19", + "session_9", + "session_17", + "session_1" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "What did Jon take a trip to Rome for?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_2", + "session_17", + "session_18", + "session_1", + "session_9", + "session_5", + "session_16", + "session_14", + "session_11" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "What is Jon working on opening?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_13", + "session_11", + "session_5", + "session_14", + "session_18", + "session_9", + "session_3", + "session_6", + "session_17" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "How does Gina describe the studio that Jon has opened?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_18", + "session_3", + "session_15", + "session_8", + "session_14", + "session_13", + "session_6", + "session_11", + "session_9" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "How does Jon feel about the opening night of his dance studio?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_11", + "session_18", + "session_8", + "session_13", + "session_17", + "session_3", + "session_10", + "session_14", + "session_6" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "How does Gina describe the feeling that dance brings?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_5", + "session_8", + "session_15", + "session_1", + "session_9", + "session_4", + "session_18", + "session_11", + "session_3" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "What does Jon plan to do at the grand opening of his dance studio?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_1", + "session_18", + "session_12", + "session_8", + "session_11", + "session_3", + "session_10", + "session_16", + "session_6" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "What does Gina say to Jon about the grand opening?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_4", + "session_11", + "session_12", + "session_13", + "session_14", + "session_6", + "session_18", + "session_3", + "session_17" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "What is the general sentiment about the upcoming grand opening?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_15", + "session_11", + "session_13", + "session_14", + "session_6", + "session_18", + "session_12", + "session_3", + "session_17" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "What did Gina make a limited edition line of?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_15", + "session_3", + "session_18", + "session_11", + "session_17", + "session_2", + "session_5", + "session_14", + "session_1" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "According to Gina, what makes Jon a perfect mentor and guide?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_13", + "session_4", + "session_3", + "session_14", + "session_6", + "session_10", + "session_2", + "session_15", + "session_18" + ], + "sample_id": "conv-30" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "What temporary job did Jon take to cover expenses?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_17", + "session_1", + "session_9", + "session_16", + "session_14", + "session_6", + "session_11", + "session_2", + "session_5" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What plans does Jon have after receiving advice at the networking event?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_16", + "session_7", + "session_14", + "session_1", + "session_10", + "session_8", + "session_4", + "session_19", + "session_12" + ], + "sample_id": "conv-30" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What offer does Gina make to Jon regarding social media?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_7", + "session_14", + "session_6", + "session_4", + "session_13", + "session_3", + "session_15", + "session_17" + ], + "sample_id": "conv-30" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_1" + ], + "question": "What is Jon's favorite style of painting?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_16", + "session_3", + "session_13", + "session_12", + "session_4", + "session_14", + "session_1", + "session_18", + "session_19" + ], + "sample_id": "conv-30" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_1" + ], + "question": "What was Jon's favorite dancing memory?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_1", + "session_9", + "session_14", + "session_4", + "session_15", + "session_11", + "session_2", + "session_18", + "session_8" + ], + "sample_id": "conv-30" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_1" + ], + "question": "What kind of dance piece did Jon's team perform to win first place?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_3", + "session_5", + "session_7", + "session_4", + "session_18", + "session_8", + "session_11", + "session_2", + "session_14" + ], + "sample_id": "conv-30" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_1" + ], + "question": "What is Gina's attitude towards participating in the dance festival?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_1", + "session_12", + "session_14", + "session_4", + "session_18", + "session_8", + "session_9", + "session_11", + "session_2" + ], + "sample_id": "conv-30" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_2" + ], + "question": "What kind of flooring is Gina looking for in her dance studio?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_8", + "session_18", + "session_3", + "session_2", + "session_4", + "session_1", + "session_15", + "session_9", + "session_5" + ], + "sample_id": "conv-30" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_3" + ], + "question": "What did Jon find for his clothing store on 1 February, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_2", + "session_3", + "session_14", + "session_1", + "session_10", + "session_6", + "session_11", + "session_5", + "session_15" + ], + "sample_id": "conv-30" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_3" + ], + "question": "What did Jon design for his store?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_16", + "session_5", + "session_11", + "session_2", + "session_14", + "session_6", + "session_1", + "session_10", + "session_15" + ], + "sample_id": "conv-30" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_3" + ], + "question": "What did Jon want his customers to feel in her store?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_14", + "session_6", + "session_8", + "session_5", + "session_11", + "session_15", + "session_10", + "session_2", + "session_1" + ], + "sample_id": "conv-30" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_3" + ], + "question": "What made Jon choose the furniture and decor for his store?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_16", + "session_7", + "session_8", + "session_5", + "session_2", + "session_6", + "session_10", + "session_15", + "session_11" + ], + "sample_id": "conv-30" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "How is Jon's store doing?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_1", + "session_4", + "session_14", + "session_5", + "session_10", + "session_6", + "session_18", + "session_8", + "session_16" + ], + "sample_id": "conv-30" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_5" + ], + "question": "What does Jon's tattoo symbolize?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_16", + "session_4", + "session_14", + "session_1", + "session_18", + "session_12", + "session_8", + "session_19", + "session_15" + ], + "sample_id": "conv-30" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_8" + ], + "question": "Why did Gina shut down her bank account?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_14", + "session_12", + "session_16", + "session_17", + "session_18", + "session_5", + "session_11", + "session_1", + "session_2" + ], + "sample_id": "conv-30" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_8" + ], + "question": "Why did Jon combine his clothing business with dance?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_14", + "session_10", + "session_2", + "session_5", + "session_3", + "session_11", + "session_1", + "session_6", + "session_15" + ], + "sample_id": "conv-30" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_9" + ], + "question": "What did Gina receive from a dance contest?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_13", + "session_4", + "session_14", + "session_1", + "session_18", + "session_8", + "session_11", + "session_2", + "session_5" + ], + "sample_id": "conv-30" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_12" + ], + "question": "What kind of professional experience did Jon get accepted for on May 23, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_4", + "session_13", + "session_3", + "session_1", + "session_2", + "session_12", + "session_11", + "session_7", + "session_14" + ], + "sample_id": "conv-30" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_12" + ], + "question": "Where is Gina's HR internship?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_11", + "session_4", + "session_19", + "session_15", + "session_16", + "session_13", + "session_14", + "session_17", + "session_1" + ], + "sample_id": "conv-30" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_12" + ], + "question": "Where is Jon's fashion internship?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_11", + "session_7", + "session_13", + "session_6", + "session_17", + "session_5", + "session_8", + "session_14", + "session_4" + ], + "sample_id": "conv-30" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_12" + ], + "question": "What book is Gina currently reading?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_4", + "session_16", + "session_13", + "session_14", + "session_1", + "session_19", + "session_15", + "session_17", + "session_11" + ], + "sample_id": "conv-30" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_13" + ], + "question": "How does Gina use the clipboard with a notepad attached to it?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_14", + "session_4", + "session_19", + "session_17", + "session_11", + "session_12", + "session_15", + "session_16", + "session_1" + ], + "sample_id": "conv-30" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_15" + ], + "question": "What did Jon take a trip to Barcelona for?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_1", + "session_9", + "session_17", + "session_2", + "session_5", + "session_7", + "session_14", + "session_16", + "session_11" + ], + "sample_id": "conv-30" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_16" + ], + "question": "What did Jon make a limited edition line of?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_15", + "session_3", + "session_18", + "session_11", + "session_17", + "session_2", + "session_5", + "session_1", + "session_14" + ], + "sample_id": "conv-30" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "What temporary job did Gina take to cover expenses?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_17", + "session_1", + "session_9", + "session_16", + "session_14", + "session_6", + "session_11", + "session_2", + "session_5" + ], + "sample_id": "conv-30" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "What plans does Gina have after receiving advice at the networking event?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_16", + "session_7", + "session_14", + "session_1", + "session_10", + "session_4", + "session_8", + "session_19", + "session_12" + ], + "sample_id": "conv-30" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_13" + ], + "question": "Who did Maria have dinner with on May 3, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_23", + "session_2", + "session_13", + "session_15", + "session_27", + "session_24", + "session_6", + "session_32", + "session_7" + ], + "sample_id": "conv-41" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_2" + ], + "question": "When did Maria donate her car?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_21", + "session_11", + "session_6", + "session_7", + "session_31", + "session_5", + "session_30", + "session_14", + "session_32" + ], + "sample_id": "conv-41" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_2" + ], + "question": "What martial arts has John done?", + "recall": 0.5, + "retrieved_ids": [ + "session_25", + "session_24", + "session_27", + "session_15", + "session_26", + "session_29", + "session_2", + "session_16", + "session_4", + "session_28" + ], + "sample_id": "conv-41" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_2", + "session_3" + ], + "question": "What type of volunteering have John and Maria both done?", + "recall": 0.0, + "retrieved_ids": [ + "session_13", + "session_27", + "session_17", + "session_24", + "session_25", + "session_15", + "session_26", + "session_29", + "session_4", + "session_10" + ], + "sample_id": "conv-41" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_3" + ], + "question": "When did John join the online support group?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_27", + "session_24", + "session_4", + "session_23", + "session_12", + "session_14", + "session_32", + "session_26", + "session_25" + ], + "sample_id": "conv-41" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_3" + ], + "question": "When did Maria go to the beach?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_19", + "session_16", + "session_13", + "session_15", + "session_1", + "session_7", + "session_9", + "session_12", + "session_27" + ], + "sample_id": "conv-41" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_14", + "session_19", + "session_2", + "session_4" + ], + "question": "Where has Maria made friends?", + "recall": 0.25, + "retrieved_ids": [ + "session_29", + "session_18", + "session_24", + "session_16", + "session_20", + "session_6", + "session_2", + "session_27", + "session_26", + "session_25" + ], + "sample_id": "conv-41" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_3", + "session_5" + ], + "question": "What items des John mention having as a child?", + "recall": 0.0, + "retrieved_ids": [ + "session_13", + "session_26", + "session_9", + "session_2", + "session_19", + "session_31", + "session_1", + "session_20", + "session_23", + "session_14" + ], + "sample_id": "conv-41" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_5" + ], + "question": "What might John's financial status be?", + "recall": 0.0, + "retrieved_ids": [ + "session_16", + "session_21", + "session_25", + "session_9", + "session_1", + "session_15", + "session_4", + "session_2", + "session_13", + "session_28" + ], + "sample_id": "conv-41" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_5", + "session_6" + ], + "question": "Who gave Maria's family money when she was younger and her family was going through tough times?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_29", + "session_21", + "session_5", + "session_7", + "session_6", + "session_18", + "session_31", + "session_30", + "session_27" + ], + "sample_id": "conv-41" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_7" + ], + "question": "When did Maria meet Jean?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_3", + "session_30", + "session_32", + "session_6", + "session_9", + "session_23", + "session_4", + "session_27", + "session_24" + ], + "sample_id": "conv-41" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_21", + "session_27", + "session_6", + "session_7" + ], + "question": "What people has Maria met and helped while volunteering?", + "recall": 0.75, + "retrieved_ids": [ + "session_21", + "session_22", + "session_7", + "session_27", + "session_5", + "session_10", + "session_8", + "session_20", + "session_24", + "session_28" + ], + "sample_id": "conv-41" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_3", + "session_8" + ], + "question": "What test has John taken multiple times?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_19", + "session_30", + "session_3", + "session_15", + "session_18", + "session_2", + "session_13", + "session_31", + "session_25" + ], + "sample_id": "conv-41" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_8" + ], + "question": "When did Maria's grandmother pass away?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_18", + "session_11", + "session_30", + "session_17", + "session_32", + "session_6", + "session_20", + "session_27", + "session_24" + ], + "sample_id": "conv-41" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_8" + ], + "question": "Would John be considered a patriotic person?", + "recall": 0.0, + "retrieved_ids": [ + "session_9", + "session_3", + "session_6", + "session_21", + "session_31", + "session_15", + "session_17", + "session_16", + "session_4", + "session_28" + ], + "sample_id": "conv-41" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_7", + "session_9" + ], + "question": "What writing classes has Maria taken?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_15", + "session_1", + "session_12", + "session_19", + "session_25", + "session_4", + "session_16", + "session_9", + "session_10" + ], + "sample_id": "conv-41" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_9" + ], + "question": "When did John get his degree?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_8", + "session_9", + "session_12", + "session_15", + "session_1", + "session_4", + "session_2", + "session_25", + "session_31" + ], + "sample_id": "conv-41" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_9" + ], + "question": "What might John's degree be in?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_16", + "session_21", + "session_25", + "session_13", + "session_1", + "session_15", + "session_4", + "session_2", + "session_28" + ], + "sample_id": "conv-41" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_7" + ], + "question": "Who did John go to yoga with?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_1", + "session_10", + "session_7", + "session_18", + "session_19", + "session_27", + "session_12", + "session_15", + "session_9" + ], + "sample_id": "conv-41" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_11", + "session_4" + ], + "question": "What damages have happened to John's car?", + "recall": 0.5, + "retrieved_ids": [ + "session_14", + "session_21", + "session_15", + "session_11", + "session_18", + "session_2", + "session_23", + "session_28", + "session_1", + "session_7" + ], + "sample_id": "conv-41" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_11" + ], + "question": "When did John take a road trip to the Pacific Northwest?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_18", + "session_8", + "session_22", + "session_1", + "session_14", + "session_30", + "session_13", + "session_16", + "session_12" + ], + "sample_id": "conv-41" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_11", + "session_12" + ], + "question": "What areas of the U.S. has John been to or is planning to go to?", + "recall": 0.0, + "retrieved_ids": [ + "session_9", + "session_1", + "session_15", + "session_14", + "session_5", + "session_2", + "session_23", + "session_4", + "session_25", + "session_17" + ], + "sample_id": "conv-41" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_12" + ], + "question": "When did John go to a convention with colleagues?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_11", + "session_7", + "session_1", + "session_15", + "session_27", + "session_9", + "session_10", + "session_31", + "session_13" + ], + "sample_id": "conv-41" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_13", + "session_2" + ], + "question": "What desserts has Maria made?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_29", + "session_27", + "session_5", + "session_15", + "session_26", + "session_6", + "session_2", + "session_32", + "session_3" + ], + "sample_id": "conv-41" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_13" + ], + "question": "When did John start boot camp with his family?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_27", + "session_6", + "session_8", + "session_18", + "session_25", + "session_30", + "session_31", + "session_2", + "session_15" + ], + "sample_id": "conv-41" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_13", + "session_8" + ], + "question": "What European countries has Maria been to?", + "recall": 0.5, + "retrieved_ids": [ + "session_27", + "session_6", + "session_8", + "session_19", + "session_9", + "session_7", + "session_4", + "session_14", + "session_12", + "session_20" + ], + "sample_id": "conv-41" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_11", + "session_14" + ], + "question": "What has Maria done to feel closer to her faith?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_14", + "session_27", + "session_31", + "session_5", + "session_25", + "session_21", + "session_30", + "session_2", + "session_7" + ], + "sample_id": "conv-41" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_15" + ], + "question": "When did John have a party with veterans?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_24", + "session_27", + "session_29", + "session_21", + "session_13", + "session_6", + "session_32", + "session_7", + "session_4" + ], + "sample_id": "conv-41" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_12", + "session_15", + "session_9" + ], + "question": "What causes does John feel passionate about supporting?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_16", + "session_10", + "session_30", + "session_12", + "session_1", + "session_15", + "session_9", + "session_2", + "session_32" + ], + "sample_id": "conv-41" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_15", + "session_16" + ], + "question": "What events is Maria planning for the homeless shelter funraiser?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_16", + "session_29", + "session_12", + "session_10", + "session_6", + "session_25", + "session_17", + "session_9", + "session_4" + ], + "sample_id": "conv-41" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_11", + "session_17", + "session_2" + ], + "question": "What shelters does Maria volunteer at?", + "recall": 0.3333, + "retrieved_ids": [ + "session_27", + "session_7", + "session_11", + "session_26", + "session_5", + "session_8", + "session_21", + "session_1", + "session_31", + "session_20" + ], + "sample_id": "conv-41" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_17" + ], + "question": "When did John get his dog Max?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_30", + "session_6", + "session_31", + "session_17", + "session_8", + "session_1", + "session_15", + "session_25", + "session_2" + ], + "sample_id": "conv-41" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_16", + "session_18" + ], + "question": "What outdoor activities has John done with his colleagues?", + "recall": 0.0, + "retrieved_ids": [ + "session_31", + "session_8", + "session_26", + "session_15", + "session_25", + "session_27", + "session_24", + "session_12", + "session_7", + "session_10" + ], + "sample_id": "conv-41" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_18", + "session_19" + ], + "question": "What types of yoga has Maria practiced?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_1", + "session_7", + "session_10", + "session_18", + "session_19", + "session_22", + "session_13", + "session_16", + "session_4" + ], + "sample_id": "conv-41" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_19" + ], + "question": "When did Maria join a gym?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_19", + "session_30", + "session_27", + "session_24", + "session_26", + "session_14", + "session_3", + "session_4", + "session_25" + ], + "sample_id": "conv-41" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_18", + "session_19" + ], + "question": "What states has Maria vacationed at?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_13", + "session_5", + "session_6", + "session_16", + "session_4", + "session_31", + "session_18", + "session_21", + "session_2" + ], + "sample_id": "conv-41" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_20", + "session_8" + ], + "question": "What music events has John attended?", + "recall": 0.5, + "retrieved_ids": [ + "session_20", + "session_5", + "session_16", + "session_22", + "session_29", + "session_24", + "session_6", + "session_10", + "session_13", + "session_3" + ], + "sample_id": "conv-41" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_15", + "session_21", + "session_24", + "session_29" + ], + "question": "What events for veterans has John participated in?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_21", + "session_24", + "session_10", + "session_3", + "session_27", + "session_16", + "session_15", + "session_6", + "session_20" + ], + "sample_id": "conv-41" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_21" + ], + "question": "When did Maria get in a car accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_21", + "session_14", + "session_2", + "session_8", + "session_6", + "session_1", + "session_27", + "session_4", + "session_12" + ], + "sample_id": "conv-41" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_21" + ], + "question": "Around which US holiday did Maria get into a car accident?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_32", + "session_14", + "session_31", + "session_12", + "session_30", + "session_11", + "session_2", + "session_28", + "session_9" + ], + "sample_id": "conv-41" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_22", + "session_8" + ], + "question": "What are the names of John's children?", + "recall": 0.5, + "retrieved_ids": [ + "session_24", + "session_6", + "session_30", + "session_31", + "session_2", + "session_16", + "session_8", + "session_15", + "session_4", + "session_28" + ], + "sample_id": "conv-41" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_22" + ], + "question": "Does John live close to a beach or the mountains?", + "recall": 0.0, + "retrieved_ids": [ + "session_18", + "session_16", + "session_20", + "session_2", + "session_3", + "session_19", + "session_32", + "session_23", + "session_9", + "session_14" + ], + "sample_id": "conv-41" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_14", + "session_23" + ], + "question": "What area was hit by a flood?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_2", + "session_29", + "session_1", + "session_9", + "session_11", + "session_21", + "session_5", + "session_14" + ], + "sample_id": "conv-41" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_23" + ], + "question": "When was John's old area hit with a flood?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_14", + "session_2", + "session_5", + "session_29", + "session_11", + "session_21", + "session_1", + "session_9", + "session_8" + ], + "sample_id": "conv-41" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_24", + "session_25", + "session_28" + ], + "question": "What activities has Maria done with her church friends?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_24", + "session_31", + "session_27", + "session_7", + "session_28", + "session_30", + "session_5", + "session_21", + "session_14" + ], + "sample_id": "conv-41" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_24", + "session_7" + ], + "question": "Would John be open to moving to another country?", + "recall": 0.5, + "retrieved_ids": [ + "session_27", + "session_31", + "session_6", + "session_8", + "session_26", + "session_3", + "session_17", + "session_24", + "session_1", + "session_12" + ], + "sample_id": "conv-41" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_25" + ], + "question": "When did Maria go hiking with her church friends?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_25", + "session_24", + "session_28", + "session_31", + "session_7", + "session_30", + "session_5", + "session_21", + "session_14" + ], + "sample_id": "conv-41" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_10", + "session_25" + ], + "question": "What exercises has John done?", + "recall": 0.3333, + "retrieved_ids": [ + "session_25", + "session_15", + "session_27", + "session_26", + "session_29", + "session_28", + "session_16", + "session_22", + "session_4", + "session_13" + ], + "sample_id": "conv-41" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_26" + ], + "question": "When did John have his first firefighter call-out?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_6", + "session_8", + "session_13", + "session_2", + "session_32", + "session_25", + "session_31", + "session_9", + "session_15" + ], + "sample_id": "conv-41" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_25", + "session_26" + ], + "question": "What food item did Maria drop off at the homeless shelter?", + "recall": 0.5, + "retrieved_ids": [ + "session_2", + "session_26", + "session_29", + "session_3", + "session_16", + "session_6", + "session_19", + "session_32", + "session_24", + "session_18" + ], + "sample_id": "conv-41" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_2", + "session_26", + "session_3", + "session_4" + ], + "question": "What attributes describe John?", + "recall": 0.0, + "retrieved_ids": [], + "sample_id": "conv-41" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_27" + ], + "question": "When did Maria start volunteering at the homeless shelter?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_1", + "session_16", + "session_12", + "session_10", + "session_25", + "session_6", + "session_17", + "session_13", + "session_23" + ], + "sample_id": "conv-41" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_21", + "session_27" + ], + "question": "Who have written notes of gratitude to Maria?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_21", + "session_27", + "session_13", + "session_7", + "session_4", + "session_6", + "session_19", + "session_31", + "session_16" + ], + "sample_id": "conv-41" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_28" + ], + "question": "When did John help renovate his hometown community center?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_6", + "session_2", + "session_32", + "session_1", + "session_15", + "session_4", + "session_31", + "session_8", + "session_25" + ], + "sample_id": "conv-41" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_28" + ], + "question": "When did Maria take up community work with her church friends?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_24", + "session_28", + "session_25", + "session_14", + "session_7", + "session_31", + "session_30", + "session_5", + "session_21" + ], + "sample_id": "conv-41" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_29" + ], + "question": "When did Maria receive a medal from the homeless shelter?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_27", + "session_15", + "session_16", + "session_1", + "session_7", + "session_26", + "session_11", + "session_3", + "session_8" + ], + "sample_id": "conv-41" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_29" + ], + "question": "When did John participate in a 5K charity run?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_10", + "session_14", + "session_6", + "session_16", + "session_7", + "session_8", + "session_27", + "session_24", + "session_21" + ], + "sample_id": "conv-41" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_29", + "session_3", + "session_6" + ], + "question": "What causes has John done events for?", + "recall": 0.6667, + "retrieved_ids": [ + "session_29", + "session_6", + "session_16", + "session_15", + "session_10", + "session_27", + "session_25", + "session_26", + "session_30", + "session_32" + ], + "sample_id": "conv-41" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_30" + ], + "question": "When did Maria get Coco?", + "recall": 1.0, + "retrieved_ids": [ + "session_30", + "session_7", + "session_6", + "session_1", + "session_27", + "session_4", + "session_32", + "session_12", + "session_8", + "session_13" + ], + "sample_id": "conv-41" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_30" + ], + "question": "When did John go on a camping trip with Max?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_30", + "session_8", + "session_22", + "session_16", + "session_1", + "session_11", + "session_12", + "session_13", + "session_17" + ], + "sample_id": "conv-41" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_30", + "session_31" + ], + "question": "What are Maria's dogs' names?", + "recall": 1.0, + "retrieved_ids": [ + "session_31", + "session_24", + "session_6", + "session_30", + "session_17", + "session_2", + "session_16", + "session_4", + "session_8", + "session_13" + ], + "sample_id": "conv-41" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_31" + ], + "question": "When did Maria adopt Shadow?", + "recall": 1.0, + "retrieved_ids": [ + "session_31", + "session_13", + "session_6", + "session_27", + "session_24", + "session_15", + "session_32", + "session_7", + "session_4", + "session_17" + ], + "sample_id": "conv-41" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_30", + "session_31" + ], + "question": "How many dogs has Maria adopted from the dog shelter she volunteers at?", + "recall": 1.0, + "retrieved_ids": [ + "session_31", + "session_27", + "session_7", + "session_5", + "session_20", + "session_29", + "session_9", + "session_21", + "session_17", + "session_30" + ], + "sample_id": "conv-41" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_30", + "session_31" + ], + "question": "How many weeks passed between Maria adopting Coco and Shadow?", + "recall": 1.0, + "retrieved_ids": [ + "session_31", + "session_30", + "session_27", + "session_20", + "session_9", + "session_8", + "session_17", + "session_6", + "session_29", + "session_11" + ], + "sample_id": "conv-41" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_11", + "session_27", + "session_32", + "session_5" + ], + "question": "What job might Maria pursue in the future?", + "recall": 0.25, + "retrieved_ids": [ + "session_25", + "session_7", + "session_6", + "session_5", + "session_3", + "session_16", + "session_1", + "session_9", + "session_21", + "session_28" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_1" + ], + "question": "What is John's main focus in local politics?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_4", + "session_30", + "session_29", + "session_7", + "session_18", + "session_6", + "session_12", + "session_32", + "session_17" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_1" + ], + "question": "What sparked John's interest in improving education and infrastructure in the community?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_1", + "session_22", + "session_4", + "session_12", + "session_14", + "session_5", + "session_23", + "session_6", + "session_2" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_1" + ], + "question": "How did the extra funding help the school shown in the photo shared by John?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_10", + "session_19", + "session_5", + "session_6", + "session_29", + "session_9", + "session_31", + "session_30", + "session_17" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_1" + ], + "question": "What type of workout class did Maria start doing in December 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_1", + "session_10", + "session_12", + "session_27", + "session_6", + "session_13", + "session_9", + "session_7", + "session_19" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What did Maria donate to a homeless shelter in December 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_16", + "session_27", + "session_1", + "session_7", + "session_26", + "session_2", + "session_8", + "session_3", + "session_32" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What kind of meal did John and his family make together in the photo shared by John?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_6", + "session_11", + "session_10", + "session_8", + "session_17", + "session_15", + "session_31", + "session_30", + "session_25" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "What kind of online group did John join?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_27", + "session_24", + "session_26", + "session_12", + "session_14", + "session_4", + "session_23", + "session_32", + "session_25" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "What kind of activities did John and his mates from the online group do as part of their service efforts?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_6", + "session_8", + "session_12", + "session_2", + "session_3", + "session_25", + "session_32", + "session_27", + "session_31" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_5" + ], + "question": "Who inspired Maria to start volunteering?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_12", + "session_27", + "session_1", + "session_16", + "session_6", + "session_25", + "session_13", + "session_17", + "session_23" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_5" + ], + "question": "Why did Maria sit with the little girl at the shelter event in February 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_29", + "session_16", + "session_21", + "session_8", + "session_20", + "session_24", + "session_10", + "session_6", + "session_19" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What did Jean go through before meeting Maria?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_4", + "session_1", + "session_9", + "session_18", + "session_14", + "session_19", + "session_23", + "session_3", + "session_30" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "Why did John decide to run for office again?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_14", + "session_15", + "session_29", + "session_6", + "session_25", + "session_28", + "session_5", + "session_10", + "session_8" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What activity did John's colleague, Rob, invite him to?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_24", + "session_6", + "session_31", + "session_10", + "session_30", + "session_21", + "session_17", + "session_15", + "session_12" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What is the name of John's one-year-old child?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_31", + "session_13", + "session_1", + "session_24", + "session_6", + "session_14", + "session_30", + "session_2", + "session_23" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "How often does John take his kids to the park?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_22", + "session_18", + "session_13", + "session_31", + "session_25", + "session_15", + "session_6", + "session_2", + "session_11" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What did Maria make for her home to remind her of a trip to England?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_19", + "session_30", + "session_21", + "session_7", + "session_31", + "session_5", + "session_17", + "session_18", + "session_23" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "Where did Maria get the idea for the castle shadow box in her home?", + "recall": 1.0, + "retrieved_ids": [ + "session_31", + "session_30", + "session_1", + "session_8", + "session_4", + "session_21", + "session_17", + "session_7", + "session_5", + "session_12" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_9" + ], + "question": "What did John receive a certificate for?", + "recall": 0.0, + "retrieved_ids": [ + "session_27", + "session_12", + "session_24", + "session_15", + "session_11", + "session_7", + "session_6", + "session_4", + "session_13", + "session_32" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_9" + ], + "question": "What areas is John particularly interested in for policymaking?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_1", + "session_30", + "session_16", + "session_5", + "session_2", + "session_14", + "session_23", + "session_25", + "session_26" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "What did Maria participate in last weekend before April 10, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_24", + "session_29", + "session_25", + "session_10", + "session_4", + "session_14", + "session_8", + "session_21", + "session_17" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "What event did John volunteer at last weekend?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_29", + "session_8", + "session_24", + "session_6", + "session_25", + "session_3", + "session_18", + "session_20", + "session_16" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_11" + ], + "question": "What did John do that put a strain on his wallet?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_31", + "session_9", + "session_6", + "session_8", + "session_19", + "session_24", + "session_25", + "session_15", + "session_2" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_11" + ], + "question": "Where did John explore on a road trip last year?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_1", + "session_8", + "session_18", + "session_22", + "session_30", + "session_13", + "session_27", + "session_25", + "session_14" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_12" + ], + "question": "What topic has John been blogging about recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_1", + "session_11", + "session_2", + "session_8", + "session_3", + "session_7", + "session_30", + "session_6", + "session_21" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_12" + ], + "question": "Why did John start blogging about politics and policies?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_1", + "session_6", + "session_17", + "session_27", + "session_23", + "session_16", + "session_4", + "session_13", + "session_7" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_12" + ], + "question": "What was the focus of John's recent research and writing on his blog?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_30", + "session_8", + "session_6", + "session_7", + "session_18", + "session_25", + "session_31", + "session_15", + "session_2" + ], + "sample_id": "conv-41" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_12" + ], + "question": "What did John attend with his colleagues in March 2023?", + "recall": 0.0, + "retrieved_ids": [], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "How often does John work out with his family?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_8", + "session_19", + "session_25", + "session_31", + "session_6", + "session_13", + "session_24", + "session_15", + "session_2" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "How has John's fitness improved since starting boot camps with his family?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_14", + "session_19", + "session_31", + "session_2", + "session_1", + "session_18", + "session_6", + "session_25", + "session_30" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "What kind of food did Maria have on her dinner spread iwth her mother?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_13", + "session_30", + "session_5", + "session_7", + "session_31", + "session_23", + "session_3", + "session_24", + "session_32" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "What activity did Maria and her mom do together in May 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_13", + "session_31", + "session_7", + "session_8", + "session_28", + "session_24", + "session_30", + "session_21", + "session_5" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_14" + ], + "question": "What did Maria do to feel closer to a community and her faith?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_14", + "session_5", + "session_6", + "session_2", + "session_32", + "session_31", + "session_30", + "session_7", + "session_21" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_14" + ], + "question": "Why did Maria join a nearby church recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_24", + "session_3", + "session_21", + "session_12", + "session_6", + "session_25", + "session_28", + "session_22", + "session_18" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "What did John host for the veterans in May 2023 as part of the project?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_12", + "session_27", + "session_24", + "session_29", + "session_21", + "session_14", + "session_32", + "session_3", + "session_28" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "What did John and the veterans do during the small party?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_21", + "session_24", + "session_4", + "session_13", + "session_19", + "session_27", + "session_29", + "session_5", + "session_11" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "What emotions did John feel during the small party with the veterans?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_21", + "session_4", + "session_5", + "session_27", + "session_24", + "session_11", + "session_13", + "session_3", + "session_29" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "What event is Maria getting ready for at the shelter on May 25, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_29", + "session_11", + "session_5", + "session_10", + "session_20", + "session_3", + "session_24", + "session_6", + "session_28" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "What does Maria need to spread the word about for the fundraiser for the volunteer shelter?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_3", + "session_29", + "session_15", + "session_7", + "session_30", + "session_23", + "session_32", + "session_20", + "session_24" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What was the name of the pet that John had to say goodbye to on 3 June, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_25", + "session_27", + "session_31", + "session_30", + "session_10", + "session_18", + "session_8", + "session_24", + "session_6" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "How long was Max a part of John's family?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_27", + "session_30", + "session_18", + "session_1", + "session_6", + "session_32", + "session_9", + "session_7", + "session_12" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "How does John plan to honor the memories of his beloved pet?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_30", + "session_6", + "session_18", + "session_8", + "session_31", + "session_2", + "session_25", + "session_29", + "session_15" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What important values does John want to teach his kids through adopting a rescue dog?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_21", + "session_27", + "session_7", + "session_8", + "session_13", + "session_25", + "session_15", + "session_31", + "session_26" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What new activity did Maria start recently, as mentioned on 3 June, 2023?", + "recall": 0.0, + "retrieved_ids": [ + "session_13", + "session_24", + "session_21", + "session_31", + "session_12", + "session_19", + "session_3", + "session_26", + "session_9", + "session_14" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What did Maria say it was like being at the waterfall in Oregon?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_10", + "session_4", + "session_27", + "session_12", + "session_13", + "session_32", + "session_17", + "session_24", + "session_8" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What does Maria say she feels when doing upside-down yoga poses?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_10", + "session_27", + "session_31", + "session_21", + "session_7", + "session_5", + "session_30", + "session_8", + "session_17" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "What exciting news did Maria share on 16 June, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_15", + "session_12", + "session_13", + "session_19", + "session_1", + "session_10", + "session_31", + "session_21", + "session_32" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "What yoga activity has Maria been trying to improve her strength and endurance?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_13", + "session_1", + "session_10", + "session_7", + "session_31", + "session_25", + "session_14", + "session_2", + "session_21" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "What did John recently get promoted to?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_7", + "session_12", + "session_11", + "session_21", + "session_2", + "session_3", + "session_4", + "session_1", + "session_6" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "What was one of the biggest challenges John faced in his journey to becoming assistant manager?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_6", + "session_13", + "session_9", + "session_8", + "session_11", + "session_26", + "session_31", + "session_21", + "session_3" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "How does John describe the support he received during his journey to becoming assistant manager?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_13", + "session_26", + "session_6", + "session_4", + "session_21", + "session_8", + "session_11", + "session_27", + "session_17" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_20" + ], + "question": "What kind of event did John and his family attend in June 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_8", + "session_16", + "session_29", + "session_10", + "session_2", + "session_20", + "session_15", + "session_31", + "session_25" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_21" + ], + "question": "Why did Maria need to help her cousin find a new place to live?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_16", + "session_30", + "session_1", + "session_25", + "session_2", + "session_7", + "session_27", + "session_31", + "session_5" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_21" + ], + "question": "What event did John participate in to show support for veterans' rights?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_15", + "session_3", + "session_27", + "session_29", + "session_24", + "session_10", + "session_5", + "session_6", + "session_20" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_21" + ], + "question": "What inspired John to join the marching event for veterans' rights?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_24", + "session_5", + "session_30", + "session_3", + "session_27", + "session_15", + "session_29", + "session_10", + "session_19" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_22" + ], + "question": "How often does John get to see sunsets like the one he shared with Maria?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_19", + "session_13", + "session_4", + "session_6", + "session_8", + "session_30", + "session_25", + "session_3", + "session_17" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "What natural disaster affected John's old area on 7 July, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_1", + "session_14", + "session_5", + "session_2", + "session_8", + "session_30", + "session_18", + "session_25", + "session_3" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "How did the flood impact the homes in John's old area?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_2", + "session_9", + "session_14", + "session_5", + "session_8", + "session_19", + "session_1", + "session_31", + "session_17" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "What motivated Maria and John to discuss potential solutions for their community on 7 July, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_15", + "session_12", + "session_2", + "session_28", + "session_32", + "session_9", + "session_11", + "session_4", + "session_8" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "What did Maria plan to do later on the evening of 7 July, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_12", + "session_29", + "session_9", + "session_25", + "session_30", + "session_4", + "session_14", + "session_23", + "session_17" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "What kind of activities did Maria do at the picnic with her church friends?", + "recall": 1.0, + "retrieved_ids": [ + "session_31", + "session_24", + "session_8", + "session_25", + "session_7", + "session_5", + "session_30", + "session_14", + "session_28", + "session_21" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "What does John appreciate about the veteran's hospital visit?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_27", + "session_4", + "session_16", + "session_15", + "session_29", + "session_21", + "session_22", + "session_28", + "session_3" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "What did John take away from visiting the veteran's hospital?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_27", + "session_15", + "session_4", + "session_18", + "session_30", + "session_8", + "session_29", + "session_21", + "session_13" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "Why did John feel inspired to join the military after the visit to the hospital?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_27", + "session_26", + "session_3", + "session_21", + "session_13", + "session_10", + "session_1", + "session_15", + "session_7" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_25" + ], + "question": "In what activity did Maria and her church friends participate in July 2023?", + "recall": 0.0, + "retrieved_ids": [ + "session_24", + "session_21", + "session_7", + "session_31", + "session_28", + "session_29", + "session_5", + "session_10", + "session_30", + "session_14" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_25" + ], + "question": "What does John think about trying new classes at the yoga studio?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_13", + "session_1", + "session_15", + "session_19", + "session_10", + "session_18", + "session_21", + "session_7", + "session_9" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_25" + ], + "question": "Which activity has John done apart from yoga at the studio?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_1", + "session_19", + "session_27", + "session_24", + "session_15", + "session_26", + "session_29", + "session_10", + "session_18" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_26" + ], + "question": "What community service did Maria mention that she was involved in on 31 July, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_26", + "session_5", + "session_9", + "session_7", + "session_6", + "session_31", + "session_30", + "session_21", + "session_3" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_27" + ], + "question": "How did Maria start volunteering at the homeless shelter?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_1", + "session_12", + "session_16", + "session_10", + "session_17", + "session_25", + "session_6", + "session_13", + "session_23" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_27" + ], + "question": "What did John do the week before August 3, 2023 involving his kids?", + "recall": 0.0, + "retrieved_ids": [ + "session_14", + "session_18", + "session_8", + "session_31", + "session_5", + "session_1", + "session_15", + "session_9", + "session_6", + "session_25" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_27" + ], + "question": "How did John describe his kids' reaction at the military memorial?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_8", + "session_21", + "session_17", + "session_24", + "session_3", + "session_15", + "session_20", + "session_31", + "session_2" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_27" + ], + "question": "Why does Maria think it's important for younger generations to visit military memorials?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_24", + "session_15", + "session_17", + "session_21", + "session_1", + "session_18", + "session_26", + "session_6", + "session_2" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_27" + ], + "question": "What does John believe is important for children regarding veterans?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_21", + "session_27", + "session_15", + "session_7", + "session_4", + "session_24", + "session_29", + "session_16", + "session_19" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_28" + ], + "question": "What happened to John's job in August 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_6", + "session_7", + "session_25", + "session_1", + "session_18", + "session_21", + "session_14", + "session_23", + "session_3" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_28" + ], + "question": "What activity did Maria take up with her friends from church in August 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_31", + "session_24", + "session_25", + "session_30", + "session_14", + "session_28", + "session_21", + "session_5", + "session_7" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_28" + ], + "question": "What did John do to help his community last year in his hometown?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_1", + "session_6", + "session_27", + "session_15", + "session_8", + "session_13", + "session_31", + "session_2", + "session_25" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_29" + ], + "question": "What cause did the 5K charity run organized by John support?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_10", + "session_16", + "session_14", + "session_1", + "session_6", + "session_8", + "session_32", + "session_7", + "session_30" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_29" + ], + "question": "Who did John work with to raise awareness and funds for victims of domestic abuse?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_12", + "session_15", + "session_5", + "session_32", + "session_16", + "session_1", + "session_24", + "session_7", + "session_4" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_29" + ], + "question": "What recognition did Maria receive at the homeless shelter in August 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_15", + "session_27", + "session_16", + "session_1", + "session_7", + "session_26", + "session_11", + "session_8", + "session_3" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_30" + ], + "question": "What is the name of Maria's puppy she got two weeks before August 11, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_30", + "session_31", + "session_14", + "session_8", + "session_24", + "session_18", + "session_5", + "session_32", + "session_7", + "session_21" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_30" + ], + "question": "What activity did John and Max enjoy together last summer?", + "recall": 1.0, + "retrieved_ids": [ + "session_30", + "session_31", + "session_8", + "session_25", + "session_18", + "session_19", + "session_13", + "session_24", + "session_21", + "session_17" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_30" + ], + "question": "How does John describe the camping trip with Max?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_30", + "session_8", + "session_17", + "session_1", + "session_22", + "session_12", + "session_13", + "session_16", + "session_11" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_30" + ], + "question": "Why does John say he feels stuck and questioning his decisions and goals?", + "recall": 1.0, + "retrieved_ids": [ + "session_30", + "session_6", + "session_31", + "session_27", + "session_10", + "session_8", + "session_19", + "session_25", + "session_17", + "session_15" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_31" + ], + "question": "What is the name of Maria's second puppy?", + "recall": 1.0, + "retrieved_ids": [ + "session_30", + "session_31", + "session_8", + "session_18", + "session_16", + "session_24", + "session_6", + "session_4", + "session_2", + "session_13" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_31" + ], + "question": "How is Maria's new puppy adjusting to its new home?", + "recall": 1.0, + "retrieved_ids": [ + "session_30", + "session_31", + "session_4", + "session_21", + "session_19", + "session_13", + "session_28", + "session_8", + "session_17", + "session_23" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_31" + ], + "question": "What is John currently doing as a volunteer in August 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_27", + "session_16", + "session_11", + "session_8", + "session_28", + "session_1", + "session_7", + "session_5", + "session_31" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_31" + ], + "question": "What activities does John's family enjoy doing together?", + "recall": 1.0, + "retrieved_ids": [ + "session_31", + "session_8", + "session_4", + "session_24", + "session_18", + "session_19", + "session_13", + "session_3", + "session_25", + "session_21" + ], + "sample_id": "conv-41" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_32" + ], + "question": "What did the donations help John's community acquire on 16 August, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_32", + "session_3", + "session_15", + "session_1", + "session_9", + "session_4", + "session_6", + "session_29", + "session_2", + "session_12" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_1" + ], + "question": "What is John's main focus in international politics?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_18", + "session_7", + "session_4", + "session_30", + "session_6", + "session_12", + "session_15", + "session_16", + "session_28" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_2" + ], + "question": "What did Maria donate to a luxury store in December 2023?", + "recall": 0.0, + "retrieved_ids": [], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_5" + ], + "question": "Who inspired John to start volunteering?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_12", + "session_27", + "session_16", + "session_1", + "session_6", + "session_25", + "session_13", + "session_17", + "session_23" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_7" + ], + "question": "Why did Maria decide to run for office again?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_14", + "session_15", + "session_29", + "session_6", + "session_25", + "session_28", + "session_5", + "session_10", + "session_8" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_7" + ], + "question": "What activity did Maria's colleague, Rob, invite her to?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_31", + "session_8", + "session_10", + "session_15", + "session_24", + "session_21", + "session_5", + "session_30", + "session_12" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_8" + ], + "question": "What is the name of Maria's one-year-old child?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_31", + "session_13", + "session_1", + "session_24", + "session_14", + "session_6", + "session_30", + "session_2", + "session_23" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_8" + ], + "question": "How often does John take his kids to the library?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_13", + "session_17", + "session_22", + "session_25", + "session_15", + "session_31", + "session_5", + "session_6", + "session_2" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_8" + ], + "question": "What did Maria make for her home to remind her of a trip to France?", + "recall": 1.0, + "retrieved_ids": [ + "session_30", + "session_19", + "session_21", + "session_31", + "session_7", + "session_5", + "session_8", + "session_17", + "session_18", + "session_23" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_8" + ], + "question": "Where did John get the idea for the castle shadow box in his home?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_12", + "session_31", + "session_30", + "session_4", + "session_6", + "session_17", + "session_15", + "session_25", + "session_2" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_9" + ], + "question": "What did Maria receive a certificate for?", + "recall": 0.0, + "retrieved_ids": [ + "session_27", + "session_12", + "session_24", + "session_11", + "session_15", + "session_6", + "session_7", + "session_4", + "session_13", + "session_32" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_9" + ], + "question": "What areas is John particularly interested in for art appreciation?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_1", + "session_26", + "session_25", + "session_24", + "session_16", + "session_5", + "session_14", + "session_2", + "session_23" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_12" + ], + "question": "Why did Maria start blogging about politics and policies?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_1", + "session_6", + "session_17", + "session_27", + "session_23", + "session_16", + "session_4", + "session_13", + "session_7" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_12" + ], + "question": "What was the focus of John's recent travel and photography blog?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_30", + "session_2", + "session_8", + "session_18", + "session_6", + "session_7", + "session_11", + "session_21", + "session_3" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_13" + ], + "question": "How often does Maria work out with her family?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_31", + "session_21", + "session_24", + "session_5", + "session_8", + "session_13", + "session_25", + "session_7", + "session_30" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_13" + ], + "question": "How has John's artistic skills improved since starting boot camps with his family?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_14", + "session_31", + "session_26", + "session_28", + "session_2", + "session_18", + "session_6", + "session_25", + "session_30" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_13" + ], + "question": "What kind of food did Maria have on her dinner spread with her father?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_13", + "session_31", + "session_5", + "session_7", + "session_30", + "session_23", + "session_3", + "session_24", + "session_32" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_14" + ], + "question": "What did John do to feel closer to a community and his faith?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_14", + "session_4", + "session_6", + "session_2", + "session_8", + "session_32", + "session_15", + "session_25", + "session_31" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_14" + ], + "question": "Why did John join a nearby church recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_24", + "session_3", + "session_21", + "session_12", + "session_25", + "session_28", + "session_6", + "session_22", + "session_18" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_17" + ], + "question": "How long was Max a part of Maria's family?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_27", + "session_30", + "session_18", + "session_1", + "session_6", + "session_32", + "session_9", + "session_7", + "session_12" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_17" + ], + "question": "How does Maria plan to honor the memories of her beloved pet?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_30", + "session_2", + "session_31", + "session_18", + "session_29", + "session_7", + "session_5", + "session_21", + "session_8" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_17" + ], + "question": "What important values does Maria want to teach her kids through adopting a rescue dog?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_21", + "session_15", + "session_7", + "session_27", + "session_31", + "session_13", + "session_5", + "session_26", + "session_30" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "What did Maria say it was like being at the desert in Oregon?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_10", + "session_4", + "session_27", + "session_12", + "session_13", + "session_32", + "session_17", + "session_24", + "session_8" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "What does John say she feels when doing upside-down yoga poses?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_10", + "session_27", + "session_31", + "session_21", + "session_7", + "session_5", + "session_30", + "session_8", + "session_17" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_19" + ], + "question": "What did Maria recently get promoted to?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_7", + "session_12", + "session_11", + "session_21", + "session_2", + "session_4", + "session_3", + "session_1", + "session_6" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_19" + ], + "question": "What was one of the biggest challenges Maria faced in her journey to becoming assistant manager?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_6", + "session_13", + "session_11", + "session_9", + "session_31", + "session_26", + "session_21", + "session_8", + "session_3" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_21" + ], + "question": "Why did John need to help his cousin find a new place to live?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_8", + "session_25", + "session_6", + "session_15", + "session_27", + "session_1", + "session_2", + "session_7", + "session_31" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_21" + ], + "question": "What event did Maria participate in to show support for veterans' rights?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_15", + "session_3", + "session_27", + "session_29", + "session_24", + "session_10", + "session_5", + "session_6", + "session_20" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_23" + ], + "question": "How did the drought impact the homes in John's old area?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_2", + "session_9", + "session_14", + "session_5", + "session_8", + "session_19", + "session_1", + "session_31", + "session_17" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_24" + ], + "question": "What does John criticize about the veteran's hospital visit?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_27", + "session_15", + "session_29", + "session_17", + "session_5", + "session_21", + "session_12", + "session_30", + "session_8" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_24" + ], + "question": "What did John take away from visiting the orphanage?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_30", + "session_8", + "session_24", + "session_27", + "session_9", + "session_15", + "session_4", + "session_13", + "session_12" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_24" + ], + "question": "Why did Maria feel inspired to join the military after the visit to the hospital?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_27", + "session_3", + "session_26", + "session_21", + "session_13", + "session_10", + "session_1", + "session_15", + "session_7" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_27" + ], + "question": "How did Maria describe her kids' reaction at the military memorial?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_27", + "session_20", + "session_5", + "session_31", + "session_24", + "session_3", + "session_7", + "session_30", + "session_8" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_27" + ], + "question": "Why does Maria think it's important for younger generations to visit art galleries?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_5", + "session_24", + "session_1", + "session_25", + "session_18", + "session_6", + "session_22", + "session_2", + "session_17" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_28" + ], + "question": "What happened to Maria's job in August 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_6", + "session_1", + "session_7", + "session_25", + "session_18", + "session_21", + "session_14", + "session_23", + "session_3" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_29" + ], + "question": "What cause did the 5K charity run organized by Maria support?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_10", + "session_16", + "session_14", + "session_1", + "session_6", + "session_8", + "session_32", + "session_7", + "session_30" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_29" + ], + "question": "Who did John work with to raise awareness and funds for animal welfare?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_12", + "session_4", + "session_31", + "session_32", + "session_5", + "session_16", + "session_1", + "session_24", + "session_7" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_29" + ], + "question": "What recognition did John receive at the homeless shelter in August 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_15", + "session_27", + "session_16", + "session_1", + "session_7", + "session_26", + "session_11", + "session_8", + "session_3" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_30" + ], + "question": "What is the name of John's puppy he got two weeks before August 11, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_30", + "session_4", + "session_14", + "session_8", + "session_18", + "session_31", + "session_32", + "session_24", + "session_6", + "session_17" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_30" + ], + "question": "How does Maria describe the camping trip with Max?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_30", + "session_8", + "session_17", + "session_1", + "session_22", + "session_12", + "session_13", + "session_16", + "session_11" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_31" + ], + "question": "What is the name of Maria's second kitten?", + "recall": 1.0, + "retrieved_ids": [ + "session_31", + "session_8", + "session_18", + "session_24", + "session_16", + "session_6", + "session_4", + "session_30", + "session_2", + "session_13" + ], + "sample_id": "conv-41" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_31" + ], + "question": "How is John's new puppy adjusting to its new home?", + "recall": 1.0, + "retrieved_ids": [ + "session_30", + "session_31", + "session_4", + "session_21", + "session_19", + "session_13", + "session_28", + "session_8", + "session_17", + "session_23" + ], + "sample_id": "conv-41" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_1" + ], + "question": "Is it likely that Nate has friends besides Joanna?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_16", + "session_15", + "session_1", + "session_6", + "session_10", + "session_4", + "session_12", + "session_14", + "session_27" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_10", + "session_20", + "session_3", + "session_4" + ], + "question": "What kind of interests do Joanna and Nate share?", + "recall": 0.4, + "retrieved_ids": [ + "session_14", + "session_6", + "session_9", + "session_26", + "session_23", + "session_17", + "session_1", + "session_4", + "session_19", + "session_25" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_1" + ], + "question": "When did Joanna first watch \"Eternal Sunshine of the Spotless Mind?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_4", + "session_1", + "session_23", + "session_25", + "session_28", + "session_2", + "session_9", + "session_17", + "session_14" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_1" + ], + "question": "When did Nate win his first video game tournament?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_1", + "session_17", + "session_14", + "session_26", + "session_28", + "session_2", + "session_10", + "session_22", + "session_15" + ], + "sample_id": "conv-42" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_2" + ], + "question": "What pets wouldn't cause any discomfort to Joanna?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_5", + "session_2", + "session_1", + "session_12", + "session_8", + "session_19", + "session_13", + "session_24", + "session_27" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_2" + ], + "question": "What are Joanna's hobbies?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_23", + "session_8", + "session_2", + "session_15", + "session_4", + "session_14", + "session_21", + "session_25", + "session_28" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_2" + ], + "question": "How long has Nate had his first two turtles?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_1", + "session_25", + "session_28", + "session_9", + "session_14", + "session_5", + "session_15", + "session_4", + "session_22" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_2" + ], + "question": "When did Joanna finish her first screenplay?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_9", + "session_12", + "session_4", + "session_14", + "session_13", + "session_3", + "session_24", + "session_22", + "session_27" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_2" + ], + "question": "When did Nate get his first two turtles?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_5", + "session_2", + "session_25", + "session_28", + "session_4", + "session_15", + "session_9", + "session_1", + "session_22" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_2" + ], + "question": "What major achievement did Joanna accomplish in January 2022?", + "recall": 0.0, + "retrieved_ids": [ + "session_14", + "session_21", + "session_22", + "session_19", + "session_10", + "session_7", + "session_11", + "session_4", + "session_16", + "session_17" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_2", + "session_3" + ], + "question": "What emotions is Joanna feeling about the screenplay she submitted?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_27", + "session_3", + "session_4", + "session_29", + "session_24", + "session_22", + "session_23", + "session_16", + "session_6" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_2", + "session_4", + "session_5" + ], + "question": "What is Joanna allergic to?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_24", + "session_5", + "session_28", + "session_21", + "session_1", + "session_15", + "session_4", + "session_14", + "session_25" + ], + "sample_id": "conv-42" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_2", + "session_5" + ], + "question": "What underlying condition might Joanna have based on her allergies?", + "recall": 0.5, + "retrieved_ids": [ + "session_2", + "session_9", + "session_21", + "session_12", + "session_24", + "session_1", + "session_3", + "session_16", + "session_17", + "session_8" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_6" + ], + "question": "When did Joanna have an audition for a writing gig?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_22", + "session_7", + "session_11", + "session_4", + "session_17", + "session_9", + "session_26", + "session_19", + "session_25" + ], + "sample_id": "conv-42" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_7" + ], + "question": "What nickname does Nate use for Joanna?", + "recall": 0.0, + "retrieved_ids": [ + "session_22", + "session_1", + "session_14", + "session_4", + "session_15", + "session_21", + "session_13", + "session_2", + "session_25", + "session_28" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_7" + ], + "question": "When did Nate get purple hair?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_23", + "session_11", + "session_6", + "session_17", + "session_4", + "session_25", + "session_13", + "session_14", + "session_28" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_7" + ], + "question": "What physical transformation did Nate undergo in April 2022?", + "recall": 0.0, + "retrieved_ids": [], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_10" + ], + "question": "What movie did Joanna watch on 1 May, 2022?", + "recall": 0.0, + "retrieved_ids": [ + "session_1", + "session_23", + "session_25", + "session_3", + "session_27", + "session_28", + "session_9", + "session_29", + "session_22", + "session_17" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_11" + ], + "question": "Which outdoor spot did Joanna visit in May?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_11", + "session_8", + "session_25", + "session_7", + "session_16", + "session_4", + "session_10", + "session_17", + "session_21" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_11", + "session_8" + ], + "question": "How many times has Joanna found new hiking trails?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_8", + "session_12", + "session_23", + "session_14", + "session_7", + "session_6", + "session_26", + "session_5", + "session_28" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_12" + ], + "question": "When did Nate adopt Max?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_13", + "session_25", + "session_7", + "session_16", + "session_11", + "session_17", + "session_4", + "session_10", + "session_21" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_12" + ], + "question": "Who was the new addition to Nate's family in May 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_16", + "session_18", + "session_22", + "session_23", + "session_3", + "session_7", + "session_15", + "session_24", + "session_28" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_12" + ], + "question": "When did Joanna start writing her third screenplay?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_28", + "session_11", + "session_4", + "session_12", + "session_27", + "session_15", + "session_9", + "session_25", + "session_24" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_14", + "session_2", + "session_24", + "session_3" + ], + "question": "Which of Joanna's screenplay were rejected from production companies?", + "recall": 0.75, + "retrieved_ids": [ + "session_14", + "session_24", + "session_16", + "session_10", + "session_13", + "session_2", + "session_27", + "session_11", + "session_4", + "session_23" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_14" + ], + "question": "When is Nate hosting a gaming party?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_16", + "session_6", + "session_10", + "session_9", + "session_23", + "session_28", + "session_27", + "session_22", + "session_1" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_14" + ], + "question": "When did Joanna hike with her buddies?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_13", + "session_28", + "session_24", + "session_11", + "session_9", + "session_19", + "session_17", + "session_7", + "session_8" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_14" + ], + "question": "When did Nate win his third tourney?", + "recall": 0.0, + "retrieved_ids": [ + "session_28", + "session_26", + "session_20", + "session_15", + "session_10", + "session_12", + "session_25", + "session_2", + "session_22", + "session_9" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_16", + "session_2" + ], + "question": "What places has Joanna submitted her work to?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_27", + "session_24", + "session_17", + "session_5", + "session_11", + "session_2", + "session_23", + "session_6", + "session_16" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_16" + ], + "question": "When did Nate make vegan icecream and share it with a vegan diet group?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_21", + "session_20", + "session_18", + "session_13", + "session_9", + "session_3", + "session_19", + "session_8", + "session_25" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_16" + ], + "question": "When is Joanna going to make Nate's ice cream for her family?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_21", + "session_8", + "session_28", + "session_18", + "session_29", + "session_4", + "session_9", + "session_24", + "session_15" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_17", + "session_18", + "session_2" + ], + "question": "What kind of writings does Joanna do?", + "recall": 0.6667, + "retrieved_ids": [ + "session_6", + "session_4", + "session_9", + "session_18", + "session_17", + "session_19", + "session_5", + "session_15", + "session_20", + "session_14" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_17" + ], + "question": "When did Nate win his fourth video game tournament?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_26", + "session_1", + "session_22", + "session_10", + "session_14", + "session_9", + "session_2", + "session_19", + "session_15" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_17" + ], + "question": "Where did Joanna travel to in July 2022?", + "recall": 0.0, + "retrieved_ids": [], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_18" + ], + "question": "When did someone write Joanna a touching letter?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_24", + "session_6", + "session_27", + "session_11", + "session_14", + "session_13", + "session_3", + "session_8", + "session_5" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_19", + "session_3" + ], + "question": "What book recommendations has Joanna given to Nate?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_9", + "session_22", + "session_4", + "session_1", + "session_3", + "session_6", + "session_17", + "session_8", + "session_15" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_19" + ], + "question": "When did Nate take time off to chill with his pets?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_5", + "session_23", + "session_2", + "session_15", + "session_29", + "session_8", + "session_9", + "session_12", + "session_22" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_19" + ], + "question": "When did Joanna share her book with her writers group?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_19", + "session_21", + "session_13", + "session_6", + "session_22", + "session_16", + "session_24", + "session_17", + "session_8" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_19" + ], + "question": "When did Nate win an international tournament?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_10", + "session_19", + "session_2", + "session_6", + "session_1", + "session_14", + "session_17", + "session_22", + "session_27" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_21" + ], + "question": "When did Joanna make a desert with almond milk?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_20", + "session_26", + "session_29", + "session_4", + "session_18", + "session_3", + "session_8", + "session_10", + "session_16" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_21" + ], + "question": "When did Nate attend a cooking show?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_16", + "session_15", + "session_10", + "session_4", + "session_29", + "session_20", + "session_3", + "session_28", + "session_7" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_21" + ], + "question": "When did Joanna's laptop crash?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_3", + "session_7", + "session_16", + "session_11", + "session_17", + "session_6", + "session_4", + "session_10", + "session_25" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_22" + ], + "question": "When did Joanna make a chocolate tart with raspberries?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_20", + "session_4", + "session_22", + "session_3", + "session_26", + "session_13", + "session_16", + "session_6", + "session_10" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_22", + "session_3" + ], + "question": "What movies have both Joanna and Nate seen?", + "recall": 0.6667, + "retrieved_ids": [ + "session_1", + "session_23", + "session_3", + "session_22", + "session_19", + "session_24", + "session_26", + "session_2", + "session_27", + "session_12" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_17", + "session_22" + ], + "question": "How long did it take for Joanna to finish writing her book?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_19", + "session_6", + "session_12", + "session_22", + "session_2", + "session_27", + "session_17", + "session_8", + "session_24" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_22" + ], + "question": "When did Nate win a lot of money in a video game tournament?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_22", + "session_26", + "session_28", + "session_1", + "session_10", + "session_14", + "session_2", + "session_27", + "session_19" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_22" + ], + "question": "When did Joanna finish up the writing for her book?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_12", + "session_19", + "session_22", + "session_2", + "session_6", + "session_17", + "session_27", + "session_24", + "session_8" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_16", + "session_23" + ], + "question": "What board games has Nate played?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_23", + "session_27", + "session_15", + "session_17", + "session_1", + "session_10", + "session_13", + "session_28", + "session_16" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_14", + "session_23" + ], + "question": "What places has Nate met new people?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_14", + "session_17", + "session_21", + "session_11", + "session_25", + "session_2", + "session_16", + "session_19", + "session_6" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_23" + ], + "question": "When did Nate go to a convention and meet new people?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_13", + "session_28", + "session_16", + "session_17", + "session_21", + "session_14", + "session_19", + "session_2", + "session_26" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_14", + "session_24" + ], + "question": "How many times has Joanna's scripts been rejected?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_26", + "session_6", + "session_5", + "session_28", + "session_29", + "session_9", + "session_24", + "session_14", + "session_17" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_13", + "session_24" + ], + "question": "What is something Nate gave to Joanna that brings her a lot of joy?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_13", + "session_2", + "session_12", + "session_29", + "session_10", + "session_28", + "session_5", + "session_3", + "session_9" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_13", + "session_24" + ], + "question": "When did Nate get Tilly for Joanna?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_23", + "session_11", + "session_6", + "session_4", + "session_17", + "session_25", + "session_13", + "session_14", + "session_28" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_15", + "session_25" + ], + "question": "How many of Joanna's writing have made it to the big screen?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_25", + "session_22", + "session_28", + "session_6", + "session_2", + "session_15", + "session_3", + "session_4", + "session_5" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_25", + "session_5" + ], + "question": "How many times has Nate taken his turtles on a walk?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_24", + "session_25", + "session_2", + "session_8", + "session_5", + "session_6", + "session_22", + "session_15", + "session_9" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_25" + ], + "question": "When was Joanna's second movie script shown on the big screens?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_5", + "session_27", + "session_3", + "session_28", + "session_29", + "session_15", + "session_17", + "session_4", + "session_10" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_11", + "session_25", + "session_26", + "session_4", + "session_7" + ], + "question": "What is Joanna inspired by?", + "recall": 0.8, + "retrieved_ids": [ + "session_7", + "session_4", + "session_11", + "session_26", + "session_22", + "session_9", + "session_17", + "session_15", + "session_6", + "session_8" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_26", + "session_5" + ], + "question": "What animal do both Nate and Joanna like?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_22", + "session_1", + "session_24", + "session_13", + "session_26", + "session_27", + "session_12", + "session_19", + "session_5" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_26" + ], + "question": "When did Joanna plan to go over to Nate's and share recipes?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_26", + "session_11", + "session_2", + "session_14", + "session_22", + "session_23", + "session_19", + "session_27", + "session_25" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_19", + "session_2", + "session_27", + "session_9" + ], + "question": "What things has Nate reccomended to Joanna?", + "recall": 0.8, + "retrieved_ids": [ + "session_2", + "session_27", + "session_19", + "session_10", + "session_8", + "session_15", + "session_3", + "session_13", + "session_23", + "session_16" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_15", + "session_27" + ], + "question": "What does Joanna do to remember happy memories?", + "recall": 0.5, + "retrieved_ids": [ + "session_27", + "session_24", + "session_13", + "session_9", + "session_29", + "session_1", + "session_22", + "session_26", + "session_23", + "session_8" + ], + "sample_id": "conv-42" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_27" + ], + "question": "What Console does Nate own?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_25", + "session_15", + "session_29", + "session_11", + "session_21", + "session_2", + "session_27", + "session_12", + "session_8" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_22", + "session_27" + ], + "question": "What mediums does Nate use to play games?", + "recall": 0.5, + "retrieved_ids": [ + "session_27", + "session_23", + "session_9", + "session_16", + "session_8", + "session_1", + "session_10", + "session_13", + "session_28", + "session_17" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_14", + "session_18" + ], + "question": "How many letters has Joanna recieved?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_18", + "session_27", + "session_14", + "session_21", + "session_1", + "session_25", + "session_28", + "session_15", + "session_4" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_10", + "session_23", + "session_27" + ], + "question": "What video games does Nate play?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_23", + "session_28", + "session_1", + "session_27", + "session_17", + "session_24", + "session_10", + "session_16", + "session_6" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_27" + ], + "question": "When did Nate win a big Valorant tourney?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_20", + "session_2", + "session_28", + "session_3", + "session_10", + "session_22", + "session_26", + "session_1", + "session_14" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_27" + ], + "question": "Which torunament did Nate win in the beginning of November 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_10", + "session_2", + "session_27", + "session_7", + "session_1", + "session_17", + "session_14", + "session_19", + "session_22" + ], + "sample_id": "conv-42" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_19", + "session_25", + "session_28", + "session_5" + ], + "question": "What alternative career might Nate consider after gaming?", + "recall": 0.5, + "retrieved_ids": [ + "session_9", + "session_19", + "session_16", + "session_21", + "session_5", + "session_27", + "session_8", + "session_12", + "session_22", + "session_13" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_12", + "session_28", + "session_8" + ], + "question": "What pets does Nate have?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_24", + "session_13", + "session_5", + "session_2", + "session_28", + "session_27", + "session_8", + "session_19", + "session_1" + ], + "sample_id": "conv-42" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_11", + "session_14", + "session_28", + "session_7" + ], + "question": "How many hikes has Joanna been on?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_11", + "session_27", + "session_10", + "session_24", + "session_28", + "session_20", + "session_8", + "session_14", + "session_7" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_28", + "session_8" + ], + "question": "How many turtles does Nate have?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_2", + "session_24", + "session_25", + "session_26", + "session_28", + "session_29", + "session_20", + "session_8", + "session_5" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_25", + "session_28" + ], + "question": "What activities does Nate do with his turtles?", + "recall": 0.5, + "retrieved_ids": [ + "session_8", + "session_26", + "session_15", + "session_2", + "session_25", + "session_5", + "session_18", + "session_9", + "session_22", + "session_12" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_11", + "session_28" + ], + "question": "What do both Joanna and Nate appreciate the beauty of?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_26", + "session_28", + "session_12", + "session_11", + "session_23", + "session_1", + "session_2", + "session_8", + "session_19" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_28" + ], + "question": "When did Joanna plan on going to Nate's to watch him play with his turtles?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_8", + "session_27", + "session_22", + "session_2", + "session_25", + "session_15", + "session_12", + "session_9", + "session_23" + ], + "sample_id": "conv-42" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_28" + ], + "question": "What state did Joanna visit in summer 2021?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_28", + "session_25", + "session_7", + "session_11", + "session_4", + "session_10", + "session_21", + "session_16", + "session_17" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_15", + "session_19", + "session_23", + "session_3" + ], + "question": "What recommendations has Nate received from Joanna?", + "recall": 0.8, + "retrieved_ids": [ + "session_1", + "session_3", + "session_9", + "session_22", + "session_19", + "session_2", + "session_12", + "session_4", + "session_15", + "session_14" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_21", + "session_3" + ], + "question": "What are Nate's favorite desserts?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_3", + "session_27", + "session_1", + "session_10", + "session_4", + "session_15", + "session_20", + "session_25", + "session_18" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_18", + "session_21" + ], + "question": "How has Nate tried to disburse his vegan ice-cream recipes?", + "recall": 0.5, + "retrieved_ids": [ + "session_21", + "session_20", + "session_16", + "session_10", + "session_22", + "session_28", + "session_15", + "session_8", + "session_4", + "session_29" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_10" + ], + "question": "When did Nate win his second tournament?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_26", + "session_2", + "session_19", + "session_1", + "session_15", + "session_5", + "session_22", + "session_9", + "session_14" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_10", + "session_14", + "session_17", + "session_19", + "session_20", + "session_22", + "session_27", + "session_6" + ], + "question": "How many video game tournaments has Nate participated in?", + "recall": 0.6667, + "retrieved_ids": [ + "session_6", + "session_1", + "session_24", + "session_22", + "session_26", + "session_28", + "session_17", + "session_14", + "session_9", + "session_20" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_12", + "session_2", + "session_4", + "session_5" + ], + "question": "How many screenplays has Joanna written?", + "recall": 0.5, + "retrieved_ids": [ + "session_6", + "session_2", + "session_16", + "session_10", + "session_4", + "session_11", + "session_15", + "session_13", + "session_3", + "session_14" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_10", + "session_14", + "session_17", + "session_19", + "session_22", + "session_27" + ], + "question": "How many tournaments has Nate won?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_1", + "session_10", + "session_22", + "session_19", + "session_27", + "session_14", + "session_17", + "session_28", + "session_24" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_19", + "session_20", + "session_21", + "session_22" + ], + "question": "What recipes has Joanna made?", + "recall": 0.6, + "retrieved_ids": [ + "session_20", + "session_11", + "session_10", + "session_21", + "session_24", + "session_4", + "session_29", + "session_16", + "session_18", + "session_8" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_3", + "session_4" + ], + "question": "What recipes has Nate made?", + "recall": 0.5, + "retrieved_ids": [ + "session_20", + "session_11", + "session_10", + "session_21", + "session_24", + "session_4", + "session_29", + "session_16", + "session_18", + "session_8" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_14", + "session_18", + "session_26" + ], + "question": "What are the skills that Nate has helped others learn?", + "recall": 0.6667, + "retrieved_ids": [ + "session_14", + "session_26", + "session_20", + "session_10", + "session_3", + "session_13", + "session_2", + "session_19", + "session_9", + "session_17" + ], + "sample_id": "conv-42" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_20", + "session_21" + ], + "question": "Was the first half of September 2022 a good month career-wise for Nate and Joanna? Answer yes or no.", + "recall": 0.5, + "retrieved_ids": [ + "session_22", + "session_14", + "session_9", + "session_16", + "session_1", + "session_4", + "session_21", + "session_3", + "session_27", + "session_19" + ], + "sample_id": "conv-42" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_29" + ], + "question": "What kind of job is Joanna beginning to preform the duties of because of her movie scripts?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_9", + "session_5", + "session_26", + "session_17", + "session_1", + "session_24", + "session_29", + "session_15", + "session_7" + ], + "sample_id": "conv-42" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_29" + ], + "question": "When did Nate take his turtles to the beach?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_28", + "session_26", + "session_5", + "session_15", + "session_24", + "session_2", + "session_9", + "session_25", + "session_22" + ], + "sample_id": "conv-42" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_29" + ], + "question": "What state did Nate visit?", + "recall": 0.0, + "retrieved_ids": [ + "session_3", + "session_6", + "session_7", + "session_11", + "session_4", + "session_10", + "session_21", + "session_25", + "session_16", + "session_17" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_1" + ], + "question": "What is one of Joanna's favorite movies?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_3", + "session_9", + "session_23", + "session_25", + "session_21", + "session_27", + "session_19", + "session_15", + "session_26" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What color did Nate choose for his hair?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_29", + "session_15", + "session_21", + "session_5", + "session_2", + "session_19", + "session_9", + "session_22", + "session_12" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_9" + ], + "question": "What is Nate's favorite movie trilogy?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_1", + "session_3", + "session_25", + "session_26", + "session_23", + "session_27", + "session_15", + "session_10", + "session_21" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_9" + ], + "question": "What is Nate's favorite book series about?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_19", + "session_27", + "session_25", + "session_21", + "session_22", + "session_17", + "session_15", + "session_6", + "session_8" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "What kind of lighting does Nate's gaming room have?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_14", + "session_19", + "session_15", + "session_10", + "session_6", + "session_4", + "session_5", + "session_20", + "session_18" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "What game was the second tournament that Nate won based on?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_1", + "session_14", + "session_22", + "session_27", + "session_17", + "session_19", + "session_6", + "session_24", + "session_5" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_12" + ], + "question": "What is Joanna's third screenplay about?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_13", + "session_11", + "session_4", + "session_10", + "session_25", + "session_3", + "session_16", + "session_12", + "session_15" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_27" + ], + "question": "What is Nate's favorite video game?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_26", + "session_23", + "session_9", + "session_27", + "session_15", + "session_21", + "session_3", + "session_25", + "session_19" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_1" + ], + "question": "What type of movies does Nate enjoy watching the most?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_22", + "session_3", + "session_29", + "session_25", + "session_23", + "session_27", + "session_19", + "session_2", + "session_16" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What did Joanna just finish last Friday on 23 January, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_27", + "session_23", + "session_16", + "session_17", + "session_22", + "session_14", + "session_21", + "session_13", + "session_12" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What genre is Joanna's first screenplay?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_4", + "session_3", + "session_14", + "session_2", + "session_11", + "session_10", + "session_9", + "session_16", + "session_6" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What are Joanna's plans for her finished screenplay in January 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_12", + "session_24", + "session_11", + "session_14", + "session_13", + "session_22", + "session_19", + "session_9", + "session_27" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "For how long has Nate had his turtles?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_28", + "session_15", + "session_24", + "session_9", + "session_25", + "session_29", + "session_22", + "session_26", + "session_5" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "What did Nate think of the coconut milk ice cream he made?", + "recall": 0.0, + "retrieved_ids": [ + "session_29", + "session_4", + "session_21", + "session_8", + "session_28", + "session_10", + "session_16", + "session_13", + "session_27", + "session_12" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "Which dairy-free dessert flavors does Nate enjoy?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_3", + "session_20", + "session_10", + "session_21", + "session_22", + "session_29", + "session_18", + "session_8", + "session_26" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "What did Joanna recently watch and recommend to Nate on February 7, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_3", + "session_5", + "session_10", + "session_23", + "session_1", + "session_15", + "session_24", + "session_19", + "session_9" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "What is \"Little Women\" about according to Joanna?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_22", + "session_5", + "session_13", + "session_2", + "session_23", + "session_27", + "session_19", + "session_15", + "session_8" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What flavor of ice cream did Nate make for his friend on 25 February, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_29", + "session_28", + "session_16", + "session_8", + "session_20", + "session_21", + "session_15", + "session_22", + "session_3" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What inspired Joanna's new screenplay on 25 February, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_11", + "session_10", + "session_13", + "session_3", + "session_14", + "session_2", + "session_26", + "session_16", + "session_15" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_5" + ], + "question": "Why does Nate like turtles as pets?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_8", + "session_2", + "session_28", + "session_27", + "session_5", + "session_19", + "session_12", + "session_13", + "session_25" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_5" + ], + "question": "How does Nate describe the process of taking care of turtles?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_26", + "session_10", + "session_19", + "session_9", + "session_7", + "session_16", + "session_6", + "session_25", + "session_3" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_6" + ], + "question": "What was Joanna's audition for?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_3", + "session_1", + "session_15", + "session_4", + "session_14", + "session_21", + "session_25", + "session_2", + "session_28" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "Why did Nate choose the hair color he did?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_29", + "session_21", + "session_19", + "session_5", + "session_28", + "session_17", + "session_12", + "session_13", + "session_27" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What are the main ingredients of the ice cream recipe shared by Nate?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_16", + "session_29", + "session_21", + "session_8", + "session_20", + "session_28", + "session_1", + "session_10", + "session_27" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_9" + ], + "question": "What is Joanna's project called in the writers group?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_1", + "session_19", + "session_6", + "session_7", + "session_11", + "session_21", + "session_23", + "session_27", + "session_16" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_9" + ], + "question": "What is Nate's favorite genre of movies?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_3", + "session_9", + "session_23", + "session_15", + "session_25", + "session_21", + "session_26", + "session_27", + "session_6" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_9" + ], + "question": "What kind of books does Nate enjoy?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_19", + "session_17", + "session_4", + "session_22", + "session_6", + "session_8", + "session_20", + "session_15", + "session_18" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_9" + ], + "question": "What kind of films does Joanna enjoy?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_4", + "session_3", + "session_2", + "session_15", + "session_29", + "session_24", + "session_16", + "session_6", + "session_18" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_9" + ], + "question": "Which activity helps Nate escape and stimulates his imagination?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_27", + "session_18", + "session_8", + "session_25", + "session_23", + "session_15", + "session_2", + "session_12", + "session_22" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "What filling did Joanna use in the cake she made recently in May 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_26", + "session_22", + "session_24", + "session_20", + "session_3", + "session_29", + "session_21", + "session_23", + "session_12" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "What kind of frosting did Joanna use on the cake she made recently in May 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_20", + "session_22", + "session_24", + "session_3", + "session_26", + "session_17", + "session_18", + "session_29", + "session_21" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_11" + ], + "question": "What does Nate feel he could do when out in cool places like Whispering Falls?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_23", + "session_5", + "session_3", + "session_13", + "session_27", + "session_17", + "session_8", + "session_6", + "session_12" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_11" + ], + "question": "What creative activity does Nate joke about pursuing after being inspired by their hikes with Jo?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_28", + "session_15", + "session_18", + "session_14", + "session_8", + "session_12", + "session_19", + "session_7", + "session_6" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_11" + ], + "question": "Who invited Nate to join her on the trails sometime?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_9", + "session_20", + "session_23", + "session_14", + "session_24", + "session_5", + "session_8", + "session_19", + "session_28" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "What did Nate do for Joanna on 25 May, 2022?", + "recall": 0.0, + "retrieved_ids": [ + "session_14", + "session_17", + "session_7", + "session_11", + "session_4", + "session_10", + "session_21", + "session_25", + "session_16", + "session_6" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "How does Nate describe the stuffed animal he got for Joanna?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_24", + "session_2", + "session_27", + "session_5", + "session_22", + "session_12", + "session_19", + "session_28", + "session_17" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_14" + ], + "question": "What event is Nate organizing in June 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_26", + "session_1", + "session_7", + "session_4", + "session_21", + "session_13", + "session_2", + "session_22", + "session_17" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_14" + ], + "question": "Who did Nate plan to invite to his gaming party in June 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_23", + "session_16", + "session_11", + "session_19", + "session_9", + "session_15", + "session_22", + "session_2", + "session_25" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_14" + ], + "question": "What special items did Nate get for everyone at his gaming party?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_11", + "session_15", + "session_9", + "session_10", + "session_5", + "session_16", + "session_22", + "session_2", + "session_20" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "What did Joanna write yesterday that appeared on the big screen?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_25", + "session_27", + "session_28", + "session_3", + "session_4", + "session_5", + "session_22", + "session_29", + "session_6" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "What superhero is Joanna a fan of?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_4", + "session_28", + "session_27", + "session_8", + "session_2", + "session_1", + "session_21", + "session_14", + "session_25" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "Which superhero toy figure does Nate share a photo of?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_28", + "session_13", + "session_11", + "session_4", + "session_18", + "session_22", + "session_9", + "session_21", + "session_25" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "What is displayed on Joanna's cork board for motivation and creativity?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_9", + "session_5", + "session_12", + "session_18", + "session_11", + "session_10", + "session_20", + "session_26", + "session_6" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "What does the photo on Joanna's cork board remind her of?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_9", + "session_13", + "session_18", + "session_7", + "session_11", + "session_24", + "session_29", + "session_25", + "session_28" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "What did Nate make and share with his vegan diet group?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_20", + "session_21", + "session_13", + "session_9", + "session_25", + "session_22", + "session_19", + "session_15", + "session_2" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "How many people attended the gaming party hosted by Nate in June 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_14", + "session_24", + "session_6", + "session_19", + "session_28", + "session_23", + "session_21", + "session_2", + "session_17" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "What recipe Nate offer to share with Joanna?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_21", + "session_28", + "session_29", + "session_18", + "session_4", + "session_16", + "session_11", + "session_8", + "session_10" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "What did Joanna plan to do with the recipe Nate promised to share?", + "recall": 0.0, + "retrieved_ids": [ + "session_22", + "session_28", + "session_14", + "session_27", + "session_11", + "session_2", + "session_23", + "session_19", + "session_25", + "session_17" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "How many video game tournaments has Nate won by July 10, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_1", + "session_27", + "session_22", + "session_17", + "session_14", + "session_10", + "session_24", + "session_26", + "session_23" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "Where did Joanna go for a road trip for research?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_4", + "session_7", + "session_12", + "session_5", + "session_16", + "session_29", + "session_6", + "session_10", + "session_21" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What did Joanna discover at the library in Woodhaven?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_3", + "session_6", + "session_7", + "session_11", + "session_4", + "session_10", + "session_21", + "session_25", + "session_16" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What specific themes are explored in Joanna's new book?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_17", + "session_9", + "session_19", + "session_26", + "session_2", + "session_6", + "session_8", + "session_12", + "session_1" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What inspired Joanna's new script in July 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_28", + "session_4", + "session_17", + "session_29", + "session_26", + "session_25", + "session_27", + "session_5", + "session_15" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What did Nate do while Joanna was on her road trip?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_4", + "session_7", + "session_24", + "session_10", + "session_8", + "session_9", + "session_14", + "session_27", + "session_29" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What does Nate do that he loves and can make money from?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_4", + "session_17", + "session_12", + "session_20", + "session_13", + "session_23", + "session_27", + "session_16", + "session_18" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "How did Joanna feel when someone wrote her a letter after reading her blog post?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_19", + "session_8", + "session_14", + "session_6", + "session_25", + "session_27", + "session_17", + "session_24", + "session_9" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What kind of impact does Joanna hope to have with her writing?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_4", + "session_9", + "session_27", + "session_18", + "session_12", + "session_24", + "session_5", + "session_26", + "session_16" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What kind of content did Joanna share that someone wrote her a letter about?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_5", + "session_14", + "session_9", + "session_22", + "session_27", + "session_28", + "session_6", + "session_24", + "session_15" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What motivates Joanna to keep writing even on tough days?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_18", + "session_12", + "session_24", + "session_9", + "session_26", + "session_13", + "session_15", + "session_4", + "session_5" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What type of ice cream does Joanna mention that Nate makes and is delicious?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_4", + "session_8", + "session_29", + "session_10", + "session_2", + "session_28", + "session_18", + "session_16", + "session_1" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "How did Nate feel about sharing his love for dairy-free desserts with Joanna?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_21", + "session_10", + "session_3", + "session_4", + "session_20", + "session_22", + "session_9", + "session_15", + "session_8" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "What did Joanna share with her writers group in August 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_21", + "session_14", + "session_13", + "session_19", + "session_24", + "session_16", + "session_6", + "session_3", + "session_25" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "How did Joanna celebrate after sharing her book with her writers group?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_9", + "session_8", + "session_21", + "session_29", + "session_16", + "session_24", + "session_6", + "session_22", + "session_17" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "How did Nate celebrate winning the international tournament?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_26", + "session_1", + "session_10", + "session_22", + "session_27", + "session_6", + "session_17", + "session_14", + "session_2" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_20" + ], + "question": "Why is Joanna experimenting with dairy-free options in her dessert recipes?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_10", + "session_21", + "session_3", + "session_7", + "session_4", + "session_18", + "session_24", + "session_9", + "session_8" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_20" + ], + "question": "What substitution does Nate suggest for butter in dairy-free baking?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_10", + "session_21", + "session_18", + "session_1", + "session_4", + "session_3", + "session_22", + "session_8", + "session_26" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_21" + ], + "question": "What type of show did Nate host where he taught vegan ice cream recipes?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_29", + "session_16", + "session_4", + "session_8", + "session_1", + "session_13", + "session_9", + "session_27", + "session_12" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_21" + ], + "question": "What is Nate's favorite dish from the cooking show he hosted?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_28", + "session_15", + "session_3", + "session_10", + "session_4", + "session_27", + "session_20", + "session_13", + "session_12" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_21" + ], + "question": "What is one of Nate's favorite dairy-free treats besides coconut milk ice cream?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_4", + "session_8", + "session_29", + "session_20", + "session_3", + "session_26", + "session_10", + "session_16", + "session_18" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_21" + ], + "question": "What dessert did Joanna share a photo of that has an almond flour crust, chocolate ganache, and fresh raspberries?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_20", + "session_18", + "session_4", + "session_3", + "session_11", + "session_10", + "session_15", + "session_9", + "session_25" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_21" + ], + "question": "What kind of cake did Joanna share a photo of that she likes making for birthdays and special days?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_18", + "session_9", + "session_25", + "session_29", + "session_15", + "session_22", + "session_20", + "session_11", + "session_24" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_21" + ], + "question": "What two main ingredients are part of the dessert Joanna shared a photo of with blueberries, coconut milk, and a gluten-free crust?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_4", + "session_20", + "session_8", + "session_3", + "session_18", + "session_26", + "session_25", + "session_10", + "session_1" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_22" + ], + "question": "What movie did Nate recently watch and enjoy on October 6, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_3", + "session_27", + "session_1", + "session_10", + "session_23", + "session_25", + "session_5", + "session_17", + "session_24" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_22" + ], + "question": "What did Joanna make for one of the ladies at her writing club?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_9", + "session_17", + "session_4", + "session_25", + "session_28", + "session_19", + "session_24", + "session_26", + "session_6" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "What game did Nate play at the game convention he attended on 9 October, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_17", + "session_16", + "session_10", + "session_27", + "session_28", + "session_13", + "session_9", + "session_1", + "session_12" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "What movie has Nate recently seen that blew his mind?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_3", + "session_27", + "session_1", + "session_24", + "session_4", + "session_15", + "session_22", + "session_9", + "session_2" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "What game has Nate been playing nonstop with a futuristic setting and gameplay on October 9, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_13", + "session_1", + "session_10", + "session_27", + "session_9", + "session_26", + "session_28", + "session_29", + "session_16" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "What did Nate share a photo of when mentioning unwinding at home?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_28", + "session_9", + "session_13", + "session_11", + "session_15", + "session_25", + "session_27", + "session_19", + "session_2" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "How did Joanna describe the classic movie he watched?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_23", + "session_25", + "session_27", + "session_28", + "session_3", + "session_29", + "session_22", + "session_12", + "session_13" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "What does Joanna recommend to make a living room comfy like hers?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_19", + "session_22", + "session_10", + "session_15", + "session_9", + "session_3", + "session_24", + "session_18", + "session_1" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "What helps Joanna stay focused and brings her joy?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_13", + "session_2", + "session_27", + "session_5", + "session_9", + "session_20", + "session_28", + "session_12", + "session_6" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "What does Joanna do while she writes?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_24", + "session_8", + "session_7", + "session_29", + "session_10", + "session_4", + "session_14", + "session_27", + "session_17" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "Why did Joanna name the stuffed animal dog Tilly?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_13", + "session_27", + "session_25", + "session_5", + "session_16", + "session_2", + "session_7", + "session_11", + "session_17" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "What does Joanna do after receiving a rejection from a production company?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_14", + "session_5", + "session_29", + "session_4", + "session_18", + "session_19", + "session_8", + "session_27", + "session_12" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "How does Nate feel about Joanna's ability to bounce back from setbacks?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_10", + "session_28", + "session_13", + "session_22", + "session_27", + "session_20", + "session_4", + "session_9", + "session_3" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "What encouragement does Nate give to Joanna after her setback?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_14", + "session_28", + "session_29", + "session_9", + "session_5", + "session_22", + "session_27", + "session_20", + "session_18" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "What does Nate rely on for cheer and joy?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_2", + "session_25", + "session_7", + "session_29", + "session_13", + "session_21", + "session_5", + "session_28", + "session_12" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "What does Joanna use to remember her dog from Michigan?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_13", + "session_9", + "session_22", + "session_26", + "session_27", + "session_14", + "session_25", + "session_8", + "session_15" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_25" + ], + "question": "What did Joanna contribute to that was shown on the big screen on the Sunday before October 25, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_27", + "session_7", + "session_15", + "session_3", + "session_5", + "session_28", + "session_4", + "session_22", + "session_2" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_25" + ], + "question": "What inspires Joanna to create drawings of her characters?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_9", + "session_2", + "session_26", + "session_4", + "session_24", + "session_28", + "session_27", + "session_19", + "session_15" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_25" + ], + "question": "Where does Joanna get her ideas for the characters from?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_9", + "session_25", + "session_27", + "session_24", + "session_4", + "session_19", + "session_5", + "session_28", + "session_1" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_25" + ], + "question": "How did Joanna feel on October 25, 2022 about seeing her characters come alive on the big screen?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_27", + "session_15", + "session_5", + "session_2", + "session_29", + "session_4", + "session_9", + "session_3", + "session_24" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_25" + ], + "question": "What type of diet do Nate's turtles have?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_25", + "session_8", + "session_1", + "session_29", + "session_24", + "session_5", + "session_26", + "session_28", + "session_16" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_26" + ], + "question": "What ingredient did Nate use to make the ice cream lactose-free?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_16", + "session_4", + "session_29", + "session_26", + "session_28", + "session_8", + "session_10", + "session_20", + "session_18" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_26" + ], + "question": "What did Joanna find in old notebooks last week that prompted her to reflect on her progress as a writer?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_21", + "session_26", + "session_24", + "session_14", + "session_17", + "session_19", + "session_4", + "session_28", + "session_20" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_27" + ], + "question": "What game is Nate currently playing and recommends to others on November 7, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_27", + "session_6", + "session_23", + "session_3", + "session_1", + "session_22", + "session_15", + "session_9", + "session_19" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_27" + ], + "question": "What did Joanna receive from her brother that brought back childhood memories?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_10", + "session_24", + "session_9", + "session_12", + "session_1", + "session_22", + "session_28", + "session_18", + "session_19" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_27" + ], + "question": "What is the type of game \"Xenoblade Chronicles\" that Nate is playing?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_27", + "session_10", + "session_23", + "session_9", + "session_8", + "session_28", + "session_13", + "session_17", + "session_16" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_28" + ], + "question": "What dish did Nate make on 9 November, 2022?", + "recall": 0.0, + "retrieved_ids": [ + "session_21", + "session_26", + "session_4", + "session_10", + "session_13", + "session_25", + "session_16", + "session_6", + "session_17", + "session_19" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_28" + ], + "question": "What project is Joanna working on in her notebook on November 9, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_10", + "session_9", + "session_26", + "session_28", + "session_24", + "session_1", + "session_6", + "session_7", + "session_18" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_28" + ], + "question": "What is Nate creating for YouTube on 9 November, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_27", + "session_22", + "session_7", + "session_1", + "session_4", + "session_2", + "session_25", + "session_14", + "session_21" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_28" + ], + "question": "What inspired Nate to start making gaming videos?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_9", + "session_27", + "session_17", + "session_6", + "session_11", + "session_4", + "session_18", + "session_15", + "session_2" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_28" + ], + "question": "What new content is Nate creating for YouTube?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_22", + "session_27", + "session_11", + "session_7", + "session_10", + "session_3", + "session_2", + "session_23", + "session_25" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_28" + ], + "question": "What advice does Joanna give to Nate about making YouTube videos?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_21", + "session_9", + "session_14", + "session_3", + "session_22", + "session_1", + "session_29", + "session_6", + "session_16" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_28" + ], + "question": "What did Joanna take a picture of near Fort Wayne last summer?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_11", + "session_6", + "session_7", + "session_15", + "session_4", + "session_27", + "session_16", + "session_19", + "session_10" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_28" + ], + "question": "What inspired Joanna to take a picture of the sunset in the field near Fort Wayne?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_7", + "session_11", + "session_8", + "session_4", + "session_26", + "session_15", + "session_22", + "session_27", + "session_17" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_28" + ], + "question": "Why did Nate get a third turtle?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_12", + "session_25", + "session_2", + "session_29", + "session_24", + "session_8", + "session_26", + "session_5", + "session_20" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_28" + ], + "question": "What does Nate want to do when he goes over to Joanna's place?", + "recall": 0.0, + "retrieved_ids": [ + "session_23", + "session_7", + "session_26", + "session_6", + "session_13", + "session_11", + "session_12", + "session_18", + "session_17", + "session_27" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_29" + ], + "question": "What did Nate take to the beach in Tampa?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_3", + "session_7", + "session_4", + "session_11", + "session_26", + "session_16", + "session_6", + "session_19", + "session_10" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_29" + ], + "question": "What does Nate love most about having turtles?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_2", + "session_22", + "session_25", + "session_28", + "session_26", + "session_5", + "session_16", + "session_24", + "session_19" + ], + "sample_id": "conv-42" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_29" + ], + "question": "What did Nate share a photo of as a part of his experimentation in November 2022?", + "recall": 0.0, + "retrieved_ids": [ + "session_25", + "session_28", + "session_15", + "session_20", + "session_26", + "session_18", + "session_9", + "session_11", + "session_22", + "session_2" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_7" + ], + "question": "What color did Joanna choose for her hair?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_21", + "session_24", + "session_29", + "session_9", + "session_5", + "session_28", + "session_17", + "session_12", + "session_19" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_9" + ], + "question": "What is Joanna's favorite movie trilogy?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_1", + "session_3", + "session_25", + "session_26", + "session_23", + "session_27", + "session_15", + "session_10", + "session_21" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_9" + ], + "question": "What is Joanna's favorite book series about?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_19", + "session_27", + "session_25", + "session_21", + "session_22", + "session_17", + "session_15", + "session_6", + "session_8" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_10" + ], + "question": "What kind of lighting does Joanna's gaming room have?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_14", + "session_19", + "session_15", + "session_10", + "session_6", + "session_4", + "session_5", + "session_20", + "session_18" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_10" + ], + "question": "What game was the second tournament that Joanna won based on?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_1", + "session_14", + "session_22", + "session_27", + "session_17", + "session_6", + "session_19", + "session_24", + "session_5" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_12" + ], + "question": "What is Nate's third screenplay about?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_13", + "session_4", + "session_11", + "session_10", + "session_25", + "session_3", + "session_16", + "session_12", + "session_15" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_1" + ], + "question": "What type of movies does Nate hate watching the most?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_3", + "session_22", + "session_23", + "session_29", + "session_25", + "session_2", + "session_27", + "session_16", + "session_19" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_2" + ], + "question": "What genre is Joanna's first novella?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_28", + "session_4", + "session_9", + "session_3", + "session_14", + "session_25", + "session_2", + "session_17", + "session_6" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_2" + ], + "question": "What are Nate's plans for his finished screenplay in January 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_15", + "session_22", + "session_11", + "session_14", + "session_12", + "session_19", + "session_13", + "session_27", + "session_9" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_2" + ], + "question": "For how long has Nate had his snakes?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_20", + "session_1", + "session_6", + "session_9", + "session_18", + "session_2", + "session_16", + "session_22", + "session_19" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_3" + ], + "question": "What did Nate think of the caramel ice cream he made?", + "recall": 0.0, + "retrieved_ids": [ + "session_21", + "session_29", + "session_4", + "session_10", + "session_28", + "session_16", + "session_8", + "session_13", + "session_27", + "session_12" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What flavor of cake did Nate make for his friend on 25 February, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_29", + "session_4", + "session_10", + "session_20", + "session_15", + "session_22", + "session_21", + "session_9", + "session_2" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_6" + ], + "question": "What was Nate's audition for?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_17", + "session_1", + "session_14", + "session_7", + "session_2", + "session_4", + "session_21", + "session_22", + "session_13" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_7" + ], + "question": "Why did Joanna choose the hair color she did?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_29", + "session_17", + "session_24", + "session_22", + "session_21", + "session_19", + "session_5", + "session_28", + "session_12" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_8" + ], + "question": "What are the main ingredients of the ice cream recipe shared by Joanna?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_16", + "session_29", + "session_21", + "session_8", + "session_20", + "session_28", + "session_1", + "session_10", + "session_27" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_9" + ], + "question": "What is Nate's project called in the writers group?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_1", + "session_6", + "session_19", + "session_7", + "session_11", + "session_21", + "session_23", + "session_27", + "session_16" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_9" + ], + "question": "Which activity helps Nate escape and numbs his mind?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_23", + "session_2", + "session_9", + "session_8", + "session_4", + "session_15", + "session_3", + "session_12", + "session_22" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_10" + ], + "question": "What filling did Nate use in the cake he made recently in May 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_10", + "session_12", + "session_20", + "session_3", + "session_26", + "session_21", + "session_23", + "session_27", + "session_13" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_14" + ], + "question": "Who did Joanna plan to invite to her gaming party in June 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_23", + "session_9", + "session_16", + "session_11", + "session_19", + "session_2", + "session_24", + "session_22", + "session_25" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_14" + ], + "question": "What special items did Joanna get for everyone at her gaming party?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_9", + "session_10", + "session_24", + "session_11", + "session_15", + "session_5", + "session_18", + "session_16", + "session_20" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_15" + ], + "question": "What supervillain is Joanna a fan of?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_28", + "session_27", + "session_8", + "session_2", + "session_1", + "session_15", + "session_14", + "session_21", + "session_25" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_15" + ], + "question": "Which superhero toy figure does Joanna share a photo of?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_13", + "session_28", + "session_11", + "session_4", + "session_18", + "session_22", + "session_9", + "session_21", + "session_25" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_16" + ], + "question": "What did Joanna make and share with her vegan diet group?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_21", + "session_23", + "session_9", + "session_20", + "session_13", + "session_25", + "session_6", + "session_19", + "session_24" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_16" + ], + "question": "How many people attended the gaming party hosted by Joanna in June 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_24", + "session_14", + "session_6", + "session_19", + "session_28", + "session_23", + "session_21", + "session_2", + "session_17" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_17" + ], + "question": "Where did Nate go for a road trip for research?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_4", + "session_7", + "session_12", + "session_5", + "session_16", + "session_29", + "session_6", + "session_10", + "session_21" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_17" + ], + "question": "What did Joanna discover at the museum in Woodhaven?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_3", + "session_6", + "session_7", + "session_11", + "session_4", + "session_10", + "session_21", + "session_25", + "session_16" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_17" + ], + "question": "What specific themes are explored in Nate's new book?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_17", + "session_9", + "session_19", + "session_26", + "session_2", + "session_6", + "session_8", + "session_12", + "session_1" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "How did Nate feel when someone wrote him a letter after reading his blog post?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_28", + "session_12", + "session_15", + "session_6", + "session_8", + "session_14", + "session_25", + "session_27", + "session_9" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "What kind of impact does Joanna hope to have with her painting?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_24", + "session_27", + "session_4", + "session_9", + "session_5", + "session_18", + "session_12", + "session_16", + "session_26" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_19" + ], + "question": "What did Nate share with his writers group in August 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_21", + "session_19", + "session_25", + "session_15", + "session_13", + "session_22", + "session_16", + "session_6", + "session_2" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_19" + ], + "question": "How did Nate celebrate after sharing his book with a writers group?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_6", + "session_9", + "session_8", + "session_16", + "session_21", + "session_22", + "session_15", + "session_17", + "session_2" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_19" + ], + "question": "How did Joanna celebrate winning the international tournament?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_26", + "session_1", + "session_10", + "session_6", + "session_22", + "session_27", + "session_17", + "session_14", + "session_2" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_20" + ], + "question": "What substitution does Nate suggest for sugar in dairy-free baking?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_10", + "session_20", + "session_21", + "session_18", + "session_1", + "session_4", + "session_3", + "session_22", + "session_26" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_21" + ], + "question": "What type of show did Joanna host where she taught vegan ice cream recipes?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_29", + "session_28", + "session_16", + "session_4", + "session_8", + "session_1", + "session_24", + "session_9", + "session_22" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_21" + ], + "question": "What is Joanna's favorite dish from the cooking show she hosted?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_28", + "session_29", + "session_15", + "session_3", + "session_24", + "session_10", + "session_4", + "session_22", + "session_20" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_21" + ], + "question": "What dessert did Nate share a photo of that has an almond flour crust, chocolate ganache, and fresh raspberries?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_20", + "session_18", + "session_4", + "session_3", + "session_11", + "session_10", + "session_15", + "session_9", + "session_25" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_21" + ], + "question": "What two main ingredients are part of the dessert Nate shared a photo of with blueberries, coconut milk, and a gluten-free crust?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_4", + "session_20", + "session_8", + "session_3", + "session_26", + "session_25", + "session_18", + "session_10", + "session_1" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_22" + ], + "question": "What movie did Joanna recently watch and enjoy on October 6, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_27", + "session_3", + "session_1", + "session_10", + "session_23", + "session_25", + "session_5", + "session_17", + "session_24" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_22" + ], + "question": "What did Nate make for one of the ladies at his writing club?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_28", + "session_4", + "session_9", + "session_25", + "session_19", + "session_2", + "session_26", + "session_6", + "session_15" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_23" + ], + "question": "What game has Joanna been playing nonstop with a futuristic setting and gameplay on October 9, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_13", + "session_1", + "session_10", + "session_27", + "session_9", + "session_26", + "session_28", + "session_29", + "session_16" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_23" + ], + "question": "How did Nate describe the classic movie he watched?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_23", + "session_25", + "session_27", + "session_28", + "session_3", + "session_29", + "session_22", + "session_12", + "session_13" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_23" + ], + "question": "What does Nate recommend to make a living room comfy like his?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_3", + "session_19", + "session_15", + "session_10", + "session_22", + "session_9", + "session_18", + "session_1", + "session_2" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_24" + ], + "question": "What helps Joanna stay distracted and brings her sadness?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_27", + "session_2", + "session_9", + "session_20", + "session_23", + "session_12", + "session_28", + "session_6", + "session_18" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_24" + ], + "question": "What does Nate do while he writes?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_7", + "session_10", + "session_4", + "session_24", + "session_14", + "session_27", + "session_17", + "session_12", + "session_13" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_24" + ], + "question": "What does Nate do after receiving a rejection from a production company?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_14", + "session_5", + "session_29", + "session_4", + "session_18", + "session_19", + "session_8", + "session_27", + "session_12" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_24" + ], + "question": "What does Joanna rely on for cheer and joy?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_2", + "session_25", + "session_7", + "session_29", + "session_13", + "session_21", + "session_5", + "session_28", + "session_12" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_24" + ], + "question": "What does Nate use to remember his dog from Michigan?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_27", + "session_13", + "session_22", + "session_26", + "session_15", + "session_9", + "session_25", + "session_8", + "session_2" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_25" + ], + "question": "What inspires Joanna to create music for her characters?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_25", + "session_2", + "session_26", + "session_4", + "session_24", + "session_28", + "session_27", + "session_19", + "session_15" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_25" + ], + "question": "What type of diet do Joanna's turtles have?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_25", + "session_8", + "session_1", + "session_29", + "session_24", + "session_26", + "session_5", + "session_28", + "session_16" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_26" + ], + "question": "What did Nate find in old notebooks last week that prompted him to reflect on her progress as a writer?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_21", + "session_26", + "session_20", + "session_24", + "session_14", + "session_28", + "session_4", + "session_12", + "session_19" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_27" + ], + "question": "What game is Joanna currently playing and recommends to others on November 7, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_27", + "session_6", + "session_23", + "session_3", + "session_1", + "session_22", + "session_15", + "session_9", + "session_19" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_27" + ], + "question": "What did Nate receive from his brother that brought back childhood memories?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_24", + "session_1", + "session_15", + "session_9", + "session_22", + "session_12", + "session_18", + "session_19", + "session_2" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_27" + ], + "question": "What is the type of game \"Xenoblade Chronicles\" that Joanna is playing?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_27", + "session_10", + "session_23", + "session_9", + "session_8", + "session_28", + "session_13", + "session_17", + "session_16" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_28" + ], + "question": "What project is Nate working on in his notebook on November 9, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_10", + "session_28", + "session_1", + "session_15", + "session_9", + "session_2", + "session_6", + "session_7", + "session_22" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_28" + ], + "question": "What is Joanna creating for YouTube on 9 November, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_3", + "session_27", + "session_1", + "session_15", + "session_4", + "session_2", + "session_25", + "session_14", + "session_21" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_28" + ], + "question": "What inspired Joanna to start making gaming videos?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_9", + "session_27", + "session_6", + "session_17", + "session_11", + "session_4", + "session_18", + "session_15", + "session_2" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_28" + ], + "question": "What new content is Nate creating for television?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_11", + "session_22", + "session_27", + "session_7", + "session_10", + "session_3", + "session_2", + "session_23", + "session_25" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_28" + ], + "question": "What did Nate take a picture of near Fort Wayne last summer?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_6", + "session_11", + "session_7", + "session_15", + "session_4", + "session_27", + "session_16", + "session_19", + "session_10" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_28" + ], + "question": "Why did Joanna get a third turtle?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_12", + "session_25", + "session_2", + "session_29", + "session_24", + "session_8", + "session_26", + "session_5", + "session_20" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_29" + ], + "question": "What did Joanna take to the beach in Tampa?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_6", + "session_7", + "session_4", + "session_11", + "session_3", + "session_26", + "session_16", + "session_19", + "session_10" + ], + "sample_id": "conv-42" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_29" + ], + "question": "What does Joanna love most about having turtles?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_2", + "session_28", + "session_22", + "session_25", + "session_26", + "session_5", + "session_16", + "session_24", + "session_19" + ], + "sample_id": "conv-42" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_11", + "session_6" + ], + "question": "what are John's goals with regards to his basketball career?", + "recall": 0.6667, + "retrieved_ids": [ + "session_11", + "session_6", + "session_19", + "session_23", + "session_7", + "session_12", + "session_21", + "session_27", + "session_5", + "session_20" + ], + "sample_id": "conv-43" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_11", + "session_6" + ], + "question": "What are John's goals for his career that are not related to his basketball skills?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_21", + "session_12", + "session_11", + "session_1", + "session_27", + "session_23", + "session_5", + "session_2", + "session_20" + ], + "sample_id": "conv-43" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_12", + "session_27" + ], + "question": "What items does John collect?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_27", + "session_24", + "session_1", + "session_2", + "session_20", + "session_8", + "session_26", + "session_9", + "session_28" + ], + "sample_id": "conv-43" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_1" + ], + "question": "Would Tim enjoy reading books by C. S. Lewis or John Greene?", + "recall": 0.0, + "retrieved_ids": [ + "session_19", + "session_4", + "session_14", + "session_22", + "session_2", + "session_11", + "session_3", + "session_26", + "session_16", + "session_7" + ], + "sample_id": "conv-43" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_11", + "session_2", + "session_20", + "session_22", + "session_26", + "session_6" + ], + "question": "What books has Tim read?", + "recall": 0.4286, + "retrieved_ids": [ + "session_22", + "session_5", + "session_14", + "session_19", + "session_11", + "session_26", + "session_15", + "session_4", + "session_28", + "session_27" + ], + "sample_id": "conv-43" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_2" + ], + "question": "Based on Tim's collections, what is a shop that he would enjoy visiting in New York city?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_2", + "session_1", + "session_11", + "session_29", + "session_16", + "session_26", + "session_27", + "session_12", + "session_3" + ], + "sample_id": "conv-43" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_3" + ], + "question": "In which month's game did John achieve a career-high score in points?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_21", + "session_23", + "session_5", + "session_27", + "session_11", + "session_20", + "session_26", + "session_9", + "session_6" + ], + "sample_id": "conv-43" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_14", + "session_3" + ], + "question": "Which geographical locations has Tim been to?", + "recall": 0.6667, + "retrieved_ids": [ + "session_25", + "session_9", + "session_1", + "session_4", + "session_18", + "session_21", + "session_10", + "session_24", + "session_14", + "session_6" + ], + "sample_id": "conv-43" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_25", + "session_3" + ], + "question": "Which outdoor gear company likely signed up John for an endorsement deal?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_3", + "session_29", + "session_21", + "session_11", + "session_1", + "session_2", + "session_16", + "session_7", + "session_23" + ], + "sample_id": "conv-43" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_25", + "session_29", + "session_3" + ], + "question": "Which endorsement deals has John been offered?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_29", + "session_3", + "session_25", + "session_11", + "session_1", + "session_2", + "session_4", + "session_23", + "session_24" + ], + "sample_id": "conv-43" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_3", + "session_5" + ], + "question": "When was John in Seattle for a game?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_8", + "session_23", + "session_6", + "session_24", + "session_28", + "session_13", + "session_5", + "session_16", + "session_12" + ], + "sample_id": "conv-43" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_2", + "session_3" + ], + "question": "What sports does John like besides basketball?", + "recall": 0.6667, + "retrieved_ids": [ + "session_16", + "session_26", + "session_3", + "session_23", + "session_6", + "session_15", + "session_5", + "session_2", + "session_28", + "session_9" + ], + "sample_id": "conv-43" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_3" + ], + "question": "What year did John start surfing?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_21", + "session_8", + "session_14", + "session_1", + "session_16", + "session_11", + "session_26", + "session_9", + "session_7" + ], + "sample_id": "conv-43" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_2", + "session_3" + ], + "question": "What does Tim do to escape reality?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_17", + "session_2", + "session_26", + "session_29", + "session_14", + "session_20", + "session_3", + "session_15", + "session_9" + ], + "sample_id": "conv-43" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_15", + "session_2", + "session_4" + ], + "question": "What kind of writing does Tim do?", + "recall": 0.6667, + "retrieved_ids": [ + "session_4", + "session_16", + "session_6", + "session_22", + "session_1", + "session_29", + "session_25", + "session_19", + "session_15", + "session_9" + ], + "sample_id": "conv-43" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_4" + ], + "question": "Who is Anthony?", + "recall": 1.0, + "retrieved_ids": [ + "session_4" + ], + "sample_id": "conv-43" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_3", + "session_5" + ], + "question": "After how many weeks did Tim reconnect with the fellow Harry Potter fan from California?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_5", + "session_11", + "session_13", + "session_27", + "session_1", + "session_26", + "session_2", + "session_6", + "session_17" + ], + "sample_id": "conv-43" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_22", + "session_23", + "session_24", + "session_3", + "session_5" + ], + "question": "How many games has John mentioned winning?", + "recall": 0.4, + "retrieved_ids": [ + "session_3", + "session_6", + "session_24", + "session_16", + "session_13", + "session_29", + "session_26", + "session_27", + "session_17", + "session_2" + ], + "sample_id": "conv-43" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_", + "session_1", + "session_2", + "session_20", + "session_26", + "session_4", + "session_5" + ], + "question": "What authors has Tim read books from?", + "recall": 0.4286, + "retrieved_ids": [ + "session_15", + "session_17", + "session_5", + "session_26", + "session_22", + "session_14", + "session_11", + "session_19", + "session_7", + "session_4" + ], + "sample_id": "conv-43" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_3", + "session_6" + ], + "question": "What is a prominent charity organization that John might want to work with and why?", + "recall": 0.5, + "retrieved_ids": [ + "session_6", + "session_11", + "session_15", + "session_28", + "session_4", + "session_29", + "session_22", + "session_16", + "session_21", + "session_26" + ], + "sample_id": "conv-43" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_3", + "session_5", + "session_6" + ], + "question": "Which city was John in before traveling to Chicago?", + "recall": 0.6667, + "retrieved_ids": [ + "session_6", + "session_27", + "session_9", + "session_11", + "session_10", + "session_29", + "session_14", + "session_21", + "session_3", + "session_26" + ], + "sample_id": "conv-43" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_3", + "session_6", + "session_9" + ], + "question": "Which US cities does John mention visiting to Tim?", + "recall": 0.6667, + "retrieved_ids": [ + "session_3", + "session_29", + "session_16", + "session_15", + "session_2", + "session_9", + "session_14", + "session_11", + "session_26", + "session_28" + ], + "sample_id": "conv-43" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_7" + ], + "question": "When did John meet with his teammates after returning from Chicago?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_12", + "session_6", + "session_7", + "session_3", + "session_1", + "session_5", + "session_27", + "session_22", + "session_20" + ], + "sample_id": "conv-43" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_7" + ], + "question": "When is Tim attending a book conference?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_14", + "session_11", + "session_19", + "session_13", + "session_5", + "session_22", + "session_27", + "session_26", + "session_4" + ], + "sample_id": "conv-43" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_6", + "session_7" + ], + "question": "Where was John between August 11 and August 15 2023?", + "recall": 0.0, + "retrieved_ids": [ + "session_22", + "session_15", + "session_13", + "session_20", + "session_3", + "session_28", + "session_1", + "session_24", + "session_26", + "session_27" + ], + "sample_id": "conv-43" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_16", + "session_7" + ], + "question": "What similar sports collectible do Tim and John own?", + "recall": 0.0, + "retrieved_ids": [ + "session_19", + "session_3", + "session_15", + "session_23", + "session_22", + "session_2", + "session_21", + "session_9", + "session_1", + "session_27" + ], + "sample_id": "conv-43" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_17", + "session_26" + ], + "question": "Which TV series does Tim mention watching?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_16", + "session_8", + "session_2", + "session_17", + "session_23", + "session_22", + "session_3", + "session_20", + "session_27" + ], + "sample_id": "conv-43" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_18" + ], + "question": "Which popular time management technique does Tim use to prepare for exams?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_9", + "session_24", + "session_18", + "session_29", + "session_5", + "session_14", + "session_12", + "session_3", + "session_15" + ], + "sample_id": "conv-43" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_8" + ], + "question": "Which popular music composer's tunes does Tim enjoy playing on the piano?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_21", + "session_28", + "session_20", + "session_12", + "session_11", + "session_19", + "session_2", + "session_22", + "session_29" + ], + "sample_id": "conv-43" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_6", + "session_9" + ], + "question": "What schools did John play basketball in and how many years was he with his team during high school?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_6", + "session_12", + "session_8", + "session_9", + "session_14", + "session_23", + "session_16", + "session_20", + "session_5" + ], + "sample_id": "conv-43" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_27", + "session_3", + "session_6", + "session_9" + ], + "question": "Which cities has John been to?", + "recall": 0.5, + "retrieved_ids": [ + "session_9", + "session_29", + "session_14", + "session_11", + "session_3", + "session_25", + "session_1", + "session_4", + "session_24", + "session_21" + ], + "sample_id": "conv-43" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_10" + ], + "question": "What month did Tim plan on going to Universal Studios?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_27", + "session_11", + "session_26", + "session_8", + "session_28", + "session_7", + "session_1", + "session_3", + "session_6" + ], + "sample_id": "conv-43" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_10" + ], + "question": "Which US states might Tim be in during September 2023 based on his plans of visiting Universal Studios?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_8", + "session_1", + "session_16", + "session_26", + "session_12", + "session_19", + "session_27", + "session_20", + "session_5" + ], + "sample_id": "conv-43" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_11" + ], + "question": "When does John plan on traveling with his team on a team trip?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_27", + "session_9", + "session_21", + "session_12", + "session_26", + "session_10", + "session_5", + "session_20", + "session_29" + ], + "sample_id": "conv-43" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_11", + "session_26", + "session_27" + ], + "question": "What could John do after his basketball career?", + "recall": 0.6667, + "retrieved_ids": [ + "session_6", + "session_11", + "session_21", + "session_14", + "session_28", + "session_5", + "session_12", + "session_27", + "session_23", + "session_20" + ], + "sample_id": "conv-43" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_12", + "session_3" + ], + "question": "What outdoor activities does John enjoy?", + "recall": 0.5, + "retrieved_ids": [ + "session_15", + "session_25", + "session_4", + "session_28", + "session_14", + "session_20", + "session_12", + "session_11", + "session_8", + "session_24" + ], + "sample_id": "conv-43" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_12", + "session_16" + ], + "question": "Who is Tim and John's favorite basketball player?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_12", + "session_3", + "session_23", + "session_7", + "session_21", + "session_19", + "session_27", + "session_8", + "session_14" + ], + "sample_id": "conv-43" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_13" + ], + "question": "Which week did Tim visit the UK for the Harry Potter Conference?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_26", + "session_1", + "session_2", + "session_27", + "session_19", + "session_29", + "session_18", + "session_7", + "session_15" + ], + "sample_id": "conv-43" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_13", + "session_18" + ], + "question": "which country has Tim visited most frequently in his travels?", + "recall": 0.0, + "retrieved_ids": [ + "session_29", + "session_27", + "session_9", + "session_21", + "session_26", + "session_25", + "session_11", + "session_12", + "session_20", + "session_5" + ], + "sample_id": "conv-43" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_14" + ], + "question": "What year did Tim go to the Smoky Mountains?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_20", + "session_1", + "session_16", + "session_21", + "session_3", + "session_8", + "session_9", + "session_13", + "session_25" + ], + "sample_id": "conv-43" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_14" + ], + "question": "Has Tim been to North Carolina and/or Tennesee states in the US?", + "recall": 0.0, + "retrieved_ids": [], + "sample_id": "conv-43" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_15", + "session_16" + ], + "question": "What kind of fiction stories does Tim write?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_22", + "session_16", + "session_15", + "session_19", + "session_6", + "session_9", + "session_29", + "session_27", + "session_26" + ], + "sample_id": "conv-43" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_15" + ], + "question": "What has John cooked?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_13", + "session_1", + "session_24", + "session_27", + "session_20", + "session_26", + "session_3", + "session_10", + "session_28" + ], + "sample_id": "conv-43" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_12", + "session_16" + ], + "question": "What does John like about Lebron James?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_16", + "session_29", + "session_2", + "session_11", + "session_15", + "session_18", + "session_10", + "session_7", + "session_4" + ], + "sample_id": "conv-43" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_16" + ], + "question": "When did John and his wife go on a European vacation?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_12", + "session_17", + "session_24", + "session_27", + "session_29", + "session_25", + "session_5", + "session_13", + "session_20" + ], + "sample_id": "conv-43" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_18" + ], + "question": "Which country was Tim visiting in the second week of November?", + "recall": 0.0, + "retrieved_ids": [ + "session_29", + "session_16", + "session_1", + "session_20", + "session_3", + "session_15", + "session_2", + "session_28", + "session_9", + "session_21" + ], + "sample_id": "conv-43" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_18" + ], + "question": "Where was Tim in the week before 16 November 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_12", + "session_25", + "session_6", + "session_18", + "session_10", + "session_1", + "session_9", + "session_21", + "session_27" + ], + "sample_id": "conv-43" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_12" + ], + "question": "When did John get married at a greenhouse?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_1", + "session_6", + "session_19", + "session_25", + "session_13", + "session_29", + "session_14", + "session_3", + "session_8" + ], + "sample_id": "conv-43" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_18" + ], + "question": "When did John get an ankle injury in 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_8", + "session_1", + "session_6", + "session_18", + "session_25", + "session_13", + "session_29", + "session_14", + "session_3" + ], + "sample_id": "conv-43" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_18", + "session_19" + ], + "question": "How many times has John injured his ankle?", + "recall": 0.5, + "retrieved_ids": [ + "session_27", + "session_2", + "session_24", + "session_29", + "session_13", + "session_12", + "session_19", + "session_17", + "session_20", + "session_5" + ], + "sample_id": "conv-43" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_18", + "session_19" + ], + "question": "Which book was John reading during his recovery from an ankle injury?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_8", + "session_15", + "session_27", + "session_12", + "session_18", + "session_5", + "session_20", + "session_1", + "session_25" + ], + "sample_id": "conv-43" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_20" + ], + "question": "What kind of yoga for building core strength might John benefit from?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_16", + "session_25", + "session_8", + "session_7", + "session_22", + "session_11", + "session_5", + "session_28", + "session_6" + ], + "sample_id": "conv-43" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_20", + "session_8" + ], + "question": "What does John do to supplement his basketball training?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_12", + "session_8", + "session_6", + "session_1", + "session_25", + "session_29", + "session_11", + "session_5", + "session_20" + ], + "sample_id": "conv-43" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_20", + "session_8" + ], + "question": "What other exercises can help John with his basketball performance?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_8", + "session_24", + "session_11", + "session_6", + "session_23", + "session_25", + "session_5", + "session_12", + "session_20" + ], + "sample_id": "conv-43" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_20" + ], + "question": "When did John take a trip to the Rocky Mountains?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_14", + "session_11", + "session_18", + "session_10", + "session_27", + "session_6", + "session_17", + "session_9", + "session_7" + ], + "sample_id": "conv-43" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_1", + "session_21" + ], + "question": "When did John start playing professionally?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_11", + "session_7", + "session_8", + "session_6", + "session_23", + "session_1", + "session_27", + "session_3", + "session_26" + ], + "sample_id": "conv-43" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_21" + ], + "question": "When did Tim start playing the violin?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_11", + "session_7", + "session_8", + "session_23", + "session_6", + "session_1", + "session_26", + "session_3", + "session_9" + ], + "sample_id": "conv-43" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_21", + "session_8" + ], + "question": "What instruments does Tim play?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_8", + "session_1", + "session_2", + "session_12", + "session_19", + "session_23", + "session_9", + "session_11", + "session_6" + ], + "sample_id": "conv-43" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_22", + "session_4" + ], + "question": "When did John attend the Harry Potter trivia?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_22", + "session_4", + "session_5", + "session_1", + "session_10", + "session_13", + "session_26", + "session_8", + "session_7" + ], + "sample_id": "conv-43" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_23", + "session_3" + ], + "question": "Which career-high performances did John achieve in 2023?", + "recall": 0.5, + "retrieved_ids": [ + "session_29", + "session_25", + "session_23", + "session_21", + "session_11", + "session_27", + "session_9", + "session_6", + "session_26", + "session_8" + ], + "sample_id": "conv-43" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_23" + ], + "question": "When did John achieve a career-high assist performance?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_21", + "session_13", + "session_8", + "session_26", + "session_11", + "session_6", + "session_25", + "session_9", + "session_27" + ], + "sample_id": "conv-43" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_11", + "session_17", + "session_19", + "session_22", + "session_4" + ], + "question": "What books has John read?", + "recall": 0.8, + "retrieved_ids": [ + "session_22", + "session_5", + "session_14", + "session_19", + "session_11", + "session_26", + "session_15", + "session_28", + "session_4", + "session_27" + ], + "sample_id": "conv-43" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_14", + "session_26" + ], + "question": "What does John do to share his knowledge?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_26", + "session_14", + "session_20", + "session_27", + "session_11", + "session_7", + "session_6", + "session_4", + "session_5" + ], + "sample_id": "conv-43" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_26" + ], + "question": "When did John organize a basketball camp for kids?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_6", + "session_22", + "session_23", + "session_11", + "session_19", + "session_28", + "session_16", + "session_8", + "session_20" + ], + "sample_id": "conv-43" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_27" + ], + "question": "Which month was John in Italy?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_28", + "session_10", + "session_7", + "session_21", + "session_3", + "session_11", + "session_26", + "session_24", + "session_20" + ], + "sample_id": "conv-43" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_26", + "session_27", + "session_8" + ], + "question": "What fantasy movies does Tim like?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_26", + "session_12", + "session_27", + "session_20", + "session_17", + "session_15", + "session_9", + "session_5", + "session_1" + ], + "sample_id": "conv-43" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_27" + ], + "question": "What is a Star Wars book that Tim might enjoy?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_19", + "session_14", + "session_22", + "session_16", + "session_4", + "session_11", + "session_20", + "session_15", + "session_2" + ], + "sample_id": "conv-43" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_15", + "session_27", + "session_4", + "session_6" + ], + "question": "What would be a good hobby related to his travel dreams for Tim to pick up?", + "recall": 0.5, + "retrieved_ids": [ + "session_11", + "session_26", + "session_6", + "session_2", + "session_27", + "session_9", + "session_1", + "session_12", + "session_21", + "session_5" + ], + "sample_id": "conv-43" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_28" + ], + "question": "What day did Tim get into his study abroad program?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_21", + "session_28", + "session_12", + "session_20", + "session_26", + "session_18", + "session_9", + "session_5", + "session_4" + ], + "sample_id": "conv-43" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_28" + ], + "question": "When will Tim leave for Ireland?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_11", + "session_27", + "session_18", + "session_10", + "session_12", + "session_20", + "session_3", + "session_26", + "session_17" + ], + "sample_id": "conv-43" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_1", + "session_27", + "session_28" + ], + "question": "Which Star Wars-related locations would Tim enjoy during his visit to Ireland?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_28", + "session_26", + "session_12", + "session_2", + "session_25", + "session_16", + "session_20", + "session_1", + "session_6" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_1" + ], + "question": "Which team did John sign with on 21 May, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_1", + "session_6", + "session_5", + "session_25", + "session_23", + "session_22", + "session_16", + "session_3", + "session_29" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_1" + ], + "question": "What is John's position on the team he signed with?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_7", + "session_6", + "session_16", + "session_14", + "session_12", + "session_3", + "session_29", + "session_27", + "session_18" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_1" + ], + "question": "What challenge did John encounter during pre-season training?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_25", + "session_8", + "session_19", + "session_18", + "session_29", + "session_14", + "session_27", + "session_28", + "session_13" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_1" + ], + "question": "What aspects of the Harry Potter universe will be discussed in John's fan project collaborations?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_5", + "session_27", + "session_11", + "session_26", + "session_3", + "session_19", + "session_8", + "session_2", + "session_10" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What forum did Tim join recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_21", + "session_29", + "session_12", + "session_27", + "session_19", + "session_4", + "session_17", + "session_10", + "session_22" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What kind of picture did Tim share as part of their Harry Potter book collection?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_27", + "session_26", + "session_1", + "session_22", + "session_7", + "session_15", + "session_20", + "session_9", + "session_25" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "What was the highest number of points John scored in a game recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_21", + "session_6", + "session_19", + "session_22", + "session_29", + "session_20", + "session_17", + "session_10", + "session_5" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "What did John celebrate at a restaurant with teammates?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_12", + "session_21", + "session_11", + "session_9", + "session_15", + "session_24", + "session_23", + "session_7", + "session_19" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "What kind of deals did John sign with Nike and Gatorade?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_2", + "session_29", + "session_1", + "session_25", + "session_16", + "session_21", + "session_7", + "session_6", + "session_23" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "Which city is John excited to have a game at?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_29", + "session_9", + "session_2", + "session_15", + "session_1", + "session_14", + "session_7", + "session_11", + "session_26" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "How long has John been surfing?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_16", + "session_24", + "session_9", + "session_14", + "session_21", + "session_15", + "session_5", + "session_22", + "session_28" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "How does John feel while surfing?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_23", + "session_11", + "session_19", + "session_15", + "session_29", + "session_28", + "session_25", + "session_24", + "session_27" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What kind of articles has Tim been writing about for the online magazine?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_25", + "session_16", + "session_6", + "session_1", + "session_29", + "session_19", + "session_15", + "session_9", + "session_13" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "Which two fantasy novels does Tim particularly enjoy writing about?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_12", + "session_4", + "session_15", + "session_19", + "session_7", + "session_28", + "session_9", + "session_6", + "session_26" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What did Anthony and John end up playing during the charity event?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_6", + "session_4", + "session_25", + "session_22", + "session_16", + "session_21", + "session_26", + "session_28", + "session_7" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_5" + ], + "question": "What did John share with the person he skyped about?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_12", + "session_11", + "session_19", + "session_26", + "session_27", + "session_7", + "session_6", + "session_5", + "session_16" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_5" + ], + "question": "How did John describe the team bond?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_5", + "session_22", + "session_9", + "session_17", + "session_25", + "session_1", + "session_23", + "session_13", + "session_3" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_6" + ], + "question": "How did John get introduced to basketball?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_3", + "session_6", + "session_19", + "session_8", + "session_22", + "session_7", + "session_16", + "session_28", + "session_23" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_6" + ], + "question": "What is John's number one goal in his basketball career?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_11", + "session_19", + "session_12", + "session_21", + "session_27", + "session_23", + "session_7", + "session_5", + "session_20" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_6" + ], + "question": "What organization is John teaming up with for his charity work?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_11", + "session_22", + "session_12", + "session_26", + "session_28", + "session_5", + "session_27", + "session_20", + "session_4" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "When did John meet back up with his teammates after his trip in August 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_7", + "session_19", + "session_21", + "session_12", + "session_3", + "session_27", + "session_1", + "session_5", + "session_20" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What did John's teammates give him when they met on Aug 15th?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_16", + "session_3", + "session_5", + "session_12", + "session_22", + "session_27", + "session_11", + "session_21", + "session_9" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "Why did John's teammates sign the basketball they gave him?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_16", + "session_12", + "session_3", + "session_15", + "session_27", + "session_8", + "session_1", + "session_10", + "session_25" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What is the main intention behind Tim wanting to attend the book conference?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_11", + "session_26", + "session_4", + "session_6", + "session_12", + "session_15", + "session_13", + "session_29", + "session_21" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What new activity has Tim started learning in August 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_27", + "session_15", + "session_8", + "session_22", + "session_11", + "session_26", + "session_7", + "session_3", + "session_24" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "Which movie's theme is Tim's favorite to play on the piano?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_27", + "session_12", + "session_20", + "session_21", + "session_26", + "session_15", + "session_17", + "session_1", + "session_4" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What special memory does \"Harry Potter and the Philosopher's Stone\" bring to Tim?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_2", + "session_22", + "session_3", + "session_13", + "session_12", + "session_15", + "session_17", + "session_28", + "session_25" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "Which movie does Tim mention they enjoy watching during Thanksgiving?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_16", + "session_12", + "session_26", + "session_20", + "session_15", + "session_1", + "session_3", + "session_27", + "session_17" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What tradition does Tim mention they love during Thanksgiving?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_16", + "session_26", + "session_15", + "session_1", + "session_25", + "session_24", + "session_14", + "session_3", + "session_18" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_9" + ], + "question": "How long did John and his high school basketball teammates play together?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_27", + "session_8", + "session_23", + "session_12", + "session_6", + "session_15", + "session_20", + "session_14", + "session_5" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_9" + ], + "question": "How was John's experience in New York City?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_1", + "session_29", + "session_14", + "session_11", + "session_3", + "session_6", + "session_26", + "session_21", + "session_10" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_9" + ], + "question": "What did John say about NYC, enticing Tim to visit?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_26", + "session_10", + "session_29", + "session_2", + "session_15", + "session_1", + "session_16", + "session_28", + "session_13" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "What kind of soup did John make recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_6", + "session_29", + "session_19", + "session_4", + "session_22", + "session_2", + "session_14", + "session_21", + "session_17" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "What spice did John add to the soup for flavor?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_1", + "session_29", + "session_22", + "session_15", + "session_8", + "session_28", + "session_23", + "session_7", + "session_13" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "What is Tim excited to see at Universal Studios?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_28", + "session_1", + "session_2", + "session_3", + "session_26", + "session_15", + "session_6", + "session_29", + "session_7" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_11" + ], + "question": "Where are John and his teammates planning to explore on a team trip?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_9", + "session_7", + "session_10", + "session_27", + "session_6", + "session_12", + "session_20", + "session_5", + "session_19" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_11" + ], + "question": "What city did Tim suggest to John for the team trip next month?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_14", + "session_7", + "session_3", + "session_26", + "session_10", + "session_9", + "session_28", + "session_6", + "session_29" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_11" + ], + "question": "What does John want to do after his basketball career?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_3", + "session_6", + "session_21", + "session_12", + "session_27", + "session_28", + "session_23", + "session_5", + "session_20" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_11" + ], + "question": "What advice did Tim give John about picking endorsements?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_2", + "session_21", + "session_29", + "session_14", + "session_12", + "session_3", + "session_19", + "session_17", + "session_7" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_11" + ], + "question": "What book recommendation did Tim give to John for the trip?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_20", + "session_5", + "session_16", + "session_4", + "session_17", + "session_28", + "session_22", + "session_13", + "session_29" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_12" + ], + "question": "What type of venue did John and his girlfriend choose for their wedding ceremony?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_15", + "session_2", + "session_27", + "session_1", + "session_14", + "session_26", + "session_5", + "session_19", + "session_20" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_12" + ], + "question": "What was the setting for John and his wife's first dance?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_27", + "session_16", + "session_22", + "session_21", + "session_8", + "session_10", + "session_24", + "session_5", + "session_17" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_12" + ], + "question": "Which basketball team does Tim support?", + "recall": 0.0, + "retrieved_ids": [ + "session_11", + "session_23", + "session_7", + "session_6", + "session_19", + "session_3", + "session_22", + "session_8", + "session_28", + "session_16" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "What passion does Tim mention connects him with people from all over the world?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_16", + "session_13", + "session_26", + "session_6", + "session_27", + "session_11", + "session_22", + "session_21", + "session_12" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "How does John describe the game season for his team?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_13", + "session_12", + "session_1", + "session_17", + "session_28", + "session_14", + "session_27", + "session_5", + "session_20" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "How does John say his team handles tough opponents?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_12", + "session_22", + "session_24", + "session_27", + "session_19", + "session_8", + "session_5", + "session_10", + "session_20" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "What motivates John's team to get better, according to John?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_13", + "session_7", + "session_21", + "session_11", + "session_26", + "session_15", + "session_16", + "session_20", + "session_18" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "What did John's team win at the end of the season?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_24", + "session_13", + "session_1", + "session_25", + "session_21", + "session_19", + "session_14", + "session_4", + "session_28" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_14" + ], + "question": "Where did Tim capture the photography of the sunset over the mountain range?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_11", + "session_13", + "session_20", + "session_8", + "session_10", + "session_5", + "session_22", + "session_16", + "session_21" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_14" + ], + "question": "How does John feel about being seen as a mentor by some of the younger players?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_23", + "session_21", + "session_19", + "session_11", + "session_25", + "session_16", + "session_12", + "session_26", + "session_17" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_14" + ], + "question": "What does John find rewarding about mentoring the younger players?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_2", + "session_21", + "session_16", + "session_12", + "session_19", + "session_29", + "session_27", + "session_4", + "session_23" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_14" + ], + "question": "What has John been able to help the younger players achieve?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_19", + "session_25", + "session_24", + "session_29", + "session_21", + "session_16", + "session_12", + "session_23", + "session_28" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "What genre is the novel that Tim is writing?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_15", + "session_6", + "session_7", + "session_16", + "session_26", + "session_19", + "session_28", + "session_9", + "session_21" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "Who is one of Tim's sources of inspiration for writing?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_16", + "session_19", + "session_6", + "session_27", + "session_26", + "session_4", + "session_22", + "session_7", + "session_21" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "What J.K. Rowling quote does Tim resonate with?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_1", + "session_12", + "session_20", + "session_3", + "session_27", + "session_26", + "session_28", + "session_18", + "session_10" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "What does John write on the whiteboard to help him stay motivated?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_7", + "session_27", + "session_19", + "session_18", + "session_13", + "session_16", + "session_4", + "session_12", + "session_6" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "What hobby is a therapy for John when away from the court?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_15", + "session_6", + "session_5", + "session_22", + "session_14", + "session_11", + "session_24", + "session_9", + "session_17" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "What type of meal does John often cook using a slow cooker?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_19", + "session_27", + "session_1", + "session_26", + "session_24", + "session_20", + "session_3", + "session_28", + "session_10" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "How will John share the honey garlic chicken recipe with the other person?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_12", + "session_10", + "session_14", + "session_19", + "session_11", + "session_26", + "session_7", + "session_27", + "session_16" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "What was Tim's huge writing issue last week,as mentioned on November 6, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_26", + "session_24", + "session_3", + "session_19", + "session_15", + "session_17", + "session_8", + "session_6", + "session_4" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "What does Tim have that serves as a reminder of hard work and is his prized possession?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_29", + "session_2", + "session_15", + "session_5", + "session_12", + "session_20", + "session_27", + "session_7", + "session_4" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "Why do Tim and John find LeBron inspiring?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_12", + "session_19", + "session_8", + "session_10", + "session_9", + "session_7", + "session_3", + "session_14", + "session_17" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "How did John describe the views during their road trip out on the European coastline?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_15", + "session_14", + "session_7", + "session_1", + "session_27", + "session_26", + "session_28", + "session_16", + "session_3" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What is one of Tim's favorite fantasy TV shows, as mentioned on November 11, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_16", + "session_3", + "session_20", + "session_15", + "session_17", + "session_27", + "session_19", + "session_28", + "session_7" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "How does Tim stay motivated during difficult study sessions?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_19", + "session_14", + "session_15", + "session_28", + "session_27", + "session_21", + "session_20", + "session_9", + "session_4" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What did Tim say about his injury on 16 November, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_12", + "session_27", + "session_19", + "session_8", + "session_4", + "session_18", + "session_16", + "session_5", + "session_20" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "What was the setback Tim faced in his writing project on 21 November, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_16", + "session_1", + "session_12", + "session_27", + "session_15", + "session_8", + "session_4", + "session_5", + "session_20" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "How did John overcome his ankle injury from last season?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_8", + "session_20", + "session_18", + "session_28", + "session_1", + "session_12", + "session_27", + "session_5", + "session_16" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "What motivated Tim to keep pushing himself to get better in writing and reading?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_26", + "session_16", + "session_15", + "session_13", + "session_20", + "session_6", + "session_21", + "session_11", + "session_4" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "How did John overcome a mistake he made during a big game in basketball?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_12", + "session_6", + "session_8", + "session_23", + "session_3", + "session_26", + "session_27", + "session_16", + "session_15" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "What book did John recently finish rereading that left him feeling inspired and hopeful about following dreams?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_29", + "session_16", + "session_11", + "session_27", + "session_6", + "session_17", + "session_22", + "session_12", + "session_23" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "How did \"The Alchemist\" impact John's perspective on following dreams?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_11", + "session_23", + "session_6", + "session_8", + "session_4", + "session_14", + "session_28", + "session_12", + "session_17" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_20" + ], + "question": "What is John trying out to improve his strength and flexibility after recovery from ankle injury?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_20", + "session_8", + "session_5", + "session_14", + "session_7", + "session_25", + "session_12", + "session_27", + "session_18" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_20" + ], + "question": "How long does John usually hold the yoga pose he shared with Tim?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_21", + "session_15", + "session_8", + "session_24", + "session_14", + "session_16", + "session_27", + "session_12", + "session_17" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_20" + ], + "question": "Where was the forest picture shared by John on December 1,2023 taken?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_20", + "session_4", + "session_11", + "session_2", + "session_7", + "session_26", + "session_27", + "session_6", + "session_12" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_21" + ], + "question": "What did Tim recently start learning in addition to being part of a travel club and working on studies?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_19", + "session_20", + "session_26", + "session_27", + "session_9", + "session_7", + "session_29", + "session_11", + "session_12" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_21" + ], + "question": "What instrument is Tim learning to play in December 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_8", + "session_15", + "session_27", + "session_12", + "session_19", + "session_20", + "session_16", + "session_26", + "session_7" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_21" + ], + "question": "How long has Tim been playing the piano for, as of December 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_21", + "session_16", + "session_15", + "session_24", + "session_9", + "session_14", + "session_5", + "session_22", + "session_28" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_22" + ], + "question": "What book did Tim just finish reading on 8th December, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_8", + "session_22", + "session_5", + "session_19", + "session_3", + "session_26", + "session_15", + "session_6", + "session_20" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_22" + ], + "question": "Which book did Tim recommend to John as a good story on 8th December, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_27", + "session_11", + "session_6", + "session_28", + "session_13", + "session_29", + "session_16", + "session_4", + "session_5" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "What is the topic of discussion between John and Tim on 11 December, 2023?", + "recall": 0.0, + "retrieved_ids": [], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "What kind of game did John have a career-high in assists in?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_4", + "session_25", + "session_13", + "session_6", + "session_9", + "session_21", + "session_27", + "session_11", + "session_26" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "What was John's way of dealing with doubts and stress when he was younger?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_8", + "session_14", + "session_9", + "session_25", + "session_16", + "session_27", + "session_20", + "session_3", + "session_12" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "How did John feel about the atmosphere during the big game against the rival team?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_12", + "session_19", + "session_24", + "session_21", + "session_6", + "session_3", + "session_16", + "session_7", + "session_28" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "How did John feel after being able to jog without pain?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_19", + "session_13", + "session_14", + "session_21", + "session_3", + "session_28", + "session_10", + "session_11", + "session_27" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_25" + ], + "question": "What kind of deal did John get in December?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_29", + "session_3", + "session_2", + "session_1", + "session_22", + "session_21", + "session_4", + "session_9", + "session_23" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_25" + ], + "question": "Where was the photoshoot done for John's gear deal?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_3", + "session_23", + "session_29", + "session_5", + "session_21", + "session_27", + "session_20", + "session_28", + "session_26" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_25" + ], + "question": "In which area has John's team seen the most growth during training?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_1", + "session_21", + "session_14", + "session_11", + "session_19", + "session_29", + "session_26", + "session_8", + "session_17" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_26" + ], + "question": "What type of seminars is John conducting?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_15", + "session_1", + "session_24", + "session_19", + "session_20", + "session_27", + "session_3", + "session_28", + "session_13" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_26" + ], + "question": "What activity did Tim do after reading the stories about the Himalayan trek?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_19", + "session_15", + "session_22", + "session_21", + "session_27", + "session_23", + "session_6", + "session_24", + "session_4" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_26" + ], + "question": "What is one cause that John supports with his influence and resources?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_27", + "session_15", + "session_23", + "session_5", + "session_12", + "session_20", + "session_11", + "session_7", + "session_2" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_26" + ], + "question": "What new fantasy TV series is Tim excited about?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_8", + "session_2", + "session_17", + "session_1", + "session_22", + "session_29", + "session_20", + "session_3", + "session_23" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_27" + ], + "question": "Which language is Tim learning?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_15", + "session_21", + "session_20", + "session_8", + "session_7", + "session_16", + "session_12", + "session_19", + "session_26" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_27" + ], + "question": "What language does Tim know besides German?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_11", + "session_5", + "session_29", + "session_22", + "session_21", + "session_16", + "session_15", + "session_28", + "session_9" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_27" + ], + "question": "What book did Tim get in Italy that inspired him to cook?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_11", + "session_15", + "session_12", + "session_22", + "session_14", + "session_16", + "session_19", + "session_6", + "session_10" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_27" + ], + "question": "What is John's favorite book series?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_20", + "session_8", + "session_17", + "session_7", + "session_22", + "session_12", + "session_27", + "session_15", + "session_23" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_27" + ], + "question": "According to John, who is his favorite character from Lord of the Rings?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_26", + "session_1", + "session_12", + "session_4", + "session_17", + "session_5", + "session_22", + "session_20", + "session_6" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_27" + ], + "question": "Why does John like Aragorn from Lord of the Rings?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_26", + "session_29", + "session_13", + "session_5", + "session_7", + "session_3", + "session_11", + "session_12", + "session_20" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_27" + ], + "question": "What kind of painting does John have in his room as a reminder?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_16", + "session_29", + "session_2", + "session_7", + "session_12", + "session_4", + "session_9", + "session_5", + "session_20" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_27" + ], + "question": "What is the painting of Aragorn a reminder for John to be in everything he does?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_29", + "session_16", + "session_2", + "session_20", + "session_3", + "session_15", + "session_17", + "session_7", + "session_12" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_27" + ], + "question": "What map does Tim show to his friend John?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_12", + "session_20", + "session_26", + "session_15", + "session_9", + "session_7", + "session_6", + "session_5", + "session_11" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_28" + ], + "question": "Where will Tim be going for a semester abroad?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_24", + "session_13", + "session_14", + "session_1", + "session_29", + "session_25", + "session_9", + "session_27", + "session_15" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_28" + ], + "question": "Which city in Ireland will Tim be staying in during his semester abroad?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_29", + "session_12", + "session_14", + "session_11", + "session_27", + "session_9", + "session_20", + "session_3", + "session_5" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_28" + ], + "question": "What charity event did John organize recently in 2024?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_22", + "session_26", + "session_10", + "session_28", + "session_29", + "session_11", + "session_19", + "session_4", + "session_7" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_29" + ], + "question": "What achievement did John share with Tim in January 2024?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_3", + "session_11", + "session_12", + "session_4", + "session_7", + "session_6", + "session_29", + "session_25", + "session_27" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_29" + ], + "question": "What was Johns's reaction to sealing the deal with the beverage company?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_21", + "session_25", + "session_3", + "session_11", + "session_27", + "session_20", + "session_26", + "session_23", + "session_28" + ], + "sample_id": "conv-43" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_29" + ], + "question": "Which city did John recommend to Tim in January 2024?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_29", + "session_28", + "session_9", + "session_13", + "session_14", + "session_22", + "session_16", + "session_3", + "session_4" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_1" + ], + "question": "Which team did Tim sign with on 21 May, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_1", + "session_6", + "session_5", + "session_25", + "session_23", + "session_22", + "session_16", + "session_3", + "session_29" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_1" + ], + "question": "What is Tim's position on the team he signed with?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_16", + "session_14", + "session_7", + "session_6", + "session_12", + "session_3", + "session_27", + "session_29", + "session_18" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_1" + ], + "question": "What challenge did Tim encounter during pre-season training?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_25", + "session_8", + "session_19", + "session_18", + "session_14", + "session_29", + "session_27", + "session_28", + "session_13" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_2" + ], + "question": "What cult did Tim join recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_2", + "session_29", + "session_12", + "session_27", + "session_19", + "session_14", + "session_17", + "session_10", + "session_22" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_3" + ], + "question": "What was the highest number of points Tim scored in a game recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_21", + "session_6", + "session_22", + "session_19", + "session_29", + "session_20", + "session_17", + "session_10", + "session_5" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_3" + ], + "question": "What did Tim celebrate at a restaurant with teammates?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_12", + "session_21", + "session_11", + "session_15", + "session_9", + "session_24", + "session_23", + "session_7", + "session_19" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_3" + ], + "question": "What kind of deals did Tim sign with Nike and Gatorade?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_2", + "session_21", + "session_29", + "session_1", + "session_25", + "session_16", + "session_7", + "session_6", + "session_23" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_3" + ], + "question": "How does Tim feel while surfing?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_23", + "session_11", + "session_19", + "session_15", + "session_29", + "session_28", + "session_25", + "session_24", + "session_27" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What kind of articles has John been writing about for the online magazine?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_25", + "session_16", + "session_6", + "session_1", + "session_29", + "session_19", + "session_15", + "session_9", + "session_13" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "Which two mystery novels does Tim particularly enjoy writing about?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_12", + "session_15", + "session_4", + "session_7", + "session_19", + "session_28", + "session_9", + "session_6", + "session_26" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What did Anthony and Tim end up playing during the charity event?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_4", + "session_6", + "session_25", + "session_22", + "session_16", + "session_21", + "session_26", + "session_28", + "session_7" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_6" + ], + "question": "How did Tim get introduced to basketball?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_3", + "session_6", + "session_19", + "session_8", + "session_22", + "session_7", + "session_16", + "session_28", + "session_23" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_6" + ], + "question": "What is Tim's number one goal in his basketball career?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_19", + "session_11", + "session_12", + "session_21", + "session_27", + "session_23", + "session_7", + "session_5", + "session_20" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_6" + ], + "question": "What organization is Tim teaming up with for his charity work?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_11", + "session_22", + "session_12", + "session_26", + "session_28", + "session_5", + "session_27", + "session_20", + "session_4" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_7" + ], + "question": "What did Tim's teammates give him when they met on Aug 15th?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_16", + "session_3", + "session_5", + "session_12", + "session_22", + "session_27", + "session_11", + "session_21", + "session_9" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_7" + ], + "question": "Why did John's teammates sign the football they gave him?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_16", + "session_15", + "session_1", + "session_12", + "session_3", + "session_27", + "session_8", + "session_10", + "session_25" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_7" + ], + "question": "What is the main intention behind John wanting to attend the book conference?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_11", + "session_4", + "session_26", + "session_6", + "session_12", + "session_15", + "session_13", + "session_29", + "session_21" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_8" + ], + "question": "What new activity has John started learning in August 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_22", + "session_21", + "session_15", + "session_8", + "session_11", + "session_26", + "session_7", + "session_3", + "session_24" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_8" + ], + "question": "What special memory does \"Fifty Shades of Grey\" bring to Tim?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_12", + "session_16", + "session_2", + "session_3", + "session_15", + "session_17", + "session_28", + "session_13", + "session_25" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_8" + ], + "question": "Which movie does John mention they enjoy watching during Thanksgiving?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_20", + "session_16", + "session_12", + "session_26", + "session_15", + "session_1", + "session_3", + "session_27", + "session_17" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_8" + ], + "question": "What tradition does Tim mention they love during Halloween?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_16", + "session_26", + "session_15", + "session_14", + "session_1", + "session_25", + "session_24", + "session_3", + "session_18" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_9" + ], + "question": "How long did Tim and his high school basketball teammates play together?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_27", + "session_8", + "session_23", + "session_12", + "session_6", + "session_15", + "session_20", + "session_14", + "session_5" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_9" + ], + "question": "How was Tim's experience in New York City?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_1", + "session_29", + "session_14", + "session_11", + "session_3", + "session_6", + "session_26", + "session_21", + "session_10" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_10" + ], + "question": "What spice did Tim add to the soup for flavor?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_1", + "session_29", + "session_22", + "session_15", + "session_8", + "session_13", + "session_28", + "session_7", + "session_25" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_10" + ], + "question": "What is Tim excited to see at Disneyland?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_2", + "session_3", + "session_26", + "session_15", + "session_28", + "session_6", + "session_10", + "session_29", + "session_7" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_11" + ], + "question": "Where are John and his teammates planning to avoid on a team trip?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_6", + "session_7", + "session_9", + "session_10", + "session_12", + "session_27", + "session_20", + "session_5", + "session_19" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_11" + ], + "question": "What does Tim want to do after his basketball career?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_3", + "session_6", + "session_21", + "session_12", + "session_27", + "session_28", + "session_23", + "session_5", + "session_20" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_12" + ], + "question": "What type of venue did John and his girlfriend choose for their breakup?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_15", + "session_2", + "session_27", + "session_1", + "session_14", + "session_26", + "session_5", + "session_19", + "session_20" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_13" + ], + "question": "What passion does John mention connects him with people from all over the world?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_16", + "session_13", + "session_26", + "session_6", + "session_27", + "session_11", + "session_22", + "session_21", + "session_12" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_13" + ], + "question": "How does Tim say his team handles tough opponents?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_12", + "session_22", + "session_24", + "session_27", + "session_19", + "session_8", + "session_5", + "session_10", + "session_20" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_14" + ], + "question": "Where did Tim capture the painting of the sunset over the mountain range?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_11", + "session_16", + "session_13", + "session_20", + "session_8", + "session_5", + "session_22", + "session_21", + "session_27" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_14" + ], + "question": "What does Tim find rewarding about mentoring the younger players?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_2", + "session_21", + "session_16", + "session_12", + "session_19", + "session_29", + "session_27", + "session_4", + "session_23" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_14" + ], + "question": "What has Tim been able to help the younger players achieve?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_19", + "session_25", + "session_24", + "session_29", + "session_21", + "session_16", + "session_12", + "session_23", + "session_28" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_15" + ], + "question": "What genre is the novel that John is writing?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_15", + "session_6", + "session_7", + "session_16", + "session_26", + "session_19", + "session_28", + "session_9", + "session_21" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_15" + ], + "question": "Who is one of Tim's sources of inspiration for painting?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_16", + "session_26", + "session_15", + "session_7", + "session_22", + "session_21", + "session_6", + "session_19", + "session_4" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_15" + ], + "question": "What does Tim write on the whiteboard to help him stay motivated?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_7", + "session_27", + "session_19", + "session_18", + "session_13", + "session_16", + "session_4", + "session_12", + "session_6" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_15" + ], + "question": "What hobby is a therapy for Tim when away from the court?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_15", + "session_6", + "session_5", + "session_22", + "session_14", + "session_11", + "session_24", + "session_9", + "session_17" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_15" + ], + "question": "What type of meal does Tim often cook using a slow cooker?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_26", + "session_19", + "session_27", + "session_12", + "session_20", + "session_3", + "session_28", + "session_18", + "session_10" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_15" + ], + "question": "How will Tim share the honey garlic chicken recipe with the other person?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_12", + "session_10", + "session_14", + "session_19", + "session_11", + "session_26", + "session_7", + "session_27", + "session_16" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_17" + ], + "question": "What is one of Tim's favorite crime TV shows, as mentioned on November 11, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_16", + "session_3", + "session_20", + "session_15", + "session_17", + "session_7", + "session_28", + "session_27", + "session_19" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_19" + ], + "question": "What was the setback Tim faced in his coding project on 21 November, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_13", + "session_1", + "session_12", + "session_24", + "session_27", + "session_26", + "session_8", + "session_5", + "session_20" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_19" + ], + "question": "How did Tim overcome his ankle injury from last season?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_8", + "session_20", + "session_18", + "session_28", + "session_1", + "session_12", + "session_27", + "session_5", + "session_16" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_19" + ], + "question": "What motivated John to keep pushing himself to get better in writing and reading?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_26", + "session_16", + "session_15", + "session_20", + "session_13", + "session_6", + "session_21", + "session_11", + "session_4" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_19" + ], + "question": "How did Tim overcome a mistake he made during a big game in basketball?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_12", + "session_6", + "session_8", + "session_23", + "session_3", + "session_26", + "session_27", + "session_16", + "session_15" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_20" + ], + "question": "What is Tim trying out to improve his strength and flexibility after recovery from ankle injury?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_20", + "session_8", + "session_14", + "session_5", + "session_7", + "session_25", + "session_12", + "session_27", + "session_18" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_21" + ], + "question": "What did John recently start learning in addition to being part of a travel club and working on studies?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_20", + "session_19", + "session_26", + "session_27", + "session_9", + "session_7", + "session_29", + "session_11", + "session_12" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_21" + ], + "question": "What instrument is John learning to play in December 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_8", + "session_15", + "session_27", + "session_12", + "session_19", + "session_20", + "session_16", + "session_26", + "session_7" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_21" + ], + "question": "How long has John been playing the piano for, as of December 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_21", + "session_16", + "session_15", + "session_24", + "session_9", + "session_14", + "session_5", + "session_22", + "session_28" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_22" + ], + "question": "What movie did Tim just finish watching on 8th December, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_26", + "session_8", + "session_20", + "session_1", + "session_22", + "session_12", + "session_19", + "session_27", + "session_15" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_23" + ], + "question": "What kind of game did Tim have a career-high in assists in?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_4", + "session_25", + "session_13", + "session_6", + "session_9", + "session_21", + "session_27", + "session_11", + "session_26" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_23" + ], + "question": "What was Tim's way of dealing with doubts and stress when he was younger?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_8", + "session_14", + "session_9", + "session_25", + "session_16", + "session_27", + "session_20", + "session_3", + "session_12" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_25" + ], + "question": "Where was the photoshoot done for John's fragrance deal?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_3", + "session_27", + "session_23", + "session_5", + "session_29", + "session_21", + "session_20", + "session_28", + "session_26" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_25" + ], + "question": "In which area has Tim's team seen the most growth during training?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_1", + "session_21", + "session_14", + "session_11", + "session_19", + "session_29", + "session_26", + "session_8", + "session_17" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_26" + ], + "question": "What type of seminars is Tim conducting?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_1", + "session_19", + "session_12", + "session_20", + "session_3", + "session_27", + "session_28", + "session_18", + "session_10" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_26" + ], + "question": "What is one cause that John opposes with his influence and resources?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_27", + "session_8", + "session_12", + "session_15", + "session_5", + "session_20", + "session_11", + "session_19", + "session_18" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_26" + ], + "question": "What new fantasy TV series is John excited about?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_2", + "session_8", + "session_17", + "session_1", + "session_29", + "session_22", + "session_20", + "session_3", + "session_23" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_27" + ], + "question": "Which language is John learning?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_15", + "session_21", + "session_20", + "session_8", + "session_7", + "session_16", + "session_12", + "session_19", + "session_26" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_27" + ], + "question": "According to John, who is his least favorite character from Lord of the Rings?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_26", + "session_1", + "session_12", + "session_4", + "session_17", + "session_5", + "session_22", + "session_20", + "session_6" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_27" + ], + "question": "Why does Tim like Aragorn from Lord of the Rings?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_26", + "session_7", + "session_13", + "session_29", + "session_5", + "session_3", + "session_12", + "session_11", + "session_20" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_27" + ], + "question": "What kind of painting does Tim have in his room as a reminder?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_16", + "session_29", + "session_2", + "session_7", + "session_12", + "session_4", + "session_9", + "session_5", + "session_20" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_27" + ], + "question": "What is the sculpture of Aragorn a reminder for John to be in everything he does?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_29", + "session_16", + "session_20", + "session_2", + "session_3", + "session_15", + "session_17", + "session_7", + "session_12" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_28" + ], + "question": "Which city in Ireland will John be staying in during his semester abroad?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_29", + "session_14", + "session_12", + "session_11", + "session_27", + "session_9", + "session_20", + "session_3", + "session_5" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_28" + ], + "question": "What charity event did Tim organize recently in 2024?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_22", + "session_26", + "session_10", + "session_28", + "session_29", + "session_11", + "session_19", + "session_4", + "session_7" + ], + "sample_id": "conv-43" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_29" + ], + "question": "What was Tims's reaction to sealing the deal with the beverage company?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_21", + "session_25", + "session_3", + "session_11", + "session_12", + "session_20", + "session_23", + "session_28", + "session_18" + ], + "sample_id": "conv-43" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_1" + ], + "question": "Which year did Audrey adopt the first three of her dogs?", + "recall": 0.0, + "retrieved_ids": [ + "session_17", + "session_2", + "session_28", + "session_26", + "session_27", + "session_11", + "session_24", + "session_15", + "session_5", + "session_19" + ], + "sample_id": "conv-44" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_1" + ], + "question": "When did Andrew start his new job as a financial analyst?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_28", + "session_26", + "session_14", + "session_4", + "session_16", + "session_25", + "session_22", + "session_21", + "session_17" + ], + "sample_id": "conv-44" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_13", + "session_19", + "session_23", + "session_25" + ], + "question": "What kind of indoor activities has Andrew pursued with his girlfriend?", + "recall": 0.5, + "retrieved_ids": [ + "session_26", + "session_24", + "session_15", + "session_27", + "session_3", + "session_14", + "session_9", + "session_6", + "session_23", + "session_13" + ], + "sample_id": "conv-44" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_13", + "session_23", + "session_25", + "session_27", + "session_3", + "session_4", + "session_6" + ], + "question": "What kind of places have Andrew and his girlfriend checked out around the city?", + "recall": 0.7143, + "retrieved_ids": [ + "session_3", + "session_26", + "session_23", + "session_15", + "session_24", + "session_13", + "session_21", + "session_25", + "session_22", + "session_6" + ], + "sample_id": "conv-44" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_3" + ], + "question": "When did Audrey make muffins for herself?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_19", + "session_12", + "session_21", + "session_6", + "session_2", + "session_18", + "session_26", + "session_16", + "session_7" + ], + "sample_id": "conv-44" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_4" + ], + "question": "When did Audrey see a hummingbird?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_21", + "session_16", + "session_7", + "session_2", + "session_3", + "session_26", + "session_18", + "session_12", + "session_6" + ], + "sample_id": "conv-44" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_2" + ], + "question": "When did Audrey adopt Pixie?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_15", + "session_26", + "session_24", + "session_5", + "session_7", + "session_12", + "session_21", + "session_16", + "session_3" + ], + "sample_id": "conv-44" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_1", + "session_2" + ], + "question": "How many years passed between Audrey adopting Pixie and her other three dogs?", + "recall": 0.5, + "retrieved_ids": [ + "session_15", + "session_2", + "session_28", + "session_11", + "session_19", + "session_20", + "session_23", + "session_24", + "session_25", + "session_5" + ], + "sample_id": "conv-44" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_2" + ], + "question": "Did Andrew have a pet dog during March 2023?", + "recall": 0.0, + "retrieved_ids": [ + "session_18", + "session_7", + "session_8", + "session_14", + "session_6", + "session_12", + "session_21", + "session_3", + "session_26", + "session_16" + ], + "sample_id": "conv-44" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_14", + "session_16", + "session_27", + "session_6" + ], + "question": "What kind of classes or groups has Audrey joined to take better care of her dogs?", + "recall": 0.6, + "retrieved_ids": [ + "session_27", + "session_19", + "session_26", + "session_6", + "session_11", + "session_13", + "session_8", + "session_14", + "session_15", + "session_2" + ], + "sample_id": "conv-44" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_6" + ], + "question": "When did Audrey's positive reinforcement training course for dogs take place?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_21", + "session_16", + "session_26", + "session_8", + "session_27", + "session_7", + "session_20", + "session_19", + "session_10" + ], + "sample_id": "conv-44" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_8" + ], + "question": "When did Andrew go rock climbing?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_17", + "session_2", + "session_12", + "session_21", + "session_16", + "session_7", + "session_6", + "session_3", + "session_26" + ], + "sample_id": "conv-44" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_14", + "session_17", + "session_8" + ], + "question": "What outdoor activities has Andrew done other than hiking in nature?", + "recall": 0.6667, + "retrieved_ids": [ + "session_14", + "session_23", + "session_24", + "session_13", + "session_27", + "session_18", + "session_8", + "session_19", + "session_25", + "session_5" + ], + "sample_id": "conv-44" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_9" + ], + "question": "When did Audrey move to a new place?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_4", + "session_7", + "session_2", + "session_28", + "session_18", + "session_12", + "session_21", + "session_3", + "session_11" + ], + "sample_id": "conv-44" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_3", + "session_9" + ], + "question": "What is something that Andrew really misses while working in the city?", + "recall": 0.5, + "retrieved_ids": [ + "session_3", + "session_18", + "session_21", + "session_10", + "session_27", + "session_23", + "session_20", + "session_12", + "session_2", + "session_22" + ], + "sample_id": "conv-44" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_7" + ], + "question": "What is a shared frustration regarding dog ownership for Audrey and Andrew?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_19", + "session_10", + "session_13", + "session_20", + "session_8", + "session_9", + "session_4", + "session_14", + "session_7" + ], + "sample_id": "conv-44" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_11" + ], + "question": "When is Andrew going to go hiking with Audrey?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_12", + "session_4", + "session_7", + "session_11", + "session_18", + "session_23", + "session_14", + "session_20", + "session_1" + ], + "sample_id": "conv-44" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_11", + "session_24", + "session_26" + ], + "question": "How many times did Audrey and Andew plan to hike together?", + "recall": 0.3333, + "retrieved_ids": [ + "session_20", + "session_23", + "session_3", + "session_16", + "session_12", + "session_18", + "session_26", + "session_2", + "session_25", + "session_28" + ], + "sample_id": "conv-44" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_11", + "session_2" + ], + "question": "Where did Audrey get Pixie from?", + "recall": 0.5, + "retrieved_ids": [ + "session_2", + "session_26", + "session_15", + "session_18", + "session_21", + "session_12", + "session_16", + "session_3", + "session_6", + "session_7" + ], + "sample_id": "conv-44" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_10", + "session_12" + ], + "question": "What is an indoor activity that Andrew would enjoy doing while make his dog happy?", + "recall": 0.0, + "retrieved_ids": [ + "session_26", + "session_24", + "session_27", + "session_23", + "session_18", + "session_9", + "session_21", + "session_8", + "session_5", + "session_25" + ], + "sample_id": "conv-44" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_10" + ], + "question": "Which meat does Audrey prefer eating more than others?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_13", + "session_10", + "session_19", + "session_15", + "session_27", + "session_22", + "session_4", + "session_2", + "session_3" + ], + "sample_id": "conv-44" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_14", + "session_6" + ], + "question": "What are the classes that Audrey took for her dogs to?", + "recall": 0.6667, + "retrieved_ids": [ + "session_6", + "session_15", + "session_21", + "session_28", + "session_14", + "session_2", + "session_19", + "session_23", + "session_7", + "session_8" + ], + "sample_id": "conv-44" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_14" + ], + "question": "Where did Andrew go during the first weekend of August 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_27", + "session_25", + "session_18", + "session_17", + "session_14", + "session_28", + "session_26", + "session_7", + "session_8" + ], + "sample_id": "conv-44" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_2", + "session_5" + ], + "question": "What are some problems that Andrew faces before he adopted Toby?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_17", + "session_27", + "session_23", + "session_5", + "session_7", + "session_2", + "session_13", + "session_14", + "session_26" + ], + "sample_id": "conv-44" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_13", + "session_2" + ], + "question": "Did Audrey and Andrew grow up with a pet dog?", + "recall": 0.5, + "retrieved_ids": [ + "session_14", + "session_12", + "session_6", + "session_19", + "session_2", + "session_21", + "session_7", + "session_3", + "session_26", + "session_16" + ], + "sample_id": "conv-44" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_17" + ], + "question": "When did Andrew and his girlfriend go fishing?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_15", + "session_26", + "session_3", + "session_6", + "session_25", + "session_24", + "session_13", + "session_21", + "session_16" + ], + "sample_id": "conv-44" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_12", + "session_16", + "session_18" + ], + "question": "What is the biggest stressor in Andrew's life besides not being able to hike frequently?", + "recall": 0.5, + "retrieved_ids": [ + "session_14", + "session_22", + "session_9", + "session_4", + "session_3", + "session_11", + "session_10", + "session_23", + "session_18", + "session_27" + ], + "sample_id": "conv-44" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_12", + "session_16", + "session_18" + ], + "question": "How does Andrew feel about his current work?", + "recall": 0.5, + "retrieved_ids": [ + "session_22", + "session_12", + "session_13", + "session_26", + "session_21", + "session_15", + "session_6", + "session_16", + "session_24", + "session_25" + ], + "sample_id": "conv-44" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_19", + "session_4" + ], + "question": "What is something that Audrey often dresses up her dogs with?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_5", + "session_27", + "session_4", + "session_14", + "session_9", + "session_18", + "session_1", + "session_15", + "session_2" + ], + "sample_id": "conv-44" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_19", + "session_2" + ], + "question": "What are the names of Audrey's dogs?", + "recall": 0.6667, + "retrieved_ids": [ + "session_1", + "session_24", + "session_26", + "session_6", + "session_28", + "session_27", + "session_5", + "session_11", + "session_23", + "session_2" + ], + "sample_id": "conv-44" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_20" + ], + "question": "When is Andrew planning to go to the beach with his girlfriend?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_26", + "session_11", + "session_21", + "session_13", + "session_6", + "session_25", + "session_24", + "session_20", + "session_16" + ], + "sample_id": "conv-44" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_14", + "session_24" + ], + "question": "What has Andrew done with his dogs?", + "recall": 0.5, + "retrieved_ids": [ + "session_13", + "session_24", + "session_15", + "session_26", + "session_25", + "session_11", + "session_6", + "session_16", + "session_21", + "session_2" + ], + "sample_id": "conv-44" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_15", + "session_23", + "session_3" + ], + "question": "What kind of tattoo does Audrey have on her arm?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_3", + "session_19", + "session_23", + "session_26", + "session_22", + "session_25", + "session_2", + "session_13", + "session_1" + ], + "sample_id": "conv-44" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_12", + "session_18", + "session_21" + ], + "question": "What can Andrew potentially do to improve his stress and accomodate his living situation with his dogs?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_24", + "session_16", + "session_21", + "session_18", + "session_13", + "session_6", + "session_15", + "session_10", + "session_25" + ], + "sample_id": "conv-44" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_12", + "session_24" + ], + "question": "How many months passed between Andrew adopting Toby and Buddy?", + "recall": 0.5, + "retrieved_ids": [ + "session_24", + "session_20", + "session_28", + "session_14", + "session_23", + "session_19", + "session_15", + "session_25", + "session_2", + "session_5" + ], + "sample_id": "conv-44" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_12", + "session_24", + "session_28" + ], + "question": "What are the names of Andrew's dogs?", + "recall": 0.6667, + "retrieved_ids": [ + "session_1", + "session_24", + "session_26", + "session_6", + "session_28", + "session_27", + "session_5", + "session_11", + "session_23", + "session_2" + ], + "sample_id": "conv-44" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_25", + "session_3" + ], + "question": "What are some foods that Audrey likes eating?", + "recall": 0.6667, + "retrieved_ids": [ + "session_10", + "session_3", + "session_18", + "session_24", + "session_9", + "session_20", + "session_19", + "session_23", + "session_26", + "session_21" + ], + "sample_id": "conv-44" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_25" + ], + "question": "When did Audrey get into an accident in the park?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_4", + "session_21", + "session_3", + "session_11", + "session_19", + "session_22", + "session_12", + "session_10", + "session_6" + ], + "sample_id": "conv-44" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_25" + ], + "question": "When did Andrew and his girlfriend go on a wine tasting trip?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_15", + "session_26", + "session_17", + "session_20", + "session_3", + "session_10", + "session_21", + "session_6", + "session_5" + ], + "sample_id": "conv-44" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_2", + "session_23" + ], + "question": "What did Audrey get wtih having so many dogs?", + "recall": 0.5, + "retrieved_ids": [ + "session_28", + "session_3", + "session_20", + "session_23", + "session_26", + "session_25", + "session_16", + "session_21", + "session_12", + "session_7" + ], + "sample_id": "conv-44" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_14", + "session_23", + "session_4" + ], + "question": "What is a good place for dogs to run around freely and meet new friends?", + "recall": 0.6667, + "retrieved_ids": [ + "session_11", + "session_5", + "session_4", + "session_27", + "session_9", + "session_23", + "session_8", + "session_28", + "session_7", + "session_12" + ], + "sample_id": "conv-44" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_19", + "session_26" + ], + "question": "What are the breeds of Audrey's dogs?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_26", + "session_5", + "session_6", + "session_27", + "session_14", + "session_11", + "session_23", + "session_10", + "session_2" + ], + "sample_id": "conv-44" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_26", + "session_6" + ], + "question": "What technique is Audrey using to discipline her dogs?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_19", + "session_16", + "session_15", + "session_27", + "session_11", + "session_6", + "session_2", + "session_5", + "session_23" + ], + "sample_id": "conv-44" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_11" + ], + "question": "Which US state do Audrey and Andrew potentially live in?", + "recall": 0.0, + "retrieved_ids": [ + "session_5", + "session_1", + "session_18", + "session_27", + "session_21", + "session_24", + "session_12", + "session_13", + "session_3", + "session_19" + ], + "sample_id": "conv-44" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_11", + "session_5" + ], + "question": "Which national park could Audrey and Andrew be referring to in their conversations?", + "recall": 0.5, + "retrieved_ids": [ + "session_7", + "session_26", + "session_5", + "session_23", + "session_14", + "session_12", + "session_27", + "session_22", + "session_24", + "session_10" + ], + "sample_id": "conv-44" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_12", + "session_24", + "session_28" + ], + "question": "How many pets will Andrew have, as of December 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_25", + "session_12", + "session_1", + "session_24", + "session_6", + "session_13", + "session_20", + "session_15", + "session_23" + ], + "sample_id": "conv-44" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_12", + "session_24" + ], + "question": "How many pets did Andrew have, as of September 2023?", + "recall": 0.5, + "retrieved_ids": [ + "session_28", + "session_25", + "session_12", + "session_20", + "session_23", + "session_7", + "session_21", + "session_16", + "session_26", + "session_3" + ], + "sample_id": "conv-44" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_24", + "session_28" + ], + "question": "How many months passed between Andrew adopting Buddy and Scout", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_20", + "session_24", + "session_14", + "session_23", + "session_25", + "session_19", + "session_2", + "session_15", + "session_5" + ], + "sample_id": "conv-44" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_15", + "session_28" + ], + "question": "What does Andrew view his pets as?", + "recall": 0.5, + "retrieved_ids": [ + "session_26", + "session_14", + "session_1", + "session_24", + "session_15", + "session_4", + "session_8", + "session_13", + "session_11", + "session_21" + ], + "sample_id": "conv-44" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_15", + "session_23" + ], + "question": "What does Audrey view her pets as?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_15", + "session_11", + "session_4", + "session_14", + "session_26", + "session_8", + "session_19", + "session_23", + "session_2" + ], + "sample_id": "conv-44" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_16", + "session_17" + ], + "question": "What is a skill that Audrey learned to take care of her dogs?", + "recall": 0.5, + "retrieved_ids": [ + "session_16", + "session_27", + "session_6", + "session_18", + "session_19", + "session_20", + "session_25", + "session_14", + "session_15", + "session_2" + ], + "sample_id": "conv-44" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_18", + "session_24", + "session_9" + ], + "question": "What items has Audrey bought or made for her dogs?", + "recall": 0.5, + "retrieved_ids": [ + "session_2", + "session_18", + "session_27", + "session_24", + "session_15", + "session_3", + "session_19", + "session_7", + "session_12", + "session_8" + ], + "sample_id": "conv-44" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_1", + "session_20", + "session_23" + ], + "question": "What is something that Andrew could do to make birdwatching hobby to fit in his city schedule?", + "recall": 0.3333, + "retrieved_ids": [ + "session_20", + "session_24", + "session_26", + "session_12", + "session_15", + "session_10", + "session_21", + "session_22", + "session_28", + "session_2" + ], + "sample_id": "conv-44" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_2", + "session_3", + "session_5", + "session_8" + ], + "question": "What is a career that Andrew could potentially pursue with his love for animals and nature?", + "recall": 0.0, + "retrieved_ids": [ + "session_26", + "session_24", + "session_14", + "session_23", + "session_7", + "session_22", + "session_6", + "session_13", + "session_15", + "session_21" + ], + "sample_id": "conv-44" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_13", + "session_23", + "session_27", + "session_4" + ], + "question": "What activity do Audrey's dogs like to do in the dog park?", + "recall": 0.6, + "retrieved_ids": [ + "session_14", + "session_8", + "session_27", + "session_28", + "session_24", + "session_5", + "session_11", + "session_23", + "session_4", + "session_18" + ], + "sample_id": "conv-44" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_28" + ], + "question": "When did Andrew make his dogs a fun indoor area?", + "recall": 0.0, + "retrieved_ids": [ + "session_9", + "session_26", + "session_24", + "session_15", + "session_13", + "session_11", + "session_20", + "session_21", + "session_16", + "session_6" + ], + "sample_id": "conv-44" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_28", + "session_5" + ], + "question": "Has Andrew moved into a new apartment for his dogs?", + "recall": 0.5, + "retrieved_ids": [ + "session_2", + "session_24", + "session_6", + "session_5", + "session_13", + "session_9", + "session_16", + "session_21", + "session_15", + "session_25" + ], + "sample_id": "conv-44" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_28" + ], + "question": "When did Andrew adopt Scout?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_2", + "session_24", + "session_5", + "session_7", + "session_12", + "session_21", + "session_16", + "session_3", + "session_26" + ], + "sample_id": "conv-44" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_25" + ], + "question": "What did Audrey eat for dinner on October 24, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_15", + "session_3", + "session_10", + "session_6", + "session_16", + "session_26", + "session_21", + "session_7", + "session_12" + ], + "sample_id": "conv-44" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_12" + ], + "question": "How long has it been since Andrew adopted his first pet, as of November 2023?", + "recall": 0.0, + "retrieved_ids": [ + "session_24", + "session_28", + "session_13", + "session_21", + "session_17", + "session_26", + "session_5", + "session_25", + "session_27", + "session_2" + ], + "sample_id": "conv-44" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_12", + "session_24", + "session_28" + ], + "question": "How many dogs does Andrew have?", + "recall": 0.3333, + "retrieved_ids": [ + "session_28", + "session_20", + "session_23", + "session_25", + "session_26", + "session_6", + "session_27", + "session_11", + "session_2", + "session_5" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_1" + ], + "question": "Which specific type of bird mesmerizes Andrew?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_26", + "session_7", + "session_17", + "session_22", + "session_9", + "session_19", + "session_20", + "session_25", + "session_14" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What did Andrew express missing about exploring nature trails with his family's dog?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_9", + "session_26", + "session_2", + "session_24", + "session_28", + "session_23", + "session_15", + "session_4", + "session_21" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "What kind of pastries did Andrew and his girlfriend have at the cafe?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_26", + "session_15", + "session_23", + "session_19", + "session_16", + "session_6", + "session_25", + "session_13", + "session_22" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "What kind of flowers does Audrey have a tattoo of?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_3", + "session_23", + "session_15", + "session_9", + "session_26", + "session_22", + "session_11", + "session_1", + "session_13" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What does Audrey do during dog playdates in the park?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_14", + "session_7", + "session_8", + "session_11", + "session_18", + "session_27", + "session_5", + "session_10", + "session_23" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_5" + ], + "question": "What type of dog was Andrew looking to adopt based on his living space?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_28", + "session_24", + "session_1", + "session_9", + "session_10", + "session_2", + "session_14", + "session_21", + "session_25" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_5" + ], + "question": "Where does Andrew want to live to give their dog a large, open space to run around?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_9", + "session_4", + "session_11", + "session_23", + "session_7", + "session_27", + "session_19", + "session_28", + "session_21" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_6" + ], + "question": "Why did Audrey sign up for a workshop about bonding with pets?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_17", + "session_12", + "session_19", + "session_14", + "session_16", + "session_2", + "session_23", + "session_21", + "session_3" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_6" + ], + "question": "How did Audrey hear about the workshop on bonding with pets?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_19", + "session_17", + "session_12", + "session_14", + "session_16", + "session_2", + "session_21", + "session_3", + "session_23" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_6" + ], + "question": "What type of training was the workshop Audrey signed up for in May 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_28", + "session_1", + "session_27", + "session_19", + "session_10", + "session_25", + "session_15", + "session_14", + "session_20" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_6" + ], + "question": "How did Audrey describe she dog he met at the pet store?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_24", + "session_21", + "session_12", + "session_13", + "session_19", + "session_14", + "session_2", + "session_26", + "session_23" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_6" + ], + "question": "Why did Audrey think positive reinforcement training is important for pets?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_27", + "session_17", + "session_26", + "session_7", + "session_19", + "session_20", + "session_4", + "session_10", + "session_16" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What challenge is Andrew facing in their search for a pet?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_18", + "session_5", + "session_14", + "session_7", + "session_10", + "session_3", + "session_11", + "session_26", + "session_21" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "How does Andrew feel about their search for a pet-friendly place?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_28", + "session_18", + "session_10", + "session_5", + "session_6", + "session_9", + "session_23", + "session_26", + "session_2" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What outdoor activities does Andrew plan on trying after the rock climbing class?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_20", + "session_16", + "session_17", + "session_14", + "session_11", + "session_23", + "session_18", + "session_6", + "session_5" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "How long does Audrey typically walk her dogs for?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_13", + "session_20", + "session_8", + "session_14", + "session_4", + "session_3", + "session_1", + "session_2", + "session_15" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_9" + ], + "question": "What did Audrey set up in the backyard for their dogs on June 26, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_11", + "session_16", + "session_6", + "session_3", + "session_26", + "session_2", + "session_12", + "session_7", + "session_21" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_9" + ], + "question": "What did Audrey and her friends stumble across during a hike a few years back, as mentioned on June 26, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_17", + "session_11", + "session_18", + "session_15", + "session_7", + "session_2", + "session_10", + "session_27", + "session_19" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "What is Audrey's favorite recipe that she shares with Andrew on 3 July, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_15", + "session_13", + "session_8", + "session_4", + "session_1", + "session_19", + "session_3", + "session_2", + "session_7" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "What dish is one of Audrey's favorite dishes that includes garlic and is shared with Andrew on 3 July, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_15", + "session_13", + "session_24", + "session_1", + "session_8", + "session_3", + "session_14", + "session_7", + "session_4" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "What did Andrew and his GF do on the Monday before July 24, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_24", + "session_25", + "session_17", + "session_27", + "session_19", + "session_7", + "session_21", + "session_20", + "session_23" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "What is the name of Audrey's childhood dog?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_24", + "session_28", + "session_6", + "session_1", + "session_27", + "session_5", + "session_11", + "session_23", + "session_2" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "What special memories does Audrey have with her childhood dog, Max?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_26", + "session_15", + "session_2", + "session_12", + "session_7", + "session_19", + "session_14", + "session_4", + "session_23" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "What are some of the personalities of Audrey's four fur babies?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_15", + "session_9", + "session_19", + "session_26", + "session_6", + "session_27", + "session_4", + "session_5", + "session_12" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_14" + ], + "question": "What type of classes did Audrey start with her pups recently on 4 August, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_16", + "session_25", + "session_6", + "session_22", + "session_1", + "session_15", + "session_19", + "session_2", + "session_8" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_14" + ], + "question": "How often does Audrey take her pups to the park for practice?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_27", + "session_14", + "session_28", + "session_25", + "session_5", + "session_1", + "session_4", + "session_2", + "session_15" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_14" + ], + "question": "How long did the trail hike that Audrey went on with her pups take?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_19", + "session_24", + "session_14", + "session_20", + "session_4", + "session_1", + "session_8", + "session_2", + "session_15" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "What advice did Audrey give to Andrew regarding grooming Toby?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_16", + "session_17", + "session_27", + "session_14", + "session_20", + "session_5", + "session_28", + "session_26", + "session_21" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What is essential to keep the dogs looking good according to Audrey?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_28", + "session_5", + "session_26", + "session_14", + "session_6", + "session_15", + "session_27", + "session_23", + "session_4" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What did Audrey organize with the neighbors' dogs?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_23", + "session_6", + "session_16", + "session_3", + "session_26", + "session_21", + "session_7", + "session_12", + "session_22" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What did Audrey do to give her dogs extra comfort as the weather cooled down?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_15", + "session_28", + "session_11", + "session_19", + "session_13", + "session_6", + "session_12", + "session_2", + "session_9" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "How does Audrey describe the new beds for her dogs?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_18", + "session_15", + "session_6", + "session_10", + "session_2", + "session_1", + "session_19", + "session_9", + "session_4" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "How did Audrey calm down her dog after the leash incident?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_28", + "session_11", + "session_16", + "session_22", + "session_3", + "session_20", + "session_7", + "session_2", + "session_15" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "How often does Audrey take her dogs for walks?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_22", + "session_27", + "session_4", + "session_14", + "session_5", + "session_1", + "session_8", + "session_2", + "session_15" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "What kind of flowers does Audrey take care of?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_20", + "session_16", + "session_9", + "session_27", + "session_3", + "session_26", + "session_13", + "session_22", + "session_23" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_20" + ], + "question": "What did Andrew learn from reading books about ecological systems?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_6", + "session_25", + "session_27", + "session_3", + "session_16", + "session_14", + "session_26", + "session_12", + "session_21" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_20" + ], + "question": "What did Andrew suggest as a way to reduce carbon footprint?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_3", + "session_5", + "session_4", + "session_26", + "session_21", + "session_22", + "session_12", + "session_1", + "session_16" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_20" + ], + "question": "How does Andrew suggest helping the planet while also training the body?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_10", + "session_22", + "session_6", + "session_27", + "session_26", + "session_1", + "session_4", + "session_19", + "session_5" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_21" + ], + "question": "What did Audrey do with her pups over the weekend before 4th October, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_25", + "session_27", + "session_21", + "session_17", + "session_23", + "session_2", + "session_7", + "session_22", + "session_15" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_22" + ], + "question": "What was the reason Audrey couldn't walk her dogs for a period of time?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_18", + "session_4", + "session_22", + "session_24", + "session_15", + "session_2", + "session_12", + "session_8", + "session_5" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_22" + ], + "question": "What type of jewelry does Audrey make?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_1", + "session_2", + "session_25", + "session_17", + "session_11", + "session_14", + "session_10", + "session_12", + "session_13" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_22" + ], + "question": "Why does Audrey make jewelry out of recycled objects?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_8", + "session_2", + "session_25", + "session_27", + "session_20", + "session_3", + "session_14", + "session_10", + "session_4" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_22" + ], + "question": "What organization does Audrey donate a portion of his profits to?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_26", + "session_24", + "session_15", + "session_13", + "session_6", + "session_25", + "session_16", + "session_18", + "session_21" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_22" + ], + "question": "How does Audrey help out the animal shelter?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_18", + "session_9", + "session_26", + "session_14", + "session_7", + "session_5", + "session_22", + "session_1", + "session_24" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "What type of games do Audrey's dogs like to play at the park?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_11", + "session_1", + "session_27", + "session_9", + "session_4", + "session_19", + "session_14", + "session_13", + "session_25" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "What did Audrey make to thank her neighbors?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_18", + "session_15", + "session_23", + "session_26", + "session_3", + "session_21", + "session_2", + "session_16", + "session_12" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "How do Audrey's dogs react to snow?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_28", + "session_6", + "session_27", + "session_11", + "session_8", + "session_4", + "session_26", + "session_2", + "session_5" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "How does Audrey describe her dogs' response to snow?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_26", + "session_27", + "session_11", + "session_19", + "session_2", + "session_15", + "session_23", + "session_6", + "session_5" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "What kind of experiences are Audrey's dogs the best companions for?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_5", + "session_3", + "session_14", + "session_26", + "session_4", + "session_19", + "session_20", + "session_13", + "session_22" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "What activity do Andrew and Buddy enjoy doing together?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_14", + "session_27", + "session_18", + "session_28", + "session_15", + "session_23", + "session_2", + "session_8", + "session_5" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "What do Andrew and Buddy like doing on walks?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_28", + "session_14", + "session_23", + "session_4", + "session_19", + "session_22", + "session_8", + "session_27", + "session_2" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_25" + ], + "question": "What cuisine did Andrew recently try at a new spot in town?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_3", + "session_27", + "session_26", + "session_23", + "session_24", + "session_6", + "session_5", + "session_15", + "session_18" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_25" + ], + "question": "Which type of sushi did Audrey suggest trying first to someone new to sushi?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_26", + "session_1", + "session_5", + "session_27", + "session_28", + "session_14", + "session_17", + "session_4", + "session_20" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_26" + ], + "question": "What type of date is Andrew going on Sunday?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_8", + "session_20", + "session_1", + "session_25", + "session_11", + "session_14", + "session_4", + "session_23", + "session_12" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_26" + ], + "question": "What did Andrew and Audrey plan to do on the Saturday after October 28, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_26", + "session_16", + "session_11", + "session_3", + "session_19", + "session_23", + "session_18", + "session_12", + "session_22" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_26" + ], + "question": "What aspect of autumn does Andrew find beautiful?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_4", + "session_5", + "session_11", + "session_7", + "session_18", + "session_1", + "session_9", + "session_3", + "session_19" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_27" + ], + "question": "What did Audrey do in November 2023 to better take care of her dogs?", + "recall": 0.0, + "retrieved_ids": [ + "session_19", + "session_13", + "session_24", + "session_6", + "session_11", + "session_26", + "session_25", + "session_14", + "session_15", + "session_2" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_27" + ], + "question": "How often does Audrey meet up with other dog owners for tips and playdates?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_27", + "session_6", + "session_14", + "session_19", + "session_28", + "session_10", + "session_5", + "session_1", + "session_18" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_27" + ], + "question": "What did Audrey share to show ways to keep dogs active in the city?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_27", + "session_20", + "session_14", + "session_23", + "session_24", + "session_12", + "session_18", + "session_16", + "session_8" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_27" + ], + "question": "What type of activities does Audrey suggest for mental stimulation of the dogs?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_14", + "session_18", + "session_1", + "session_26", + "session_8", + "session_5", + "session_4", + "session_20", + "session_25" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_28" + ], + "question": "What is Andrew planning to do with Scout, Toby, and Buddy?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_23", + "session_24", + "session_27", + "session_14", + "session_20", + "session_18", + "session_16", + "session_12", + "session_2" + ], + "sample_id": "conv-44" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_28" + ], + "question": "What did Andrew get for Scout to create a safe and fun space for them?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_4", + "session_11", + "session_9", + "session_12", + "session_2", + "session_13", + "session_5", + "session_19", + "session_21" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_1" + ], + "question": "Which specific type of bird mesmerizes Audrey?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_26", + "session_7", + "session_22", + "session_17", + "session_9", + "session_19", + "session_20", + "session_25", + "session_14" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_3" + ], + "question": "What kind of flowers does Andrew have a tattoo of?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_3", + "session_23", + "session_15", + "session_9", + "session_26", + "session_20", + "session_22", + "session_1", + "session_13" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_5" + ], + "question": "What type of dog was Audrey looking to adopt based on her living space?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_2", + "session_21", + "session_1", + "session_19", + "session_10", + "session_24", + "session_15", + "session_14", + "session_25" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_6" + ], + "question": "Why did Audrey sign up for a workshop about car maintenance?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_19", + "session_12", + "session_2", + "session_18", + "session_26", + "session_21", + "session_16", + "session_7", + "session_3" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_6" + ], + "question": "How did Andrew hear about the workshop on bonding with pets?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_19", + "session_16", + "session_17", + "session_12", + "session_14", + "session_2", + "session_21", + "session_3", + "session_23" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_6" + ], + "question": "What type of training was the workshop Andrew signed up for in May 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_28", + "session_1", + "session_27", + "session_19", + "session_10", + "session_25", + "session_15", + "session_14", + "session_20" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_6" + ], + "question": "How did Andrew describe the dog he met at the pet store?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_24", + "session_12", + "session_13", + "session_14", + "session_3", + "session_21", + "session_16", + "session_26", + "session_23" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_7" + ], + "question": "What challenge is Audrey facing in their search for a pet?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_8", + "session_5", + "session_14", + "session_7", + "session_10", + "session_3", + "session_11", + "session_26", + "session_21" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_8" + ], + "question": "What indoor activities does Andrew plan on trying after the rock climbing class?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_20", + "session_16", + "session_9", + "session_6", + "session_17", + "session_5", + "session_14", + "session_23", + "session_18" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_9" + ], + "question": "What did Andrew set up in the backyard for their dogs on June 26, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_11", + "session_16", + "session_6", + "session_3", + "session_26", + "session_2", + "session_12", + "session_7", + "session_21" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_13" + ], + "question": "What did Audrey and her GF do on the Monday before July 24, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_25", + "session_17", + "session_19", + "session_24", + "session_15", + "session_2", + "session_7", + "session_20", + "session_23" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_13" + ], + "question": "What is the name of Andrew's childhood dog?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_24", + "session_28", + "session_6", + "session_1", + "session_27", + "session_5", + "session_11", + "session_23", + "session_2" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_13" + ], + "question": "What special memories does Andrew have with his childhood dog, Max?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_26", + "session_15", + "session_12", + "session_24", + "session_23", + "session_2", + "session_25", + "session_6", + "session_21" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_13" + ], + "question": "What are some of the personalities of Andrew's four fur babies?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_15", + "session_9", + "session_19", + "session_26", + "session_6", + "session_27", + "session_4", + "session_5", + "session_12" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_14" + ], + "question": "What type of classes did Andrew start with his pups recently on 4 August, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_28", + "session_6", + "session_24", + "session_14", + "session_16", + "session_1", + "session_15", + "session_22", + "session_8" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_17" + ], + "question": "What is essential to keep the dogs looking good according to Andrew?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_28", + "session_14", + "session_5", + "session_26", + "session_6", + "session_15", + "session_27", + "session_23", + "session_4" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "What did Audrey organize with the neighbors' cats?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_23", + "session_6", + "session_16", + "session_3", + "session_26", + "session_21", + "session_22", + "session_12", + "session_7" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "What did Andrew do to give his dogs extra comfort as the weather cooled down?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_15", + "session_6", + "session_28", + "session_11", + "session_12", + "session_13", + "session_25", + "session_9", + "session_21" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "How does Andrew describe the new beds for his dogs?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_16", + "session_24", + "session_18", + "session_28", + "session_13", + "session_10", + "session_6", + "session_21", + "session_25" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_19" + ], + "question": "How did Andrew calm down his dog after the leash incident?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_14", + "session_28", + "session_11", + "session_4", + "session_22", + "session_3", + "session_16", + "session_20", + "session_7" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_19" + ], + "question": "How often does Andrew take his dogs for walks?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_24", + "session_13", + "session_4", + "session_14", + "session_5", + "session_1", + "session_27", + "session_15", + "session_21" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_19" + ], + "question": "What kind of vegetables does Audrey take care of?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_17", + "session_22", + "session_16", + "session_27", + "session_3", + "session_26", + "session_20", + "session_13", + "session_23" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_20" + ], + "question": "What did Andrew learn from reading books about economic systems?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_6", + "session_25", + "session_27", + "session_3", + "session_16", + "session_14", + "session_26", + "session_12", + "session_21" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_22" + ], + "question": "What was the reason Andrew couldn't walk his dogs for a period of time?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_4", + "session_22", + "session_16", + "session_13", + "session_15", + "session_12", + "session_6", + "session_25", + "session_21" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_22" + ], + "question": "What type of jewelry does Andrew make?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_1", + "session_2", + "session_25", + "session_17", + "session_11", + "session_14", + "session_10", + "session_12", + "session_13" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_22" + ], + "question": "Why does Andrew make jewelry out of recycled objects?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_8", + "session_2", + "session_25", + "session_27", + "session_20", + "session_3", + "session_14", + "session_10", + "session_4" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_23" + ], + "question": "What type of games do Andrew's dogs like to play at the park?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_11", + "session_1", + "session_27", + "session_9", + "session_4", + "session_19", + "session_14", + "session_13", + "session_25" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_23" + ], + "question": "What did Andrew make to thank his neighbors?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_24", + "session_15", + "session_23", + "session_26", + "session_25", + "session_13", + "session_6", + "session_21", + "session_16" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_23" + ], + "question": "How do Andrew's dogs react to snow?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_28", + "session_6", + "session_27", + "session_11", + "session_8", + "session_4", + "session_26", + "session_2", + "session_5" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_23" + ], + "question": "How does Andrew describe his dogs' response to snow?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_17", + "session_24", + "session_13", + "session_15", + "session_23", + "session_21", + "session_6", + "session_16", + "session_25" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_23" + ], + "question": "What kind of experiences are Audrey's cats the best companions for?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_4", + "session_5", + "session_3", + "session_14", + "session_26", + "session_19", + "session_20", + "session_13", + "session_22" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_24" + ], + "question": "What activity do Audrey and Buddy enjoy doing together?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_14", + "session_27", + "session_18", + "session_28", + "session_15", + "session_23", + "session_2", + "session_8", + "session_5" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_25" + ], + "question": "What type of drink did Andrew recently try at a new spot in town?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_23", + "session_3", + "session_1", + "session_27", + "session_24", + "session_6", + "session_5", + "session_18", + "session_14" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_25" + ], + "question": "Which type of pizza did Audrey suggest trying first to someone new to Italian cuisine?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_26", + "session_1", + "session_5", + "session_27", + "session_28", + "session_14", + "session_17", + "session_4", + "session_20" + ], + "sample_id": "conv-44" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_27" + ], + "question": "How often does Andrew meet up with other dog owners for tips and playdates?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_27", + "session_6", + "session_14", + "session_19", + "session_28", + "session_10", + "session_5", + "session_1", + "session_18" + ], + "sample_id": "conv-44" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_1" + ], + "question": "What are John's suspected health problems?", + "recall": 0.0, + "retrieved_ids": [], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_1" + ], + "question": "Which recreational activity was James pursuing on March 16, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_14", + "session_23", + "session_3", + "session_17", + "session_18", + "session_28", + "session_31", + "session_1", + "session_8" + ], + "sample_id": "conv-47" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_21", + "session_23" + ], + "question": "Which places or events have John and James planned to meet at?", + "recall": 0.3333, + "retrieved_ids": [ + "session_10", + "session_7", + "session_11", + "session_2", + "session_20", + "session_12", + "session_9", + "session_21", + "session_6", + "session_8" + ], + "sample_id": "conv-47" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_2" + ], + "question": "Do both James and John have pets?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_20", + "session_29", + "session_13", + "session_2", + "session_18", + "session_9", + "session_11", + "session_23", + "session_1" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_3" + ], + "question": "When did John resume playing drums in his adulthood?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_4", + "session_14", + "session_21", + "session_3", + "session_26", + "session_23", + "session_30", + "session_17", + "session_28" + ], + "sample_id": "conv-47" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_3", + "session_4" + ], + "question": "What are John and James' favorite games?", + "recall": 0.5, + "retrieved_ids": [ + "session_19", + "session_9", + "session_3", + "session_1", + "session_10", + "session_14", + "session_28", + "session_27", + "session_20", + "session_8" + ], + "sample_id": "conv-47" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_5" + ], + "question": "Does James live in Connecticut?", + "recall": 0.0, + "retrieved_ids": [ + "session_29", + "session_10", + "session_20", + "session_31", + "session_8", + "session_14", + "session_18", + "session_17", + "session_28", + "session_1" + ], + "sample_id": "conv-47" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_5" + ], + "question": "In which state is the shelter from which James adopted the puppy?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_5", + "session_10", + "session_21", + "session_31", + "session_17", + "session_28", + "session_8", + "session_1", + "session_14" + ], + "sample_id": "conv-47" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_5" + ], + "question": "How many pets does James have?", + "recall": 0.5, + "retrieved_ids": [ + "session_2", + "session_18", + "session_9", + "session_7", + "session_28", + "session_12", + "session_31", + "session_1", + "session_6", + "session_17" + ], + "sample_id": "conv-47" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_5" + ], + "question": "What are the names of James's dogs?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_1", + "session_31", + "session_30", + "session_19", + "session_8", + "session_14", + "session_17", + "session_5", + "session_10" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_5" + ], + "question": "When did James adopt Ned?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_5", + "session_15", + "session_12", + "session_13", + "session_4", + "session_30", + "session_10", + "session_28", + "session_23" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_6" + ], + "question": "How was John feeling on April 10, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_2", + "session_14", + "session_1", + "session_23", + "session_6", + "session_11", + "session_31", + "session_17", + "session_12" + ], + "sample_id": "conv-47" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_6" + ], + "question": "Did James have a girlfriend during April 2022?", + "recall": 0.0, + "retrieved_ids": [ + "session_23", + "session_12", + "session_13", + "session_22", + "session_2", + "session_14", + "session_10", + "session_28", + "session_30", + "session_4" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_6" + ], + "question": "When did James visit Italy?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_16", + "session_31", + "session_12", + "session_13", + "session_4", + "session_30", + "session_10", + "session_28", + "session_23" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_8" + ], + "question": "When did James buy himself a new adventure book?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_8", + "session_2", + "session_14", + "session_10", + "session_21", + "session_30", + "session_24", + "session_23", + "session_7" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_8" + ], + "question": "When did James start playing Civilization VI?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_21", + "session_22", + "session_14", + "session_26", + "session_24", + "session_30", + "session_23", + "session_13", + "session_15" + ], + "sample_id": "conv-47" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_8" + ], + "question": "What is the game with different colored cards that was John talking about with James?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_20", + "session_26", + "session_9", + "session_4", + "session_29", + "session_11", + "session_7", + "session_13", + "session_16" + ], + "sample_id": "conv-47" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_8" + ], + "question": "What is the board game where you have to find the imposter that John mentions to James?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_2", + "session_24", + "session_31", + "session_8", + "session_17", + "session_28", + "session_11", + "session_7", + "session_15" + ], + "sample_id": "conv-47" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_14", + "session_8" + ], + "question": "Which books has John recommended to James?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_24", + "session_3", + "session_25", + "session_8", + "session_18", + "session_26", + "session_30", + "session_17", + "session_28" + ], + "sample_id": "conv-47" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_9" + ], + "question": "Was James feeling lonely before meeting Samantha?", + "recall": 0.0, + "retrieved_ids": [ + "session_6", + "session_17", + "session_19", + "session_23", + "session_29", + "session_21", + "session_24", + "session_2", + "session_25", + "session_28" + ], + "sample_id": "conv-47" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_29" + ], + "question": "How many charity tournaments has John organized till date?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_28", + "session_19", + "session_12", + "session_6", + "session_7", + "session_25", + "session_29", + "session_18", + "session_31" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_10" + ], + "question": "When did John first organize a charity tournament with his friends?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_18", + "session_23", + "session_4", + "session_14", + "session_27", + "session_30", + "session_17", + "session_29", + "session_21" + ], + "sample_id": "conv-47" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_29" + ], + "question": "Who or which organizations have been the beneficiaries of John's charity tournaments?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_11", + "session_18", + "session_16", + "session_30", + "session_12", + "session_4", + "session_15", + "session_3", + "session_29" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_13" + ], + "question": "When will John start his new job?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_14", + "session_23", + "session_4", + "session_21", + "session_18", + "session_10", + "session_29", + "session_12", + "session_17" + ], + "sample_id": "conv-47" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_13", + "session_27" + ], + "question": "What kind of games has James tried to develop?", + "recall": 0.3333, + "retrieved_ids": [ + "session_1", + "session_3", + "session_24", + "session_4", + "session_28", + "session_17", + "session_26", + "session_16", + "session_22", + "session_15" + ], + "sample_id": "conv-47" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_13" + ], + "question": "Are John and James fans of the same football team?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_9", + "session_2", + "session_4", + "session_14", + "session_20", + "session_27", + "session_30", + "session_3", + "session_1" + ], + "sample_id": "conv-47" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_16", + "session_17", + "session_6" + ], + "question": "Which countries has James visited?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_16", + "session_31", + "session_17", + "session_3", + "session_18", + "session_28", + "session_14", + "session_8", + "session_1" + ], + "sample_id": "conv-47" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_13", + "session_23" + ], + "question": "What kind of classes has James joined?", + "recall": 0.5, + "retrieved_ids": [ + "session_20", + "session_23", + "session_24", + "session_26", + "session_21", + "session_16", + "session_15", + "session_1", + "session_17", + "session_8" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_15" + ], + "question": "When did James volunteer at an organization?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_11", + "session_26", + "session_28", + "session_20", + "session_10", + "session_27", + "session_19", + "session_29", + "session_18" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_16" + ], + "question": "When did James depart for his trip to Canada?", + "recall": 0.0, + "retrieved_ids": [ + "session_17", + "session_4", + "session_14", + "session_21", + "session_31", + "session_30", + "session_23", + "session_10", + "session_13", + "session_28" + ], + "sample_id": "conv-47" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_16" + ], + "question": "Which country did James book tickets for in July 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_24", + "session_6", + "session_25", + "session_14", + "session_17", + "session_8", + "session_13", + "session_30", + "session_28" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_16" + ], + "question": "How many days did James plan to spend on his trip in Canada?", + "recall": 0.0, + "retrieved_ids": [ + "session_17", + "session_7", + "session_31", + "session_21", + "session_4", + "session_23", + "session_6", + "session_14", + "session_30", + "session_12" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_16" + ], + "question": "Where was James at on July 12, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_9", + "session_14", + "session_18", + "session_17", + "session_31", + "session_3", + "session_28", + "session_8", + "session_1" + ], + "sample_id": "conv-47" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_17" + ], + "question": "Did John and James study together?", + "recall": 1.0, + "retrieved_ids": [ + "session_30", + "session_4", + "session_6", + "session_29", + "session_10", + "session_15", + "session_17", + "session_2", + "session_20", + "session_1" + ], + "sample_id": "conv-47" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_16", + "session_17" + ], + "question": "Which countries did James visit in July 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_6", + "session_31", + "session_17", + "session_10", + "session_13", + "session_30", + "session_4", + "session_23", + "session_28" + ], + "sample_id": "conv-47" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_17" + ], + "question": "What additional country did James visit during his trip to Canada?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_16", + "session_31", + "session_6", + "session_14", + "session_13", + "session_21", + "session_30", + "session_12", + "session_11" + ], + "sample_id": "conv-47" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_17" + ], + "question": "Who is Jill?", + "recall": 1.0, + "retrieved_ids": [ + "session_17" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_17" + ], + "question": "When did John spend time with his sister and dogs?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_21", + "session_23", + "session_7", + "session_31", + "session_4", + "session_14", + "session_30", + "session_9", + "session_29" + ], + "sample_id": "conv-47" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_18", + "session_4" + ], + "question": "What happened to John's job situation in 2022?", + "recall": 0.5, + "retrieved_ids": [ + "session_18", + "session_17", + "session_13", + "session_15", + "session_27", + "session_29", + "session_12", + "session_9", + "session_25", + "session_10" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_18" + ], + "question": "When did John start his job in IT?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_14", + "session_23", + "session_4", + "session_18", + "session_21", + "session_10", + "session_29", + "session_12", + "session_17" + ], + "sample_id": "conv-47" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_14", + "session_17", + "session_2" + ], + "question": "What kind of tricks do James's pets know?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_17", + "session_26", + "session_9", + "session_25", + "session_1", + "session_15", + "session_16", + "session_18", + "session_14" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_19" + ], + "question": "When did James meet Samantha?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_19", + "session_29", + "session_2", + "session_4", + "session_13", + "session_21", + "session_30", + "session_6", + "session_28" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_19" + ], + "question": "When did James take his 3 dogs to the beach?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_23", + "session_2", + "session_4", + "session_19", + "session_21", + "session_31", + "session_17", + "session_1", + "session_10" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_20" + ], + "question": "When did John plan his next meeting with his siblings?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_4", + "session_20", + "session_8", + "session_22", + "session_23", + "session_6", + "session_14", + "session_17", + "session_2" + ], + "sample_id": "conv-47" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_21" + ], + "question": "Why didn't John want to go to Starbucks?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_6", + "session_28", + "session_1", + "session_7", + "session_25", + "session_4", + "session_12", + "session_18", + "session_23" + ], + "sample_id": "conv-47" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_21", + "session_23" + ], + "question": "What kind of beer does McGee's bar serve?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_23", + "session_29", + "session_3", + "session_26", + "session_16", + "session_24", + "session_15", + "session_22", + "session_28" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_21" + ], + "question": "When did John and James meet at McGee's bar?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_23", + "session_21", + "session_2", + "session_10", + "session_13", + "session_4", + "session_30", + "session_6", + "session_28" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_23" + ], + "question": "When did James ask Samantha to be his girlfriend?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_26", + "session_19", + "session_29", + "session_4", + "session_14", + "session_21", + "session_10", + "session_7", + "session_17" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_23" + ], + "question": "When did James, Samantha and John go to the baseball game together?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_29", + "session_4", + "session_19", + "session_20", + "session_10", + "session_6", + "session_2", + "session_1", + "session_15" + ], + "sample_id": "conv-47" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_20", + "session_23" + ], + "question": "What gaming equipments did John buy or refurbish?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_20", + "session_10", + "session_18", + "session_26", + "session_30", + "session_4", + "session_28", + "session_13", + "session_31" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_23" + ], + "question": "When did James start taking cooking classes?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_30", + "session_14", + "session_13", + "session_26", + "session_24", + "session_8", + "session_17", + "session_1", + "session_10" + ], + "sample_id": "conv-47" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_19", + "session_24", + "session_30", + "session_5", + "session_8" + ], + "question": "Which new games did John start play during the course of the conversation with James?", + "recall": 0.0, + "retrieved_ids": [ + "session_13", + "session_21", + "session_12", + "session_15", + "session_23", + "session_2", + "session_6", + "session_25", + "session_14", + "session_22" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_25" + ], + "question": "When did John start working on his 2D Adventure mobile game?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_21", + "session_1", + "session_3", + "session_14", + "session_30", + "session_23", + "session_7", + "session_4", + "session_17" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_27", + "session_6" + ], + "question": "How long did it take for James to complete his Witcher-inspired game?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_4", + "session_5", + "session_1", + "session_13", + "session_21", + "session_6", + "session_27", + "session_19", + "session_22" + ], + "sample_id": "conv-47" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_27", + "session_28" + ], + "question": "What kind of programming-related events has John hosted?", + "recall": 0.0, + "retrieved_ids": [ + "session_20", + "session_15", + "session_26", + "session_24", + "session_16", + "session_18", + "session_22", + "session_1", + "session_10", + "session_9" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_27" + ], + "question": "When did John and his programming friends host an online programming competition?", + "recall": 0.0, + "retrieved_ids": [ + "session_20", + "session_4", + "session_14", + "session_18", + "session_10", + "session_21", + "session_23", + "session_3", + "session_17", + "session_12" + ], + "sample_id": "conv-47" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_17", + "session_28" + ], + "question": "Which of James's family members have visited him in the last year?", + "recall": 0.5, + "retrieved_ids": [ + "session_6", + "session_1", + "session_5", + "session_31", + "session_16", + "session_18", + "session_20", + "session_14", + "session_17", + "session_21" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_28" + ], + "question": "When did James' mother and her friend visit him?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_6", + "session_31", + "session_23", + "session_9", + "session_5", + "session_16", + "session_14", + "session_21", + "session_19" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_28" + ], + "question": "When did James try Cyberpunk 2077 game?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_26", + "session_19", + "session_4", + "session_5", + "session_30", + "session_31", + "session_9", + "session_1", + "session_18" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_29" + ], + "question": "When did John and his gaming friends organize the charity tournament?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_4", + "session_18", + "session_23", + "session_3", + "session_14", + "session_29", + "session_21", + "session_12", + "session_17" + ], + "sample_id": "conv-47" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_29" + ], + "question": "What games has John played with his friends at charity tournaments?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_3", + "session_29", + "session_4", + "session_23", + "session_17", + "session_21", + "session_12", + "session_16", + "session_14" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_29" + ], + "question": "What was James' big moment with Samantha in October 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_24", + "session_25", + "session_3", + "session_19", + "session_23", + "session_18", + "session_12", + "session_17", + "session_1" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_19", + "session_29" + ], + "question": "How long did James and Samantha date for before deciding to move in together?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_8", + "session_24", + "session_6", + "session_19", + "session_25", + "session_17", + "session_1", + "session_23", + "session_30" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_30" + ], + "question": "When did James, his family and his dogs start on a road trip together?", + "recall": 1.0, + "retrieved_ids": [ + "session_30", + "session_17", + "session_31", + "session_1", + "session_20", + "session_14", + "session_21", + "session_23", + "session_2", + "session_4" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_17", + "session_30" + ], + "question": "How long did John practice chess for before winning the chess tournament?", + "recall": 1.0, + "retrieved_ids": [ + "session_30", + "session_16", + "session_17", + "session_24", + "session_3", + "session_28", + "session_8", + "session_6", + "session_13", + "session_12" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_31" + ], + "question": "When did James and his family visit Mark and Josh?", + "recall": 1.0, + "retrieved_ids": [ + "session_31", + "session_4", + "session_16", + "session_20", + "session_21", + "session_30", + "session_17", + "session_6", + "session_23", + "session_2" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_31" + ], + "question": "When did John work with a game developer on a project?", + "recall": 1.0, + "retrieved_ids": [ + "session_31", + "session_26", + "session_22", + "session_3", + "session_25", + "session_28", + "session_1", + "session_4", + "session_27", + "session_17" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_1" + ], + "question": "What programming languages has James worked with?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_11", + "session_6", + "session_20", + "session_28", + "session_26", + "session_21", + "session_12", + "session_22", + "session_2" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_1" + ], + "question": "What type of mobile application does James plan to build with John?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_11", + "session_30", + "session_9", + "session_25", + "session_20", + "session_14", + "session_3", + "session_6", + "session_2" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_1" + ], + "question": "How does James plan to make his dog-sitting app unique?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_4", + "session_23", + "session_8", + "session_21", + "session_31", + "session_10", + "session_2", + "session_14", + "session_17" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What has John mostly found with the metal detector so far?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_25", + "session_23", + "session_29", + "session_11", + "session_22", + "session_20", + "session_17", + "session_8", + "session_10" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What did James offer to do for John regarding pets?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_17", + "session_21", + "session_26", + "session_15", + "session_9", + "session_7", + "session_25", + "session_1", + "session_13" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "What instrument is John learning to play as of 27 March, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_30", + "session_24", + "session_23", + "session_28", + "session_21", + "session_22", + "session_25", + "session_8", + "session_12" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "How long has John been playing the drums as of 27 March, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_3", + "session_8", + "session_4", + "session_28", + "session_5", + "session_14", + "session_16", + "session_6", + "session_13" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "What game did John play in an intense tournament at the gaming convention in March 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_30", + "session_8", + "session_4", + "session_20", + "session_29", + "session_17", + "session_16", + "session_10", + "session_12" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What game was James playing in the online gaming tournament in April 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_30", + "session_4", + "session_27", + "session_16", + "session_3", + "session_20", + "session_29", + "session_17", + "session_10", + "session_12" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "How does James communicate with his gaming team?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_27", + "session_23", + "session_21", + "session_13", + "session_20", + "session_2", + "session_9", + "session_14", + "session_17" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What advice did James receive from the famous players he met at the tournament?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_30", + "session_3", + "session_12", + "session_18", + "session_27", + "session_19", + "session_23", + "session_21", + "session_14" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_5" + ], + "question": "What did James adopt in April 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_21", + "session_31", + "session_30", + "session_12", + "session_4", + "session_23", + "session_28", + "session_10", + "session_13" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_5" + ], + "question": "What is the name of the pup that was adopted by James?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_5", + "session_31", + "session_19", + "session_15", + "session_14", + "session_8", + "session_28", + "session_1", + "session_17" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_6" + ], + "question": "Why did James embody the appearance of the game character from the woman he saw during a walk?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_14", + "session_2", + "session_13", + "session_19", + "session_12", + "session_1", + "session_21", + "session_18", + "session_23" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_6" + ], + "question": "What inspired James to create the game character in the virtual world?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_27", + "session_11", + "session_14", + "session_19", + "session_12", + "session_22", + "session_25", + "session_1", + "session_28" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_6" + ], + "question": "Which country did James visit in 2021?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_16", + "session_31", + "session_17", + "session_10", + "session_30", + "session_13", + "session_4", + "session_23", + "session_28" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_6" + ], + "question": "What impresses John about Japan?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_3", + "session_21", + "session_26", + "session_15", + "session_18", + "session_9", + "session_25", + "session_8", + "session_17" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What kind of assignment was giving John a hard time at work?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_20", + "session_16", + "session_15", + "session_22", + "session_24", + "session_3", + "session_1", + "session_14", + "session_26" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_9" + ], + "question": "What breed is Daisy, one of James' dogs?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_8", + "session_1", + "session_30", + "session_9", + "session_19", + "session_31", + "session_17", + "session_15", + "session_10" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_9" + ], + "question": "What type of pizza is James' favorite?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_19", + "session_3", + "session_14", + "session_1", + "session_8", + "session_10", + "session_18", + "session_17", + "session_28" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_9" + ], + "question": "What type of pizza is John's favorite?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_19", + "session_3", + "session_14", + "session_1", + "session_10", + "session_13", + "session_4", + "session_25", + "session_8" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "What did John organize with his friends on May 8, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_23", + "session_13", + "session_21", + "session_27", + "session_4", + "session_17", + "session_20", + "session_14", + "session_10" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "What did John and his friends do with the remaining money after helping the dog shelter?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_1", + "session_21", + "session_23", + "session_17", + "session_12", + "session_14", + "session_5", + "session_29", + "session_4" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "What was the main goal of the money raised from the charity tournament organized by John and his friends in May 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_12", + "session_18", + "session_29", + "session_3", + "session_4", + "session_9", + "session_20", + "session_1", + "session_13" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_11" + ], + "question": "What did the system John created help the charitable foundation with?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_6", + "session_30", + "session_3", + "session_22", + "session_14", + "session_28", + "session_1", + "session_27", + "session_21" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_11" + ], + "question": "What did John create for the charitable foundation that helped generate reports for analysis?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_6", + "session_30", + "session_22", + "session_4", + "session_1", + "session_27", + "session_21", + "session_24", + "session_3" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_12" + ], + "question": "What did John receive for achieving second place in the tournament?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_16", + "session_15", + "session_30", + "session_3", + "session_11", + "session_7", + "session_4", + "session_25", + "session_9" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "What project is James working on in his game design course?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_9", + "session_12", + "session_25", + "session_13", + "session_6", + "session_23", + "session_4", + "session_14", + "session_17" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "Who does James support in football matches?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_31", + "session_30", + "session_29", + "session_11", + "session_18", + "session_4", + "session_15", + "session_22", + "session_23" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "Which football club does John support?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_17", + "session_14", + "session_30", + "session_29", + "session_18", + "session_15", + "session_22", + "session_31", + "session_23" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "What disagreement do James and John have about their football teams?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_28", + "session_9", + "session_20", + "session_27", + "session_30", + "session_31", + "session_4", + "session_11", + "session_22" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_14" + ], + "question": "What is Max good at doing according to James?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_30", + "session_1", + "session_29", + "session_25", + "session_10", + "session_15", + "session_23", + "session_11", + "session_20" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "What is the main focus of the organization that James volunteered with?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_11", + "session_10", + "session_29", + "session_18", + "session_26", + "session_9", + "session_1", + "session_27", + "session_19" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "Will there be an interview required to volunteer with the organization James volunteered for?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_11", + "session_13", + "session_8", + "session_26", + "session_19", + "session_10", + "session_27", + "session_25", + "session_18" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "How did John relax in his free time on 9 July, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_21", + "session_1", + "session_23", + "session_4", + "session_3", + "session_25", + "session_2", + "session_14", + "session_17" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "What did James enjoy doing on cold winter days?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_7", + "session_26", + "session_29", + "session_30", + "session_17", + "session_5", + "session_14", + "session_2", + "session_27" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "What new hobby did James become interested in on 9 July, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_23", + "session_2", + "session_10", + "session_18", + "session_3", + "session_13", + "session_24", + "session_8", + "session_28" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "Where did James plan to visit after Toronto?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_18", + "session_1", + "session_5", + "session_10", + "session_27", + "session_31", + "session_6", + "session_20", + "session_8" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "When did James plan to return from his trip to Toronto and Vancouver?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_4", + "session_18", + "session_20", + "session_14", + "session_21", + "session_31", + "session_30", + "session_17", + "session_23" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What online game did John start playing recently for improving strategy?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_30", + "session_20", + "session_8", + "session_27", + "session_3", + "session_2", + "session_25", + "session_16", + "session_13" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What made John leave his IT job?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_4", + "session_29", + "session_12", + "session_23", + "session_10", + "session_21", + "session_13", + "session_16", + "session_7" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "Which game tournaments does John plan to organize besides CS:GO?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_10", + "session_3", + "session_12", + "session_11", + "session_4", + "session_16", + "session_23", + "session_6", + "session_29" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What happened to James's puppy during the recent visit to the clinic?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_25", + "session_14", + "session_2", + "session_6", + "session_13", + "session_27", + "session_16", + "session_31", + "session_12" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "What game genre did John start exploring instead of shooters?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_30", + "session_3", + "session_24", + "session_14", + "session_28", + "session_25", + "session_2", + "session_27", + "session_7" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "Which RPG game is John playing and enjoying on 10 August, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_27", + "session_5", + "session_19", + "session_26", + "session_4", + "session_3", + "session_14", + "session_2", + "session_23" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "What aspect of \"The Witcher 3\" does John find immersive?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_19", + "session_14", + "session_28", + "session_2", + "session_27", + "session_6", + "session_26", + "session_5", + "session_11" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "Whose phone number did James receive during the beach outing?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_12", + "session_2", + "session_14", + "session_4", + "session_13", + "session_22", + "session_8", + "session_30", + "session_28" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "What is James planning to do after receiving Samantha's phone number?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_20", + "session_8", + "session_10", + "session_29", + "session_23", + "session_16", + "session_18", + "session_1", + "session_12" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_20" + ], + "question": "What is John organizing with his siblings?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_20", + "session_22", + "session_4", + "session_23", + "session_17", + "session_14", + "session_15", + "session_10", + "session_18" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_21" + ], + "question": "What type of beer does John not like?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_29", + "session_30", + "session_9", + "session_24", + "session_23", + "session_7", + "session_20", + "session_10", + "session_3" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_22" + ], + "question": "What were some difficulties James faced during the development of his game?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_25", + "session_4", + "session_12", + "session_23", + "session_14", + "session_21", + "session_17", + "session_2", + "session_13" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_22" + ], + "question": "What has John been teaching his siblings?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_21", + "session_25", + "session_4", + "session_23", + "session_20", + "session_17", + "session_14", + "session_26", + "session_3" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_22" + ], + "question": "What kind of programs are John's siblings making?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_22", + "session_3", + "session_1", + "session_21", + "session_26", + "session_16", + "session_15", + "session_24", + "session_17" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "Which company's headphones did John choose for gaming?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_18", + "session_13", + "session_10", + "session_26", + "session_30", + "session_4", + "session_28", + "session_31", + "session_17" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "What did James and Samantha discover they both enjoy at McGee's bar?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_23", + "session_14", + "session_7", + "session_17", + "session_21", + "session_11", + "session_13", + "session_19", + "session_20" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "How much does James pay per cooking class?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_16", + "session_8", + "session_31", + "session_18", + "session_17", + "session_12", + "session_1", + "session_10", + "session_25" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "What did James learn to make in the cooking class besides omelette and meringue?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_21", + "session_12", + "session_8", + "session_22", + "session_17", + "session_28", + "session_1", + "session_6", + "session_10" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "Why did James sign up for a cooking class?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_1", + "session_21", + "session_8", + "session_17", + "session_12", + "session_10", + "session_13", + "session_4", + "session_28" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "What did James prepare for the first time in the cooking class?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_6", + "session_30", + "session_18", + "session_15", + "session_8", + "session_17", + "session_1", + "session_27", + "session_10" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "What is the name of the board game John tried in September 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_31", + "session_28", + "session_24", + "session_8", + "session_17", + "session_21", + "session_19", + "session_9", + "session_1", + "session_5" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "Where does James get his ideas from?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_24", + "session_4", + "session_31", + "session_23", + "session_14", + "session_7", + "session_17", + "session_28", + "session_1" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "What kind of dream did James have recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_15", + "session_3", + "session_20", + "session_27", + "session_13", + "session_26", + "session_16", + "session_6", + "session_2" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "What kind of music does John like?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_16", + "session_22", + "session_3", + "session_20", + "session_1", + "session_13", + "session_26", + "session_15", + "session_2" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "What instrument did James used to play when he was younger?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_3", + "session_20", + "session_28", + "session_21", + "session_30", + "session_23", + "session_4", + "session_18", + "session_14" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "What did John use to play when he was younger to let off steam?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_30", + "session_20", + "session_12", + "session_21", + "session_23", + "session_27", + "session_18", + "session_16", + "session_14" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_25" + ], + "question": "What career milestone did John achieve recently in September 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_2", + "session_15", + "session_18", + "session_11", + "session_12", + "session_26", + "session_9", + "session_3", + "session_14" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_25" + ], + "question": "What type of game is John's upcoming mobile game?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_25", + "session_9", + "session_1", + "session_21", + "session_27", + "session_4", + "session_20", + "session_8", + "session_5" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_25" + ], + "question": "What does John do to stay informed and constantly learn about game design?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_28", + "session_21", + "session_20", + "session_29", + "session_30", + "session_3", + "session_17", + "session_1", + "session_16" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_26" + ], + "question": "What kind of gig was John offered at the game dev non-profit organization?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_15", + "session_24", + "session_25", + "session_11", + "session_20", + "session_2", + "session_16", + "session_7", + "session_13" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_26" + ], + "question": "What does John feel about starting the journey as a programming mentor for game developers?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_14", + "session_13", + "session_3", + "session_28", + "session_8", + "session_1", + "session_16", + "session_27", + "session_22" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_26" + ], + "question": "What kind of games is James excited to play with his new video card?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_1", + "session_24", + "session_20", + "session_21", + "session_22", + "session_4", + "session_8", + "session_14", + "session_23" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_27" + ], + "question": "What inspired James to create his game?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_27", + "session_22", + "session_25", + "session_11", + "session_14", + "session_4", + "session_23", + "session_6", + "session_17" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_28" + ], + "question": "What sparked James' passion for gaming when he was a kid?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_6", + "session_11", + "session_29", + "session_18", + "session_21", + "session_12", + "session_23", + "session_17", + "session_14" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_28" + ], + "question": "What did James lose progress on due to a power outage?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_1", + "session_11", + "session_12", + "session_3", + "session_7", + "session_8", + "session_5", + "session_25", + "session_20" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_29" + ], + "question": "What games were played at the gaming tournament organized by John on 31 October, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_29", + "session_30", + "session_4", + "session_3", + "session_18", + "session_11", + "session_16", + "session_12", + "session_15" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_29" + ], + "question": "What was the purpose of the gaming tournament organized by John on 31 October, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_29", + "session_16", + "session_18", + "session_30", + "session_4", + "session_3", + "session_10", + "session_15", + "session_12" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_29" + ], + "question": "What decision did James and Samantha make on 31 October, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_19", + "session_22", + "session_23", + "session_18", + "session_8", + "session_31", + "session_11", + "session_4", + "session_17" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_29" + ], + "question": "Where did James and Samantha decide to live together on 31 October, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_6", + "session_10", + "session_20", + "session_1", + "session_23", + "session_19", + "session_15", + "session_9", + "session_22" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_29" + ], + "question": "Why did James and Samantha choose an apartment near McGee's bar?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_29", + "session_30", + "session_19", + "session_21", + "session_13", + "session_20", + "session_17", + "session_28", + "session_1" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_30" + ], + "question": "What game is John hooked on playing on 5 November, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_20", + "session_30", + "session_28", + "session_26", + "session_24", + "session_3", + "session_4", + "session_8", + "session_29" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_30" + ], + "question": "What did John suggest James practice before playing FIFA 23 together?", + "recall": 1.0, + "retrieved_ids": [ + "session_30", + "session_17", + "session_29", + "session_21", + "session_24", + "session_6", + "session_8", + "session_28", + "session_25", + "session_31" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_31" + ], + "question": "What project did John work on with a game developer by 7 November, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_31", + "session_26", + "session_22", + "session_3", + "session_25", + "session_28", + "session_1", + "session_17", + "session_27", + "session_4" + ], + "sample_id": "conv-47" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_31" + ], + "question": "What is the name of John's cousin's dog?", + "recall": 1.0, + "retrieved_ids": [ + "session_31", + "session_21", + "session_19", + "session_14", + "session_8", + "session_30", + "session_17", + "session_1", + "session_10", + "session_5" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_5" + ], + "question": "What did John adopt in April 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_21", + "session_31", + "session_30", + "session_12", + "session_23", + "session_4", + "session_28", + "session_13", + "session_10" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_5" + ], + "question": "What is the name of the kitten that was adopted by James?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_19", + "session_31", + "session_8", + "session_14", + "session_18", + "session_5", + "session_17", + "session_28", + "session_1" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_6" + ], + "question": "What inspired John to create the game character in the virtual world?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_27", + "session_11", + "session_14", + "session_28", + "session_19", + "session_12", + "session_22", + "session_25", + "session_1" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_6" + ], + "question": "Which country did John visit in 2021?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_16", + "session_31", + "session_17", + "session_10", + "session_13", + "session_30", + "session_4", + "session_23", + "session_28" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_7" + ], + "question": "What kind of assignment was giving James a hard time at work?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_20", + "session_16", + "session_15", + "session_22", + "session_24", + "session_3", + "session_1", + "session_14", + "session_26" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_10" + ], + "question": "What did James and his friends do with the remaining money after helping the dog shelter?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_1", + "session_21", + "session_23", + "session_17", + "session_12", + "session_14", + "session_5", + "session_29", + "session_4" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_10" + ], + "question": "What was the main goal of the money raised from the political campaign organized by John and his friends in May 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_21", + "session_12", + "session_29", + "session_19", + "session_9", + "session_20", + "session_1", + "session_15", + "session_13" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_11" + ], + "question": "What did the system John created help the illegal organization with?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_15", + "session_3", + "session_28", + "session_20", + "session_14", + "session_19", + "session_27", + "session_18", + "session_10" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_11" + ], + "question": "What did James create for the charitable foundation that helped generate reports for analysis?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_6", + "session_30", + "session_22", + "session_4", + "session_1", + "session_27", + "session_21", + "session_24", + "session_3" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_13" + ], + "question": "Who does James support in cricket matches?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_14", + "session_29", + "session_11", + "session_4", + "session_18", + "session_15", + "session_31", + "session_22", + "session_23" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_14" + ], + "question": "What is Max good at doing according to John?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_30", + "session_1", + "session_29", + "session_25", + "session_10", + "session_15", + "session_11", + "session_23", + "session_20" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_15" + ], + "question": "Will there be a background check required to volunteer with the organization James volunteered for?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_28", + "session_11", + "session_2", + "session_8", + "session_26", + "session_10", + "session_19", + "session_29", + "session_18" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_16" + ], + "question": "How did James relax in his free time on 9 July, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_16", + "session_1", + "session_4", + "session_23", + "session_3", + "session_25", + "session_2", + "session_14", + "session_17" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_16" + ], + "question": "What new hobby did John become interested in on 9 July, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_10", + "session_23", + "session_2", + "session_18", + "session_3", + "session_13", + "session_24", + "session_8", + "session_28" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_16" + ], + "question": "When did John plan to return from his trip to Toronto and Vancouver?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_4", + "session_18", + "session_20", + "session_14", + "session_21", + "session_31", + "session_30", + "session_17", + "session_23" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "What made James leave his IT job?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_4", + "session_29", + "session_12", + "session_23", + "session_10", + "session_21", + "session_13", + "session_16", + "session_7" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "Which game tournaments does James plan to organize besides CS:GO?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_10", + "session_3", + "session_11", + "session_12", + "session_4", + "session_16", + "session_23", + "session_6", + "session_29" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "What happened to James's kitten during the recent visit to the clinic?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_14", + "session_25", + "session_2", + "session_6", + "session_13", + "session_27", + "session_16", + "session_31", + "session_12" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_19" + ], + "question": "What aspect of \"The Witcher 3\" does John find boring?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_25", + "session_27", + "session_19", + "session_6", + "session_28", + "session_11", + "session_7", + "session_2", + "session_15" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_19" + ], + "question": "What is John planning to do after receiving Samantha's phone number?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_20", + "session_8", + "session_10", + "session_29", + "session_23", + "session_16", + "session_18", + "session_1", + "session_12" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_22" + ], + "question": "What has James been teaching his siblings?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_21", + "session_25", + "session_4", + "session_23", + "session_20", + "session_17", + "session_14", + "session_26", + "session_3" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_23" + ], + "question": "How much does James pay per dance class?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_16", + "session_8", + "session_15", + "session_18", + "session_17", + "session_12", + "session_31", + "session_1", + "session_25" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_23" + ], + "question": "What did James learn to make in the chemistry class besides omelette and meringue?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_12", + "session_8", + "session_21", + "session_22", + "session_17", + "session_28", + "session_1", + "session_3", + "session_6" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_23" + ], + "question": "Why did James sign up for a ballet class?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_1", + "session_21", + "session_8", + "session_17", + "session_12", + "session_10", + "session_13", + "session_4", + "session_28" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_23" + ], + "question": "What did John prepare for the first time in the cooking class?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_6", + "session_30", + "session_18", + "session_15", + "session_8", + "session_17", + "session_1", + "session_27", + "session_10" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_24" + ], + "question": "What is the name of the board game James tried in September 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_31", + "session_28", + "session_24", + "session_8", + "session_17", + "session_21", + "session_19", + "session_9", + "session_1", + "session_5" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_24" + ], + "question": "Where does John get his ideas from?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_24", + "session_4", + "session_31", + "session_23", + "session_14", + "session_7", + "session_17", + "session_28", + "session_1" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_24" + ], + "question": "What did James use to play when he was younger to let off steam?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_30", + "session_20", + "session_12", + "session_21", + "session_23", + "session_27", + "session_18", + "session_16", + "session_14" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_25" + ], + "question": "What does James do to stay informed and constantly learn about game design?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_28", + "session_21", + "session_20", + "session_29", + "session_30", + "session_3", + "session_17", + "session_1", + "session_16" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_26" + ], + "question": "What kind of gig was James offered at the game dev non-profit organization?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_15", + "session_24", + "session_25", + "session_11", + "session_20", + "session_2", + "session_16", + "session_7", + "session_13" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_26" + ], + "question": "What does James feel about starting the journey as a programming mentor for game developers?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_14", + "session_13", + "session_3", + "session_28", + "session_8", + "session_1", + "session_16", + "session_27", + "session_22" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_27" + ], + "question": "What inspired James to create his painting?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_27", + "session_22", + "session_11", + "session_25", + "session_14", + "session_17", + "session_4", + "session_23", + "session_6" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_29" + ], + "question": "What games were played at the gaming tournament organized by James on 31 October, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_29", + "session_30", + "session_4", + "session_3", + "session_18", + "session_11", + "session_16", + "session_12", + "session_15" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_29" + ], + "question": "What was the purpose of the gaming tournament organized by James on 31 October, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_29", + "session_16", + "session_18", + "session_30", + "session_4", + "session_3", + "session_10", + "session_15", + "session_12" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_29" + ], + "question": "What decision did John and Samantha make on 31 October, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_22", + "session_19", + "session_23", + "session_18", + "session_8", + "session_31", + "session_11", + "session_4", + "session_17" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_29" + ], + "question": "Where did John and Samantha decide to live together on 31 October, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_6", + "session_10", + "session_20", + "session_1", + "session_23", + "session_19", + "session_15", + "session_9", + "session_22" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_29" + ], + "question": "Why did John and Samantha choose an apartment near McGee's bar?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_29", + "session_30", + "session_19", + "session_21", + "session_13", + "session_20", + "session_17", + "session_28", + "session_1" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_30" + ], + "question": "What game is James hooked on playing on 5 November, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_20", + "session_30", + "session_28", + "session_26", + "session_24", + "session_3", + "session_4", + "session_8", + "session_29" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_31" + ], + "question": "What project did James work on with a game developer by 7 November, 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_31", + "session_26", + "session_22", + "session_3", + "session_25", + "session_28", + "session_1", + "session_17", + "session_27", + "session_4" + ], + "sample_id": "conv-47" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_31" + ], + "question": "What is the name of James's cousin's dog?", + "recall": 1.0, + "retrieved_ids": [ + "session_31", + "session_21", + "session_19", + "session_8", + "session_14", + "session_30", + "session_17", + "session_1", + "session_10", + "session_5" + ], + "sample_id": "conv-47" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_1" + ], + "question": "What kind of project was Jolene working on in the beginning of January 2023?", + "recall": 0.0, + "retrieved_ids": [ + "session_23", + "session_3", + "session_12", + "session_7", + "session_22", + "session_17", + "session_6", + "session_14", + "session_19", + "session_16" + ], + "sample_id": "conv-48" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_2", + "session_6" + ], + "question": "Which of Deborah`s family and friends have passed away?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_22", + "session_1", + "session_24", + "session_12", + "session_6", + "session_9", + "session_30", + "session_13", + "session_28" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_1" + ], + "question": "When did Deborah`s mother pass away?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_22", + "session_2", + "session_4", + "session_30", + "session_13", + "session_24", + "session_3", + "session_8", + "session_5" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_1" + ], + "question": "When did Jolene`s mother pass away?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_22", + "session_2", + "session_4", + "session_30", + "session_13", + "session_24", + "session_3", + "session_8", + "session_5" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_1" + ], + "question": "When did Jolene's mom gift her a pendant?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_4", + "session_1", + "session_30", + "session_12", + "session_28", + "session_20", + "session_2", + "session_29", + "session_23" + ], + "sample_id": "conv-48" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_1" + ], + "question": "In what country did Jolene's mother buy her the pendant?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_4", + "session_23", + "session_2", + "session_22", + "session_15", + "session_14", + "session_16", + "session_28", + "session_29" + ], + "sample_id": "conv-48" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1" + ], + "question": "What symbolic gifts do Deborah and Jolene have from their mothers?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_4", + "session_1", + "session_5", + "session_11", + "session_23", + "session_19", + "session_2", + "session_28", + "session_14" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_1" + ], + "question": "Which country were Jolene and her mother visiting in 2010?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_1", + "session_22", + "session_2", + "session_4", + "session_12", + "session_29", + "session_19", + "session_14", + "session_6" + ], + "sample_id": "conv-48" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_15", + "session_2", + "session_6" + ], + "question": "What helped Deborah find peace when grieving deaths of her loved ones?", + "recall": 0.75, + "retrieved_ids": [ + "session_1", + "session_28", + "session_4", + "session_12", + "session_2", + "session_29", + "session_19", + "session_15", + "session_16", + "session_14" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_2" + ], + "question": "When did Deborah's father pass away?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_22", + "session_2", + "session_30", + "session_13", + "session_7", + "session_3", + "session_24", + "session_8", + "session_5" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_2" + ], + "question": "When was Deborah's parents' wedding?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_24", + "session_7", + "session_20", + "session_26", + "session_14", + "session_25", + "session_23", + "session_13", + "session_29" + ], + "sample_id": "conv-48" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_19", + "session_2", + "session_23", + "session_28" + ], + "question": "Is Deborah married?", + "recall": 0.75, + "retrieved_ids": [ + "session_7", + "session_28", + "session_20", + "session_14", + "session_25", + "session_26", + "session_23", + "session_13", + "session_2", + "session_29" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_2" + ], + "question": "When did Deborah receive an appreciation letter from her community?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_2", + "session_19", + "session_1", + "session_16", + "session_28", + "session_29", + "session_3", + "session_21", + "session_27" + ], + "sample_id": "conv-48" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_19", + "session_2", + "session_4", + "session_6" + ], + "question": "What places give Deborah peace?", + "recall": 0.75, + "retrieved_ids": [ + "session_6", + "session_2", + "session_23", + "session_1", + "session_28", + "session_22", + "session_4", + "session_17", + "session_18", + "session_20" + ], + "sample_id": "conv-48" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_12", + "session_2", + "session_29" + ], + "question": "What were Deborah's mother's hobbies?", + "recall": 0.6667, + "retrieved_ids": [ + "session_2", + "session_19", + "session_22", + "session_4", + "session_27", + "session_1", + "session_9", + "session_13", + "session_23", + "session_12" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What pets does Jolene have?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_14", + "session_28", + "session_15", + "session_22", + "session_16", + "session_29", + "session_25", + "session_9", + "session_20" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What are the names of Jolene's snakes?", + "recall": 0.0, + "retrieved_ids": [], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_2" + ], + "question": "When did Jolene buy her pet Seraphim?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_2", + "session_28", + "session_8", + "session_22", + "session_16", + "session_15", + "session_4", + "session_1", + "session_29" + ], + "sample_id": "conv-48" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_2" + ], + "question": "In what country did Jolene buy snake Seraphim?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_8", + "session_14", + "session_28", + "session_23", + "session_15", + "session_5", + "session_3", + "session_12", + "session_22" + ], + "sample_id": "conv-48" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_2" + ], + "question": "How many times has Jolene been to France?", + "recall": 0.5, + "retrieved_ids": [ + "session_23", + "session_1", + "session_9", + "session_16", + "session_4", + "session_19", + "session_6", + "session_14", + "session_8", + "session_7" + ], + "sample_id": "conv-48" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_15", + "session_19", + "session_2", + "session_20" + ], + "question": "Which games have Jolene and her partner played together?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_12", + "session_2", + "session_7", + "session_15", + "session_3", + "session_24", + "session_27", + "session_16", + "session_20" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_2" + ], + "question": "When do Jolene and her partner plan to complete the game \"Walking Dead\"?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_22", + "session_19", + "session_12", + "session_15", + "session_16", + "session_24", + "session_20", + "session_27", + "session_25" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_3" + ], + "question": "When did Deborah meet Anna?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_15", + "session_10", + "session_4", + "session_26", + "session_8", + "session_19", + "session_1", + "session_7", + "session_27" + ], + "sample_id": "conv-48" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_2", + "session_3" + ], + "question": "Why did Jolene sometimes put off doing yoga?", + "recall": 0.0, + "retrieved_ids": [ + "session_13", + "session_7", + "session_11", + "session_15", + "session_1", + "session_18", + "session_14", + "session_8", + "session_27", + "session_29" + ], + "sample_id": "conv-48" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_14", + "session_4" + ], + "question": "What new yoga poses did Deborah try?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_16", + "session_2", + "session_22", + "session_14", + "session_8", + "session_20", + "session_23", + "session_28", + "session_3" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What are Jolene's favorite books?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_2", + "session_4", + "session_18", + "session_15", + "session_11", + "session_24", + "session_19", + "session_26", + "session_16" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_4" + ], + "question": "Which book did Jolene read in January 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_4", + "session_26", + "session_18", + "session_2", + "session_24", + "session_3", + "session_5", + "session_8", + "session_19" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_4" + ], + "question": "When was Jolene in Bogota?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_3", + "session_10", + "session_25", + "session_20", + "session_9", + "session_14", + "session_2", + "session_23", + "session_29" + ], + "sample_id": "conv-48" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_4" + ], + "question": "In what country was Jolene during summer 2022?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_23", + "session_16", + "session_6", + "session_25", + "session_11", + "session_1", + "session_12", + "session_18", + "session_13" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_5" + ], + "question": "When did Jolene have a mini-retreat to reflect on her career?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_5", + "session_15", + "session_22", + "session_14", + "session_27", + "session_30", + "session_25", + "session_17", + "session_21" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_6" + ], + "question": "When did Jolene have a dinner and drinks with her friends?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_12", + "session_4", + "session_28", + "session_23", + "session_7", + "session_29", + "session_24", + "session_9", + "session_2" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_6" + ], + "question": "When was the last photo of Deborah and Karlie taken?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_23", + "session_30", + "session_24", + "session_1", + "session_25", + "session_14", + "session_7", + "session_20", + "session_28" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_6" + ], + "question": "When was Deborah in Bali?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_8", + "session_26", + "session_14", + "session_25", + "session_23", + "session_2", + "session_13", + "session_20", + "session_29" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "How long have Jolene and her partner been together?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_7", + "session_19", + "session_30", + "session_9", + "session_20", + "session_13", + "session_28", + "session_2", + "session_11" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_7" + ], + "question": "Which year did Jolene and her partner start dating?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_7", + "session_16", + "session_30", + "session_2", + "session_15", + "session_18", + "session_24", + "session_29", + "session_26" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_7" + ], + "question": "When did Deborah go for her first morning jog in a nearby park?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_4", + "session_19", + "session_15", + "session_3", + "session_12", + "session_1", + "session_13", + "session_23", + "session_5" + ], + "sample_id": "conv-48" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_13", + "session_21", + "session_22", + "session_24", + "session_25", + "session_26", + "session_8" + ], + "question": "How old is Jolene?", + "recall": 0.4286, + "retrieved_ids": [ + "session_22", + "session_1", + "session_24", + "session_28", + "session_23", + "session_29", + "session_4", + "session_2", + "session_25", + "session_20" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_8" + ], + "question": "When did Jolene take Seraphim to the park?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_2", + "session_28", + "session_4", + "session_7", + "session_19", + "session_15", + "session_5", + "session_3", + "session_14" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_9" + ], + "question": "When did Deborah start the yoga class in the neighborhood?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_7", + "session_24", + "session_11", + "session_18", + "session_23", + "session_29", + "session_8", + "session_26", + "session_5" + ], + "sample_id": "conv-48" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_18" + ], + "question": "What time management techniques do Deborah and Jolene use?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_18", + "session_14", + "session_9", + "session_8", + "session_25", + "session_6", + "session_19", + "session_12", + "session_16" + ], + "sample_id": "conv-48" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_10" + ], + "question": "Does Deborah live close to the beach or the mountains?", + "recall": 0.0, + "retrieved_ids": [ + "session_4", + "session_2", + "session_6", + "session_29", + "session_19", + "session_24", + "session_28", + "session_25", + "session_3", + "session_13" + ], + "sample_id": "conv-48" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_11", + "session_28" + ], + "question": "What ways do Deborah and Jolene use to enhance their yoga practice?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_22", + "session_11", + "session_4", + "session_26", + "session_21", + "session_10", + "session_23", + "session_5", + "session_17" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_11" + ], + "question": "What music pieces does Deborah listen to during her yoga practice?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_15", + "session_28", + "session_30", + "session_16", + "session_8", + "session_19", + "session_7", + "session_21", + "session_9" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_12" + ], + "question": "When did Deborah go for a bicycle ride with Anna?", + "recall": 0.0, + "retrieved_ids": [ + "session_27", + "session_3", + "session_10", + "session_4", + "session_8", + "session_19", + "session_15", + "session_7", + "session_2", + "session_23" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_12" + ], + "question": "When did Deborah go to an art show with Anna?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_4", + "session_15", + "session_3", + "session_10", + "session_5", + "session_8", + "session_19", + "session_22", + "session_17" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_13" + ], + "question": "When did Jolene finish her robotics project?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_1", + "session_12", + "session_5", + "session_14", + "session_16", + "session_13", + "session_28", + "session_27", + "session_2" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_12", + "session_13", + "session_3" + ], + "question": "How long did Jolene work on the robotics project given to her by her Professor?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_12", + "session_20", + "session_13", + "session_22", + "session_11", + "session_7", + "session_9", + "session_28", + "session_19" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_13" + ], + "question": "When did Jolene do yoga at Talkeetna?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_8", + "session_3", + "session_26", + "session_24", + "session_22", + "session_30", + "session_7", + "session_2", + "session_23" + ], + "sample_id": "conv-48" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_13" + ], + "question": "Which US state did Jolene visit during her internship?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_1", + "session_13", + "session_12", + "session_16", + "session_23", + "session_2", + "session_29", + "session_6", + "session_21" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "How long has Jolene been doing yoga and meditation?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_13", + "session_7", + "session_28", + "session_11", + "session_19", + "session_22", + "session_16", + "session_9", + "session_12" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_13" + ], + "question": "Which year did Jolene start practicing yoga?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_16", + "session_23", + "session_13", + "session_9", + "session_22", + "session_18", + "session_20", + "session_28", + "session_15" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_14" + ], + "question": "When did Jolene buy a new aquarium for Seraphim?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_8", + "session_2", + "session_28", + "session_24", + "session_3", + "session_25", + "session_5", + "session_16", + "session_23" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_16" + ], + "question": "When did Jolene lose a lot of progress in her work?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_4", + "session_1", + "session_16", + "session_7", + "session_27", + "session_2", + "session_28", + "session_29", + "session_6" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_16", + "session_28" + ], + "question": "When did Jolene adopt her snake Susie?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_2", + "session_28", + "session_8", + "session_15", + "session_4", + "session_12", + "session_22", + "session_1", + "session_29" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_16", + "session_2" + ], + "question": "Which pet did Jolene adopt first - Susie or Seraphim?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_28", + "session_2", + "session_14", + "session_7", + "session_8", + "session_3", + "session_24", + "session_5", + "session_10" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_16", + "session_2" + ], + "question": "Which pet did Jolene adopt more recently - Susie or Seraphim?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_2", + "session_16", + "session_14", + "session_27", + "session_8", + "session_22", + "session_15", + "session_25", + "session_18" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_18" + ], + "question": "When did Deborah lead a meditation session during the sunset?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_22", + "session_18", + "session_4", + "session_25", + "session_27", + "session_13", + "session_1", + "session_10", + "session_19" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_19" + ], + "question": "When did Jolene gift her partner a new console?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_2", + "session_3", + "session_20", + "session_23", + "session_29", + "session_12", + "session_30", + "session_28", + "session_15" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "What games does Jolene recommend for Deborah?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_24", + "session_13", + "session_2", + "session_12", + "session_16", + "session_20", + "session_11", + "session_15", + "session_27" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "What do Deborah and her husband do together?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_2", + "session_15", + "session_1", + "session_23", + "session_29", + "session_4", + "session_16", + "session_14", + "session_28" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_21" + ], + "question": "When did Deborah go to a yoga retreat near her mom's place?", + "recall": 1.0, + "retrieved_ids": [ + "session_30", + "session_4", + "session_1", + "session_21", + "session_19", + "session_14", + "session_26", + "session_25", + "session_5", + "session_27" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_22" + ], + "question": "What projects is Jolene planning for next year?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_1", + "session_2", + "session_22", + "session_30", + "session_16", + "session_14", + "session_21", + "session_29", + "session_26" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_22" + ], + "question": "Where did Deborah get her cats?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_15", + "session_14", + "session_28", + "session_4", + "session_1", + "session_27", + "session_16", + "session_2", + "session_29" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_22" + ], + "question": "How old are Deborah's cats?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_4", + "session_1", + "session_24", + "session_28", + "session_23", + "session_29", + "session_15", + "session_27", + "session_2" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15", + "session_22" + ], + "question": "Does Deborah like cats?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_27", + "session_15", + "session_5", + "session_14", + "session_11", + "session_12", + "session_8", + "session_20", + "session_29" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_23" + ], + "question": "Which country was Jolene located in during the last week of August 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_12", + "session_18", + "session_16", + "session_23", + "session_2", + "session_4", + "session_13", + "session_26", + "session_11" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_23" + ], + "question": "When did Jolene and her partner return home from Rio de Janeiro?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_22", + "session_12", + "session_30", + "session_2", + "session_26", + "session_18", + "session_29", + "session_19", + "session_24" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "What was Jolene doing with her partner in Rio de Janeiro?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_19", + "session_15", + "session_30", + "session_24", + "session_2", + "session_28", + "session_12", + "session_16", + "session_29" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_23" + ], + "question": "When did Deborah visit Brazil?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_6", + "session_1", + "session_14", + "session_29", + "session_24", + "session_3", + "session_5", + "session_8", + "session_2" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "Have Deborah and Jolene been to Rio de Janeiro?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_21", + "session_24", + "session_18", + "session_13", + "session_7", + "session_6", + "session_14", + "session_16", + "session_20" + ], + "sample_id": "conv-48" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_23" + ], + "question": "Is the friend who wrote Deborah the motivational quote no longer alive?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_12", + "session_6", + "session_9", + "session_18", + "session_24", + "session_30", + "session_7", + "session_4", + "session_15" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_24" + ], + "question": "When did Deborah go to a community meetup?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_19", + "session_3", + "session_22", + "session_4", + "session_1", + "session_9", + "session_16", + "session_27", + "session_29" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "When did Jolene's parents give her first console?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_10", + "session_7", + "session_3", + "session_24", + "session_5", + "session_20", + "session_28", + "session_19", + "session_29" + ], + "sample_id": "conv-48" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_2", + "session_24" + ], + "question": "Did Jolene teach herself how to play the console?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_3", + "session_24", + "session_19", + "session_15", + "session_12", + "session_20", + "session_27", + "session_9", + "session_10" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_26" + ], + "question": "What do Deborah and Jolene plan to try when they meet in a new cafe?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_22", + "session_5", + "session_23", + "session_3", + "session_17", + "session_27", + "session_7", + "session_1", + "session_30" + ], + "sample_id": "conv-48" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_27" + ], + "question": "What card game is Deborah talking about?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_24", + "session_20", + "session_2", + "session_19", + "session_15", + "session_3", + "session_12", + "session_16", + "session_7" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_29" + ], + "question": "When did Jolene and her partner try scuba diving lessons?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_10", + "session_2", + "session_16", + "session_12", + "session_30", + "session_23", + "session_19", + "session_15", + "session_28" + ], + "sample_id": "conv-48" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_27", + "session_29" + ], + "question": "Where did Jolene and her partner find a cool diving spot?", + "recall": 0.5, + "retrieved_ids": [ + "session_23", + "session_19", + "session_15", + "session_29", + "session_12", + "session_30", + "session_2", + "session_4", + "session_14", + "session_28" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_2" + ], + "question": "Where did Jolene and her partner spend most of September 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_12", + "session_7", + "session_22", + "session_4", + "session_19", + "session_14", + "session_6", + "session_26", + "session_29" + ], + "sample_id": "conv-48" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_28", + "session_29" + ], + "question": "Has Deborah tried surfing?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_28", + "session_29", + "session_14", + "session_18", + "session_26", + "session_13", + "session_25", + "session_23", + "session_2" + ], + "sample_id": "conv-48" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_29" + ], + "question": "Has Jolene tried surfing?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_28", + "session_29", + "session_14", + "session_18", + "session_23", + "session_25", + "session_9", + "session_20", + "session_2" + ], + "sample_id": "conv-48" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_29" + ], + "question": "When did the Deboran and Jolene agree to go surfing?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_10", + "session_27", + "session_28", + "session_8", + "session_24", + "session_5", + "session_30", + "session_23", + "session_2" + ], + "sample_id": "conv-48" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_2", + "session_3", + "session_4", + "session_6" + ], + "question": "Which locations does Deborah practice her yoga at?", + "recall": 0.0, + "retrieved_ids": [ + "session_16", + "session_20", + "session_28", + "session_15", + "session_13", + "session_17", + "session_26", + "session_11", + "session_7", + "session_27" + ], + "sample_id": "conv-48" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_13", + "session_21" + ], + "question": "What kind of professional activities does Jolene participate in to gain more experience in her field?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_21", + "session_23", + "session_19", + "session_15", + "session_5", + "session_12", + "session_6", + "session_7", + "session_3" + ], + "sample_id": "conv-48" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_17", + "session_3", + "session_4" + ], + "question": "What kind of engineering projects has Jolene worked on?", + "recall": 0.5, + "retrieved_ids": [ + "session_3", + "session_13", + "session_22", + "session_12", + "session_17", + "session_5", + "session_19", + "session_7", + "session_6", + "session_23" + ], + "sample_id": "conv-48" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_15", + "session_4" + ], + "question": "Which community activities have Deborah and Anna participated in?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_19", + "session_3", + "session_22", + "session_4", + "session_12", + "session_8", + "session_10", + "session_26", + "session_11" + ], + "sample_id": "conv-48" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_2", + "session_23", + "session_4" + ], + "question": "What gifts has Deborah received?", + "recall": 0.6667, + "retrieved_ids": [ + "session_19", + "session_21", + "session_2", + "session_3", + "session_26", + "session_25", + "session_14", + "session_13", + "session_23", + "session_29" + ], + "sample_id": "conv-48" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_23", + "session_6" + ], + "question": "Which countries has Deborah traveled to?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_6", + "session_27", + "session_2", + "session_20", + "session_25", + "session_26", + "session_14", + "session_13", + "session_29" + ], + "sample_id": "conv-48" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_12", + "session_15", + "session_28", + "session_29" + ], + "question": "What activities does Deborah pursue besides practicing and teaching yoga?", + "recall": 0.5, + "retrieved_ids": [ + "session_22", + "session_13", + "session_19", + "session_2", + "session_4", + "session_14", + "session_11", + "session_15", + "session_26", + "session_12" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What are the names of Jolene's snakes?", + "recall": 0.0, + "retrieved_ids": [], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What are Jolene's favorite books?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_2", + "session_4", + "session_18", + "session_15", + "session_11", + "session_24", + "session_19", + "session_26", + "session_16" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_11" + ], + "question": "What music pieces does Deborah listen to during her yoga practice?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_15", + "session_28", + "session_30", + "session_16", + "session_8", + "session_19", + "session_7", + "session_21", + "session_9" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "What games does Jolene recommend for Deborah?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_24", + "session_13", + "session_2", + "session_12", + "session_16", + "session_20", + "session_11", + "session_15", + "session_27" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_22" + ], + "question": "What projects is Jolene planning for next year?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_1", + "session_2", + "session_22", + "session_30", + "session_16", + "session_14", + "session_21", + "session_29", + "session_26" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_22" + ], + "question": "Where did Deborah get her cats?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_15", + "session_14", + "session_28", + "session_4", + "session_1", + "session_27", + "session_16", + "session_2", + "session_29" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_22" + ], + "question": "How old are Deborah's cats?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_4", + "session_1", + "session_24", + "session_28", + "session_23", + "session_29", + "session_15", + "session_27", + "session_2" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "What was Jolene doing with her partner in Rio de Janeiro?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_19", + "session_15", + "session_30", + "session_24", + "session_2", + "session_28", + "session_12", + "session_16", + "session_29" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "Have Deborah and Jolene been to Rio de Janeiro?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_21", + "session_24", + "session_18", + "session_13", + "session_7", + "session_6", + "session_14", + "session_16", + "session_20" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "When did Jolene's parents give her first console?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_10", + "session_7", + "session_3", + "session_24", + "session_5", + "session_20", + "session_29", + "session_28", + "session_19" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_26" + ], + "question": "What do Deborah and Jolene plan to try when they meet in a new cafe?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_22", + "session_5", + "session_23", + "session_3", + "session_17", + "session_27", + "session_7", + "session_1", + "session_30" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_1" + ], + "question": "What project did Jolene finish last week before 23 January, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_14", + "session_5", + "session_22", + "session_12", + "session_27", + "session_16", + "session_13", + "session_28", + "session_26" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "When did Jolene buy her pet snake?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_15", + "session_22", + "session_14", + "session_16", + "session_8", + "session_28", + "session_1", + "session_12", + "session_29" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "What project was Jolene working on as of 1 February, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_13", + "session_3", + "session_1", + "session_12", + "session_7", + "session_25", + "session_14", + "session_22", + "session_16" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "Where did Deborah meet her new neighbor Anna?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_19", + "session_4", + "session_29", + "session_1", + "session_12", + "session_7", + "session_9", + "session_26", + "session_27" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "What activity did Jolene and her partner plan to do together instead of resuming yoga?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_22", + "session_15", + "session_3", + "session_30", + "session_2", + "session_26", + "session_12", + "session_13", + "session_11" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What milestone did Jolene achieve recently on 4 February, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_10", + "session_25", + "session_14", + "session_13", + "session_4", + "session_15", + "session_18", + "session_27", + "session_5" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What is Jolene's favorite book which she mentioned on 4 February, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_15", + "session_29", + "session_4", + "session_20", + "session_2", + "session_18", + "session_16", + "session_24", + "session_26" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What does Deborah bring with her whenever she comes to reflect on her mom?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_15", + "session_29", + "session_19", + "session_22", + "session_1", + "session_14", + "session_23", + "session_21", + "session_7" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_5" + ], + "question": "What new outlook did Jolene gain after her mini retreat on 9 February, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_14", + "session_13", + "session_28", + "session_19", + "session_27", + "session_12", + "session_18", + "session_7", + "session_21" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_5" + ], + "question": "What cool stuff did Jolene accomplish at the retreat on 9 February, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_21", + "session_17", + "session_14", + "session_23", + "session_25", + "session_4", + "session_27", + "session_12", + "session_16" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_5" + ], + "question": "What idea did Jolene have to help underprivileged kids learn about STEM subjects on 9 February, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_21", + "session_13", + "session_24", + "session_29", + "session_22", + "session_17", + "session_26", + "session_25", + "session_30" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_5" + ], + "question": "How does Jolene plan to involve local engineers in her idea of teaching STEM to underprivileged kids?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_4", + "session_18", + "session_29", + "session_22", + "session_17", + "session_30", + "session_13", + "session_15", + "session_21" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_6" + ], + "question": "What gave Deborah peace in the garden she visited?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_1", + "session_14", + "session_29", + "session_23", + "session_4", + "session_20", + "session_5", + "session_21", + "session_3" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_6" + ], + "question": "Why did Deborah spend time in the garden?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_7", + "session_4", + "session_30", + "session_26", + "session_14", + "session_29", + "session_8", + "session_2", + "session_19" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "How did Jolene and her partner initially meet?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_22", + "session_12", + "session_3", + "session_1", + "session_26", + "session_30", + "session_19", + "session_2", + "session_27" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What activity does Deborah incorporate into her daily routine after going for a morning jog in the park?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_19", + "session_15", + "session_13", + "session_10", + "session_4", + "session_12", + "session_28", + "session_22", + "session_26" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "According to Jolene, what does exercise help her to feel?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_15", + "session_9", + "session_28", + "session_29", + "session_4", + "session_14", + "session_1", + "session_16", + "session_2" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What did Deb share a photo of, which brought a smile to Jolene's face?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_25", + "session_9", + "session_2", + "session_23", + "session_29", + "session_6", + "session_20", + "session_4", + "session_7" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What is one of Jolene's favorite dishes?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_29", + "session_11", + "session_24", + "session_19", + "session_4", + "session_20", + "session_18", + "session_16", + "session_15" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What picture did Jolene share related to feeling overwhelmed?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_8", + "session_15", + "session_4", + "session_30", + "session_24", + "session_28", + "session_17", + "session_7", + "session_10" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What did Jolene and Deb discuss as a helpful strategy for studying and time management?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_21", + "session_8", + "session_10", + "session_26", + "session_9", + "session_15", + "session_19", + "session_24", + "session_27" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_9" + ], + "question": "What did Jolene ask Deb to help with on 13 March, 2023?", + "recall": 0.0, + "retrieved_ids": [ + "session_13", + "session_12", + "session_25", + "session_8", + "session_5", + "session_21", + "session_20", + "session_26", + "session_30", + "session_7" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "What method does Deb suggest Jolene to try for organizing tasks based on importance and urgency?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_24", + "session_18", + "session_15", + "session_16", + "session_27", + "session_1", + "session_8", + "session_22", + "session_19" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "What did Jolene and Anna discuss while watching the sunset by the sea?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_19", + "session_4", + "session_23", + "session_22", + "session_8", + "session_3", + "session_14", + "session_18", + "session_7" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "How does Jolene plan to pursue her dream of learning to surf?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_23", + "session_19", + "session_29", + "session_22", + "session_4", + "session_13", + "session_26", + "session_28", + "session_2" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_11" + ], + "question": "What did Deborah buy to enhance her yoga practice besides the props?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_28", + "session_22", + "session_14", + "session_20", + "session_15", + "session_26", + "session_16", + "session_21", + "session_17" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_11" + ], + "question": "What type of music does Deborah find helpful during her yoga practice?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_20", + "session_13", + "session_28", + "session_19", + "session_22", + "session_15", + "session_30", + "session_16", + "session_6" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_11" + ], + "question": "Who are the musicians mentioned by Jolene that she enjoys listening to during her yoga practice?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_22", + "session_19", + "session_16", + "session_20", + "session_11", + "session_12", + "session_9", + "session_7", + "session_21" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_11" + ], + "question": "What album does Deborah recommend for meditation and deep relaxation?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_13", + "session_20", + "session_25", + "session_2", + "session_8", + "session_28", + "session_4", + "session_7", + "session_19" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_12" + ], + "question": "Which show did Deborah go to with a friend on 9 April, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_23", + "session_4", + "session_5", + "session_22", + "session_6", + "session_9", + "session_24", + "session_28", + "session_7" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_12" + ], + "question": "What does Deborah find comforting about going to art shows?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_14", + "session_4", + "session_2", + "session_29", + "session_22", + "session_16", + "session_6", + "session_17", + "session_5" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_12" + ], + "question": "How does Jolene describe the time spent with her snakes and partner?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_12", + "session_19", + "session_22", + "session_2", + "session_30", + "session_13", + "session_23", + "session_28", + "session_8" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_12" + ], + "question": "What does Jolene enjoy doing with her partner after a long day?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_12", + "session_28", + "session_7", + "session_20", + "session_13", + "session_9", + "session_10", + "session_18", + "session_11" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "What is Jolene currently doing in June 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_7", + "session_10", + "session_20", + "session_13", + "session_15", + "session_4", + "session_26", + "session_11", + "session_2" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_14" + ], + "question": "For how long has Jolene had Seraphim as a pet?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_14", + "session_2", + "session_11", + "session_8", + "session_16", + "session_13", + "session_20", + "session_15", + "session_22" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_14" + ], + "question": "How does Jolene feel when spending time with Seraphim?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_2", + "session_8", + "session_29", + "session_19", + "session_4", + "session_28", + "session_6", + "session_26", + "session_7" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_14" + ], + "question": "Which new yoga pose did Deborah share a photo of?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_28", + "session_14", + "session_20", + "session_2", + "session_4", + "session_9", + "session_22", + "session_24", + "session_16" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "What group activity did Deborah start with Anna?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_10", + "session_19", + "session_2", + "session_12", + "session_8", + "session_4", + "session_3", + "session_11", + "session_26" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "What made being part of the running group easy for Deborah to stay motivated?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_13", + "session_25", + "session_8", + "session_2", + "session_9", + "session_10", + "session_18", + "session_1", + "session_3" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "Why did Jolene decide to get a snake as a pet?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_16", + "session_7", + "session_14", + "session_2", + "session_8", + "session_22", + "session_6", + "session_12", + "session_29" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "What is the favorite game Jolene plays with her partner?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_2", + "session_15", + "session_24", + "session_7", + "session_20", + "session_27", + "session_3", + "session_12", + "session_16" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "What activity does Deborah do with her cats?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_15", + "session_13", + "session_2", + "session_19", + "session_12", + "session_28", + "session_11", + "session_26", + "session_27" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "How does Jolene describe the feeling of finding her snake snuggled under the bed after it got out?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_10", + "session_22", + "session_12", + "session_18", + "session_8", + "session_28", + "session_19", + "session_4", + "session_2" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "Why does Deborah take her cats out for a run in the park every day?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_7", + "session_19", + "session_4", + "session_1", + "session_14", + "session_8", + "session_22", + "session_3", + "session_27" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "How did Jolene come to have her pet, Susie?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_28", + "session_2", + "session_15", + "session_29", + "session_1", + "session_14", + "session_19", + "session_22", + "session_4" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "What activities have been helping Jolene stay distracted during tough times?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_13", + "session_15", + "session_25", + "session_2", + "session_22", + "session_12", + "session_11", + "session_26", + "session_19" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "What kind of yoga routine does Deborah recommend to Jolene?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_6", + "session_13", + "session_3", + "session_26", + "session_7", + "session_12", + "session_11", + "session_25", + "session_16" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What did Jolene design inspired by their love for space and engines?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_11", + "session_15", + "session_28", + "session_23", + "session_4", + "session_22", + "session_5", + "session_10", + "session_3" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What journal has Jolene been using to help track tasks and stay organized?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_10", + "session_20", + "session_8", + "session_22", + "session_11", + "session_21", + "session_15", + "session_19", + "session_26" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "What game did Jolene recommend for being calming and cute?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_15", + "session_13", + "session_20", + "session_2", + "session_11", + "session_27", + "session_24", + "session_12", + "session_16" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "What game did Jolene suggest as an awesome open-world game for the Nintendo Switch?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_24", + "session_5", + "session_4", + "session_15", + "session_29", + "session_27", + "session_16", + "session_2", + "session_17" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "What did Deborah and her husband use to play to bond and make memories?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_2", + "session_28", + "session_15", + "session_4", + "session_23", + "session_24", + "session_12", + "session_27", + "session_7" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "What is special about the bench at the park near Deborah's house?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_1", + "session_4", + "session_29", + "session_8", + "session_23", + "session_2", + "session_28", + "session_20", + "session_3" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "What did Deborah and her mom chat about at their special bench in the park?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_1", + "session_4", + "session_2", + "session_20", + "session_23", + "session_28", + "session_29", + "session_22", + "session_7" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "What feeling does Deborah get when she thinks about the time spent with her mom at their special spot?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_22", + "session_19", + "session_23", + "session_29", + "session_1", + "session_28", + "session_15", + "session_2", + "session_13" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_20" + ], + "question": "What habits does Jolene practice to feel balanced?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_11", + "session_22", + "session_19", + "session_26", + "session_16", + "session_30", + "session_7", + "session_13", + "session_27" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_20" + ], + "question": "Which yoga pose is Jolene a fan of for rest and calmness?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_28", + "session_4", + "session_22", + "session_11", + "session_14", + "session_24", + "session_16", + "session_19", + "session_25" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_20" + ], + "question": "How long has Jolene been doing yoga?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_13", + "session_7", + "session_28", + "session_19", + "session_22", + "session_9", + "session_11", + "session_12", + "session_16" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_21" + ], + "question": "What did Jolene participate in recently that provided her with a rewarding experience?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_4", + "session_21", + "session_29", + "session_15", + "session_25", + "session_14", + "session_19", + "session_27", + "session_16" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_21" + ], + "question": "How did Jolene feel after receiving positive feedback at the virtual conference?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_30", + "session_18", + "session_10", + "session_13", + "session_28", + "session_19", + "session_2", + "session_3", + "session_12" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_21" + ], + "question": "What kind of event did Jolene present at recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_21", + "session_29", + "session_19", + "session_23", + "session_6", + "session_24", + "session_15", + "session_3", + "session_12" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_22" + ], + "question": "What did Jolene's mom stress the value of, which she wants to keep in mind for her engineering projects?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_13", + "session_20", + "session_2", + "session_8", + "session_23", + "session_30", + "session_17", + "session_19", + "session_15" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_22" + ], + "question": "What type of projects is Jolene interested in getting involved in the future?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_18", + "session_29", + "session_3", + "session_12", + "session_16", + "session_6", + "session_9", + "session_15", + "session_26" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_22" + ], + "question": "How did Deborah get Luna, one of her cats?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_1", + "session_16", + "session_14", + "session_28", + "session_15", + "session_4", + "session_27", + "session_29", + "session_2" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_22" + ], + "question": "How old is Max?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_4", + "session_1", + "session_24", + "session_28", + "session_29", + "session_23", + "session_2" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "What type of classes did Jolene and her partner check out during their trip to Rio de Janeiro on 30 August, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_12", + "session_22", + "session_4", + "session_30", + "session_11", + "session_28", + "session_26", + "session_6", + "session_24" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "What type of place does Jolene visit to meditate?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_23", + "session_28", + "session_1", + "session_29", + "session_11", + "session_14", + "session_20", + "session_22", + "session_17" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "What was the new plant Jolene got used as a reminder for on 30 August, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_25", + "session_8", + "session_29", + "session_4", + "session_3", + "session_14", + "session_28", + "session_22", + "session_30" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "Why did Jolene get the new plant on 30 August, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_28", + "session_4", + "session_3", + "session_14", + "session_24", + "session_8", + "session_16", + "session_29", + "session_22" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "What has Jolene been focusing on lately besides studying?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_24", + "session_21", + "session_19", + "session_25", + "session_20", + "session_8", + "session_15", + "session_22", + "session_27" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "How did Deborah's mom support her yoga practice when she first started?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_24", + "session_16", + "session_23", + "session_15", + "session_7", + "session_10", + "session_29", + "session_5", + "session_3" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "What was the video game console that Jolene's parents got her at age 10?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_2", + "session_15", + "session_20", + "session_19", + "session_29", + "session_3", + "session_12", + "session_16", + "session_7" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "What was one of Jolene's favorite games to play with her mom on the nintendo wii game system?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_2", + "session_24", + "session_7", + "session_20", + "session_12", + "session_15", + "session_16", + "session_27", + "session_17" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_25" + ], + "question": "What course did Jolene sign up for on 6 September 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_7", + "session_8", + "session_5", + "session_9", + "session_24", + "session_2", + "session_3", + "session_26", + "session_30" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_26" + ], + "question": "Why did Jolene have to reschedule their meeting with Deborah on September 8, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_1", + "session_11", + "session_27", + "session_23", + "session_28", + "session_22", + "session_7", + "session_3", + "session_26" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_27" + ], + "question": "Where did Jolene and her partner travel for a few weeks in September 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_2", + "session_27", + "session_4", + "session_8", + "session_1", + "session_12", + "session_20", + "session_19", + "session_5" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_27" + ], + "question": "What was the main focus of the session that stood out to Jolene during the retreat?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_27", + "session_14", + "session_11", + "session_13", + "session_4", + "session_30", + "session_25", + "session_23", + "session_7" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_27" + ], + "question": "How did Jolene feel about her progress in practicing mindfulness and gratitude?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_18", + "session_19", + "session_15", + "session_13", + "session_12", + "session_20", + "session_7", + "session_22", + "session_26" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_27" + ], + "question": "What positive change did Jolene experience during the retreat?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_4", + "session_30", + "session_25", + "session_13", + "session_15", + "session_22", + "session_27", + "session_5", + "session_2" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_27" + ], + "question": "What did Jolene recently play that she described to Deb?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_29", + "session_12", + "session_25", + "session_24", + "session_10", + "session_2", + "session_4", + "session_18", + "session_19" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_28" + ], + "question": "What did Deborah do with their mom's old friends?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_23", + "session_4", + "session_24", + "session_22", + "session_1", + "session_2", + "session_29", + "session_6", + "session_12" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_28" + ], + "question": "Where did Deborah get married?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_28", + "session_12", + "session_14", + "session_30", + "session_3", + "session_9", + "session_8", + "session_24", + "session_22" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_28" + ], + "question": "What does yoga on the beach provide for Deborah?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_6", + "session_24", + "session_28", + "session_19", + "session_4", + "session_11", + "session_25", + "session_29", + "session_7" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_28" + ], + "question": "How does Jolene describe their home room?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_23", + "session_22", + "session_30", + "session_1", + "session_26", + "session_11", + "session_18", + "session_16", + "session_5" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_29" + ], + "question": "What new activity did Deborah and her neighbor organize for the community on 16 September, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_19", + "session_29", + "session_22", + "session_8", + "session_15", + "session_3", + "session_9", + "session_10", + "session_18" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_29" + ], + "question": "What was Deborah's mom passionate about?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_2", + "session_23", + "session_24", + "session_28", + "session_29", + "session_22", + "session_15", + "session_4", + "session_12" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_29" + ], + "question": "What food did Deborah's mom make for her on birthdays?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_19", + "session_20", + "session_8", + "session_28", + "session_2", + "session_23", + "session_4", + "session_12", + "session_1" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_29" + ], + "question": "What kind of cookies did Jolene used to bake with someone close to her?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_15", + "session_6", + "session_19", + "session_4", + "session_12", + "session_2", + "session_23", + "session_13", + "session_3" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_29" + ], + "question": "What outdoor activity did Jolene suggest doing together with Deborah?", + "recall": 0.0, + "retrieved_ids": [ + "session_19", + "session_15", + "session_7", + "session_2", + "session_13", + "session_26", + "session_16", + "session_12", + "session_11", + "session_8" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_30" + ], + "question": "What activity did Deborah enjoy at the music festival with their pals on September 20, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_30", + "session_11", + "session_28", + "session_19", + "session_22", + "session_26", + "session_12", + "session_24", + "session_5", + "session_23" + ], + "sample_id": "conv-48" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_30" + ], + "question": "What did Deborah find freeing at the music festival?", + "recall": 1.0, + "retrieved_ids": [ + "session_30", + "session_11", + "session_28", + "session_19", + "session_9", + "session_12", + "session_24", + "session_8", + "session_29", + "session_26" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_2" + ], + "question": "What are the names of Deborah's snakes?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_2", + "session_15", + "session_12", + "session_29", + "session_25", + "session_14", + "session_8", + "session_13", + "session_23" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What are Deborah's favorite books?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_2", + "session_4", + "session_18", + "session_15", + "session_11", + "session_24", + "session_19", + "session_26", + "session_16" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_22" + ], + "question": "Where did Deborah get her dogs?", + "recall": 0.0, + "retrieved_ids": [ + "session_15", + "session_12", + "session_14", + "session_28", + "session_4", + "session_1", + "session_16", + "session_2", + "session_29", + "session_19" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_22" + ], + "question": "How old are Jolene's cats?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_4", + "session_1", + "session_24", + "session_28", + "session_23", + "session_15", + "session_29", + "session_27", + "session_2" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_24" + ], + "question": "When did Deborah's parents give her first console?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_10", + "session_7", + "session_3", + "session_24", + "session_29", + "session_5", + "session_20", + "session_28", + "session_19" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_2" + ], + "question": "When did Jolene release her pet snake?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_15", + "session_22", + "session_14", + "session_16", + "session_8", + "session_28", + "session_29", + "session_12", + "session_27" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_3" + ], + "question": "Where did Jolene meet her new friend Anna?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_4", + "session_8", + "session_28", + "session_19", + "session_7", + "session_15", + "session_1", + "session_26", + "session_27" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What is Deborah's favorite book which she mentioned on 4 February, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_15", + "session_29", + "session_4", + "session_20", + "session_2", + "session_18", + "session_16", + "session_24", + "session_26" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_5" + ], + "question": "What cool stuff did Deborah accomplish at the retreat on 9 February, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_21", + "session_17", + "session_14", + "session_23", + "session_16", + "session_25", + "session_4", + "session_27", + "session_12" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_5" + ], + "question": "How does Deborah plan to involve local engineers in her idea of teaching STEM to underprivileged kids?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_4", + "session_18", + "session_29", + "session_22", + "session_17", + "session_30", + "session_13", + "session_15", + "session_21" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_6" + ], + "question": "What gave Deborah anxiety in the garden she visited?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_1", + "session_20", + "session_14", + "session_29", + "session_23", + "session_4", + "session_5", + "session_21", + "session_3" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_6" + ], + "question": "Why did Jolene spend time in the garden?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_7", + "session_4", + "session_30", + "session_26", + "session_14", + "session_29", + "session_8", + "session_2", + "session_19" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_7" + ], + "question": "How did Jolene and her rival initially meet?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_1", + "session_16", + "session_7", + "session_3", + "session_26", + "session_28", + "session_29", + "session_2", + "session_27" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_7" + ], + "question": "What activity does Jolene incorporate into her daily routine after going for a morning jog in the park?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_13", + "session_19", + "session_15", + "session_10", + "session_4", + "session_12", + "session_28", + "session_22", + "session_26" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_10" + ], + "question": "What method does Jolene suggest Deborah to try for organizing tasks based on importance and urgency?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_24", + "session_18", + "session_16", + "session_19", + "session_15", + "session_27", + "session_1", + "session_8", + "session_22" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_10" + ], + "question": "How does Jolene plan to pursue her dream of climbing mountains?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_22", + "session_4", + "session_16", + "session_13", + "session_10", + "session_29", + "session_30", + "session_28", + "session_2" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_11" + ], + "question": "Who are the authors mentioned by Jolene that she enjoys reading during her yoga practice?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_22", + "session_16", + "session_20", + "session_15", + "session_2", + "session_11", + "session_12", + "session_4", + "session_29" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_12" + ], + "question": "Which show did Jolene go to with a friend on 9 April, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_23", + "session_5", + "session_4", + "session_22", + "session_6", + "session_9", + "session_24", + "session_28", + "session_7" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_12" + ], + "question": "What does Deborah find comforting about going to horror movie screenings?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_8", + "session_6", + "session_12", + "session_2", + "session_28", + "session_22", + "session_16", + "session_26", + "session_5" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_12" + ], + "question": "How does Deborah describe the time spent with her snakes and partner?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_12", + "session_19", + "session_22", + "session_2", + "session_30", + "session_13", + "session_23", + "session_28", + "session_8" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_14" + ], + "question": "For how long has Jolene had Lucifer as a pet?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_14", + "session_11", + "session_16", + "session_2", + "session_13", + "session_9", + "session_20", + "session_15", + "session_22" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_14" + ], + "question": "How does Deborah feel when spending time with Seraphim?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_2", + "session_8", + "session_29", + "session_19", + "session_4", + "session_28", + "session_6", + "session_26", + "session_7" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_15" + ], + "question": "What made being part of the running group easy for Jolene to stay motivated?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_13", + "session_25", + "session_8", + "session_2", + "session_9", + "session_10", + "session_18", + "session_1", + "session_3" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_15" + ], + "question": "Why did Jolene decide to get a tarantula as a pet?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_7", + "session_14", + "session_8", + "session_16", + "session_28", + "session_22", + "session_6", + "session_2", + "session_29" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_16" + ], + "question": "How did Deborah come to have her pet, Susie?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_28", + "session_2", + "session_15", + "session_29", + "session_1", + "session_14", + "session_19", + "session_22", + "session_4" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_17" + ], + "question": "What did Deborah design inspired by their love for space and engines?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_15", + "session_11", + "session_28", + "session_23", + "session_4", + "session_22", + "session_5", + "session_10", + "session_3" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "What journal has Deborah been using to help track tasks and stay organized?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_10", + "session_20", + "session_8", + "session_22", + "session_11", + "session_21", + "session_15", + "session_19", + "session_26" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_19" + ], + "question": "What game did Jolene recommend to Deborah for being thrilling and intense?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_13", + "session_2", + "session_19", + "session_15", + "session_12", + "session_27", + "session_24", + "session_21", + "session_11" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_19" + ], + "question": "What game did Deborah suggest as an awesome open-world game for the Nintendo Switch?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_24", + "session_5", + "session_4", + "session_15", + "session_29", + "session_27", + "session_16", + "session_2", + "session_17" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_19" + ], + "question": "What is special about the bench at the park near Jolene's house?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_1", + "session_4", + "session_29", + "session_8", + "session_23", + "session_28", + "session_2", + "session_20", + "session_3" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_19" + ], + "question": "What did Jolene and her mom chat about at their special bench in the park?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_1", + "session_4", + "session_2", + "session_20", + "session_29", + "session_23", + "session_28", + "session_22", + "session_7" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_21" + ], + "question": "How did Deborah feel after receiving positive feedback at the virtual conference?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_30", + "session_18", + "session_10", + "session_13", + "session_28", + "session_19", + "session_2", + "session_3", + "session_12" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_21" + ], + "question": "What kind of event did Deborah present at recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_21", + "session_23", + "session_29", + "session_19", + "session_6", + "session_24", + "session_15", + "session_3", + "session_12" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_22" + ], + "question": "What did Deborah's mom stress the value of, which she wants to keep in mind for her engineering projects?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_13", + "session_20", + "session_8", + "session_2", + "session_23", + "session_30", + "session_17", + "session_19", + "session_15" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_22" + ], + "question": "What type of projects is Deborah interested in getting involved in the future?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_18", + "session_29", + "session_3", + "session_12", + "session_16", + "session_6", + "session_9", + "session_15", + "session_26" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_22" + ], + "question": "How did Jolene get Luna, one of her cats?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_14", + "session_1", + "session_16", + "session_28", + "session_15", + "session_4", + "session_27", + "session_29", + "session_2" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_23" + ], + "question": "What type of classes did Deborah and her partner check out during their trip to Rio de Janeiro on 30 August, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_12", + "session_22", + "session_4", + "session_30", + "session_11", + "session_28", + "session_26", + "session_6", + "session_24" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_23" + ], + "question": "Why did Deborah get the new plant on 30 August, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_28", + "session_4", + "session_3", + "session_14", + "session_24", + "session_8", + "session_16", + "session_29", + "session_22" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_24" + ], + "question": "How did Jolene's mom support her yoga practice when she first started?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_24", + "session_16", + "session_23", + "session_15", + "session_7", + "session_10", + "session_29", + "session_5", + "session_3" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_24" + ], + "question": "What was the video game console that Deborah's parents got her at age 10?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_2", + "session_15", + "session_20", + "session_19", + "session_29", + "session_3", + "session_12", + "session_16", + "session_7" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_24" + ], + "question": "What was one of Deborah's favorite games to play with her mom on the PlayStation game system?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_2", + "session_24", + "session_7", + "session_12", + "session_16", + "session_20", + "session_15", + "session_27", + "session_17" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_27" + ], + "question": "Where did Deborah and her partner travel for a few weeks in September 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_2", + "session_27", + "session_8", + "session_4", + "session_1", + "session_12", + "session_20", + "session_19", + "session_5" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_28" + ], + "question": "What did Jolene do with their mom's old friends?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_23", + "session_4", + "session_24", + "session_22", + "session_6", + "session_1", + "session_2", + "session_29", + "session_12" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_28" + ], + "question": "Where did Jolene get married?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_28", + "session_12", + "session_14", + "session_30", + "session_3", + "session_9", + "session_24", + "session_8", + "session_22" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_29" + ], + "question": "What new activity did Jolene and her neighbor organize for the community on 16 September, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_29", + "session_19", + "session_22", + "session_8", + "session_15", + "session_3", + "session_9", + "session_10", + "session_18" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_29" + ], + "question": "What food did Jolene's mom make for her on holidays?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_20", + "session_19", + "session_28", + "session_2", + "session_23", + "session_29", + "session_4", + "session_12", + "session_1" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_29" + ], + "question": "What kind of cookies did Deborah used to bake with someone close to her?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_15", + "session_6", + "session_19", + "session_4", + "session_12", + "session_2", + "session_23", + "session_13", + "session_3" + ], + "sample_id": "conv-48" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_30" + ], + "question": "What activity did Jolene enjoy at the music festival with their pals on September 20, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_30", + "session_11", + "session_28", + "session_19", + "session_22", + "session_26", + "session_12", + "session_24", + "session_5", + "session_23" + ], + "sample_id": "conv-48" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_18", + "session_22" + ], + "question": "What kind of car does Evan drive?", + "recall": 0.6667, + "retrieved_ids": [ + "session_10", + "session_20", + "session_18", + "session_19", + "session_9", + "session_17", + "session_8", + "session_16", + "session_22", + "session_25" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_18" + ], + "question": "What kinds of things did Evan have broken?", + "recall": 0.5, + "retrieved_ids": [ + "session_3", + "session_7", + "session_20", + "session_11", + "session_21", + "session_23", + "session_19", + "session_16", + "session_8", + "session_1" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_2" + ], + "question": "Where has Evan been on roadtrips with his family?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_8", + "session_1", + "session_11", + "session_20", + "session_15", + "session_19", + "session_2", + "session_23", + "session_5" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1" + ], + "question": "How many Prius has Evan owned?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_1", + "session_21", + "session_22", + "session_8", + "session_11", + "session_24", + "session_2", + "session_12", + "session_15" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_1" + ], + "question": "Which hobby did Sam take up in May 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_13", + "session_20", + "session_17", + "session_2", + "session_15", + "session_11", + "session_10", + "session_8", + "session_24" + ], + "sample_id": "conv-49" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_2" + ], + "question": "Which country was Evan visiting in May 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_21", + "session_15", + "session_16", + "session_12", + "session_4", + "session_1", + "session_23", + "session_2", + "session_17" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_2" + ], + "question": "How many roadtrips did Evan take in May 2023?", + "recall": 0.5, + "retrieved_ids": [ + "session_8", + "session_24", + "session_1", + "session_15", + "session_13", + "session_7", + "session_25", + "session_9", + "session_21", + "session_11" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_10", + "session_13", + "session_2", + "session_20", + "session_21", + "session_7" + ], + "question": "What new hobbies did Sam consider trying?", + "recall": 0.7143, + "retrieved_ids": [ + "session_13", + "session_1", + "session_20", + "session_17", + "session_11", + "session_8", + "session_4", + "session_10", + "session_3", + "session_2" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_8" + ], + "question": "What hobby did Evan start practicing a few years ago that he enjoys?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_8", + "session_20", + "session_21", + "session_2", + "session_1", + "session_4", + "session_10", + "session_12", + "session_24" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_2" + ], + "question": "When did Evan go to Jasper with his family?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_5", + "session_1", + "session_15", + "session_11", + "session_8", + "session_7", + "session_20", + "session_19", + "session_23" + ], + "sample_id": "conv-49" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_19", + "session_2" + ], + "question": "Which type of vacation would Evan prefer with his family, walking tours in metropolitan cities or camping trip in the outdoors?", + "recall": 0.5, + "retrieved_ids": [ + "session_11", + "session_25", + "session_8", + "session_13", + "session_19", + "session_15", + "session_6", + "session_7", + "session_20", + "session_24" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_12", + "session_13", + "session_14", + "session_15", + "session_16", + "session_17", + "session_2", + "session_24", + "session_25", + "session_3", + "session_5", + "session_6", + "session_7", + "session_8" + ], + "question": "What health issue did Sam face that motivated him to change his lifestyle?", + "recall": 0.4, + "retrieved_ids": [ + "session_7", + "session_5", + "session_6", + "session_3", + "session_24", + "session_25", + "session_18", + "session_20", + "session_9", + "session_22" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_2" + ], + "question": "When did Sam first go to the doctor and find out he had a weight problem?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_12", + "session_7", + "session_24", + "session_20", + "session_17", + "session_21", + "session_19", + "session_4", + "session_18" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_3" + ], + "question": "When did Evan have his sudden heart palpitation incident that really shocked him up?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_7", + "session_17", + "session_24", + "session_13", + "session_1", + "session_15", + "session_20", + "session_11", + "session_8" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_22", + "session_23", + "session_3", + "session_5" + ], + "question": "What is Evan's favorite food?", + "recall": 0.5, + "retrieved_ids": [ + "session_3", + "session_11", + "session_16", + "session_8", + "session_2", + "session_4", + "session_24", + "session_23", + "session_15", + "session_12" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_3" + ], + "question": "What kind of unhealthy snacks does Sam enjoy eating?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_11", + "session_4", + "session_20", + "session_22", + "session_19", + "session_10", + "session_7", + "session_3", + "session_16" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_22", + "session_3" + ], + "question": "What recurring issue frustrates Sam at the grocery store?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_11", + "session_24", + "session_14", + "session_3", + "session_25", + "session_18", + "session_9", + "session_6", + "session_5" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_4" + ], + "question": "When did Sam's friends mock him for being overweight?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_17", + "session_8", + "session_16", + "session_4", + "session_11", + "session_1", + "session_7", + "session_22", + "session_20" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_22", + "session_24", + "session_3", + "session_4" + ], + "question": "What kind of healthy food suggestions has Evan given to Sam?", + "recall": 0.75, + "retrieved_ids": [ + "session_24", + "session_7", + "session_20", + "session_13", + "session_18", + "session_3", + "session_19", + "session_16", + "session_22", + "session_10" + ], + "sample_id": "conv-49" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_12", + "session_14", + "session_22", + "session_3", + "session_8", + "session_9" + ], + "question": "Considering their conversations and personal growth, what advice might Evan and Sam give to someone facing a major life transition or challenge?", + "recall": 0.6667, + "retrieved_ids": [ + "session_11", + "session_14", + "session_20", + "session_16", + "session_23", + "session_8", + "session_12", + "session_9", + "session_24", + "session_25" + ], + "sample_id": "conv-49" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_14", + "session_2", + "session_3", + "session_4", + "session_5", + "session_7", + "session_8", + "session_9" + ], + "question": "In light of the health and dietary changes discussed, what would be an appropriate gift for both Evan and Sam to encourage their healthy lifestyles?", + "recall": 0.5, + "retrieved_ids": [ + "session_23", + "session_4", + "session_15", + "session_7", + "session_2", + "session_8", + "session_11", + "session_6", + "session_16", + "session_21" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_23", + "session_5" + ], + "question": "How does Evan describe the woman and his feelings for her that he met in Canada?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_21", + "session_7", + "session_20", + "session_11", + "session_23", + "session_2", + "session_1", + "session_6", + "session_24" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_5" + ], + "question": "When Evan did meet his future wife?", + "recall": 0.0, + "retrieved_ids": [ + "session_25", + "session_7", + "session_20", + "session_18", + "session_13", + "session_1", + "session_15", + "session_11", + "session_8", + "session_24" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_4" + ], + "question": "When did Sam start working out at the gym?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_14", + "session_1", + "session_15", + "session_17", + "session_11", + "session_8", + "session_24", + "session_7", + "session_13" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_5" + ], + "question": "What significant event happened in Sam's life towards the end of summer 2023?", + "recall": 0.0, + "retrieved_ids": [ + "session_19", + "session_6", + "session_11", + "session_16", + "session_25", + "session_23", + "session_24", + "session_1", + "session_14", + "session_21" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_5" + ], + "question": "Which year did Evan start taking care of his health seriously?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_12", + "session_4", + "session_11", + "session_5", + "session_17", + "session_2", + "session_8", + "session_1", + "session_20" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_5" + ], + "question": "What motivates Evan to take care of his health?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_5", + "session_6", + "session_2", + "session_17", + "session_12", + "session_14", + "session_20", + "session_16", + "session_4" + ], + "sample_id": "conv-49" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_5" + ], + "question": "What electronic device could Evan gift Sam to help him keep up with his fitness goals?", + "recall": 0.0, + "retrieved_ids": [ + "session_7", + "session_17", + "session_3", + "session_16", + "session_24", + "session_12", + "session_20", + "session_9", + "session_14", + "session_2" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_11", + "session_6" + ], + "question": "What kind of writing does Sam do to relax and cope with his health issues?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_7", + "session_11", + "session_16", + "session_25", + "session_6", + "session_5", + "session_18", + "session_3", + "session_22" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_5", + "session_6" + ], + "question": "Who did Evan meet on his trip to Canada, and who did he come back from Canada with?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_6", + "session_5", + "session_16", + "session_20", + "session_21", + "session_18", + "session_7", + "session_3", + "session_24" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_6" + ], + "question": "When Evan get back from a vacation with his SO?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_19", + "session_17", + "session_1", + "session_15", + "session_7", + "session_20", + "session_6", + "session_22", + "session_8" + ], + "sample_id": "conv-49" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_9" + ], + "question": "How might Evan and Sam's experiences with health and lifestyle changes influence their approach to stress and challenges?", + "recall": 0.0, + "retrieved_ids": [ + "session_2", + "session_8", + "session_14", + "session_18", + "session_25", + "session_11", + "session_5", + "session_21", + "session_23", + "session_22" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_21", + "session_6" + ], + "question": "What recurring frustration does Evan experience?", + "recall": 0.0, + "retrieved_ids": [ + "session_14", + "session_24", + "session_22", + "session_2", + "session_25", + "session_11", + "session_18", + "session_9", + "session_3", + "session_5" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_24", + "session_6" + ], + "question": "What is the recurring dream that Sam keeps having?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_6", + "session_17", + "session_11", + "session_23", + "session_14", + "session_15", + "session_1", + "session_7", + "session_3" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_20", + "session_7" + ], + "question": "What accidents has Evan's son faced lately?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_7", + "session_25", + "session_12", + "session_9", + "session_3", + "session_17", + "session_24", + "session_16", + "session_22" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_7" + ], + "question": "When was Evan's son injured at soccer?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_20", + "session_1", + "session_15", + "session_12", + "session_4", + "session_2", + "session_17", + "session_23", + "session_16" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_23", + "session_7", + "session_8" + ], + "question": "What kind of foods or recipes has Sam recommended to Evan?", + "recall": 0.6667, + "retrieved_ids": [ + "session_7", + "session_11", + "session_16", + "session_3", + "session_10", + "session_8", + "session_9", + "session_22", + "session_2", + "session_5" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_11", + "session_18", + "session_3", + "session_7", + "session_8" + ], + "question": "What kind of healthy meals did Sam start eating after getting a health scare?", + "recall": 0.6667, + "retrieved_ids": [ + "session_7", + "session_4", + "session_8", + "session_24", + "session_3", + "session_14", + "session_11", + "session_17", + "session_5", + "session_22" + ], + "sample_id": "conv-49" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_22" + ], + "question": "What role does nature and the outdoors play in Evan and Sam's mental well-being?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_13", + "session_8", + "session_14", + "session_25", + "session_22", + "session_11", + "session_6", + "session_9", + "session_15" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_2", + "session_7" + ], + "question": "How many months lapsed between Sam's first and second doctor's appointment?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_7", + "session_8", + "session_9", + "session_23", + "session_17", + "session_21", + "session_11", + "session_16", + "session_2" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_8" + ], + "question": "When did Evan start taking painting classes?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_11", + "session_1", + "session_10", + "session_20", + "session_21", + "session_24", + "session_13", + "session_7", + "session_2" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_8" + ], + "question": "Which classes did Evan join in mid-August 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_21", + "session_9", + "session_13", + "session_15", + "session_1", + "session_7", + "session_25", + "session_11", + "session_24" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_8" + ], + "question": "How did Evan get into painting?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_11", + "session_13", + "session_10", + "session_9", + "session_20", + "session_21", + "session_8", + "session_22", + "session_2" + ], + "sample_id": "conv-49" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_12", + "session_2", + "session_7" + ], + "question": "How often does Sam get health checkups?", + "recall": 0.6667, + "retrieved_ids": [ + "session_25", + "session_16", + "session_17", + "session_12", + "session_7", + "session_5", + "session_3", + "session_14", + "session_8", + "session_4" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_20", + "session_21", + "session_8" + ], + "question": "What kind of subjects does Evan enjoy painting?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_21", + "session_11", + "session_13", + "session_8", + "session_25", + "session_1", + "session_19", + "session_10", + "session_16" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_8", + "session_9" + ], + "question": "Which places in Canada was Evan visiting in July 2023?", + "recall": 0.5, + "retrieved_ids": [ + "session_21", + "session_6", + "session_5", + "session_23", + "session_11", + "session_8", + "session_16", + "session_12", + "session_15", + "session_17" + ], + "sample_id": "conv-49" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_21" + ], + "question": "How do Evan and Sam use creative outlets to cope with life's challenges?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_20", + "session_10", + "session_21", + "session_2", + "session_6", + "session_1", + "session_8", + "session_16", + "session_18" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_8" + ], + "question": "When did Evan go skiing in Banff?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_19", + "session_23", + "session_25", + "session_1", + "session_13", + "session_15", + "session_7", + "session_11", + "session_24" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_21", + "session_22", + "session_8", + "session_9" + ], + "question": "What new diet and lifestyle change did Sam adopt over time?", + "recall": 0.5, + "retrieved_ids": [ + "session_4", + "session_9", + "session_24", + "session_6", + "session_3", + "session_5", + "session_14", + "session_2", + "session_18", + "session_22" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_11", + "session_7", + "session_9" + ], + "question": "Who was injured in Evan's family?", + "recall": 0.3333, + "retrieved_ids": [ + "session_19", + "session_15", + "session_1", + "session_23", + "session_2", + "session_5", + "session_11", + "session_8", + "session_4", + "session_12" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_25", + "session_4", + "session_6", + "session_8", + "session_9" + ], + "question": "What kind of hobbies does Evan pursue?", + "recall": 0.1667, + "retrieved_ids": [ + "session_11", + "session_20", + "session_21", + "session_13", + "session_1", + "session_19", + "session_10", + "session_7", + "session_16", + "session_2" + ], + "sample_id": "conv-49" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_14", + "session_4" + ], + "question": "What challenges does Sam face in his quest for a healthier lifestyle, and how does he address them?", + "recall": 0.0, + "retrieved_ids": [ + "session_20", + "session_8", + "session_2", + "session_9", + "session_7", + "session_24", + "session_11", + "session_16", + "session_22", + "session_25" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_10" + ], + "question": "Which activity do Evan and Sam plan on doing together during September 2023?", + "recall": 0.0, + "retrieved_ids": [ + "session_16", + "session_6", + "session_13", + "session_17", + "session_24", + "session_4", + "session_19", + "session_25", + "session_9", + "session_23" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_10" + ], + "question": "When did Evan and Sam decide to paint together?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_1", + "session_11", + "session_16", + "session_10", + "session_5", + "session_8", + "session_21", + "session_20", + "session_2" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_11", + "session_3", + "session_9" + ], + "question": "What personal health incidents does Evan face in 2023?", + "recall": 0.6667, + "retrieved_ids": [ + "session_16", + "session_7", + "session_3", + "session_25", + "session_12", + "session_4", + "session_17", + "session_5", + "session_24", + "session_9" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_11", + "session_14" + ], + "question": "What recurring adventure does Evan have with strangers?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_13", + "session_11", + "session_5", + "session_14", + "session_2", + "session_4", + "session_23", + "session_15", + "session_12" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_11", + "session_14" + ], + "question": "What is Sam's persistent problem with his phone?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_20", + "session_11", + "session_23", + "session_14", + "session_18", + "session_5", + "session_12", + "session_22", + "session_2" + ], + "sample_id": "conv-49" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_13" + ], + "question": "Which US state was Sam travelling in during October 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_17", + "session_25", + "session_24", + "session_16", + "session_22", + "session_18", + "session_13", + "session_5", + "session_23" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_12" + ], + "question": "When did Evan start lifting weights?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_20", + "session_4", + "session_11", + "session_1", + "session_25", + "session_18", + "session_2", + "session_16", + "session_8" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_13" + ], + "question": "When did Sam and his friend decide to try kayaking?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_8", + "session_1", + "session_13", + "session_22", + "session_17", + "session_25", + "session_7", + "session_5", + "session_11" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_13" + ], + "question": "Which new activity does Sam take up in October 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_9", + "session_20", + "session_8", + "session_13", + "session_15", + "session_18", + "session_4", + "session_11", + "session_10" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_13" + ], + "question": "What kind of stress was Sam dealing with in October 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_18", + "session_20", + "session_11", + "session_16", + "session_21", + "session_19", + "session_7", + "session_6", + "session_5" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_14", + "session_17", + "session_3" + ], + "question": "What health scares did Sam and Evan experience?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_25", + "session_17", + "session_3", + "session_7", + "session_5", + "session_15", + "session_14", + "session_22", + "session_1" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_14" + ], + "question": "When was Sam in the ER?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_2", + "session_1", + "session_15", + "session_23", + "session_12", + "session_4", + "session_17", + "session_5", + "session_8" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_12", + "session_14", + "session_2", + "session_7" + ], + "question": "Which ailment does Sam have to face due to his weight?", + "recall": 0.75, + "retrieved_ids": [ + "session_20", + "session_25", + "session_12", + "session_18", + "session_4", + "session_7", + "session_2", + "session_16", + "session_23", + "session_9" + ], + "sample_id": "conv-49" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_16" + ], + "question": "Does Evan live close to a beach or mountains?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_25", + "session_20", + "session_22", + "session_2", + "session_24", + "session_3", + "session_9", + "session_8", + "session_23" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_16" + ], + "question": "When did Evan lose his job?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_7", + "session_20", + "session_13", + "session_1", + "session_15", + "session_6", + "session_11", + "session_8", + "session_24" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_16" + ], + "question": "When did Evan and Sam planned a trip to the beach together?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_1", + "session_13", + "session_9", + "session_24", + "session_3", + "session_19", + "session_4", + "session_17", + "session_23" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_18" + ], + "question": "What was Sam doing on December 4, 2023?", + "recall": 0.0, + "retrieved_ids": [ + "session_10", + "session_2", + "session_7", + "session_20", + "session_11", + "session_17", + "session_16", + "session_8", + "session_9", + "session_21" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_19", + "session_21" + ], + "question": "Which two significant life events occur in Evan's life in December 2023 with his partner?", + "recall": 0.5, + "retrieved_ids": [ + "session_19", + "session_5", + "session_23", + "session_20", + "session_11", + "session_25", + "session_16", + "session_1", + "session_7", + "session_24" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_21", + "session_5" + ], + "question": "How long did Evan and his partner date before getting married?", + "recall": 0.5, + "retrieved_ids": [ + "session_21", + "session_22", + "session_24", + "session_7", + "session_10", + "session_20", + "session_4", + "session_1", + "session_19", + "session_25" + ], + "sample_id": "conv-49" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_21" + ], + "question": "Which major holiday season conincides with Evan's wedding?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_19", + "session_21", + "session_15", + "session_12", + "session_1", + "session_4", + "session_2", + "session_17", + "session_16" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_20", + "session_22" + ], + "question": "Which activity did Sam resume in December 2023 after a long time?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_10", + "session_5", + "session_6", + "session_11", + "session_8", + "session_25", + "session_22", + "session_7", + "session_24" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_19" + ], + "question": "When is Evan planning a big family reunion?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_25", + "session_1", + "session_8", + "session_23", + "session_10", + "session_15", + "session_18", + "session_4", + "session_16" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_20" + ], + "question": "When did Evan's son fall off his bike?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_7", + "session_10", + "session_17", + "session_21", + "session_23", + "session_11", + "session_1", + "session_6", + "session_8" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_23" + ], + "question": "When did Evan announce his marriage to his extended family?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_2", + "session_1", + "session_7", + "session_15", + "session_20", + "session_19", + "session_22", + "session_11", + "session_8" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_20" + ], + "question": "When did Evan finish the painting that's hanging in the exhibit?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_21", + "session_1", + "session_10", + "session_13", + "session_11", + "session_8", + "session_2", + "session_17", + "session_24" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_23", + "session_24" + ], + "question": "How does Evan spend his time with his bride after the wedding?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_20", + "session_25", + "session_8", + "session_24", + "session_10", + "session_5", + "session_23", + "session_2", + "session_22" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_21", + "session_22", + "session_23" + ], + "question": "Who did Evan tell about his marriage?", + "recall": 0.6667, + "retrieved_ids": [ + "session_22", + "session_7", + "session_20", + "session_14", + "session_2", + "session_23", + "session_11", + "session_18", + "session_1", + "session_8" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_23" + ], + "question": "When will Evan and his partner have their honeymoon in Canada?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_24", + "session_6", + "session_25", + "session_5", + "session_21", + "session_7", + "session_20", + "session_19", + "session_11" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_24" + ], + "question": "When did Evan have a drunken night with his friends?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_25", + "session_20", + "session_24", + "session_4", + "session_1", + "session_22", + "session_6", + "session_17", + "session_8" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_10", + "session_11", + "session_16", + "session_18", + "session_2", + "session_24", + "session_8" + ], + "question": "What is a stress reliever for Evan?", + "recall": 0.875, + "retrieved_ids": [ + "session_11", + "session_24", + "session_2", + "session_13", + "session_18", + "session_10", + "session_8", + "session_15", + "session_14", + "session_1" + ], + "sample_id": "conv-49" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_13", + "session_16", + "session_18" + ], + "question": "What is a stress reliever for Sam?", + "recall": 0.75, + "retrieved_ids": [ + "session_11", + "session_24", + "session_2", + "session_13", + "session_18", + "session_10", + "session_8", + "session_15", + "session_14", + "session_1" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_1" + ], + "question": "What type of car did Evan get after his old Prius broke down?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_18", + "session_10", + "session_8", + "session_25", + "session_7", + "session_20", + "session_22", + "session_17", + "session_2" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_1" + ], + "question": "How did Evan get into watercolor painting?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_11", + "session_8", + "session_9", + "session_13", + "session_10", + "session_20", + "session_21", + "session_22", + "session_2" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_1" + ], + "question": "What did Evan start doing a few years back as a stress-buster?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_11", + "session_8", + "session_4", + "session_19", + "session_15", + "session_12", + "session_2", + "session_17", + "session_5" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_1" + ], + "question": "What advice did Evan give Sam about finding a passion?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_21", + "session_20", + "session_24", + "session_8", + "session_13", + "session_7", + "session_12", + "session_11", + "session_15" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "Where did Evan take his family for a road trip on 24 May, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_2", + "session_5", + "session_11", + "session_9", + "session_7", + "session_19", + "session_8", + "session_20", + "session_23" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "What did Evan find relaxing about his road trip to Jasper?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_18", + "session_11", + "session_8", + "session_16", + "session_9", + "session_25", + "session_7", + "session_13", + "session_20" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "What habit is Sam trying to change in terms of diet?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_10", + "session_4", + "session_15", + "session_2", + "session_17", + "session_23", + "session_6", + "session_9", + "session_24" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "What new suggestion did Evan give to Sam regarding his soda and candy consumption?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_6", + "session_13", + "session_4", + "session_20", + "session_24", + "session_18", + "session_7", + "session_10", + "session_9" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "What did Sam agree to try instead of soda and candy?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_4", + "session_23", + "session_1", + "session_6", + "session_8", + "session_15", + "session_25", + "session_11", + "session_24" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "What frustrating issue did Sam face at the supermarket?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_18", + "session_9", + "session_8", + "session_25", + "session_1", + "session_11", + "session_5", + "session_6", + "session_22" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What novel is Evan reading that he finds gripping?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_25", + "session_1", + "session_10", + "session_21", + "session_12", + "session_2", + "session_22", + "session_20", + "session_24" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What kind of water does Evan suggest Sam try as an alternative to soda?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_20", + "session_3", + "session_13", + "session_7", + "session_10", + "session_18", + "session_19", + "session_16", + "session_25" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_5" + ], + "question": "What does the smartwatch help Evan with?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_5", + "session_13", + "session_6", + "session_10", + "session_16", + "session_15", + "session_2", + "session_22", + "session_25" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_5" + ], + "question": "What does the bonsai tree symbolize for Evan?", + "recall": 0.0, + "retrieved_ids": [], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_5" + ], + "question": "Why did Evan decide to get the bonsai tree?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_5", + "session_1", + "session_9", + "session_15", + "session_13", + "session_7", + "session_25", + "session_8", + "session_24" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_6" + ], + "question": "According to Sam, what is more important than perfection?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_15", + "session_4", + "session_17", + "session_19", + "session_23", + "session_24", + "session_22", + "session_9", + "session_16" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_6" + ], + "question": "What did Evan suggest Sam to check out for insights into his dream?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_9", + "session_1", + "session_7", + "session_22", + "session_20", + "session_24", + "session_12", + "session_13", + "session_18" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_6" + ], + "question": "What did Evan mention he had been searching for fruitlessly for half an hour?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_14", + "session_13", + "session_20", + "session_24", + "session_11", + "session_8", + "session_1", + "session_9", + "session_15" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What class is Sam taking to learn how to make healthier meals?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_8", + "session_24", + "session_11", + "session_2", + "session_10", + "session_6", + "session_22", + "session_18", + "session_3" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What dish did Sam make on 18 August, 2023 that turned out flavorful?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_10", + "session_23", + "session_11", + "session_2", + "session_1", + "session_8", + "session_14", + "session_4", + "session_3" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What kind of recipe did Evan request from Sam on 19 August, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_1", + "session_19", + "session_20", + "session_8", + "session_11", + "session_21", + "session_16", + "session_23", + "session_22" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What food did Sam share a photo of on 19 August, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_19", + "session_16", + "session_11", + "session_7", + "session_6", + "session_8", + "session_22", + "session_25", + "session_1" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What type of painting classes did Evan start taking in 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_11", + "session_24", + "session_1", + "session_10", + "session_20", + "session_21", + "session_7", + "session_13", + "session_2" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What did Evan start painting years ago due to being inspired by a friend's gift?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_11", + "session_1", + "session_4", + "session_20", + "session_21", + "session_15", + "session_2", + "session_16", + "session_12" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What nature concept do watercolor painting classes emphasize according to Evan?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_1", + "session_13", + "session_20", + "session_11", + "session_25", + "session_10", + "session_21", + "session_9", + "session_7" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What type of landscapes does Evan love painting the most?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_21", + "session_2", + "session_13", + "session_1", + "session_20", + "session_23", + "session_11", + "session_10", + "session_24" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What fun activity did Evan mention doing in July 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_20", + "session_6", + "session_11", + "session_13", + "session_4", + "session_8", + "session_24", + "session_14", + "session_9" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_9" + ], + "question": "What injury did Evan suffer from in August 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_9", + "session_1", + "session_13", + "session_8", + "session_11", + "session_7", + "session_24", + "session_15", + "session_25" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_9" + ], + "question": "What sports activity has Evan been doing to stay active while dealing with the knee injury?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_13", + "session_5", + "session_20", + "session_6", + "session_24", + "session_12", + "session_9", + "session_8", + "session_18" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_9" + ], + "question": "What suggestion did Sam give to Evan to help with his knee issue?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_22", + "session_7", + "session_3", + "session_18", + "session_9", + "session_11", + "session_6", + "session_5", + "session_25" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "What did Evan suggest Sam try as a calming hobby?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_13", + "session_1", + "session_17", + "session_9", + "session_7", + "session_11", + "session_24", + "session_16", + "session_25" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "What did Evan recommend Sam acquire to get started with painting?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_11", + "session_1", + "session_9", + "session_21", + "session_13", + "session_8", + "session_2", + "session_5", + "session_22" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_11" + ], + "question": "What activity does Evan do to keep himself busy while healing his knee?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_20", + "session_23", + "session_8", + "session_19", + "session_7", + "session_9", + "session_24", + "session_12", + "session_13" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_11" + ], + "question": "What painting did Evan share with Sam in October?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_13", + "session_1", + "session_7", + "session_19", + "session_16", + "session_11", + "session_21", + "session_22", + "session_6" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_11" + ], + "question": "What kind of writing does Sam enjoy as a form of expression?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_23", + "session_21", + "session_19", + "session_20", + "session_7", + "session_8", + "session_16", + "session_12", + "session_2" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_11" + ], + "question": "What electronics issue has been frustrating Sam lately?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_9", + "session_18", + "session_17", + "session_7", + "session_6", + "session_11", + "session_5", + "session_25", + "session_22" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_12" + ], + "question": "What activity did Evan start one year ago?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_4", + "session_20", + "session_11", + "session_12", + "session_1", + "session_21", + "session_15", + "session_5", + "session_2" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_12" + ], + "question": "What advice did Evan give to Sam to avoid injuries while starting weightlifting?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_24", + "session_20", + "session_13", + "session_1", + "session_23", + "session_19", + "session_11", + "session_8", + "session_15" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "Where did Sam and his mate plan to try kayaking?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_20", + "session_10", + "session_23", + "session_19", + "session_24", + "session_7", + "session_15", + "session_25", + "session_16" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_14" + ], + "question": "What digestive issue did Sam experience lately?", + "recall": 0.0, + "retrieved_ids": [ + "session_5", + "session_25", + "session_11", + "session_3", + "session_6", + "session_22", + "session_2", + "session_18", + "session_1", + "session_8" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_14" + ], + "question": "What adventurous theme is emerging in Evan's life as mentioned by Sam?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_3", + "session_6", + "session_20", + "session_17", + "session_5", + "session_9", + "session_13", + "session_11", + "session_24" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_14" + ], + "question": "What does Evan mention about his progress at the gym to Sam?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_4", + "session_17", + "session_5", + "session_20", + "session_7", + "session_6", + "session_3", + "session_12", + "session_15" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "How did Evan start his transformation journey two years ago?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_11", + "session_15", + "session_12", + "session_4", + "session_1", + "session_20", + "session_2", + "session_5", + "session_7" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "What gift did Evan receive from a close friend?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_20", + "session_1", + "session_4", + "session_17", + "session_24", + "session_8", + "session_22", + "session_15", + "session_11" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "Why had Evan been going through a tough time lately?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_7", + "session_16", + "session_17", + "session_20", + "session_5", + "session_2", + "session_15", + "session_23", + "session_13" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "How does Evan describe the island he grew up on?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_5", + "session_20", + "session_15", + "session_7", + "session_11", + "session_3", + "session_24", + "session_2", + "session_14" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What was the main reason for Evan's frustration with his new Prius breaking down?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_1", + "session_20", + "session_11", + "session_7", + "session_10", + "session_22", + "session_8", + "session_3", + "session_9" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "How did Sam suggest Evan view the setback with his broken Prius?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_1", + "session_13", + "session_7", + "session_9", + "session_20", + "session_22", + "session_3", + "session_11", + "session_6" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What did Sam suggest Evan try for stress relief and flexibility?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_10", + "session_13", + "session_8", + "session_15", + "session_1", + "session_7", + "session_24", + "session_9", + "session_6" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What did Sam offer Evan regarding yoga?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_18", + "session_6", + "session_7", + "session_2", + "session_1", + "session_13", + "session_11", + "session_8", + "session_15" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "What news did Evan share with Sam on 9th December 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_19", + "session_8", + "session_16", + "session_15", + "session_22", + "session_1", + "session_6", + "session_11", + "session_24" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "What family event is Evan planning for next summer?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_23", + "session_16", + "session_11", + "session_24", + "session_8", + "session_15", + "session_25", + "session_10", + "session_20" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "What is the motto of Evan's family?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_1", + "session_23", + "session_5", + "session_11", + "session_8", + "session_15", + "session_2", + "session_4", + "session_12" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_20" + ], + "question": "According to Evan, what is important for Sam to believe in concerning his weight?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_12", + "session_7", + "session_16", + "session_17", + "session_2", + "session_24", + "session_21", + "session_11", + "session_9" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_20" + ], + "question": "Who helped Evan get the painting published in the exhibition?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_13", + "session_10", + "session_21", + "session_11", + "session_12", + "session_1", + "session_2", + "session_6", + "session_8" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_21" + ], + "question": "What did Sam recently start enjoying to clear his head?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_3", + "session_8", + "session_20", + "session_11", + "session_9", + "session_7", + "session_13", + "session_19", + "session_17" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_21" + ], + "question": "What did Sam suggest Evan should do with his keys?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_9", + "session_20", + "session_6", + "session_12", + "session_18", + "session_1", + "session_21", + "session_11", + "session_16" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_21" + ], + "question": "How did Evan feel when he painted the piece with the bird flying over it?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_24", + "session_10", + "session_13", + "session_5", + "session_6", + "session_20", + "session_11", + "session_1", + "session_8" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_21" + ], + "question": "What did Evan suggest Sam should keep doing to find his own version of love?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_16", + "session_1", + "session_20", + "session_6", + "session_11", + "session_2", + "session_7", + "session_9", + "session_18" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_21" + ], + "question": "How did Evan describe the process of creating the painting with the bird flying over it?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_11", + "session_2", + "session_24", + "session_19", + "session_5", + "session_8", + "session_6", + "session_23", + "session_3" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_22" + ], + "question": "What did Evan want to share with his work friends?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_8", + "session_16", + "session_22", + "session_20", + "session_24", + "session_17", + "session_19", + "session_1", + "session_6" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_22" + ], + "question": "What did Evan share with Sam after their hiking trip?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_16", + "session_1", + "session_7", + "session_20", + "session_23", + "session_11", + "session_5", + "session_8", + "session_6" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_22" + ], + "question": "What did Evan offer to share with Sam after talking about healthy snacks?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_7", + "session_24", + "session_8", + "session_2", + "session_5", + "session_4", + "session_10", + "session_13", + "session_6" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "What did Evan and his partner share with their extended family on January 5, 2024?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_19", + "session_7", + "session_11", + "session_22", + "session_8", + "session_20", + "session_24", + "session_16", + "session_25" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "What was Evan limiting himself to on his new diet?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_3", + "session_4", + "session_20", + "session_17", + "session_23", + "session_9", + "session_24", + "session_15", + "session_10" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "What sports activity did Evan and his partner try in a recent weekend?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_23", + "session_20", + "session_24", + "session_14", + "session_22", + "session_7", + "session_19", + "session_2", + "session_25" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "What advice did Evan suggest Sam seek from a doctor?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_7", + "session_1", + "session_18", + "session_20", + "session_17", + "session_9", + "session_12", + "session_2", + "session_13" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "What suggestions did Evan give for low-impact exercises?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_13", + "session_9", + "session_3", + "session_6", + "session_25", + "session_18", + "session_10", + "session_4", + "session_7" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "What movie did Sam watch that motivated him to keep up with his routine?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_24", + "session_12", + "session_5", + "session_16", + "session_20", + "session_17", + "session_9", + "session_15", + "session_25" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "What activity helped Evan with stress and flexibility?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_10", + "session_11", + "session_2", + "session_13", + "session_24", + "session_8", + "session_15", + "session_4", + "session_9" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_25" + ], + "question": "What did Evan share a photo of that was taken on a camping trip?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_19", + "session_25", + "session_22", + "session_24", + "session_1", + "session_7", + "session_6", + "session_11", + "session_8" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_25" + ], + "question": "Why did Evan apologize to his partner?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_19", + "session_7", + "session_20", + "session_24", + "session_23", + "session_15", + "session_1", + "session_11", + "session_8" + ], + "sample_id": "conv-49" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_25" + ], + "question": "How does Evan describe being out on the water while kayaking and watching the sunset?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_13", + "session_11", + "session_16", + "session_21", + "session_24", + "session_3", + "session_19", + "session_7", + "session_4" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_1" + ], + "question": "What type of car did Sam get after his old Prius broke down?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_18", + "session_10", + "session_8", + "session_25", + "session_7", + "session_20", + "session_22", + "session_17", + "session_2" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_1" + ], + "question": "How did Sam get into watercolor painting?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_11", + "session_8", + "session_9", + "session_13", + "session_10", + "session_20", + "session_21", + "session_22", + "session_2" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_1" + ], + "question": "What did Sam start doing a few years back as a stress-buster?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_11", + "session_8", + "session_19", + "session_4", + "session_15", + "session_12", + "session_2", + "session_17", + "session_5" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_2" + ], + "question": "Where did Sam take his family for a road trip on 24 May, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_2", + "session_5", + "session_7", + "session_11", + "session_9", + "session_19", + "session_8", + "session_20", + "session_23" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_2" + ], + "question": "What did Sam find relaxing about his road trip to Jasper?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_18", + "session_11", + "session_8", + "session_16", + "session_9", + "session_25", + "session_7", + "session_13", + "session_20" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_3" + ], + "question": "What habit is Evan trying to change in terms of diet?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_10", + "session_4", + "session_15", + "session_2", + "session_17", + "session_23", + "session_6", + "session_9", + "session_24" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_3" + ], + "question": "What frustrating issue did Evan face at the supermarket?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_18", + "session_9", + "session_8", + "session_25", + "session_1", + "session_11", + "session_5", + "session_6", + "session_22" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What novel is Sam reading that he finds gripping?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_5", + "session_1", + "session_10", + "session_21", + "session_12", + "session_2", + "session_22", + "session_20", + "session_24" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_5" + ], + "question": "What does the smartwatch help Sam with?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_15", + "session_13", + "session_6", + "session_10", + "session_5", + "session_16", + "session_2", + "session_22", + "session_25" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_5" + ], + "question": "Why did Sam decide to get the bonsai tree?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_11", + "session_5", + "session_9", + "session_15", + "session_13", + "session_7", + "session_8", + "session_24", + "session_25" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_6" + ], + "question": "What did Sam mention he had been searching for fruitlessly for half an hour?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_13", + "session_14", + "session_20", + "session_24", + "session_11", + "session_8", + "session_1", + "session_9", + "session_15" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_7" + ], + "question": "What class is Evan taking to learn how to make healthier meals?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_8", + "session_24", + "session_11", + "session_2", + "session_10", + "session_6", + "session_22", + "session_18", + "session_3" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_7" + ], + "question": "What dish did Sam make on 18 August, 2023 that turned out bland?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_10", + "session_23", + "session_11", + "session_2", + "session_24", + "session_15", + "session_14", + "session_1", + "session_8" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_8" + ], + "question": "What food did Evan share a photo of on 19 August, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_19", + "session_16", + "session_11", + "session_7", + "session_6", + "session_8", + "session_22", + "session_25", + "session_1" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_8" + ], + "question": "What did Evan start sculpting years ago due to being inspired by a friend's gift?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_11", + "session_16", + "session_20", + "session_4", + "session_1", + "session_21", + "session_15", + "session_12", + "session_2" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_8" + ], + "question": "What nature concept do watercolor painting classes emphasize according to Sam?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_1", + "session_13", + "session_20", + "session_11", + "session_25", + "session_10", + "session_21", + "session_9", + "session_7" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_8" + ], + "question": "What type of landscapes does Sam love painting the most?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_21", + "session_2", + "session_13", + "session_1", + "session_20", + "session_23", + "session_11", + "session_10", + "session_24" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_9" + ], + "question": "What sports activity has Sam been doing to stay active while dealing with the knee injury?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_13", + "session_5", + "session_20", + "session_6", + "session_24", + "session_12", + "session_9", + "session_8", + "session_18" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_11" + ], + "question": "What activity does Sam do to keep himself busy while healing his knee?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_20", + "session_23", + "session_8", + "session_19", + "session_7", + "session_9", + "session_24", + "session_12", + "session_13" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_11" + ], + "question": "What kind of writing does Evan enjoy as a form of expression?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_23", + "session_21", + "session_19", + "session_20", + "session_7", + "session_8", + "session_16", + "session_12", + "session_2" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_11" + ], + "question": "What electronics issue has been frustrating Evan lately?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_18", + "session_9", + "session_17", + "session_7", + "session_6", + "session_11", + "session_5", + "session_25", + "session_22" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_12" + ], + "question": "What activity did Evan quit one year ago?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_1", + "session_4", + "session_11", + "session_12", + "session_15", + "session_2", + "session_3", + "session_16", + "session_14" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_13" + ], + "question": "Where did Sam and his mate plan to try skydiving?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_20", + "session_19", + "session_10", + "session_23", + "session_24", + "session_7", + "session_15", + "session_16", + "session_25" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_14" + ], + "question": "What digestive issue did Evan experience lately?", + "recall": 0.0, + "retrieved_ids": [ + "session_5", + "session_11", + "session_25", + "session_3", + "session_6", + "session_22", + "session_2", + "session_18", + "session_1", + "session_8" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_15" + ], + "question": "How did Sam start his transformation journey two years ago?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_11", + "session_15", + "session_12", + "session_4", + "session_1", + "session_20", + "session_2", + "session_5", + "session_7" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_16" + ], + "question": "What gift did Sam receive from a close friend?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_20", + "session_1", + "session_4", + "session_17", + "session_24", + "session_8", + "session_22", + "session_15", + "session_11" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_17" + ], + "question": "How does Sam describe the island he grew up on?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_15", + "session_20", + "session_3", + "session_7", + "session_11", + "session_5", + "session_24", + "session_2", + "session_14" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_18" + ], + "question": "What was the main reason for Evan's frustration with his new Prius getting stolen?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_7", + "session_12", + "session_22", + "session_1", + "session_20", + "session_11", + "session_13", + "session_3", + "session_9" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_19" + ], + "question": "What family event is Sam planning for next summer?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_23", + "session_16", + "session_11", + "session_24", + "session_8", + "session_15", + "session_25", + "session_10", + "session_20" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_19" + ], + "question": "What is the motto of Sam's family?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_1", + "session_23", + "session_5", + "session_11", + "session_8", + "session_15", + "session_2", + "session_4", + "session_17" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_20" + ], + "question": "Who helped Sam get the painting published in the exhibition?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_12", + "session_13", + "session_10", + "session_21", + "session_11", + "session_1", + "session_2", + "session_6", + "session_8" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_21" + ], + "question": "How did Sam feel when he painted the piece with the bird flying over it?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_24", + "session_10", + "session_13", + "session_5", + "session_6", + "session_20", + "session_11", + "session_1", + "session_8" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_21" + ], + "question": "How did Sam describe the process of creating the painting with the bird flying over it?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_11", + "session_2", + "session_24", + "session_19", + "session_5", + "session_8", + "session_6", + "session_23", + "session_3" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_23" + ], + "question": "What did Evan and his partner keep from their extended family on January 5, 2024?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_8", + "session_19", + "session_11", + "session_1", + "session_7", + "session_20", + "session_24", + "session_15", + "session_25" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_23" + ], + "question": "What was Sam limiting himself to on his new diet?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_3", + "session_4", + "session_20", + "session_17", + "session_23", + "session_9", + "session_24", + "session_15", + "session_10" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_24" + ], + "question": "What dance activity did Evan and his partner try in a recent weekend?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_20", + "session_23", + "session_9", + "session_14", + "session_22", + "session_7", + "session_19", + "session_25", + "session_2" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_24" + ], + "question": "What suggestions did Evan give for high-impact exercises?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_24", + "session_6", + "session_13", + "session_18", + "session_10", + "session_9", + "session_25", + "session_4", + "session_7" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_24" + ], + "question": "What movie did Evan watch that motivated him to keep up with his routine?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_24", + "session_12", + "session_5", + "session_16", + "session_20", + "session_17", + "session_9", + "session_15", + "session_25" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_24" + ], + "question": "What activity hindered Evan's stress and flexibility?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_10", + "session_11", + "session_8", + "session_2", + "session_13", + "session_4", + "session_24", + "session_15", + "session_9" + ], + "sample_id": "conv-49" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_25" + ], + "question": "What did Sam share a photo of that was taken on a camping trip?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_19", + "session_25", + "session_22", + "session_24", + "session_1", + "session_7", + "session_6", + "session_11", + "session_8" + ], + "sample_id": "conv-49" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_3" + ], + "question": "When did Calvin first travel to Tokyo?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_15", + "session_14", + "session_6", + "session_24", + "session_7", + "session_8", + "session_2", + "session_26", + "session_9" + ], + "sample_id": "conv-50" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_2" + ], + "question": "What items did Calvin buy in March 2023?", + "recall": 0.0, + "retrieved_ids": [], + "sample_id": "conv-50" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_2" + ], + "question": "When did Dave see Aerosmith perform live?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_23", + "session_18", + "session_7", + "session_24", + "session_29", + "session_22", + "session_10", + "session_15", + "session_9" + ], + "sample_id": "conv-50" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_2", + "session_23" + ], + "question": "Which bands has Dave enjoyed listening to?", + "recall": 0.5, + "retrieved_ids": [ + "session_28", + "session_2", + "session_15", + "session_19", + "session_3", + "session_9", + "session_1", + "session_8", + "session_25", + "session_18" + ], + "sample_id": "conv-50" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_3" + ], + "question": "Which country do Calvin and Dave want to meet in?", + "recall": 0.0, + "retrieved_ids": [ + "session_15", + "session_8", + "session_13", + "session_17", + "session_1", + "session_27", + "session_26", + "session_21", + "session_24", + "session_25" + ], + "sample_id": "conv-50" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_4", + "session_5" + ], + "question": "What are Dave's dreams?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_19", + "session_12", + "session_2", + "session_26", + "session_15", + "session_22", + "session_24", + "session_5", + "session_11" + ], + "sample_id": "conv-50" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_3", + "session_4" + ], + "question": "Which types of cars does Dave like the most?", + "recall": 0.3333, + "retrieved_ids": [ + "session_13", + "session_9", + "session_2", + "session_17", + "session_22", + "session_12", + "session_30", + "session_4", + "session_7", + "session_25" + ], + "sample_id": "conv-50" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_4" + ], + "question": "Does Dave's shop employ a lot of people?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_9", + "session_23", + "session_27", + "session_28", + "session_6", + "session_3", + "session_19", + "session_12", + "session_18" + ], + "sample_id": "conv-50" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_4" + ], + "question": "When did Dave start his car maintenance shop?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_23", + "session_6", + "session_9", + "session_20", + "session_1", + "session_28", + "session_18", + "session_29", + "session_24" + ], + "sample_id": "conv-50" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_6" + ], + "question": "When did a mishap occur with Calvin's musical gear and favorite mic?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_23", + "session_30", + "session_19", + "session_2", + "session_3", + "session_5", + "session_10", + "session_26", + "session_16" + ], + "sample_id": "conv-50" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_6" + ], + "question": "When did Calvin's place get flooded in Tokyo?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_14", + "session_8", + "session_3", + "session_24", + "session_1", + "session_7", + "session_11", + "session_15", + "session_27" + ], + "sample_id": "conv-50" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_6", + "session_9" + ], + "question": "What mishaps has Calvin run into?", + "recall": 0.5, + "retrieved_ids": [ + "session_7", + "session_19", + "session_22", + "session_27", + "session_26", + "session_3", + "session_25", + "session_28", + "session_14", + "session_9" + ], + "sample_id": "conv-50" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_6", + "session_7" + ], + "question": "When was Calvin's concert in Tokyo?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_15", + "session_28", + "session_6", + "session_3", + "session_19", + "session_14", + "session_7", + "session_5", + "session_25" + ], + "sample_id": "conv-50" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_7" + ], + "question": "Would Calvin enjoy performing at the Hollywood Bowl?", + "recall": 0.0, + "retrieved_ids": [ + "session_24", + "session_2", + "session_25", + "session_14", + "session_1", + "session_10", + "session_27", + "session_18", + "session_20", + "session_19" + ], + "sample_id": "conv-50" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_8" + ], + "question": "When did Calvin meet with the creative team for his new album?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_27", + "session_28", + "session_25", + "session_1", + "session_5", + "session_9", + "session_21", + "session_29", + "session_15" + ], + "sample_id": "conv-50" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_1", + "session_8" + ], + "question": "Why does Dave regularly visit parks?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_3", + "session_1", + "session_18", + "session_26", + "session_10", + "session_6", + "session_24", + "session_30", + "session_28" + ], + "sample_id": "conv-50" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_8" + ], + "question": "When did Dave take a trip to mountainous regions?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_29", + "session_20", + "session_3", + "session_12", + "session_6", + "session_11", + "session_1", + "session_17", + "session_9" + ], + "sample_id": "conv-50" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_9" + ], + "question": "When did Calvin have a car incident?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_21", + "session_28", + "session_14", + "session_9", + "session_3", + "session_12", + "session_2", + "session_26", + "session_4" + ], + "sample_id": "conv-50" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_6", + "session_9" + ], + "question": "How many times has Calvin had to deal with insurance paperwork?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_25", + "session_15", + "session_6", + "session_1", + "session_18", + "session_2", + "session_23", + "session_11", + "session_22" + ], + "sample_id": "conv-50" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_12", + "session_24", + "session_3" + ], + "question": "Which places or events has Calvin visited in Tokyo?", + "recall": 0.6667, + "retrieved_ids": [ + "session_24", + "session_6", + "session_3", + "session_7", + "session_27", + "session_15", + "session_14", + "session_1", + "session_8", + "session_25" + ], + "sample_id": "conv-50" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_12", + "session_26" + ], + "question": "Who inspired Dave's passion for car engineering?", + "recall": 0.5, + "retrieved_ids": [ + "session_26", + "session_13", + "session_4", + "session_5", + "session_25", + "session_11", + "session_22", + "session_3", + "session_20", + "session_7" + ], + "sample_id": "conv-50" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_12", + "session_27" + ], + "question": "Does Calvin wish to become more popular?", + "recall": 0.5, + "retrieved_ids": [ + "session_3", + "session_15", + "session_4", + "session_27", + "session_19", + "session_1", + "session_21", + "session_24", + "session_25", + "session_6" + ], + "sample_id": "conv-50" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_12", + "session_18" + ], + "question": "Does Calvin want to expand his brand?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_18", + "session_17", + "session_13", + "session_1", + "session_9", + "session_26", + "session_6", + "session_29", + "session_24" + ], + "sample_id": "conv-50" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_13", + "session_3", + "session_4", + "session_5" + ], + "question": "What is Dave's main passion?", + "recall": 0.25, + "retrieved_ids": [ + "session_23", + "session_6", + "session_11", + "session_26", + "session_22", + "session_4", + "session_28", + "session_14", + "session_16", + "session_29" + ], + "sample_id": "conv-50" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_13", + "session_20", + "session_22" + ], + "question": "Can Dave work with engines?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_21", + "session_20", + "session_3", + "session_7", + "session_26", + "session_4", + "session_25", + "session_13", + "session_5" + ], + "sample_id": "conv-50" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_15" + ], + "question": "When did Dave host a card-playing night with his friends?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_29", + "session_6", + "session_19", + "session_9", + "session_12", + "session_24", + "session_16", + "session_7", + "session_14" + ], + "sample_id": "conv-50" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_15" + ], + "question": "When did Calvin record a podcast with his friends?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_29", + "session_26", + "session_19", + "session_23", + "session_9", + "session_11", + "session_6", + "session_10", + "session_25" + ], + "sample_id": "conv-50" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_16" + ], + "question": "Which city was Calvin visiting in August 2023?", + "recall": 0.0, + "retrieved_ids": [ + "session_24", + "session_3", + "session_1", + "session_18", + "session_27", + "session_15", + "session_6", + "session_10", + "session_8", + "session_26" + ], + "sample_id": "conv-50" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_5", + "session_7" + ], + "question": "What does Calvin do to relax?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_8", + "session_11", + "session_7", + "session_30", + "session_28", + "session_21", + "session_24", + "session_13", + "session_2" + ], + "sample_id": "conv-50" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_27", + "session_5", + "session_8" + ], + "question": "What are Dave's hobbies other than fixing cars?", + "recall": 0.3333, + "retrieved_ids": [ + "session_7", + "session_19", + "session_12", + "session_5", + "session_17", + "session_21", + "session_13", + "session_22", + "session_18", + "session_10" + ], + "sample_id": "conv-50" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_2", + "session_28" + ], + "question": "What kind of music does Dave listen to?", + "recall": 0.3333, + "retrieved_ids": [ + "session_13", + "session_27", + "session_18", + "session_12", + "session_4", + "session_11", + "session_28", + "session_23", + "session_15", + "session_6" + ], + "sample_id": "conv-50" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_14", + "session_17" + ], + "question": "Where was Dave in the last two weeks of August 2023?", + "recall": 0.5, + "retrieved_ids": [ + "session_24", + "session_30", + "session_6", + "session_9", + "session_28", + "session_17", + "session_8", + "session_2", + "session_23", + "session_7" + ], + "sample_id": "conv-50" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_14", + "session_17" + ], + "question": "Where did Dave return from with new knowledge of different techniques of car restoration?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_1", + "session_28", + "session_25", + "session_17", + "session_5", + "session_4", + "session_9", + "session_12", + "session_22" + ], + "sample_id": "conv-50" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_14", + "session_17" + ], + "question": "What was Dave doing in San Francisco?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_10", + "session_13", + "session_17", + "session_22", + "session_4", + "session_5", + "session_28", + "session_19", + "session_20" + ], + "sample_id": "conv-50" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_17" + ], + "question": "When did Dave return from San Francisco?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_17", + "session_1", + "session_12", + "session_28", + "session_15", + "session_3", + "session_2", + "session_9", + "session_6" + ], + "sample_id": "conv-50" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_17" + ], + "question": "When did Calvin book flight tickets to Boston?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_8", + "session_24", + "session_15", + "session_3", + "session_26", + "session_2", + "session_21", + "session_30", + "session_29" + ], + "sample_id": "conv-50" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_18" + ], + "question": "When was Calvin's album released?", + "recall": 1.0, + "retrieved_ids": [ + "session_18", + "session_28", + "session_27", + "session_8", + "session_5", + "session_16", + "session_24", + "session_2", + "session_30", + "session_13" + ], + "sample_id": "conv-50" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [ + "session_16", + "session_18", + "session_7" + ], + "question": "Does Calvin love music tours?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_15", + "session_18", + "session_16", + "session_25", + "session_11", + "session_24", + "session_3", + "session_14", + "session_9" + ], + "sample_id": "conv-50" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_19" + ], + "question": "When did Dave have a great jam session with his band?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_2", + "session_20", + "session_9", + "session_28", + "session_8", + "session_3", + "session_29", + "session_15", + "session_24" + ], + "sample_id": "conv-50" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [], + "question": "Would Dave prefer working on a Dodge Charger or a Subaru Forester?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_10", + "session_25", + "session_4", + "session_14", + "session_1", + "session_2", + "session_20", + "session_18", + "session_19" + ], + "sample_id": "conv-50" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_20" + ], + "question": "What was the artists Calvin used to listen to when he was a kid?", + "recall": 0.0, + "retrieved_ids": [ + "session_28", + "session_12", + "session_10", + "session_22", + "session_7", + "session_29", + "session_15", + "session_2", + "session_24", + "session_9" + ], + "sample_id": "conv-50" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_12", + "session_20" + ], + "question": "Which of their family member do Calvin and Dave have nostalgic memories about?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_20", + "session_4", + "session_17", + "session_19", + "session_25", + "session_28", + "session_2", + "session_12", + "session_18" + ], + "sample_id": "conv-50" + }, + { + "category": 3, + "category_name": "Temporal-inference", + "evidence_ids": [], + "question": "Based on the conversation, did Calvin and Dave have a meeting in Boston between August and November 2023? Answer in yes or no.", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_7", + "session_30", + "session_21", + "session_27", + "session_15", + "session_25", + "session_1", + "session_22", + "session_11" + ], + "sample_id": "conv-50" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_21" + ], + "question": "Which city was Calvin at on October 3, 2023?", + "recall": 0.0, + "retrieved_ids": [ + "session_27", + "session_24", + "session_15", + "session_6", + "session_1", + "session_3", + "session_10", + "session_18", + "session_2", + "session_28" + ], + "sample_id": "conv-50" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_21" + ], + "question": "When did Calvin met with local artists in Boston?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_3", + "session_21", + "session_30", + "session_1", + "session_8", + "session_29", + "session_24", + "session_10", + "session_5" + ], + "sample_id": "conv-50" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_21" + ], + "question": "What shared activities do Dave and Calvin have?", + "recall": 0.0, + "retrieved_ids": [ + "session_23", + "session_2", + "session_10", + "session_15", + "session_18", + "session_28", + "session_25", + "session_17", + "session_19", + "session_6" + ], + "sample_id": "conv-50" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_19", + "session_21", + "session_22" + ], + "question": "What is Dave's favorite activity?", + "recall": 0.3333, + "retrieved_ids": [ + "session_23", + "session_30", + "session_10", + "session_2", + "session_3", + "session_5", + "session_19", + "session_6", + "session_26", + "session_16" + ], + "sample_id": "conv-50" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_22", + "session_3" + ], + "question": "How many car shows has Dave attended?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_26", + "session_2", + "session_22", + "session_18", + "session_3", + "session_25", + "session_19", + "session_29", + "session_30" + ], + "sample_id": "conv-50" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_22" + ], + "question": "What was Dave doing in the first weekend of October 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_8", + "session_19", + "session_16", + "session_26", + "session_22", + "session_3", + "session_4", + "session_28", + "session_13" + ], + "sample_id": "conv-50" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_12", + "session_22" + ], + "question": "When Dave was a child, what did he and his father do in the garage?", + "recall": 0.5, + "retrieved_ids": [ + "session_9", + "session_22", + "session_29", + "session_18", + "session_5", + "session_21", + "session_25", + "session_7", + "session_2", + "session_24" + ], + "sample_id": "conv-50" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_23" + ], + "question": "When did Calvin buy his second Ferrari?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_12", + "session_7", + "session_9", + "session_29", + "session_6", + "session_28", + "session_15", + "session_14", + "session_3" + ], + "sample_id": "conv-50" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_15", + "session_24" + ], + "question": "When did Calvin and Frank Ocean start collaborating?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_25", + "session_23", + "session_29", + "session_15", + "session_3", + "session_9", + "session_7", + "session_1", + "session_11" + ], + "sample_id": "conv-50" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_24" + ], + "question": "When did Calvin plan on travelling to Tokyo the second time?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_7", + "session_12", + "session_15", + "session_1", + "session_6", + "session_5", + "session_14", + "session_24", + "session_8" + ], + "sample_id": "conv-50" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_25", + "session_29" + ], + "question": "Who supports Calvin in tough times?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_30", + "session_23", + "session_22", + "session_6", + "session_11", + "session_29", + "session_16", + "session_20", + "session_12" + ], + "sample_id": "conv-50" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_25", + "session_28" + ], + "question": "What does help Calvin stay connected to the creative process?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_30", + "session_5", + "session_25", + "session_6", + "session_27", + "session_11", + "session_7", + "session_9", + "session_8" + ], + "sample_id": "conv-50" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_26" + ], + "question": "When did Calvin visit some of the sights in Boston with a former high school friend?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_29", + "session_8", + "session_7", + "session_18", + "session_11", + "session_6", + "session_24", + "session_5", + "session_25" + ], + "sample_id": "conv-50" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_14", + "session_26" + ], + "question": "Which cities did Dave travel to in 2023?", + "recall": 0.5, + "retrieved_ids": [ + "session_3", + "session_27", + "session_8", + "session_15", + "session_1", + "session_6", + "session_18", + "session_24", + "session_10", + "session_14" + ], + "sample_id": "conv-50" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_27" + ], + "question": "Which hobby did Dave pick up in October 2023?", + "recall": 0.0, + "retrieved_ids": [ + "session_13", + "session_21", + "session_2", + "session_22", + "session_16", + "session_19", + "session_12", + "session_18", + "session_5", + "session_7" + ], + "sample_id": "conv-50" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_12", + "session_25", + "session_26" + ], + "question": "Which events in Dave's life inspired him to take up auto engineering?", + "recall": 0.3333, + "retrieved_ids": [ + "session_25", + "session_9", + "session_13", + "session_1", + "session_3", + "session_5", + "session_24", + "session_30", + "session_23", + "session_27" + ], + "sample_id": "conv-50" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_2", + "session_23" + ], + "question": "How many Ferraris does Calvin own?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_22", + "session_18", + "session_23", + "session_4", + "session_2", + "session_1", + "session_11", + "session_12", + "session_29" + ], + "sample_id": "conv-50" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_16", + "session_4" + ], + "question": "What gifts has Calvin received from his artist friends?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_4", + "session_11", + "session_7", + "session_25", + "session_15", + "session_16", + "session_21", + "session_10", + "session_9" + ], + "sample_id": "conv-50" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_14", + "session_20", + "session_21" + ], + "question": "How long did Dave's work on the Ford Mustang take?", + "recall": 0.6667, + "retrieved_ids": [ + "session_21", + "session_20", + "session_3", + "session_6", + "session_11", + "session_12", + "session_4", + "session_29", + "session_26", + "session_9" + ], + "sample_id": "conv-50" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_14", + "session_17" + ], + "question": "How long was the car modification workshop in San Francisco?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_17", + "session_13", + "session_4", + "session_26", + "session_6", + "session_5", + "session_9", + "session_18", + "session_12" + ], + "sample_id": "conv-50" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_16" + ], + "question": "What style of guitars does Calvin own?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_22", + "session_4", + "session_11", + "session_28", + "session_25", + "session_13", + "session_9", + "session_2", + "session_21" + ], + "sample_id": "conv-50" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_10", + "session_11", + "session_15", + "session_6" + ], + "question": "What activities has Dave participated in with his friends?", + "recall": 1.0, + "retrieved_ids": [ + "session_29", + "session_23", + "session_10", + "session_21", + "session_15", + "session_26", + "session_11", + "session_25", + "session_9", + "session_6" + ], + "sample_id": "conv-50" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_27" + ], + "question": "When did Dave take a photo of a Boston clock tower?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_15", + "session_8", + "session_3", + "session_30", + "session_21", + "session_29", + "session_24", + "session_28", + "session_9" + ], + "sample_id": "conv-50" + }, + { + "category": 1, + "category_name": "Single-hop", + "evidence_ids": [ + "session_13", + "session_20", + "session_25", + "session_27" + ], + "question": "Do all of Dave's car restoration projects go smoothly?", + "recall": 0.5, + "retrieved_ids": [ + "session_14", + "session_25", + "session_6", + "session_4", + "session_7", + "session_21", + "session_28", + "session_27", + "session_12", + "session_22" + ], + "sample_id": "conv-50" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_28" + ], + "question": "Where was Calvin located in the last week of October 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_30", + "session_6", + "session_9", + "session_28", + "session_17", + "session_8", + "session_2", + "session_23", + "session_7", + "session_16" + ], + "sample_id": "conv-50" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_28" + ], + "question": "When did Dave find the car he repaired and started sharing in his blog?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_1", + "session_9", + "session_6", + "session_29", + "session_25", + "session_7", + "session_11", + "session_23", + "session_24" + ], + "sample_id": "conv-50" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_30" + ], + "question": "When did Dave buy a vintage camera?", + "recall": 1.0, + "retrieved_ids": [ + "session_30", + "session_19", + "session_27", + "session_20", + "session_6", + "session_28", + "session_15", + "session_14", + "session_3", + "session_9" + ], + "sample_id": "conv-50" + }, + { + "category": 2, + "category_name": "Temporal", + "evidence_ids": [ + "session_30" + ], + "question": "When did Calvin attend a gala in Boston?", + "recall": 1.0, + "retrieved_ids": [ + "session_30", + "session_19", + "session_1", + "session_26", + "session_24", + "session_15", + "session_3", + "session_2", + "session_21", + "session_29" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_1" + ], + "question": "How long did Calvin plan to stay in Japan?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_10", + "session_3", + "session_5", + "session_12", + "session_6", + "session_14", + "session_9", + "session_11", + "session_20" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_2" + ], + "question": "Which band was Dave's favorite at the music festival in April 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_3", + "session_19", + "session_23", + "session_5", + "session_15", + "session_6", + "session_9", + "session_24", + "session_28" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "Where did Calvin attend a music festival in April 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_24", + "session_28", + "session_19", + "session_26", + "session_2", + "session_3", + "session_23", + "session_1", + "session_30" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_3" + ], + "question": "What advice did Calvin receive from the producer at the music festival?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_15", + "session_26", + "session_2", + "session_5", + "session_24", + "session_28", + "session_23", + "session_29", + "session_6" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What is Dave's new business venture as of 1 May, 2023?", + "recall": 0.0, + "retrieved_ids": [], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What type of cars does Dave work on at his shop?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_4", + "session_28", + "session_6", + "session_23", + "session_22", + "session_12", + "session_25", + "session_13", + "session_29" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What did Calvin receive as a gift from another artist?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_28", + "session_25", + "session_30", + "session_10", + "session_27", + "session_12", + "session_21", + "session_16", + "session_29" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_4" + ], + "question": "What was the necklace Calvin received meant to remind him of?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_20", + "session_22", + "session_12", + "session_18", + "session_24", + "session_11", + "session_9", + "session_19", + "session_23" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_5" + ], + "question": "What does Dave do when he feels his creativity is frozen?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_30", + "session_28", + "session_2", + "session_8", + "session_24", + "session_9", + "session_29", + "session_27", + "session_6" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_5" + ], + "question": "How does Calvin plan to jumpstart his inspiration?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_1", + "session_9", + "session_3", + "session_10", + "session_11", + "session_28", + "session_26", + "session_12", + "session_29" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_6" + ], + "question": "What did Calvin manage to save during the flood incident?", + "recall": 1.0, + "retrieved_ids": [ + "session_6", + "session_26", + "session_20", + "session_24", + "session_25", + "session_19", + "session_29", + "session_9", + "session_15", + "session_14" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_6" + ], + "question": "What did Dave open in May 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_5", + "session_28", + "session_6", + "session_2", + "session_15", + "session_26", + "session_3", + "session_14", + "session_9" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What gives Dave a sense of achievement and purpose?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_17", + "session_12", + "session_26", + "session_8", + "session_11", + "session_4", + "session_21", + "session_16", + "session_5" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_7" + ], + "question": "What fuels Calvin's soul?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_7", + "session_28", + "session_16", + "session_20", + "session_2", + "session_18", + "session_24", + "session_30", + "session_13" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_8" + ], + "question": "What is Dave doing to relax on weekends?", + "recall": 1.0, + "retrieved_ids": [ + "session_8", + "session_2", + "session_5", + "session_11", + "session_28", + "session_19", + "session_16", + "session_7", + "session_3", + "session_21" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_9" + ], + "question": "What sports activity is Calvin planning to try after the tour with Frank Ocean?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_24", + "session_9", + "session_25", + "session_15", + "session_7", + "session_23", + "session_5", + "session_1", + "session_10" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "What was Calvin excited to do after getting his car fixed on 7 July, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_18", + "session_10", + "session_24", + "session_20", + "session_2", + "session_3", + "session_29", + "session_19", + "session_21" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_10" + ], + "question": "What did Calvin and his friends arrange for in the park?", + "recall": 1.0, + "retrieved_ids": [ + "session_10", + "session_11", + "session_29", + "session_21", + "session_15", + "session_1", + "session_8", + "session_3", + "session_30", + "session_9" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_11" + ], + "question": "What kind of music has Calvin been creating lately?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_27", + "session_18", + "session_28", + "session_25", + "session_23", + "session_4", + "session_13", + "session_10", + "session_12" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_11" + ], + "question": "How does Calvin describe his process of adding electronic elements to his songs?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_14", + "session_15", + "session_9", + "session_6", + "session_5", + "session_28", + "session_29", + "session_20", + "session_22" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_12" + ], + "question": "What car brand does Calvin own that he is proud of?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_22", + "session_18", + "session_14", + "session_28", + "session_17", + "session_21", + "session_2", + "session_24", + "session_29" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_12" + ], + "question": "What is Calvin's biggest current goal?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_11", + "session_22", + "session_26", + "session_25", + "session_13", + "session_28", + "session_21", + "session_23", + "session_5" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_12" + ], + "question": "What is Dave's advice to Calvin regarding his dreams?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_5", + "session_24", + "session_9", + "session_3", + "session_2", + "session_4", + "session_12", + "session_28", + "session_29" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "What workshop did Dave get picked for on 11 August, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_24", + "session_2", + "session_3", + "session_14", + "session_11", + "session_28", + "session_16", + "session_9", + "session_6" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "What kind of modifications has Dave been working on in the car mod workshop?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_22", + "session_11", + "session_23", + "session_27", + "session_18", + "session_12", + "session_28", + "session_17", + "session_14" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "What type of car did Dave work on during the workshop?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_20", + "session_13", + "session_28", + "session_14", + "session_19", + "session_12", + "session_4", + "session_24", + "session_29" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_13" + ], + "question": "What does Dave say is important for making his custom cars unique?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_4", + "session_13", + "session_23", + "session_22", + "session_5", + "session_19", + "session_29", + "session_16", + "session_9" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_14" + ], + "question": "How did the audience in Tokyo react when Calvin sang one of his songs?", + "recall": 1.0, + "retrieved_ids": [ + "session_14", + "session_16", + "session_7", + "session_6", + "session_15", + "session_3", + "session_24", + "session_9", + "session_20", + "session_29" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "How did Calvin meet Frank Ocean?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_3", + "session_24", + "session_9", + "session_23", + "session_25", + "session_7", + "session_27", + "session_1", + "session_21" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "Where did Calvin and Frank Ocean record a song together?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_25", + "session_24", + "session_3", + "session_7", + "session_23", + "session_20", + "session_9", + "session_11", + "session_19" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_15" + ], + "question": "What did Calvin and his friends record in August 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_29", + "session_6", + "session_24", + "session_11", + "session_25", + "session_9", + "session_19", + "session_10", + "session_23" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "Where did Calvin start shooting a video for his new album?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_28", + "session_29", + "session_24", + "session_9", + "session_27", + "session_19", + "session_8", + "session_18", + "session_5" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "What design is featured on Calvin's guitar?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_4", + "session_20", + "session_9", + "session_2", + "session_18", + "session_30", + "session_24", + "session_28", + "session_13" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "Why did Calvin get his guitar customized with a shiny finish?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_9", + "session_4", + "session_3", + "session_29", + "session_13", + "session_27", + "session_22", + "session_19", + "session_5" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_16" + ], + "question": "What color glow did Calvin customize his guitar with?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_9", + "session_27", + "session_14", + "session_29", + "session_15", + "session_22", + "session_24", + "session_5", + "session_13" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "Where did Dave come back from with insights on car modification on 1st September 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_9", + "session_4", + "session_28", + "session_12", + "session_21", + "session_5", + "session_11", + "session_13", + "session_14" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What emotion does Dave mention feeling when he sees the relief of someone whose car he fixed?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_17", + "session_9", + "session_7", + "session_30", + "session_19", + "session_26", + "session_29", + "session_15", + "session_24" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_17" + ], + "question": "What did Calvin book a flight ticket for on 1st September 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_12", + "session_8", + "session_15", + "session_3", + "session_2", + "session_28", + "session_14", + "session_9", + "session_6" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What is Calvin excited about after the tour?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_18", + "session_9", + "session_15", + "session_3", + "session_2", + "session_20", + "session_7", + "session_25", + "session_16" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_18" + ], + "question": "What plans do Calvin and Dave have for when Calvin visits Boston?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_1", + "session_8", + "session_18", + "session_26", + "session_24", + "session_10", + "session_12", + "session_5", + "session_6" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "Which Disney movie did Dave mention as one of his favorites?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_2", + "session_9", + "session_26", + "session_29", + "session_7", + "session_3", + "session_10", + "session_6", + "session_5" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_19" + ], + "question": "How does Dave feel about the reactions of people when they see the finished restoration project?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_25", + "session_19", + "session_4", + "session_27", + "session_21", + "session_12", + "session_3", + "session_16", + "session_13" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_20" + ], + "question": "What activity did Calvin enjoy during his summer drives?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_24", + "session_26", + "session_5", + "session_29", + "session_2", + "session_12", + "session_9", + "session_23", + "session_10" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_20" + ], + "question": "Which song from the childhood of Calvin brings back memories of a road trip with his dad?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_12", + "session_29", + "session_11", + "session_25", + "session_10", + "session_9", + "session_22", + "session_26", + "session_5" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_21" + ], + "question": "What project did Calvin work on to chill out?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_21", + "session_25", + "session_26", + "session_7", + "session_13", + "session_27", + "session_1", + "session_14", + "session_8" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_21" + ], + "question": "What car did Dave work on in the junkyard?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_6", + "session_12", + "session_4", + "session_26", + "session_2", + "session_3", + "session_28", + "session_14", + "session_9" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_21" + ], + "question": "What does Dave find satisfying about restoring old cars?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_28", + "session_22", + "session_12", + "session_17", + "session_19", + "session_4", + "session_11", + "session_25", + "session_29" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_21" + ], + "question": "What do Calvin and Dave use to reach their goals?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_2", + "session_22", + "session_18", + "session_26", + "session_3", + "session_11", + "session_17", + "session_21", + "session_23" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_22" + ], + "question": "What does working on cars represent for Dave?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_22", + "session_4", + "session_12", + "session_5", + "session_25", + "session_13", + "session_21", + "session_16", + "session_28" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_22" + ], + "question": "What does Dave aim to do with his passion for cars?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_9", + "session_26", + "session_4", + "session_11", + "session_28", + "session_14", + "session_29", + "session_23", + "session_16" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "What did Calvin recently get that is a \"masterpiece on wheels\"?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_13", + "session_22", + "session_1", + "session_2", + "session_8", + "session_28", + "session_25", + "session_26", + "session_27" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "Who headlined the music festival that Dave attended in October?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_15", + "session_19", + "session_26", + "session_2", + "session_3", + "session_24", + "session_28", + "session_30", + "session_1" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "How does Calvin stay motivated when faced with setbacks?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_21", + "session_14", + "session_12", + "session_3", + "session_11", + "session_28", + "session_18", + "session_4", + "session_29" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_23" + ], + "question": "What activity does Dave find fulfilling, similar to Calvin's passion for music festivals?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_26", + "session_28", + "session_3", + "session_24", + "session_11", + "session_15", + "session_8", + "session_2", + "session_10" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "Where did Calvin and Dave meet Frank Ocean to start collaborating?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_24", + "session_23", + "session_25", + "session_1", + "session_9", + "session_3", + "session_7", + "session_27", + "session_21" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "Which part of Tokyo is described as Tokyo's Times Square by Calvin?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_25", + "session_15", + "session_7", + "session_14", + "session_3", + "session_6", + "session_26", + "session_10", + "session_13" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "What specific location in Tokyo does Calvin mention being excited to explore?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_8", + "session_14", + "session_15", + "session_7", + "session_3", + "session_6", + "session_16", + "session_5", + "session_19" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_24" + ], + "question": "What dish does Dave recommend Calvin to try in Tokyo?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_24", + "session_1", + "session_9", + "session_15", + "session_7", + "session_14", + "session_6", + "session_5", + "session_25" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_25" + ], + "question": "What does Calvin find energizing during the tour?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_26", + "session_24", + "session_7", + "session_28", + "session_1", + "session_29", + "session_20", + "session_19", + "session_11" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_25" + ], + "question": "How does Calvin balance his job and personal life?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_9", + "session_14", + "session_13", + "session_22", + "session_29", + "session_30", + "session_11", + "session_24", + "session_28" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_25" + ], + "question": "What inspired Calvin's recent music?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_28", + "session_27", + "session_30", + "session_11", + "session_1", + "session_8", + "session_26", + "session_7", + "session_9" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_25" + ], + "question": "How does Calvin describe his music in relation to capturing feelings?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_23", + "session_9", + "session_29", + "session_25", + "session_12", + "session_19", + "session_30", + "session_27", + "session_16" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_25" + ], + "question": "Why did Dave start working on cars?", + "recall": 0.0, + "retrieved_ids": [ + "session_4", + "session_6", + "session_11", + "session_18", + "session_1", + "session_20", + "session_29", + "session_28", + "session_16", + "session_24" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_25" + ], + "question": "What is the toughest part of car restoration according to Dave?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_27", + "session_26", + "session_4", + "session_7", + "session_13", + "session_14", + "session_10", + "session_22", + "session_12" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_25" + ], + "question": "What does Calvin believe makes an artist create something extraordinary?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_12", + "session_4", + "session_7", + "session_13", + "session_28", + "session_10", + "session_2", + "session_11", + "session_23" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_25" + ], + "question": "When did Dave sell the car he restored last year?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_24", + "session_27", + "session_4", + "session_14", + "session_28", + "session_2", + "session_9", + "session_7", + "session_29" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_26" + ], + "question": "When did Calvin first get interested in cars?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_6", + "session_28", + "session_26", + "session_14", + "session_9", + "session_11", + "session_24", + "session_20", + "session_30" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_26" + ], + "question": "How did Calvin feel about performing with someone he admires?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_24", + "session_14", + "session_7", + "session_15", + "session_2", + "session_28", + "session_17", + "session_29", + "session_9" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_26" + ], + "question": "What realization did the nightclub experience bring to Calvin?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_12", + "session_30", + "session_25", + "session_2", + "session_14", + "session_21", + "session_15", + "session_9", + "session_29" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_26" + ], + "question": "What do Dave and Calvin agree on regarding their pursuits?", + "recall": 0.0, + "retrieved_ids": [ + "session_9", + "session_2", + "session_1", + "session_30", + "session_4", + "session_28", + "session_21", + "session_17", + "session_14", + "session_29" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_27" + ], + "question": "Which city is featured in the photograph Dave showed Calvin?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_1", + "session_6", + "session_3", + "session_24", + "session_18", + "session_10", + "session_14", + "session_2", + "session_15" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_28" + ], + "question": "What did Calvin do recently at his Japanese house?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_1", + "session_30", + "session_16", + "session_9", + "session_11", + "session_5", + "session_3", + "session_10", + "session_29" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_28" + ], + "question": "What did Dave recently start a blog about?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_1", + "session_25", + "session_20", + "session_27", + "session_8", + "session_11", + "session_24", + "session_6", + "session_18" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_28" + ], + "question": "What is Dave's way to share his passion with others?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_28", + "session_23", + "session_4", + "session_9", + "session_14", + "session_26", + "session_11", + "session_17", + "session_29" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_28" + ], + "question": "What type of videos does Calvin usually watch on his television?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_24", + "session_5", + "session_9", + "session_19", + "session_16", + "session_29", + "session_2", + "session_30", + "session_13" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_28" + ], + "question": "What type of music has Dave been getting into lately?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_27", + "session_11", + "session_25", + "session_14", + "session_10", + "session_29", + "session_1", + "session_8", + "session_17" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_28" + ], + "question": "What tools does Calvin use to boost his motivation for music?", + "recall": 1.0, + "retrieved_ids": [ + "session_30", + "session_28", + "session_9", + "session_29", + "session_4", + "session_11", + "session_21", + "session_22", + "session_18", + "session_12" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_28" + ], + "question": "What type of content does Dave post on his blog that inspired others to start their own DIY projects?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_25", + "session_14", + "session_4", + "session_11", + "session_18", + "session_6", + "session_29", + "session_9", + "session_2" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_28" + ], + "question": "What kind of impact does Dave's blog on car mods have on people?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_12", + "session_23", + "session_27", + "session_3", + "session_18", + "session_25", + "session_13", + "session_22", + "session_11" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_29" + ], + "question": "Who did Calvin invite to see him perform in Boston on 13 November, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_24", + "session_9", + "session_30", + "session_26", + "session_29", + "session_15", + "session_2", + "session_3", + "session_6" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_30" + ], + "question": "What hobby did Calvin take up recently?", + "recall": 0.0, + "retrieved_ids": [ + "session_7", + "session_25", + "session_22", + "session_27", + "session_12", + "session_18", + "session_5", + "session_13", + "session_19", + "session_21" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_30" + ], + "question": "What new item did Dave buy recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_30", + "session_1", + "session_8", + "session_26", + "session_7", + "session_25", + "session_28", + "session_9", + "session_11" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_30" + ], + "question": "What type of photos does Dave like to capture with his new camera?", + "recall": 1.0, + "retrieved_ids": [ + "session_30", + "session_27", + "session_10", + "session_9", + "session_29", + "session_25", + "session_28", + "session_16", + "session_23", + "session_19" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_30" + ], + "question": "What event did Calvin attend in Boston?", + "recall": 1.0, + "retrieved_ids": [ + "session_1", + "session_27", + "session_26", + "session_30", + "session_19", + "session_2", + "session_15", + "session_3", + "session_25", + "session_23" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_30" + ], + "question": "What did Calvin discuss with the cool artist he met at the gala?", + "recall": 1.0, + "retrieved_ids": [ + "session_30", + "session_27", + "session_3", + "session_2", + "session_8", + "session_24", + "session_7", + "session_21", + "session_29", + "session_15" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_30" + ], + "question": "Where did Dave take a stunning photo of a waterfall?", + "recall": 1.0, + "retrieved_ids": [ + "session_30", + "session_4", + "session_9", + "session_28", + "session_27", + "session_10", + "session_15", + "session_6", + "session_18", + "session_11" + ], + "sample_id": "conv-50" + }, + { + "category": 4, + "category_name": "Open-domain", + "evidence_ids": [ + "session_30" + ], + "question": "What positive impact does Calvin mention nature has on tough times?", + "recall": 1.0, + "retrieved_ids": [ + "session_30", + "session_25", + "session_12", + "session_11", + "session_6", + "session_23", + "session_8", + "session_7", + "session_19", + "session_29" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_2" + ], + "question": "Which DJ was Dave's favorite at the music festival in April 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_2", + "session_10", + "session_3", + "session_15", + "session_19", + "session_6", + "session_5", + "session_24", + "session_26" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_3" + ], + "question": "What advice did Calvin receive from the chef at the music festival?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_15", + "session_26", + "session_2", + "session_5", + "session_24", + "session_28", + "session_29", + "session_23", + "session_6" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What is Calvin's new business venture as of 1 May, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_5", + "session_1", + "session_2", + "session_8", + "session_30", + "session_13", + "session_17", + "session_29", + "session_3", + "session_4" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What type of cars does Calvin work on at his shop?", + "recall": 1.0, + "retrieved_ids": [ + "session_9", + "session_4", + "session_28", + "session_6", + "session_23", + "session_22", + "session_12", + "session_25", + "session_13", + "session_29" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What did Dave receive as a gift from another artist?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_25", + "session_28", + "session_30", + "session_10", + "session_27", + "session_12", + "session_21", + "session_29", + "session_16" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_4" + ], + "question": "What was the necklace Dave received meant to remind him of?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_20", + "session_22", + "session_12", + "session_24", + "session_18", + "session_11", + "session_9", + "session_19", + "session_23" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_6" + ], + "question": "What did Calvin open in May 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_4", + "session_5", + "session_28", + "session_2", + "session_6", + "session_15", + "session_26", + "session_3", + "session_14", + "session_9" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_7" + ], + "question": "What gives Calvin a sense of achievement and purpose?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_17", + "session_12", + "session_4", + "session_26", + "session_8", + "session_11", + "session_21", + "session_16", + "session_5" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_9" + ], + "question": "What sports activity is Dave planning to try after the tour with Frank Ocean?", + "recall": 1.0, + "retrieved_ids": [ + "session_3", + "session_24", + "session_9", + "session_25", + "session_15", + "session_7", + "session_23", + "session_5", + "session_1", + "session_10" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_11" + ], + "question": "How does Calvin describe his process of adding acoustic elements to his songs?", + "recall": 1.0, + "retrieved_ids": [ + "session_11", + "session_14", + "session_15", + "session_9", + "session_6", + "session_5", + "session_28", + "session_29", + "session_20", + "session_22" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_12" + ], + "question": "What clothing brand does Calvin own that he is proud of?", + "recall": 1.0, + "retrieved_ids": [ + "session_12", + "session_22", + "session_18", + "session_14", + "session_28", + "session_17", + "session_21", + "session_2", + "session_24", + "session_29" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_13" + ], + "question": "What workshop did Calvin get picked for on 11 August, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_24", + "session_2", + "session_29", + "session_14", + "session_11", + "session_28", + "session_16", + "session_9", + "session_6" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_13" + ], + "question": "What kind of modifications has Calvin been working on in the car mod workshop?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_22", + "session_11", + "session_23", + "session_27", + "session_18", + "session_12", + "session_28", + "session_17", + "session_14" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_13" + ], + "question": "What type of car did Calvin work on during the workshop?", + "recall": 1.0, + "retrieved_ids": [ + "session_13", + "session_26", + "session_20", + "session_28", + "session_14", + "session_19", + "session_12", + "session_4", + "session_24", + "session_29" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_15" + ], + "question": "What did Dave and his friends record in August 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_15", + "session_6", + "session_29", + "session_24", + "session_11", + "session_25", + "session_9", + "session_19", + "session_10", + "session_23" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_16" + ], + "question": "Where did Dave start shooting a video for his new album?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_28", + "session_29", + "session_24", + "session_9", + "session_27", + "session_19", + "session_8", + "session_18", + "session_5" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_16" + ], + "question": "What design is featured on Dave's guitar?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_4", + "session_9", + "session_22", + "session_28", + "session_30", + "session_15", + "session_12", + "session_13", + "session_6" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_16" + ], + "question": "Why did Dave get his guitar customized with a shiny finish?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_4", + "session_9", + "session_3", + "session_29", + "session_13", + "session_27", + "session_22", + "session_19", + "session_5" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_16" + ], + "question": "What color glow did Dave customize his guitar with?", + "recall": 1.0, + "retrieved_ids": [ + "session_16", + "session_9", + "session_27", + "session_14", + "session_29", + "session_15", + "session_22", + "session_24", + "session_5", + "session_13" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_17" + ], + "question": "Where did Calvin come back from with insights on car modification on 1st September 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_9", + "session_4", + "session_28", + "session_12", + "session_21", + "session_5", + "session_11", + "session_13", + "session_14" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_17" + ], + "question": "What emotion does Calvin mention feeling when he sees the relief of someone whose car he fixed?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_25", + "session_9", + "session_7", + "session_30", + "session_19", + "session_26", + "session_29", + "session_15", + "session_24" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_17" + ], + "question": "What did Dave book a flight ticket for on 1st September 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_17", + "session_12", + "session_8", + "session_15", + "session_3", + "session_2", + "session_28", + "session_14", + "session_9", + "session_6" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_19" + ], + "question": "Which horror movie did Dave mention as one of his favorites?", + "recall": 1.0, + "retrieved_ids": [ + "session_19", + "session_2", + "session_9", + "session_26", + "session_29", + "session_7", + "session_3", + "session_10", + "session_6", + "session_5" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_20" + ], + "question": "Which song from the childhood of Dave brings back memories of a road trip with his dad?", + "recall": 1.0, + "retrieved_ids": [ + "session_20", + "session_12", + "session_29", + "session_11", + "session_25", + "session_10", + "session_9", + "session_22", + "session_26", + "session_5" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_21" + ], + "question": "What car did Calvin work on in the junkyard?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_6", + "session_12", + "session_4", + "session_26", + "session_2", + "session_3", + "session_28", + "session_14", + "session_9" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_21" + ], + "question": "What does Dave find satisfying about destroying old cars?", + "recall": 1.0, + "retrieved_ids": [ + "session_21", + "session_4", + "session_28", + "session_17", + "session_1", + "session_12", + "session_7", + "session_25", + "session_29", + "session_11" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_22" + ], + "question": "What does working on boats represent for Dave?", + "recall": 0.0, + "retrieved_ids": [ + "session_25", + "session_16", + "session_4", + "session_12", + "session_5", + "session_13", + "session_21", + "session_20", + "session_28", + "session_19" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_22" + ], + "question": "What does Dave aim to do with his passion for cooking?", + "recall": 1.0, + "retrieved_ids": [ + "session_22", + "session_9", + "session_29", + "session_26", + "session_11", + "session_4", + "session_28", + "session_14", + "session_16", + "session_23" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_23" + ], + "question": "What did Calvin recently get that is a \"masterpiece on canvas\"?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_9", + "session_22", + "session_1", + "session_8", + "session_28", + "session_25", + "session_13", + "session_27", + "session_26" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_23" + ], + "question": "Who headlined the music festival that Calvin attended in October?", + "recall": 1.0, + "retrieved_ids": [ + "session_23", + "session_15", + "session_28", + "session_19", + "session_26", + "session_2", + "session_3", + "session_24", + "session_30", + "session_1" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_24" + ], + "question": "Which part of Tokyo is described as Tokyo's Times Square by Dave?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_25", + "session_15", + "session_7", + "session_14", + "session_3", + "session_6", + "session_26", + "session_10", + "session_13" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_24" + ], + "question": "What specific location in Tokyo does Calvin mention being excited to avoid?", + "recall": 1.0, + "retrieved_ids": [ + "session_24", + "session_15", + "session_7", + "session_3", + "session_18", + "session_14", + "session_6", + "session_16", + "session_1", + "session_19" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_25" + ], + "question": "When did Calvin sell the car he restored last year?", + "recall": 1.0, + "retrieved_ids": [ + "session_25", + "session_24", + "session_27", + "session_4", + "session_14", + "session_28", + "session_2", + "session_9", + "session_7", + "session_29" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_26" + ], + "question": "When did Calvin first get interested in motorcycles?", + "recall": 1.0, + "retrieved_ids": [ + "session_2", + "session_24", + "session_14", + "session_28", + "session_26", + "session_6", + "session_9", + "session_11", + "session_20", + "session_30" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_26" + ], + "question": "What realization did the nightclub experience bring to Dave?", + "recall": 1.0, + "retrieved_ids": [ + "session_26", + "session_8", + "session_12", + "session_30", + "session_25", + "session_14", + "session_21", + "session_15", + "session_9", + "session_29" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_28" + ], + "question": "What did Dave do recently at his Japanese house?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_1", + "session_30", + "session_16", + "session_9", + "session_11", + "session_5", + "session_3", + "session_10", + "session_29" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_28" + ], + "question": "What did Calvin recently start a blog about?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_1", + "session_25", + "session_20", + "session_27", + "session_8", + "session_11", + "session_24", + "session_6", + "session_18" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_28" + ], + "question": "What type of videos does Dave usually watch on his television?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_13", + "session_5", + "session_9", + "session_19", + "session_16", + "session_29", + "session_4", + "session_30", + "session_6" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_28" + ], + "question": "What type of art has Dave been getting into lately?", + "recall": 0.0, + "retrieved_ids": [ + "session_27", + "session_14", + "session_25", + "session_29", + "session_19", + "session_1", + "session_30", + "session_13", + "session_17", + "session_16" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_28" + ], + "question": "What type of content does Dave post on his blog that inspired others to start their own cooking projects?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_25", + "session_14", + "session_4", + "session_11", + "session_18", + "session_6", + "session_29", + "session_2", + "session_9" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_28" + ], + "question": "What kind of impact does Dave's blog on vegan recipes have on people?", + "recall": 1.0, + "retrieved_ids": [ + "session_28", + "session_4", + "session_12", + "session_27", + "session_3", + "session_18", + "session_25", + "session_23", + "session_11", + "session_13" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_29" + ], + "question": "Who did Dave invite to see him perform in Boston on 13 November, 2023?", + "recall": 1.0, + "retrieved_ids": [ + "session_7", + "session_24", + "session_9", + "session_30", + "session_26", + "session_29", + "session_15", + "session_3", + "session_2", + "session_6" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_30" + ], + "question": "What new item did Calvin buy recently?", + "recall": 1.0, + "retrieved_ids": [ + "session_27", + "session_30", + "session_1", + "session_8", + "session_26", + "session_7", + "session_25", + "session_28", + "session_9", + "session_11" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_30" + ], + "question": "What type of photos does Calvin like to capture with his new camera?", + "recall": 1.0, + "retrieved_ids": [ + "session_30", + "session_27", + "session_10", + "session_9", + "session_29", + "session_25", + "session_28", + "session_16", + "session_23", + "session_19" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_30" + ], + "question": "What did Dave discuss with the cool artist he met at the gala?", + "recall": 1.0, + "retrieved_ids": [ + "session_30", + "session_27", + "session_3", + "session_2", + "session_8", + "session_24", + "session_7", + "session_21", + "session_29", + "session_15" + ], + "sample_id": "conv-50" + }, + { + "category": 5, + "category_name": "Adversarial", + "evidence_ids": [ + "session_30" + ], + "question": "Where did Calvin take a stunning photo of a waterfall?", + "recall": 1.0, + "retrieved_ids": [ + "session_30", + "session_4", + "session_9", + "session_28", + "session_27", + "session_10", + "session_15", + "session_6", + "session_18", + "session_11" + ], + "sample_id": "conv-50" + } + ], + "runtime_seconds": 157.701, + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/locomo_old_brainctl_cmd_session.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/locomo_old_brainctl_cmd_session.json new file mode 100644 index 0000000..57a2a2f --- /dev/null +++ b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/locomo_old_brainctl_cmd_session.json @@ -0,0 +1,38 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\locomo_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\locomo_old_brainctl_cmd_session.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" + }, + "benchmark": "locomo", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "measured": false, + "metadata": {}, + "metrics": { + "avg_recall": 0.9217, + "cat_1_recall": 0.7614, + "cat_2_recall": 0.92, + "cat_3_recall": 0.6616, + "cat_4_recall": 0.9738, + "cat_5_recall": 0.9821, + "perfect_rate": 0.8817, + "top_k": 10, + "zero_rate": 0.0438 + }, + "mode": "cmd_session", + "notes": [ + "granularity=session", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": 0.9217, + "reference_kind": "historical", + "runtime_seconds": 445.74, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_mempalace_raw_session.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_mempalace_raw_session.json new file mode 100644 index 0000000..291cd74 --- /dev/null +++ b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_mempalace_raw_session.json @@ -0,0 +1,33 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\longmemeval_mempalace_raw_session.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "measured": false, + "metadata": {}, + "metrics": { + "ndcg_at_10": 0.8948, + "ndcg_at_5": 0.893, + "r_at_10": 0.983, + "r_at_5": 0.966 + }, + "mode": "raw_session", + "notes": [ + "top_k=10", + "Runs MemPalace benchmark module raw session retrieval logic directly." + ], + "primary_metric": "r_at_5", + "primary_metric_value": 0.966, + "reference_kind": "historical", + "runtime_seconds": 695.36, + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "mempalace" +} \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_new_brainctl_brain.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_new_brainctl_brain.json new file mode 100644 index 0000000..ba42b30 --- /dev/null +++ b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_new_brainctl_brain.json @@ -0,0 +1,12197 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\longmemeval_new_brainctl_brain.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "measured": true, + "metadata": {}, + "metrics": { + "ndcg_at_10": 0.872, + "ndcg_at_5": 0.8678, + "r_at_10": 0.9894, + "r_at_5": 0.9574 + }, + "mode": "brain", + "notes": [ + "top_k=10", + "Legacy 470-question session-level slice." + ], + "primary_metric": "r_at_5", + "primary_metric_value": 0.9574, + "reference_kind": "measured", + "rows": [ + { + "answer_session_ids": [ + "answer_280352e9" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e47becba", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_280352e9", + "02bd2b90_3", + "ultrachat_214101", + "5dac7cc2_1", + "sharegpt_Cr2tc1f_0", + "f6859b48_2" + ] + }, + { + "answer_session_ids": [ + "answer_40a90d51" + ], + "ndcg_at_10": 0.4307, + "ndcg_at_5": 0.4307, + "question_id": "118b2229", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "ultrachat_392094", + "db73b7e4_4", + "afe04238_1", + "answer_40a90d51", + "5dce60dd", + "a0b8b0ad_6", + "db50c0f6", + "3ea9f765", + "d600c646", + "sharegpt_i1iuP70_0" + ] + }, + { + "answer_session_ids": [ + "answer_d61669c7" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "51a45a95", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_d61669c7", + "98c198fb", + "217debf7", + "6e0b1800_2", + "5c44d9fe_1", + "07ba9acd_2", + "sharegpt_CyJ3dal_0", + "sharegpt_hChsWOp_128", + "sharegpt_3vxz2Zr_0", + "ultrachat_282235" + ] + }, + { + "answer_session_ids": [ + "answer_355c48bb" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "58bf7951", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_355c48bb", + "ultrachat_304942", + "ultrachat_268434", + "ultrachat_155546", + "sharegpt_gmLF3PE_17", + "7e17ae56_5", + "sharegpt_M84blrA_53", + "sharegpt_RkFlOeC_19", + "sharegpt_OFiElWf_37", + "sharegpt_CvVLg2J_18" + ] + }, + { + "answer_session_ids": [ + "answer_3e012175" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "1e043500", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_3e012175", + "01bd9def", + "ultrachat_227993", + "7b1d1f43_1", + "8020d945_2", + "e132259f", + "sharegpt_bph7DTk_7", + "ultrachat_271928", + "4d0d7984", + "sharegpt_6QUDIXG_24" + ] + }, + { + "answer_session_ids": [ + "answer_f6168136" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "c5e8278d", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_f6168136", + "89b1028d_1", + "64767b7f", + "aaf71ce2_2", + "d777e7da_1", + "fc6549b2", + "sharegpt_WPGrlAK_0", + "sharegpt_EXJAjmw_0", + "ultrachat_474903", + "3fa35683_7" + ] + }, + { + "answer_session_ids": [ + "answer_9398da02" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "6ade9755", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_9398da02", + "7c5ab02a_2", + "ec8fa65d_2", + "ultrachat_84599", + "ultrachat_393203", + "910ed479_3", + "ultrachat_270925", + "ultrachat_462", + "28621d6a_2", + "sharegpt_YsgacQi_0" + ] + }, + { + "answer_session_ids": [ + "answer_feb5200f" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "6f9b354f", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_feb5200f", + "a79f1a04_1", + "0c0f2449_2", + "ultrachat_549317", + "60207b41_1", + "54d06eeb", + "sharegpt_IuLT3VI_9", + "sharegpt_VAELSB3_0", + "sharegpt_kRjBUsA_79", + "ultrachat_224845" + ] + }, + { + "answer_session_ids": [ + "answer_59547700" + ], + "ndcg_at_10": 0.6309, + "ndcg_at_5": 0.6309, + "question_id": "58ef2f1c", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "13f3da23_3", + "answer_59547700", + "e6ab6a7b_2", + "ultrachat_5212", + "e82ce7cd_1", + "ultrachat_246324", + "ultrachat_477344", + "dbcfab21", + "b546cb39_2", + "403ee298_1" + ] + }, + { + "answer_session_ids": [ + "answer_c3567066" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "f8c5f88b", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c3567066", + "68ace657_1", + "sharegpt_TKj3yO8_0", + "91880423", + "15232887_1", + "ultrachat_360085", + "ultrachat_412973", + "7ff89315_3", + "58de8f9b", + "3b5205b6_1" + ] + }, + { + "answer_session_ids": [ + "answer_235eb6fb" + ], + "ndcg_at_10": 0.6309, + "ndcg_at_5": 0.6309, + "question_id": "5d3d2817", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "ultrachat_384575", + "answer_235eb6fb", + "670ca40e", + "sharegpt_ipLglky_48", + "sharegpt_8zsFRKh_0", + "e93efd9a_1", + "cef33b28_2", + "ultrachat_106114", + "sharegpt_bTDMt3m_64", + "25d1830a_1" + ] + }, + { + "answer_session_ids": [ + "answer_7cb94507" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "7527f7e2", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_7cb94507", + "864a563d_3", + "sharegpt_cRrfdsc_0", + "sharegpt_rraH1Q7_0", + "44bbd8cd_2", + "d81d9846", + "sharegpt_bTDMt3m_0", + "bee872da_2", + "ultrachat_62919", + "18a06652_3" + ] + }, + { + "answer_session_ids": [ + "answer_e05e4612" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "c960da58", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e05e4612", + "89d3053f", + "ultrachat_228940", + "ultrachat_36718", + "ultrachat_109577", + "ultrachat_129096", + "sharegpt_9iUZNsL_0", + "47db3b56_1", + "b76006cb_3", + "f01feb31" + ] + }, + { + "answer_session_ids": [ + "answer_94030872" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "3b6f954b", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_94030872", + "c271c071", + "6d3f5c68_4", + "ultrachat_475284", + "c566edbb_3", + "90b1c6d4_2", + "sharegpt_kpnwrGe_12", + "ultrachat_298929", + "4ed9cd1b_1", + "f999b05c_2" + ] + }, + { + "answer_session_ids": [ + "answer_f38f679b" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "726462e0", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_f38f679b", + "9c05acfe_2", + "741c6781_1", + "082889ba_1", + "5b83c26e_1", + "6ef99651_2", + "ce144f08", + "cdd9704d_1", + "7e4dab66_1", + "322a96e8" + ] + }, + { + "answer_session_ids": [ + "answer_c63c0458" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "94f70d80", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c63c0458", + "67045503", + "58de8f9b", + "sharegpt_6fTnCnw_76", + "ultrachat_380138", + "ultrachat_172828", + "sharegpt_mJ7MnQS_31", + "ec628e8d_2", + "sharegpt_ZvjRGRN_0", + "9a59023e_1" + ] + }, + { + "answer_session_ids": [ + "answer_fea2e4d3" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "66f24dbb", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_fea2e4d3", + "89452526_2", + "8d46b8fa_1", + "798e4ba3_2", + "21c6c41d", + "ca3a4e4f_1", + "sharegpt_83fkMsQ_0", + "ultrachat_445260", + "sharegpt_ZcfatzD_0", + "76299a49_1" + ] + }, + { + "answer_session_ids": [ + "answer_679840f8" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "ad7109d1", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_679840f8", + "22aa7cca_3", + "3dc02d26_2", + "d92cc997", + "ultrachat_74974", + "495cf518_1", + "sharegpt_Ck9R9HX_0", + "1d6744b5_1", + "fb721812_2", + "ultrachat_365256" + ] + }, + { + "answer_session_ids": [ + "answer_82a04f59" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "af8d2e46", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_82a04f59", + "33c3b457", + "73d5f10c", + "ultrachat_560731", + "266d604c", + "32a67b35", + "ultrachat_284576", + "13047a02", + "d5a6f455_2", + "ultrachat_210554" + ] + }, + { + "answer_session_ids": [ + "answer_8f276838" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "dccbc061", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_8f276838", + "0f417163", + "sharegpt_nc62Spr_7", + "ultrachat_278202", + "959e6cb0_3", + "b2a885b9", + "3c614701", + "8d74c5f6", + "sharegpt_MKMWjX0_25", + "ultrachat_519037" + ] + }, + { + "answer_session_ids": [ + "answer_761acef8" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "c8c3f81d", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_761acef8", + "66e94928", + "19ec83c5_1", + "eb5d5a93_1", + "sharegpt_pRqHb1o_0", + "eb403c80_2", + "9c5f1314_1", + "2b16f3d6_1", + "25432e10", + "998616b4_4" + ] + }, + { + "answer_session_ids": [ + "answer_8ad8a34f" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "8ebdbe50", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_8ad8a34f", + "b53f447a_1", + "27d378b2_3", + "90f16f14_2", + "01e806f0_1", + "39911f94", + "90a5e1f3_1", + "sharegpt_QxC97cL_0", + "d4f7a065_2", + "326f41d8" + ] + }, + { + "answer_session_ids": [ + "answer_e623ae87" + ], + "ndcg_at_10": 0.301, + "ndcg_at_5": 0.0, + "question_id": "6b168ec8", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 0.0, + "top_session_ids": [ + "ultrachat_360618", + "ultrachat_377741", + "33dff20c_3", + "ultrachat_249928", + "ultrachat_499691", + "dded1725_3", + "sharegpt_D9kTAIU_0", + "02f0738e_1", + "answer_e623ae87", + "f5b33470_abs" + ] + }, + { + "answer_session_ids": [ + "answer_723bf11f" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "75499fd8", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_723bf11f", + "5328c3c2_2", + "990b60df_1", + "18b0d357_1", + "eb0d8dc1_7", + "3e7df334_1", + "ddf0f116_3", + "sharegpt_1kE6haW_276", + "sharegpt_yPCVYhX_0", + "168feef7_1" + ] + }, + { + "answer_session_ids": [ + "answer_1e6d4567" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "21436231", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_1e6d4567", + "0d0a89dc_1", + "7a7a4bf0_4", + "ultrachat_172276", + "dc378711_1", + "sharegpt_6byuqlt_0", + "sharegpt_DGiqvVv_169", + "d0785ee9_1", + "3197d998_6", + "6bd0b406_1" + ] + }, + { + "answer_session_ids": [ + "answer_cb742a61" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "95bcc1c8", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_cb742a61", + "3919227f", + "ultrachat_320154", + "c9763bff", + "396238f9", + "abeda028", + "99e5e380", + "56266c38_1", + "sharegpt_uATA5m5_41", + "ultrachat_203404" + ] + }, + { + "answer_session_ids": [ + "answer_c6fd8ebd" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "0862e8bf", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c6fd8ebd", + "0566c21b", + "2442d5b9", + "sharegpt_3BkB8g5_0", + "sharegpt_2I1TxKW_0", + "ultrachat_47999", + "sharegpt_A7KJApk_21", + "sharegpt_KOaA4jf_0", + "sharegpt_isQJkCz_17", + "sharegpt_sCFErnY_22" + ] + }, + { + "answer_session_ids": [ + "answer_69811d4a" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "853b0a1d", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_69811d4a", + "sharegpt_n3MgsgL_0", + "c70f9f9c", + "3fa35683_3", + "a8b4bfb3_2", + "f71bf532_1", + "66a6a097_2", + "ebb5bc7c_2", + "451120d3_2", + "sharegpt_B7mSlfQ_55" + ] + }, + { + "answer_session_ids": [ + "answer_6fe9fb49" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "a06e4cfe", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_6fe9fb49", + "ee43db4a", + "3c2e6725", + "84fb50bb_3", + "de64539a_2", + "8dd2d3cc", + "af257b0b_2", + "0c891c0b_1", + "ultrachat_442201", + "ultrachat_231287" + ] + }, + { + "answer_session_ids": [ + "answer_55161935" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "37d43f65", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_55161935", + "3dcf1e0b_2", + "63587769", + "ultrachat_370801", + "ultrachat_525148", + "sharegpt_MOpCbrr_0", + "sharegpt_brc2wJS_264", + "4050ebff_5", + "sharegpt_ErOTMZ3_35", + "sharegpt_afbMhMS_45" + ] + }, + { + "answer_session_ids": [ + "answer_645b0329" + ], + "ndcg_at_10": 0.5, + "ndcg_at_5": 0.5, + "question_id": "b86304ba", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "ea8bb4f8_2", + "sharegpt_xGoJZ6Z_0", + "answer_645b0329", + "ultrachat_328696", + "ultrachat_10525", + "3b73120a_1", + "e831a29f_1", + "fe1e4351_1", + "91074ce7", + "091aa510_1" + ] + }, + { + "answer_session_ids": [ + "answer_0df6aa4b" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "d52b4f67", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_0df6aa4b", + "d31c3ec8_3", + "da1797c4_1", + "ultrachat_258008", + "18a06652_6", + "62a6d083", + "974fe383_2", + "sharegpt_IqVsDp7_89", + "sharegpt_MM7pEKH_0", + "sharegpt_cn3iUbp_7" + ] + }, + { + "answer_session_ids": [ + "answer_986de8c3" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "25e5aa4f", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_986de8c3", + "18807892_4", + "48f804a3", + "ultrachat_381447", + "a459d477", + "ca5ff3b4", + "13ad258b", + "sharegpt_8cFNcVG_0", + "ultrachat_225537", + "087d2b0a_2" + ] + }, + { + "answer_session_ids": [ + "answer_0714183a" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "caf9ead2", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_0714183a", + "f684ac4c_2", + "fc167d00", + "sharegpt_3D3oQC0_213", + "44713827_2", + "9f8bdd23_1", + "48a72204_1", + "ultrachat_30245", + "fcc6d66d_1", + "ultrachat_532107" + ] + }, + { + "answer_session_ids": [ + "answer_c8354ae9" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "8550ddae", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c8354ae9", + "a7189b75", + "71cf5aeb_3", + "e10dfffb_1", + "4f234e2c", + "9deebbc2_2", + "8ec23b2c", + "295dc1ab_1", + "c7a15bdc", + "sharegpt_ipLglky_42" + ] + }, + { + "answer_session_ids": [ + "answer_9cddca88" + ], + "ndcg_at_10": 0.4307, + "ndcg_at_5": 0.4307, + "question_id": "60d45044", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "fcc6d66d_1", + "4ebf9095_1", + "af4d4ecd_2", + "answer_9cddca88", + "26bc645b_5", + "a76e7e3c_2", + "fec059c5", + "bda4a421", + "0c0f2449_1", + "446173bb_1" + ] + }, + { + "answer_session_ids": [ + "answer_57fc1954" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "3f1e9474", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_57fc1954", + "sharegpt_wWUI4SO_7", + "ultrachat_177477", + "e7497c07_2", + "ultrachat_462827", + "3aac691d_2", + "cdf068b1_1", + "4f23a396_1", + "205f348b_2", + "954d4451_2" + ] + }, + { + "answer_session_ids": [ + "answer_dc11c1eb" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "86b68151", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_dc11c1eb", + "sharegpt_beV15ZO_0", + "d8454317_4", + "3d72c0c0", + "ultrachat_182718", + "0abaca24_1", + "sharegpt_rnL5VQO_0", + "4e671700", + "5ace87df_1", + "eed7b3ac_2" + ] + }, + { + "answer_session_ids": [ + "answer_0dd4d99a" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "577d4d32", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_0dd4d99a", + "ultrachat_125084", + "88925167_1", + "6e2cca63_2", + "449b918c_1", + "2c0d04d6_1", + "2ef53f61_2", + "sharegpt_pcoHTby_85", + "b960cbd3", + "sharegpt_101i5OR_0" + ] + }, + { + "answer_session_ids": [ + "answer_ed1982fc" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "ec81a493", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ed1982fc", + "6f55019d", + "ultrachat_447961", + "d2fc150a_1", + "sharegpt_ST22P5t_0", + "ultrachat_27488", + "ab4643a2_4", + "ultrachat_413665", + "ultrachat_492664", + "616fdd27" + ] + }, + { + "answer_session_ids": [ + "answer_586de428" + ], + "ndcg_at_10": 0.6309, + "ndcg_at_5": 0.6309, + "question_id": "15745da0", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "07d33eaa", + "answer_586de428", + "64a97d9b", + "sharegpt_IzpzBot_0", + "1f5370f7_4", + "1c662b7b_1", + "0d73afc8", + "ultrachat_383425", + "sharegpt_iF4hlNs_5", + "3cec6e2b_2" + ] + }, + { + "answer_session_ids": [ + "answer_5ca6cd28" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e01b8e2f", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_5ca6cd28", + "97338ddd_1", + "8acfa731", + "sharegpt_jyMHY1J_31", + "f9d0bc67", + "ultrachat_433104", + "d31c3ec8_1", + "e74be4c6_4", + "8be2c3f1", + "1aaf8057_3" + ] + }, + { + "answer_session_ids": [ + "answer_e6143162" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "bc8a6e93", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e6143162", + "f62bfe2b_1", + "8c1da8f9_1", + "b869d7ed_2", + "a1b3dd27_1", + "sharegpt_qyWsgp8_13", + "552c7fd0", + "c6bed037_1", + "sharegpt_jl0GWjL_0", + "ultrachat_453950" + ] + }, + { + "answer_session_ids": [ + "answer_f1fbb330" + ], + "ndcg_at_10": 0.5, + "ndcg_at_5": 0.5, + "question_id": "ccb36322", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "bdcee74e_3", + "6d52ee93_1", + "answer_f1fbb330", + "c38b33ae", + "d99bab55_1", + "04a0b385", + "8b1019b8_1", + "0e349efa", + "06d1a1a0", + "f79a48dd_1" + ] + }, + { + "answer_session_ids": [ + "answer_530960c1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "001be529", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_530960c1", + "0f71db75_1", + "9becef17_3", + "sharegpt_YQ4UlQM_83", + "sharegpt_PKXxlMF_0", + "ultrachat_351532", + "sharegpt_ETJwE0s_0", + "sharegpt_o3cfukb_0", + "ultrachat_95472", + "ultrachat_39884" + ] + }, + { + "answer_session_ids": [ + "answer_5cc9b056" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "b320f3f8", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_5cc9b056", + "d3a3f421_2", + "514a7856_2", + "b89f68c4_4", + "8bb8bc0d", + "aae4411b_2", + "40628271_1", + "5e4bb245_2", + "sharegpt_4FCMpJR_0", + "1b71c896_2" + ] + }, + { + "answer_session_ids": [ + "answer_5ff494b9" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "19b5f2b3", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_5ff494b9", + "f379d356_1", + "ultrachat_440262", + "ultrachat_463145", + "ultrachat_302261", + "ultrachat_8325", + "ultrachat_268389", + "ultrachat_68139", + "fbe6fa2c_1", + "ultrachat_138083" + ] + }, + { + "answer_session_ids": [ + "answer_2952aee4" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "4fd1909e", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_2952aee4", + "2479cbae", + "94c582bb_2", + "ultrachat_186237", + "sharegpt_hO4FoVt_0", + "fef185b1", + "ultrachat_472809", + "sharegpt_bTDMt3m_0", + "sharegpt_FNyKOSt_0", + "sharegpt_xX1L0kg_29" + ] + }, + { + "answer_session_ids": [ + "answer_47ffab4c" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "545bd2b5", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_47ffab4c", + "b7f69a7d_1", + "5fb3b5ac", + "a79f1a04_1", + "700c16e2_1", + "cc021f81_2", + "6e672b84_2", + "d1a1b9ea_1", + "4404a52a_1", + "sharegpt_nuYnG53_0" + ] + }, + { + "answer_session_ids": [ + "answer_15d63a22" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "8a137a7f", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_15d63a22", + "77908e43_1", + "fcff2dc4_3", + "20507d15_2", + "ultrachat_90882", + "ultrachat_186116", + "ultrachat_382344", + "sharegpt_sj5ZvVB_0", + "ultrachat_151889", + "ultrachat_192894" + ] + }, + { + "answer_session_ids": [ + "answer_bbdc7b0a" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "76d63226", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_bbdc7b0a", + "6a2f9452_2", + "adc47df1_1", + "a5ab6ba7_3", + "f27e27f9_2", + "cb289226_1", + "sharegpt_sRhcMiu_13", + "ultrachat_328229", + "ultrachat_180710", + "50136b31" + ] + }, + { + "answer_session_ids": [ + "answer_96b8c9ee" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "86f00804", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_96b8c9ee", + "sharegpt_ErOTMZ3_277", + "137d8c6a_2", + "0424a40a_2", + "35c5419d_abs_3", + "6e672b84_3", + "sharegpt_PQ71V9I_7", + "60153a02_1", + "fc41f919", + "sharegpt_sCFErnY_22" + ] + }, + { + "answer_session_ids": [ + "answer_7bdcbd23" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "8e9d538c", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_7bdcbd23", + "ultrachat_549592", + "3cdad629_1", + "0bd928bf_2", + "ultrachat_26072", + "a200b713_4", + "2deed26f", + "ultrachat_232141", + "sharegpt_cADPA4R_0", + "sharegpt_KtWBRv1_4" + ] + }, + { + "answer_session_ids": [ + "answer_e40b054e" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "311778f1", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e40b054e", + "5db56d24_2", + "8e6e058b_3", + "sharegpt_vXNQZ2I_0", + "ultrachat_267100", + "d09360d3_1", + "31903f54", + "95cec590_1", + "27fb983c_2", + "ultrachat_316997" + ] + }, + { + "answer_session_ids": [ + "answer_f442ccbe" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "c19f7a0b", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_f442ccbe", + "58ab5fc5", + "eb336de0_3", + "sharegpt_6ZeW9Vb_0", + "c9292210_2", + "1b0d77b0", + "0e829b8f", + "195c8fe6_3", + "8bb8bc0d", + "81ba81ab" + ] + }, + { + "answer_session_ids": [ + "answer_83c13ff9" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "4100d0a0", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_83c13ff9", + "8d068598", + "ea3db78e_1", + "sharegpt_eV0krsR_0", + "b9890c15_1", + "ultrachat_130655", + "f8ab60d7", + "0f6b9698_1", + "665c09b1_1", + "7fc9c056" + ] + }, + { + "answer_session_ids": [ + "answer_7cc5362f" + ], + "ndcg_at_10": 0.5, + "ndcg_at_5": 0.5, + "question_id": "29f2956b", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "6d6ffac5", + "88361fdd", + "answer_7cc5362f", + "ultrachat_238255", + "c4ed2287_3", + "d50a8a33_1", + "sharegpt_jZOf9E5_0", + "715b1b28", + "eb0d8dc1_5", + "sharegpt_UCVg3jO_0" + ] + }, + { + "answer_session_ids": [ + "answer_d01949bf" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "1faac195", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_d01949bf", + "b2543a58_3", + "8064b6ca", + "9d3d54f7", + "ultrachat_283922", + "f2ccf83b", + "94f460be_1", + "ultrachat_12436", + "sharegpt_NhF2Neg_0", + "48d385f0_1" + ] + }, + { + "answer_session_ids": [ + "answer_39b12014" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "faba32e5", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_39b12014", + "67045503", + "69f3fc12_1", + "c6c5bb8b_4", + "770d11f5", + "sharegpt_FNyKOSt_0", + "sharegpt_benxw0S_9", + "ultrachat_205840", + "d3da4592_2", + "ultrachat_250588" + ] + }, + { + "answer_session_ids": [ + "answer_f5b33470" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "f4f1d8a4", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_f5b33470", + "2bfea7af", + "3aac691d_2", + "9614d065_1", + "eb682e52_2", + "6f051087_2", + "ecb24dd8_1", + "76ea88ba_2", + "ultrachat_67942", + "ultrachat_576354" + ] + }, + { + "answer_session_ids": [ + "answer_304511ce" + ], + "ndcg_at_10": 0.5, + "ndcg_at_5": 0.5, + "question_id": "c14c00dd", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "c77250d2", + "7cd7c296_3", + "answer_304511ce", + "ffd66315_2", + "ultrachat_272210", + "183b6085", + "cd1f5147", + "sharegpt_gU0qvZu_0", + "sharegpt_FfqJ2xI_0", + "sharegpt_vXWOUqx_11" + ] + }, + { + "answer_session_ids": [ + "answer_93e1bd22" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "36580ce8", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_93e1bd22", + "ultrachat_53784", + "9b38bda6_2", + "ultrachat_272552", + "ultrachat_270984", + "ultrachat_32917", + "sharegpt_FpTfRvR_0", + "ultrachat_299779", + "ultrachat_149170", + "sharegpt_ZbUcxUa_0" + ] + }, + { + "answer_session_ids": [ + "answer_19c24c11" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "3d86fd0a", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_19c24c11", + "cf8e352f_1", + "sharegpt_d7bP5fb_0", + "69674f25", + "d1d3fd12_2", + "1c1a2b7f_2", + "06d1a1a0", + "091ba7f5_1", + "5a13d047_1", + "7e4dab66_2" + ] + }, + { + "answer_session_ids": [ + "answer_787e6a6d" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "a82c026e", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_787e6a6d", + "d9727262_4", + "f18ebe36_4", + "3009ba66_2", + "ccaabf0b_1", + "516b0aae_1", + "a6c41826_3", + "daa32134", + "c71e1c7a_2", + "bdf735de_1" + ] + }, + { + "answer_session_ids": [ + "answer_afa9873b_2", + "answer_afa9873b_3", + "answer_afa9873b_1" + ], + "ndcg_at_10": 0.9675, + "ndcg_at_5": 0.9675, + "question_id": "0a995998", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_afa9873b_1", + "answer_afa9873b_2", + "85846900_2", + "answer_afa9873b_3", + "439f9ed8_3", + "e6790684_1", + "fa46a10e", + "3f05c474", + "sharegpt_ZzArUrr_0", + "ultrachat_331531" + ] + }, + { + "answer_session_ids": [ + "answer_ec904b3c_1", + "answer_ec904b3c_4", + "answer_ec904b3c_3", + "answer_ec904b3c_2" + ], + "ndcg_at_10": 0.8772, + "ndcg_at_5": 0.8772, + "question_id": "6d550036", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ec904b3c_1", + "2e4430d8_2", + "0bab76de", + "answer_ec904b3c_4", + "ultrachat_184799", + "a9981dc6_3", + "sharegpt_UbMVpdp_93", + "f6246b5f", + "dae3906e_2", + "sharegpt_J7ZAFLd_0" + ] + }, + { + "answer_session_ids": [ + "answer_593bdffd_4", + "answer_593bdffd_1", + "answer_593bdffd_3", + "answer_593bdffd_2" + ], + "ndcg_at_10": 0.5508, + "ndcg_at_5": 0.5013, + "question_id": "gpt4_59c863d7", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "9ef698bc_2", + "de5f2701_1", + "a7d014e4_1", + "answer_593bdffd_1", + "answer_593bdffd_4", + "answer_593bdffd_2", + "2f09d4c8", + "ultrachat_126967", + "eb47739f_2", + "ultrachat_59708" + ] + }, + { + "answer_session_ids": [ + "answer_a8b4290f_3", + "answer_a8b4290f_1", + "answer_a8b4290f_2" + ], + "ndcg_at_10": 0.6395, + "ndcg_at_5": 0.6509, + "question_id": "b5ef892d", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "ultrachat_565056", + "answer_a8b4290f_3", + "sharegpt_XWqXdom_0", + "answer_a8b4290f_1", + "ultrachat_53385", + "35dcacdc_2", + "ultrachat_367439", + "ultrachat_375427", + "answer_a8b4290f_2", + "e552e1f9_2" + ] + }, + { + "answer_session_ids": [ + "answer_86c505e7_1", + "answer_86c505e7_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e831120c", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_86c505e7_2", + "answer_86c505e7_1", + "sharegpt_sDfu38Q_0", + "8fb3fe3a_1", + "a63ad8e3_2", + "ultrachat_310037", + "7c82a6c3", + "6387e969", + "sharegpt_hvdBqMM_0", + "sharegpt_d8EB6Ze_0" + ] + }, + { + "answer_session_ids": [ + "answer_c2204106_2", + "answer_c2204106_3", + "answer_c2204106_1" + ], + "ndcg_at_10": 0.8529, + "ndcg_at_5": 0.8529, + "question_id": "3a704032", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c2204106_1", + "7712aded", + "86e830d2_3", + "answer_c2204106_2", + "answer_c2204106_3", + "78764b10_2", + "ultrachat_501733", + "sharegpt_MRCLhIj_0", + "ultrachat_357807", + "sharegpt_HrxBbBK_0" + ] + }, + { + "answer_session_ids": [ + "answer_2880eb6c_2", + "answer_2880eb6c_4", + "answer_2880eb6c_1", + "answer_2880eb6c_3" + ], + "ndcg_at_10": 0.7606, + "ndcg_at_5": 0.7606, + "question_id": "gpt4_d84a3211", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "18d68208_1", + "answer_2880eb6c_4", + "answer_2880eb6c_3", + "answer_2880eb6c_2", + "answer_2880eb6c_1", + "4889cd54_1", + "4a32d58c_2", + "ultrachat_83526", + "sharegpt_OHd1RQ7_37", + "13c55c69_1" + ] + }, + { + "answer_session_ids": [ + "answer_526354c8_1", + "answer_526354c8_3", + "answer_526354c8_2" + ], + "ndcg_at_10": 0.9325, + "ndcg_at_5": 1.0, + "question_id": "aae3761f", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_526354c8_2", + "answer_526354c8_1", + "b192ae00_2", + "ultrachat_266173", + "sharegpt_8I9vH7n_7", + "answer_526354c8_3", + "3d85d91e_2", + "sharegpt_ynLoh9N_0", + "2a86d0da_2", + "ultrachat_92484" + ] + }, + { + "answer_session_ids": [ + "answer_55a6940c_3", + "answer_55a6940c_1", + "answer_55a6940c_2" + ], + "ndcg_at_10": 0.4716, + "ndcg_at_5": 0.0, + "question_id": "gpt4_f2262a51", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 0.0, + "top_session_ids": [ + "c0d6fa6f_2", + "dd81b163_1", + "sharegpt_eoEbthf_0", + "ultrachat_268434", + "sharegpt_BWMyoNr_0", + "answer_55a6940c_3", + "answer_55a6940c_2", + "answer_55a6940c_1", + "sharegpt_hChsWOp_128", + "sharegpt_ab6IEma_0" + ] + }, + { + "answer_session_ids": [ + "answer_f9de4602_2", + "answer_f9de4602_1" + ], + "ndcg_at_10": 0.8316, + "ndcg_at_5": 1.0, + "question_id": "dd2973ad", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_f9de4602_2", + "sharegpt_O6NO7Vo_9", + "e5f08e40", + "41743aae_1", + "0f73eee7_2", + "answer_f9de4602_1", + "ultrachat_26627", + "e720dbe2", + "32c0dae1_3", + "b192ae00_2" + ] + }, + { + "answer_session_ids": [ + "answer_56d02cab_3", + "answer_56d02cab_4", + "answer_56d02cab_2", + "answer_56d02cab_1" + ], + "ndcg_at_10": 0.9047, + "ndcg_at_5": 0.9047, + "question_id": "c4a1ceb8", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_56d02cab_3", + "40c77045", + "answer_56d02cab_1", + "answer_56d02cab_4", + "answer_56d02cab_2", + "ba9f938b_2", + "cef33b28_1", + "sharegpt_1sxNkyQ_67", + "0b64c6cb_2", + "sharegpt_WanXxRQ_0" + ] + }, + { + "answer_session_ids": [ + "answer_cf9e3940_2", + "answer_cf9e3940_1", + "answer_cf9e3940_3" + ], + "ndcg_at_10": 0.6462, + "ndcg_at_5": 0.6509, + "question_id": "gpt4_a56e767c", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "88c8df0e_3", + "answer_cf9e3940_3", + "f56e6152_1", + "answer_cf9e3940_2", + "ultrachat_257906", + "ultrachat_104129", + "d75245ea", + "answer_cf9e3940_1", + "dac54d1e_1", + "71621be0" + ] + }, + { + "answer_session_ids": [ + "answer_a4204937_2", + "answer_a4204937_1" + ], + "ndcg_at_10": 0.8772, + "ndcg_at_5": 0.8772, + "question_id": "6cb6f249", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_a4204937_1", + "9b6db1a9_2", + "ultrachat_404221", + "answer_a4204937_2", + "5edfe253_1", + "f584ba36_2", + "67620dbc_2", + "ultrachat_374192", + "cbe70f21", + "sharegpt_IkYYvSd_79" + ] + }, + { + "answer_session_ids": [ + "answer_c65042d7_3", + "answer_c65042d7_2", + "answer_c65042d7_1" + ], + "ndcg_at_10": 0.9066, + "ndcg_at_5": 1.0, + "question_id": "46a3abf7", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c65042d7_3", + "answer_c65042d7_2", + "09f2c6c5", + "28eb86ab", + "a95bedca", + "ultrachat_406179", + "c1e598cd_1", + "4fe91719_2", + "answer_c65042d7_1", + "e224317f_2" + ] + }, + { + "answer_session_ids": [ + "answer_ef74281f_2", + "answer_ef74281f_1", + "answer_ef74281f_3" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "36b9f61e", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ef74281f_2", + "answer_ef74281f_1", + "answer_ef74281f_3", + "e05baf83_2", + "3159942d_2", + "fc6f9c59", + "5829ab38", + "3ceb6783_2", + "fd6f60f0_3", + "85cd56c7_1" + ] + }, + { + "answer_session_ids": [ + "answer_8d015d9d_3", + "answer_8d015d9d_5", + "answer_8d015d9d_2", + "answer_8d015d9d_4", + "answer_8d015d9d_1" + ], + "ndcg_at_10": 0.9609, + "ndcg_at_5": 0.9829, + "question_id": "28dc39ac", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_8d015d9d_2", + "answer_8d015d9d_3", + "answer_8d015d9d_1", + "a7cd1c0f_1", + "answer_8d015d9d_4", + "bfa499f7", + "ultrachat_452868", + "answer_8d015d9d_5", + "5854eebc_1", + "95578ea8_2" + ] + }, + { + "answer_session_ids": [ + "answer_e7b0637e_2", + "answer_e7b0637e_1", + "answer_e7b0637e_3" + ], + "ndcg_at_10": 0.5574, + "ndcg_at_5": 0.5438, + "question_id": "gpt4_2f8be40d", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "ultrachat_217382", + "1d6e01e8", + "answer_e7b0637e_2", + "ultrachat_304474", + "answer_e7b0637e_1", + "sharegpt_L64rHOA_0", + "81b971b8_2", + "cb1f962f_4", + "answer_e7b0637e_3", + "dfe646a7_2" + ] + }, + { + "answer_session_ids": [ + "answer_fa526fc0_4", + "answer_fa526fc0_3", + "answer_fa526fc0_1", + "answer_fa526fc0_2" + ], + "ndcg_at_10": 0.8118, + "ndcg_at_5": 0.8772, + "question_id": "2e6d26dc", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_fa526fc0_4", + "sharegpt_vXNQZ2I_0", + "ultrachat_541982", + "answer_fa526fc0_3", + "7a3cbde3", + "14a520c8_2", + "answer_fa526fc0_1", + "answer_fa526fc0_2", + "sharegpt_IigLRfw_0", + "sharegpt_Wt2YDZs_35" + ] + }, + { + "answer_session_ids": [ + "answer_8858d9dc_3", + "answer_8858d9dc_1", + "answer_8858d9dc_4", + "answer_8858d9dc_2" + ], + "ndcg_at_10": 0.6509, + "ndcg_at_5": 0.6509, + "question_id": "gpt4_15e38248", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "sharegpt_mm9tc5g_0", + "answer_8858d9dc_2", + "1e7f3172_5", + "answer_8858d9dc_4", + "530742d1", + "d4ab49f1", + "ec616e7e_5", + "15acfb99", + "sharegpt_01VDd0u_0", + "8c9bc932_2" + ] + }, + { + "answer_session_ids": [ + "answer_733e443a_3", + "answer_733e443a_4", + "answer_733e443a_2", + "answer_733e443a_1" + ], + "ndcg_at_10": 0.4306, + "ndcg_at_5": 0.3869, + "question_id": "88432d0a", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "f777f641", + "1c8832b4_2", + "c9d35c00_2", + "d1f6080f_1", + "answer_733e443a_2", + "ultrachat_122782", + "ultrachat_351103", + "answer_733e443a_3", + "sharegpt_HpkiAOb_9", + "sharegpt_tESxx1y_0" + ] + }, + { + "answer_session_ids": [ + "answer_990c8992_2", + "answer_990c8992_1", + "answer_990c8992_3" + ], + "ndcg_at_10": 0.8278, + "ndcg_at_5": 0.8772, + "question_id": "80ec1f4f", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_990c8992_2", + "f6ce25ef", + "6884f11d", + "answer_990c8992_1", + "b909542d_1", + "d22c92b2", + "answer_990c8992_3", + "ultrachat_318441", + "ultrachat_497101", + "sharegpt_1kE6haW_52" + ] + }, + { + "answer_session_ids": [ + "answer_5a0d28f8_4", + "answer_5a0d28f8_2", + "answer_5a0d28f8_3", + "answer_5a0d28f8_1" + ], + "ndcg_at_10": 0.9439, + "ndcg_at_5": 0.9675, + "question_id": "d23cf73b", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_5a0d28f8_4", + "answer_5a0d28f8_2", + "66369fde_2", + "answer_5a0d28f8_3", + "b124e4a1_1", + "answer_5a0d28f8_1", + "ultrachat_224488", + "ultrachat_283390", + "ultrachat_456819", + "ultrachat_375734" + ] + }, + { + "answer_session_ids": [ + "answer_a679a86a_5", + "answer_a679a86a_4", + "answer_a679a86a_2", + "answer_a679a86a_3", + "answer_a679a86a_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_7fce9456", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_a679a86a_1", + "answer_a679a86a_5", + "answer_a679a86a_4", + "answer_a679a86a_3", + "541ecc45_3", + "1da409cd_1", + "sharegpt_OjaM9hX_225", + "191e1832", + "67154fb2_1", + "288e7d30" + ] + }, + { + "answer_session_ids": [ + "answer_c008e5df_1", + "answer_c008e5df_2", + "answer_c008e5df_3" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "d682f1a2", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c008e5df_3", + "answer_c008e5df_1", + "sharegpt_E7VUcfj_0", + "7332a779_2", + "9de29219_1", + "d7f9742a_2", + "sharegpt_LFPdgVp_22", + "e30307d8_2", + "58335f60_2", + "ba0ae49c_2" + ] + }, + { + "answer_session_ids": [ + "answer_a21f3697_1", + "answer_a21f3697_2", + "answer_a21f3697_3" + ], + "ndcg_at_10": 0.8711, + "ndcg_at_5": 0.9197, + "question_id": "7024f17c", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_a21f3697_1", + "sharegpt_rshCiS5_1", + "answer_a21f3697_2", + "4f838497_3", + "8c652eb0_1", + "answer_a21f3697_3", + "3c12ff0c", + "f8476198", + "d7281662_2", + "445e06ce_2" + ] + }, + { + "answer_session_ids": [ + "answer_203bf3fa_1", + "answer_203bf3fa_3", + "answer_203bf3fa_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_5501fe77", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_203bf3fa_1", + "answer_203bf3fa_3", + "bc5827b5", + "ultrachat_163385", + "021c0d09_2", + "7a2c618c_1", + "6dfb33f1_2", + "sharegpt_x7apNZ0_0", + "631e4016", + "13b79e39_3" + ] + }, + { + "answer_session_ids": [ + "answer_6a3b5c13_3", + "answer_6a3b5c13_1", + "answer_6a3b5c13_2", + "answer_6a3b5c13_4" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_2ba83207", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_6a3b5c13_1", + "answer_6a3b5c13_2", + "answer_6a3b5c13_4", + "answer_6a3b5c13_3", + "ultrachat_568233", + "cc3c5fa9_2", + "ultrachat_385963", + "ultrachat_224880", + "ea336da0", + "bc6de190_1" + ] + }, + { + "answer_session_ids": [ + "answer_eaa8e3ef_1", + "answer_eaa8e3ef_2" + ], + "ndcg_at_10": 0.8066, + "ndcg_at_5": 1.0, + "question_id": "2318644b", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_eaa8e3ef_2", + "920f808b", + "c4ea8219_1", + "e491f2d4_2", + "7d02ef5c_5", + "6ec23dc5_1", + "fc733b20_2", + "answer_eaa8e3ef_1", + "fcae5c39_2", + "7ff89315_3" + ] + }, + { + "answer_session_ids": [ + "answer_901a6763_2", + "answer_901a6763_4", + "answer_901a6763_1", + "answer_901a6763_3" + ], + "ndcg_at_10": 0.4914, + "ndcg_at_5": 0.4307, + "question_id": "2ce6a0f2", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "a1714d2c_2", + "c1634164_2", + "ultrachat_355740", + "answer_901a6763_4", + "23ddce03_1", + "600077b0_3", + "ultrachat_36844", + "answer_901a6763_2", + "answer_901a6763_1", + "3e90815f" + ] + }, + { + "answer_session_ids": [ + "answer_2504635e_3", + "answer_2504635e_2", + "answer_2504635e_1" + ], + "ndcg_at_10": 0.7929, + "ndcg_at_5": 1.0, + "question_id": "gpt4_d12ceb0e", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_2504635e_2", + "73f4798f", + "sharegpt_dxirwR4_25", + "2fe5510e_3", + "2f23dd1a", + "answer_2504635e_1", + "answer_2504635e_3", + "ultrachat_55863", + "d8e33f5c_abs_2", + "sharegpt_dIgdYPv_13" + ] + }, + { + "answer_session_ids": [ + "answer_39900a0a_3", + "answer_39900a0a_2", + "answer_39900a0a_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "00ca467f", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_39900a0a_3", + "answer_39900a0a_1", + "answer_39900a0a_2", + "e4cb6c56", + "ultrachat_145200", + "ultrachat_239705", + "07942c06", + "4951c0b9_1", + "36828c66_1", + "sharegpt_3cVjZ2b_16" + ] + }, + { + "answer_session_ids": [ + "answer_05d808e6_1", + "answer_05d808e6_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "b3c15d39", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_05d808e6_1", + "answer_05d808e6_2", + "5d4ff8db_1", + "a16a4158", + "sharegpt_H4fpIsr_9", + "sharegpt_inuIr9J_119", + "8d25b813_1", + "d298714d_1", + "sharegpt_V2j1zkI_0", + "sharegpt_5n9NBC8_0" + ] + }, + { + "answer_session_ids": [ + "answer_02b63d04_1", + "answer_02b63d04_5", + "answer_02b63d04_2", + "answer_02b63d04_3", + "answer_02b63d04_4" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_31ff4165", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_02b63d04_1", + "answer_02b63d04_5", + "answer_02b63d04_3", + "35201d43", + "sharegpt_tKmxch5_0", + "e2393b63_2", + "c625cff4_3", + "2d0b80f1", + "6aa87dc3", + "ultrachat_555273" + ] + }, + { + "answer_session_ids": [ + "answer_3e5fea0e_1", + "answer_3e5fea0e_2" + ], + "ndcg_at_10": 0.6053, + "ndcg_at_5": 0.6309, + "question_id": "eeda8a6d", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "sharegpt_PTZ2ime_0", + "answer_3e5fea0e_1", + "ea67b29e_1", + "cf7aad73_3", + "f3745025_1", + "answer_3e5fea0e_2", + "91c71874_1", + "sharegpt_JsCcnH4_0", + "sharegpt_wrXNAYQ_0", + "sharegpt_khNFc2W_0" + ] + }, + { + "answer_session_ids": [ + "answer_8f6b938d_1", + "answer_8f6b938d_3", + "answer_8f6b938d_4", + "answer_8f6b938d_2" + ], + "ndcg_at_10": 0.7397, + "ndcg_at_5": 0.7328, + "question_id": "2788b940", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "544fe66c_2", + "answer_8f6b938d_4", + "answer_8f6b938d_2", + "answer_8f6b938d_1", + "ff6e824b_1", + "sharegpt_vgBnRfE_0", + "answer_8f6b938d_3", + "7033540c_2", + "3f863722", + "sharegpt_xFriTfz_35" + ] + }, + { + "answer_session_ids": [ + "answer_e0956e0a_1", + "answer_e0956e0a_2" + ], + "ndcg_at_10": 0.525, + "ndcg_at_5": 0.5, + "question_id": "60bf93ed", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "a200b713_2", + "388c5534_3", + "answer_e0956e0a_1", + "4fe27c66_4", + "0e297c49", + "answer_e0956e0a_2", + "6414f676_4", + "ultrachat_7648", + "e9590b7d", + "sharegpt_f40Yvlz_0" + ] + }, + { + "answer_session_ids": [ + "answer_fcff2dc4_2", + "answer_fcff2dc4_1", + "answer_fcff2dc4_3" + ], + "ndcg_at_10": 0.8194, + "ndcg_at_5": 0.8772, + "question_id": "9d25d4e0", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_fcff2dc4_3", + "464f3821", + "a4b774c4", + "answer_fcff2dc4_1", + "37beb8da", + "ca555919_3", + "46d9d476", + "answer_fcff2dc4_2", + "39911f94", + "41abc171_1" + ] + }, + { + "answer_session_ids": [ + "answer_1de862d6_1", + "answer_1de862d6_3", + "answer_1de862d6_2" + ], + "ndcg_at_10": 0.906, + "ndcg_at_5": 0.906, + "question_id": "129d1232", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_1de862d6_2", + "69a9211c_2", + "answer_1de862d6_1", + "answer_1de862d6_3", + "5e4bb245_2", + "1ab255ba_1", + "sharegpt_De0lanR_0", + "sharegpt_jBwwg7B_0", + "bf3aebdb_1", + "sharegpt_YfIEYo1_0" + ] + }, + { + "answer_session_ids": [ + "answer_e7fe8c8b_1", + "answer_e7fe8c8b_2", + "answer_e7fe8c8b_3" + ], + "ndcg_at_10": 0.8711, + "ndcg_at_5": 0.9197, + "question_id": "60472f9c", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e7fe8c8b_1", + "b1815b81", + "answer_e7fe8c8b_2", + "7b74d32d_3", + "sharegpt_kF8hHMt_45", + "answer_e7fe8c8b_3", + "ultrachat_334324", + "ultrachat_126217", + "ultrachat_470951", + "sharegpt_5BPO9KJ_0" + ] + }, + { + "answer_session_ids": [ + "answer_3826dc55_1", + "answer_3826dc55_3", + "answer_3826dc55_5", + "answer_3826dc55_2", + "answer_3826dc55_4" + ], + "ndcg_at_10": 0.7157, + "ndcg_at_5": 0.7123, + "question_id": "gpt4_194be4b3", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "ultrachat_58180", + "answer_3826dc55_3", + "answer_3826dc55_5", + "31403c51", + "answer_3826dc55_4", + "ultrachat_385467", + "ultrachat_474217", + "answer_3826dc55_1", + "sharegpt_KwbJJ66_12", + "sharegpt_VO7dTG3_0" + ] + }, + { + "answer_session_ids": [ + "answer_cc021f81_2", + "answer_cc021f81_3", + "answer_cc021f81_1" + ], + "ndcg_at_10": 0.906, + "ndcg_at_5": 0.906, + "question_id": "a9f6b44c", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_cc021f81_3", + "837f258b", + "answer_cc021f81_1", + "answer_cc021f81_2", + "d4912cd9", + "ca896bdc_1", + "sharegpt_AqgYTct_0", + "ec830058_1", + "ultrachat_279504", + "b7e36cc5_2" + ] + }, + { + "answer_session_ids": [ + "answer_5cdf9bd2_2", + "answer_5cdf9bd2_1", + "answer_5cdf9bd2_3", + "answer_5cdf9bd2_4" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "d851d5ba", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_5cdf9bd2_2", + "answer_5cdf9bd2_1", + "answer_5cdf9bd2_4", + "answer_5cdf9bd2_3", + "sharegpt_0gZVK1A_15", + "68ace657_2", + "sharegpt_wduYbsX_0", + "sharegpt_dxirwR4_25", + "sharegpt_x7apNZ0_0", + "d77d4ac9_1" + ] + }, + { + "answer_session_ids": [ + "answer_4cef8a3c_3", + "answer_4cef8a3c_1", + "answer_4cef8a3c_2" + ], + "ndcg_at_10": 0.9675, + "ndcg_at_5": 0.9675, + "question_id": "5a7937c8", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_4cef8a3c_3", + "answer_4cef8a3c_2", + "3b38be11_2", + "answer_4cef8a3c_1", + "ultrachat_288747", + "ultrachat_377263", + "b6bc5b09_2", + "a76e7e3c_4", + "b70ac29f_2", + "07417b21" + ] + }, + { + "answer_session_ids": [ + "answer_728deb4d_5", + "answer_728deb4d_2", + "answer_728deb4d_3", + "answer_728deb4d_1", + "answer_728deb4d_4" + ], + "ndcg_at_10": 0.9709, + "ndcg_at_5": 1.0, + "question_id": "gpt4_ab202e7f", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_728deb4d_4", + "answer_728deb4d_5", + "answer_728deb4d_3", + "answer_728deb4d_2", + "ultrachat_433876", + "sharegpt_q3qJzui_0", + "29bc69b3", + "cdcbdf13", + "answer_728deb4d_1", + "sharegpt_xD2q9ER_14" + ] + }, + { + "answer_session_ids": [ + "answer_6350aa4f_1", + "answer_6350aa4f_2", + "answer_6350aa4f_3", + "answer_6350aa4f_4" + ], + "ndcg_at_10": 0.9709, + "ndcg_at_5": 1.0, + "question_id": "gpt4_e05b82a6", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_6350aa4f_2", + "answer_6350aa4f_4", + "answer_6350aa4f_3", + "88e01130_1", + "b405d3d8_1", + "answer_6350aa4f_1", + "aed6b1b6_1", + "386f6df9", + "ultrachat_43272", + "ultrachat_256330" + ] + }, + { + "answer_session_ids": [ + "answer_826d51da_3", + "answer_826d51da_4", + "answer_826d51da_2", + "answer_826d51da_1" + ], + "ndcg_at_10": 0.7614, + "ndcg_at_5": 1.0, + "question_id": "gpt4_731e37d7", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_826d51da_1", + "07820595_1", + "5365840b_2", + "267a912c", + "c566edbb_3", + "sharegpt_Rwql31f_62", + "answer_826d51da_2", + "7e4aa7c2_1", + "baa5b468_3", + "answer_826d51da_3" + ] + }, + { + "answer_session_ids": [ + "answer_60e8941a_2", + "answer_60e8941a_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "edced276", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_60e8941a_2", + "answer_60e8941a_1", + "1da409cd_1", + "f584ba36_2", + "59a6150d_3", + "726fa34a_1", + "5726dc37_2", + "f5e28561_4", + "93f23301_1", + "e4bbcd5f_4" + ] + }, + { + "answer_session_ids": [ + "answer_e0585cb5_2", + "answer_e0585cb5_1" + ], + "ndcg_at_10": 0.5, + "ndcg_at_5": 0.5, + "question_id": "10d9b85a", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "c51583cd_3", + "cbd1fe79_2", + "answer_e0585cb5_2", + "8dd2fca2", + "84889496_1", + "aa6afba8", + "d576152e_1", + "answer_e0585cb5_1", + "faac3c60", + "5cbfaf3e_4" + ] + }, + { + "answer_session_ids": [ + "answer_b6018747_2", + "answer_b6018747_4", + "answer_b6018747_1", + "answer_b6018747_3" + ], + "ndcg_at_10": 0.7107, + "ndcg_at_5": 0.6934, + "question_id": "e3038f8c", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "sharegpt_hgGAUvu_0", + "answer_b6018747_2", + "answer_b6018747_4", + "85846900_2", + "a3d8e134_2", + "answer_b6018747_3", + "answer_b6018747_1", + "sharegpt_s72sGRc_0", + "4629287b", + "sharegpt_WxCMnMO_5" + ] + }, + { + "answer_session_ids": [ + "answer_23759615_2", + "answer_23759615_3", + "answer_23759615_1" + ], + "ndcg_at_10": 0.9066, + "ndcg_at_5": 1.0, + "question_id": "2b8f3739", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_23759615_3", + "answer_23759615_1", + "dd25eeb5_6", + "c075835c", + "67d6f18f_1", + "285525b2_1", + "ec8691f5", + "sharegpt_ZH6IAa2_11", + "answer_23759615_2", + "ce6cae1b_3" + ] + }, + { + "answer_session_ids": [ + "answer_2bd23659_3", + "answer_2bd23659_2", + "answer_2bd23659_4", + "answer_2bd23659_1" + ], + "ndcg_at_10": 0.906, + "ndcg_at_5": 0.906, + "question_id": "1a8a66a6", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_2bd23659_3", + "54aa4875", + "answer_2bd23659_1", + "answer_2bd23659_2", + "1e91a236_1", + "dc71023e", + "sharegpt_lsiJM0Y_0", + "d6792df8", + "ebb5d262", + "ultrachat_287097" + ] + }, + { + "answer_session_ids": [ + "answer_923c0221_1", + "answer_923c0221_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "c2ac3c61", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_923c0221_2", + "answer_923c0221_1", + "0c483a3c", + "8e4e215b", + "sharegpt_2BSHRQc_25", + "6009ba4e_5", + "9f803df5_1", + "ultrachat_412857", + "ultrachat_38171", + "0b64c6cb_1" + ] + }, + { + "answer_session_ids": [ + "answer_7726e7e9_1", + "answer_7726e7e9_3", + "answer_7726e7e9_2" + ], + "ndcg_at_10": 0.8385, + "ndcg_at_5": 0.8772, + "question_id": "bf659f65", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_7726e7e9_1", + "f2835879_2", + "9d480413_5", + "answer_7726e7e9_2", + "ultrachat_465222", + "answer_7726e7e9_3", + "258645ba_3", + "71619051_1", + "ultrachat_377765", + "sharegpt_SBJezEx_4" + ] + }, + { + "answer_session_ids": [ + "answer_35c5419d_3", + "answer_35c5419d_2", + "answer_35c5419d_1" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "gpt4_372c3eed", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_35c5419d_3", + "4d04b866", + "answer_35c5419d_1", + "ultrachat_400797", + "ultrachat_290513", + "ultrachat_424492", + "0d396995_1", + "bc1f1051_5", + "c95f88fd_1", + "sharegpt_d1pTYei_39" + ] + }, + { + "answer_session_ids": [ + "answer_669318cf_2", + "answer_669318cf_1", + "answer_669318cf_3" + ], + "ndcg_at_10": 0.9469, + "ndcg_at_5": 0.9469, + "question_id": "gpt4_2f91af09", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_669318cf_2", + "answer_669318cf_1", + "22b3af37_1", + "ultrachat_42946", + "answer_669318cf_3", + "51c82c8e_2", + "d75245ea", + "541ecc45_1", + "fd6f60f0_2", + "e7e416f5_2" + ] + }, + { + "answer_session_ids": [ + "answer_da3c1266_3", + "answer_da3c1266_4", + "answer_da3c1266_1", + "answer_da3c1266_5", + "answer_da3c1266_2" + ], + "ndcg_at_10": 0.9829, + "ndcg_at_5": 0.9829, + "question_id": "81507db6", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_da3c1266_2", + "answer_da3c1266_4", + "answer_da3c1266_5", + "5c2f41a0_1", + "answer_da3c1266_1", + "9071ba70_3", + "ultrachat_162129", + "88c09142_2", + "42fa05d3", + "058f17e5_1" + ] + }, + { + "answer_session_ids": [ + "answer_edb03329" + ], + "ndcg_at_10": 0.6309, + "ndcg_at_5": 0.6309, + "question_id": "8a2466db", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "6a5b5a78", + "answer_edb03329", + "6dcf5fa0_1", + "3392c0c7", + "6be54739_3", + "d8b3e1c8_2", + "3d0c9f89_1", + "76967d22", + "sharegpt_MKMWjX0_25", + "sharegpt_osTHjYi_0" + ] + }, + { + "answer_session_ids": [ + "answer_555dfb94" + ], + "ndcg_at_10": 0.3869, + "ndcg_at_5": 0.3869, + "question_id": "06878be2", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "a6074da9_1", + "ca929779_1", + "3e9fce53_1", + "465e7ef9", + "answer_555dfb94", + "8fcfbe43_2", + "sharegpt_GSC090N_6", + "ultrachat_448784", + "79d53fd0_2", + "sharegpt_InRLwN7_0" + ] + }, + { + "answer_session_ids": [ + "answer_d87a6ef8" + ], + "ndcg_at_10": 0.0, + "ndcg_at_5": 0.0, + "question_id": "75832dbd", + "question_type": "single-session-preference", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 0.0, + "r_at_5": 0.0, + "top_session_ids": [ + "d49b1a24_1", + "1ef63c66", + "c884b67a_1", + "sharegpt_xni9yYO_6", + "ultrachat_165893", + "sharegpt_FYqt26U_0", + "47ec1674_2", + "11e9d591_1", + "sharegpt_fK7pfNf_0", + "ultrachat_93325" + ] + }, + { + "answer_session_ids": [ + "answer_d586e9cd" + ], + "ndcg_at_10": 0.5, + "ndcg_at_5": 0.5, + "question_id": "0edc2aef", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "f20e72e4_1", + "6a747f2e", + "answer_d586e9cd", + "08ca1f31_2", + "sharegpt_AsgRx0A_0", + "53dc1394", + "3c3a9042_3", + "8464304d_2", + "c6c3a982_1", + "52939d20" + ] + }, + { + "answer_session_ids": [ + "answer_9b182436" + ], + "ndcg_at_10": 0.5, + "ndcg_at_5": 0.5, + "question_id": "35a27287", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "ultrachat_267380", + "465f0d1b_1", + "answer_9b182436", + "58ae7c4f_3", + "880ee13e", + "e4a1b565_2", + "203bf3fa_2", + "c9b1c309_4", + "ultrachat_422353", + "7128c070_2" + ] + }, + { + "answer_session_ids": [ + "answer_0250ae1c" + ], + "ndcg_at_10": 0.5, + "ndcg_at_5": 0.5, + "question_id": "32260d93", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "c4d370d3_2", + "04bef53b", + "answer_0250ae1c", + "573dc24b_1", + "3f787a78_4", + "ultrachat_26485", + "ultrachat_171173", + "sharegpt_H4jw5s7_39", + "sharegpt_8jxRz9U_0", + "sharegpt_cMHMPNy_29" + ] + }, + { + "answer_session_ids": [ + "answer_6dc4305e" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "195a1a1b", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_6dc4305e", + "ultrachat_396124", + "9f12daa1_1", + "1b71c896_2", + "d03b6a05_2", + "ultrachat_576932", + "9b90460f_3", + "29695e1c_1", + "87fff4b4_1", + "ultrachat_199480" + ] + }, + { + "answer_session_ids": [ + "answer_8549e5e0" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "afdc33df", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_8549e5e0", + "0b0bc8ae_2", + "22f9f163_3", + "b6558bfb", + "afa0c2e7_2", + "d03b6a05_3", + "43b53ac4_1", + "ultrachat_381452", + "ultrachat_512490", + "e58109ed_1" + ] + }, + { + "answer_session_ids": [ + "answer_2fc6aabb" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "caf03d32", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_2fc6aabb", + "ec806228", + "6b137bce_4", + "eb0d8dc1_6", + "4374f9a6_2", + "ultrachat_501938", + "sharegpt_3D7Qpuq_14", + "af631aa3_1", + "bd6d0687", + "147e93c4" + ] + }, + { + "answer_session_ids": [ + "answer_f7b22c66" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "54026fce", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_f7b22c66", + "ba4db050_1", + "fab585c2_1", + "670c49e6", + "ultrachat_166270", + "ef5d4aa7_1", + "1c95e152_1", + "sharegpt_jyHBcl1_0", + "sharegpt_DnGdKtp_0", + "sharegpt_8YTuqXO_0" + ] + }, + { + "answer_session_ids": [ + "answer_92d5f7cd" + ], + "ndcg_at_10": 0.0, + "ndcg_at_5": 0.0, + "question_id": "06f04340", + "question_type": "single-session-preference", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 0.0, + "r_at_5": 0.0, + "top_session_ids": [ + "6e6fbb6b", + "91223fd5_1", + "sharegpt_h4ZC1fl_203", + "66bfa1db", + "0844dea6", + "42924d15", + "8b156015_2", + "728deb4d_4", + "sharegpt_MkLNumZ_0", + "sharegpt_tb5H6IH_3" + ] + }, + { + "answer_session_ids": [ + "answer_f6502d0f" + ], + "ndcg_at_10": 0.3869, + "ndcg_at_5": 0.3869, + "question_id": "6b7dfb22", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "574df152_1", + "9ee69ca6_3", + "377f8fd9_1", + "56859d37", + "answer_f6502d0f", + "be581797", + "ultrachat_443160", + "fc69fd44_3", + "603a8580_2", + "ultrachat_336051" + ] + }, + { + "answer_session_ids": [ + "answer_719502eb" + ], + "ndcg_at_10": 0.6309, + "ndcg_at_5": 0.6309, + "question_id": "1a1907b4", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "9585e0c6_1", + "answer_719502eb", + "4f3b36d4_5", + "3dced3e9_1", + "e42e7876_5", + "4e3da326_1", + "ultrachat_183428", + "ultrachat_490921", + "097c93d8", + "sharegpt_buGcwO5_0" + ] + }, + { + "answer_session_ids": [ + "answer_b10dce5e" + ], + "ndcg_at_10": 0.3562, + "ndcg_at_5": 0.0, + "question_id": "09d032c9", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 0.0, + "top_session_ids": [ + "b21bd3e2", + "3fc2244f", + "612e23f1", + "af631aa3_2", + "d3971322_1", + "answer_b10dce5e", + "26d9aaaf", + "9d1999d7_1", + "e8bfacec_2", + "sharegpt_e9sAtcZ_63" + ] + }, + { + "answer_session_ids": [ + "answer_772472c8" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "38146c39", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_772472c8", + "8064b6ca", + "909d57ca_2", + "381b80a3", + "5263736d_1", + "87aaee77", + "5806662d_1", + "a86b30e4", + "sharegpt_yBVQLaX_15", + "ultrachat_233519" + ] + }, + { + "answer_session_ids": [ + "answer_7c0ade93" + ], + "ndcg_at_10": 0.6309, + "ndcg_at_5": 0.6309, + "question_id": "d24813b1", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "3124bb28_1", + "answer_7c0ade93", + "084802f9_3", + "61a46ff7_3", + "c2a34674_1", + "ultrachat_494027", + "446173bb_1", + "1bc08e44", + "128f4e4d_3", + "sharegpt_tntUB4J_0" + ] + }, + { + "answer_session_ids": [ + "answer_1bde8d3b" + ], + "ndcg_at_10": 0.3562, + "ndcg_at_5": 0.0, + "question_id": "57f827a0", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 0.0, + "top_session_ids": [ + "d0f42e3f", + "1bff1675", + "e85e0eaf_1", + "2ef55f49_1", + "d05d6391_1", + "answer_1bde8d3b", + "1fc5074c_2", + "5150a4e9_2", + "6dd592c3", + "963e896f_1" + ] + }, + { + "answer_session_ids": [ + "answer_5e613445" + ], + "ndcg_at_10": 0.3155, + "ndcg_at_5": 0.0, + "question_id": "95228167", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 0.0, + "top_session_ids": [ + "b916da64_2", + "3d2e66f2_2", + "385a60c8_3", + "99e0fd9f_3", + "084802f9_2", + "5d8c99d3_3", + "7e17ae56_3", + "answer_5e613445", + "e2691068", + "c34b6a1c_1" + ] + }, + { + "answer_session_ids": [ + "answer_f3164f2c" + ], + "ndcg_at_10": 0.5, + "ndcg_at_5": 0.5, + "question_id": "505af2f5", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "7d52ca48_2", + "4f2d6be4_2", + "answer_f3164f2c", + "5e4bb245_1", + "af631aa3_1", + "0e193841_8", + "sharegpt_cjulkGS_15", + "ultrachat_208471", + "sharegpt_kjeGJvK_11", + "9f8fa5e4_1" + ] + }, + { + "answer_session_ids": [ + "answer_8ee04a2e" + ], + "ndcg_at_10": 0.6309, + "ndcg_at_5": 0.6309, + "question_id": "75f70248", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "2d74df23_4", + "answer_8ee04a2e", + "53dc1394", + "75671e3f_1", + "204862e4_2", + "ff67236f_3", + "6f689aee", + "ultrachat_27339", + "sharegpt_ijZ2Bps_0", + "ultrachat_309088" + ] + }, + { + "answer_session_ids": [ + "answer_b0fac439" + ], + "ndcg_at_10": 0.4307, + "ndcg_at_5": 0.4307, + "question_id": "d6233ab6", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "e419b7c3_4", + "94bc18df_3", + "87cfeb28_2", + "answer_b0fac439", + "911f859b_5", + "f916c63a_2", + "ultrachat_457634", + "sharegpt_xni9yYO_6", + "b45e05f6", + "ultrachat_329160" + ] + }, + { + "answer_session_ids": [ + "answer_4d3be2ab" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "1da05512", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_4d3be2ab", + "f2565b3b_2", + "sharegpt_eSTYwMj_14", + "ce919391_2", + "6b93018e_2", + "6dfa3dc6_2", + "ce3ad11f_2", + "5aa88956_2", + "ultrachat_159846", + "6de70f46_1" + ] + }, + { + "answer_session_ids": [ + "answer_a1e169b1" + ], + "ndcg_at_10": 0.3869, + "ndcg_at_5": 0.3869, + "question_id": "fca70973", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "fc005760", + "f504b269", + "5d5e80c5_1", + "99b061c5_1", + "answer_a1e169b1", + "2f2884ad_1", + "ultrachat_3370", + "312bf939", + "sharegpt_lSeiu11_0", + "sharegpt_fPvtTbw_0" + ] + }, + { + "answer_session_ids": [ + "answer_8414cc57" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "b6025781", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_8414cc57", + "f20c73f5_3", + "1a693999_4", + "3a1ea336_1", + "d6956a2e_1", + "1d6744b5_1", + "612c8368_2", + "6446f6e6", + "ebe60348_1", + "2ef53f61_1" + ] + }, + { + "answer_session_ids": [ + "answer_8f15ac24" + ], + "ndcg_at_10": 0.5, + "ndcg_at_5": 0.5, + "question_id": "a89d7624", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "99f2f5b1_1", + "02f70006_1", + "answer_8f15ac24", + "a479a7d6_4", + "fd778d8e", + "39a056ae", + "082fc07f_1", + "168776ed_2", + "bef3247f_1", + "sharegpt_G3RxeFJ_94" + ] + }, + { + "answer_session_ids": [ + "answer_30f63ddb" + ], + "ndcg_at_10": 0.3869, + "ndcg_at_5": 0.3869, + "question_id": "b0479f84", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "6af947bf_1", + "0cf86cbe_2", + "fbfda981", + "457b1adb_1", + "answer_30f63ddb", + "94cffbd2_2", + "9090b1f0_2", + "ddf41a24", + "d1bb5323_1", + "1cde7ee4_1" + ] + }, + { + "answer_session_ids": [ + "answer_e6b6353d" + ], + "ndcg_at_10": 0.3155, + "ndcg_at_5": 0.0, + "question_id": "1d4e3b97", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 0.0, + "top_session_ids": [ + "ecb24dd8_2", + "809cabca_1", + "c51583cd_3", + "03633a51", + "fb2368b8_3", + "1b320137_1", + "12b6f0f1_3", + "answer_e6b6353d", + "5cd6ab1b", + "ultrachat_8041" + ] + }, + { + "answer_session_ids": [ + "answer_d03098f9" + ], + "ndcg_at_10": 0.3333, + "ndcg_at_5": 0.0, + "question_id": "07b6f563", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 0.0, + "top_session_ids": [ + "dcc7f430_1", + "66ab6260", + "5e7b1c98_1", + "13b79e39_2", + "0ac1a6f9_2", + "6af947bf_1", + "answer_d03098f9", + "bbccfe79", + "55c267aa", + "39358a85_2" + ] + }, + { + "answer_session_ids": [ + "answer_8da8c7e0" + ], + "ndcg_at_10": 0.3869, + "ndcg_at_5": 0.3869, + "question_id": "1c0ddc50", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "2aa70c9c_1", + "5439cbf0_2", + "c7ca6dff", + "b33e89b5_1", + "answer_8da8c7e0", + "a0aa5035", + "ultrachat_339938", + "2566382f_2", + "6cd203f7_1", + "0a7d5bf6_2" + ] + }, + { + "answer_session_ids": [ + "answer_cebb7159" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "0a34ad58", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_cebb7159", + "fb048b5d_1", + "9ca2a195_1", + "f849dd04", + "121efb3b_2", + "4b495a04_1", + "f2b8e6a8_1", + "58ad78d6_2", + "ultrachat_122225", + "f7e682c3_5" + ] + }, + { + "answer_session_ids": [ + "answer_5237bb0b_2", + "answer_5237bb0b_1" + ], + "ndcg_at_10": 0.8503, + "ndcg_at_5": 0.8503, + "question_id": "d3ab962e", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_5237bb0b_2", + "ultrachat_564044", + "3329e5e8_1", + "sharegpt_6fTnCnw_49", + "answer_5237bb0b_1", + "aa235649_1", + "ultrachat_164834", + "ultrachat_155769", + "sharegpt_CDacKfJ_0", + "ultrachat_248741" + ] + }, + { + "answer_session_ids": [ + "answer_bf633415_2", + "answer_bf633415_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "2311e44b", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_bf633415_2", + "answer_bf633415_1", + "b5b8f8f9_3", + "5854eebc_2", + "ultrachat_360918", + "c824c711_1", + "84fedd9f_2", + "c6bed037_1", + "sharegpt_YAiYuAD_7", + "2714885f_1" + ] + }, + { + "answer_session_ids": [ + "answer_4fb01417_2", + "answer_4fb01417_1" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "cc06de0d", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_4fb01417_1", + "01551a39_2", + "answer_4fb01417_2", + "55033f6f", + "d5eab084_2", + "1dbe77da_1", + "ultrachat_104440", + "66ffbb8b_1", + "670c49e6", + "92d74e73" + ] + }, + { + "answer_session_ids": [ + "answer_c69ee1f9_2", + "answer_c69ee1f9_1" + ], + "ndcg_at_10": 0.8503, + "ndcg_at_5": 0.8503, + "question_id": "a11281a2", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c69ee1f9_2", + "ultrachat_55839", + "bb2180e9_3", + "c4c4d85a", + "answer_c69ee1f9_1", + "b8adda7f_1", + "28209b6a_5", + "d95e64a7_4", + "9126833b", + "60f10fd3" + ] + }, + { + "answer_session_ids": [ + "answer_50940cb7_2", + "answer_50940cb7_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "4f54b7c9", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_50940cb7_1", + "answer_50940cb7_2", + "bcfd4ab7", + "dde142b2_1", + "ultrachat_518898", + "8a834aee_1", + "b89f68c4_1", + "49a91ed8", + "663337ce_1", + "ultrachat_473718" + ] + }, + { + "answer_session_ids": [ + "answer_dcb18b9b_2", + "answer_dcb18b9b_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "85fa3a3f", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_dcb18b9b_2", + "answer_dcb18b9b_1", + "8f707521_1", + "71110dbd_1", + "0d66562c_1", + "c6c06e6e", + "93ff2f73", + "d31c3ec8_1", + "b7fd3b48", + "sharegpt_3WzI4bW_0" + ] + }, + { + "answer_session_ids": [ + "answer_353d3c6d_2", + "answer_353d3c6d_1" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "9aaed6a3", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_353d3c6d_2", + "c846422a_1", + "answer_353d3c6d_1", + "819dfad7_2", + "3f9f66f1_1", + "2e89bd03_1", + "11e942da_2", + "sharegpt_HpdNUlz_9", + "1bf761b8_2", + "73726c40_1" + ] + }, + { + "answer_session_ids": [ + "answer_d8454588_1", + "answer_d8454588_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "1f2b8d4f", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_d8454588_2", + "answer_d8454588_1", + "d3ee5958_2", + "c9b1c309_4", + "c8c64252", + "b5c147c7_2", + "9e96d5b9_1", + "0bd59f15_3", + "23759615_1", + "sharegpt_OHIxTQm_22" + ] + }, + { + "answer_session_ids": [ + "answer_4eb6d671_2", + "answer_4eb6d671_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e6041065", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_4eb6d671_2", + "answer_4eb6d671_1", + "d79173aa_1", + "b73a4d04_1", + "49f2e396_2", + "4a006b77", + "ec616e7e_4", + "2a86d0da_2", + "bdcee74e_3", + "b7dca8f1_1" + ] + }, + { + "answer_session_ids": [ + "answer_58820c75_1", + "answer_58820c75_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "51c32626", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_58820c75_1", + "answer_58820c75_2", + "c051e740_2", + "sharegpt_hzwc0HI_13", + "84d7650f_1", + "809cbce9_1", + "sharegpt_ErOTMZ3_59", + "932ea3bf_1", + "ultrachat_273746", + "sharegpt_jBafAKm_0" + ] + }, + { + "answer_session_ids": [ + "answer_85a77c48_2", + "answer_85a77c48_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "d905b33f", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_85a77c48_1", + "answer_85a77c48_2", + "8cfa91c3_2", + "2d364fb0", + "a9b7f49e_1", + "368276ab_3", + "dd4a0e15", + "6423d6a4_2", + "7613d5ec_2", + "sharegpt_ektieAO_13" + ] + }, + { + "answer_session_ids": [ + "answer_80323f3f_1", + "answer_80323f3f_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "7405e8b1", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_80323f3f_1", + "answer_80323f3f_2", + "f2f2a606_3", + "a6b00510_4", + "6c1c5bc3_2", + "c578da1f_1", + "6e983235_1", + "3a4012a5_1", + "8f8f2ced_2", + "643d285a_3" + ] + }, + { + "answer_session_ids": [ + "answer_e9bb9500_2", + "answer_e9bb9500_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "f35224e0", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e9bb9500_2", + "answer_e9bb9500_1", + "96da07f9_4", + "e184e4c3_1", + "9393001b_3", + "ultrachat_215582", + "4648f214_2", + "42234f98", + "ultrachat_558221", + "sharegpt_Pqhvz5n_0" + ] + }, + { + "answer_session_ids": [ + "answer_743f03a1_2", + "answer_743f03a1_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "6456829e", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_743f03a1_1", + "answer_743f03a1_2", + "c44d1533_2", + "ultrachat_490336", + "ultrachat_518760", + "a8c73552", + "898cfd46_2", + "ultrachat_555576", + "3304f57b_3", + "ultrachat_247641" + ] + }, + { + "answer_session_ids": [ + "answer_feb5f98a_1", + "answer_feb5f98a_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "a4996e51", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_feb5f98a_2", + "answer_feb5f98a_1", + "252715f8_3", + "e9fba4f8", + "c1dac02f", + "082b7e52_2", + "d1a61aa1_3", + "ultrachat_506025", + "sharegpt_KjF70Iy_0", + "eea3e4be_2" + ] + }, + { + "answer_session_ids": [ + "answer_c8cc60d6_2", + "answer_c8cc60d6_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "3c1045c8", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c8cc60d6_2", + "ultrachat_561782", + "ultrachat_20893", + "b52f03a1_1", + "62541047", + "sharegpt_oIB2JG9_0", + "4f3b36d4_3", + "ultrachat_299649", + "sharegpt_GYqnAhC_190", + "cf422c07" + ] + }, + { + "answer_session_ids": [ + "answer_e552e1f9_1", + "answer_e552e1f9_2" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "60036106", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e552e1f9_2", + "sharegpt_moARQa7_0", + "answer_e552e1f9_1", + "sharegpt_ckvrgF1_99", + "c575291e_2", + "e9cb7125", + "ultrachat_466668", + "23a7f0ec", + "763f028a", + "ultrachat_134557" + ] + }, + { + "answer_session_ids": [ + "answer_3be95d43_1", + "answer_3be95d43_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "681a1674", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_3be95d43_2", + "answer_3be95d43_1", + "ultrachat_64650", + "fdd25ea1", + "50e8f2b5_1", + "d81e853b", + "sharegpt_8Kzmn95_0", + "ultrachat_436833", + "7d33fcb4_2", + "9e411500_1" + ] + }, + { + "answer_session_ids": [ + "answer_6702277b_1", + "answer_6702277b_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e25c3b8d", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_6702277b_2", + "answer_6702277b_1", + "sharegpt_40qcQLS_17", + "sharegpt_WYQSZ3n_0", + "90fea364_1", + "sharegpt_Nk10TjE_29", + "824634d9_1", + "ff201786_1", + "4256f981_2", + "ultrachat_363088" + ] + }, + { + "answer_session_ids": [ + "answer_6efce493_1", + "answer_6efce493_2" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "4adc0475", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_6efce493_1", + "bb27df5e_2", + "answer_6efce493_2", + "fc6f9c59", + "4fc2cf8c_1", + "2880eb6c_3", + "0bde5dac", + "sharegpt_wcTU09Q_31", + "debc34d7_2", + "ee659010_2" + ] + }, + { + "answer_session_ids": [ + "answer_9ef115d4_1", + "answer_9ef115d4_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "4bc144e2", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_9ef115d4_1", + "answer_9ef115d4_2", + "4f23a396_2", + "d75245ea", + "ultrachat_380790", + "42bcee92_2", + "099c1b6c_3", + "60207b41_3", + "26d9aaaf", + "2e2ae792_1" + ] + }, + { + "answer_session_ids": [ + "answer_f7fd1029_2", + "answer_f7fd1029_1" + ], + "ndcg_at_10": 0.8772, + "ndcg_at_5": 0.8772, + "question_id": "ef66a6e5", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_f7fd1029_1", + "48941fec_4", + "74c2d3a1_1", + "answer_f7fd1029_2", + "3f05a03e_1", + "41e778e4_2", + "afb58af0", + "295645f5_1", + "67045503", + "5273d4b8" + ] + }, + { + "answer_session_ids": [ + "answer_688f9a3f_1", + "answer_688f9a3f_2" + ], + "ndcg_at_10": 0.6934, + "ndcg_at_5": 0.6934, + "question_id": "5025383b", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "22f9af5f", + "answer_688f9a3f_2", + "answer_688f9a3f_1", + "f40fa3ed", + "d10f0307_1", + "2d21f921", + "ultrachat_50563", + "ultrachat_150500", + "sharegpt_BmS3AX0_10", + "ultrachat_137351" + ] + }, + { + "answer_session_ids": [ + "answer_17dc2f5b_2", + "answer_17dc2f5b_1" + ], + "ndcg_at_10": 0.5, + "ndcg_at_5": 0.5, + "question_id": "a1cc6108", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "298a07de_5", + "fce56dfb_4", + "answer_17dc2f5b_2", + "7285299a_4", + "6dfc99bf_1", + "001cefa7_2", + "fb328ace_1" + ] + }, + { + "answer_session_ids": [ + "answer_66c23110_1", + "answer_66c23110_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "9ee3ecd6", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_66c23110_2", + "answer_66c23110_1", + "fd56c5bd_2", + "a9eda378", + "6d9d5941", + "ultrachat_282862", + "fa0fa74d_2", + "ultrachat_96128", + "sharegpt_6cz1Sq6_264", + "42b45f6e_1" + ] + }, + { + "answer_session_ids": [ + "answer_419d21d5_2", + "answer_419d21d5_1" + ], + "ndcg_at_10": 0.8316, + "ndcg_at_5": 1.0, + "question_id": "3fdac837", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_419d21d5_2", + "92b06116_1", + "a98ff421_7", + "ultrachat_526748", + "c85ef24e_1", + "answer_419d21d5_1", + "ac8a2cd0", + "ultrachat_293546", + "88c8df0e_3", + "22210456_1" + ] + }, + { + "answer_session_ids": [ + "answer_5404a208_1", + "answer_5404a208_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "91b15a6e", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_5404a208_1", + "answer_5404a208_2", + "ultrachat_15125", + "f9a304e9_2", + "6169ef55_1", + "ultrachat_217922", + "7cfeb89c_2", + "ultrachat_338803", + "ultrachat_326780", + "ultrachat_222016" + ] + }, + { + "answer_session_ids": [ + "answer_a37bdf22_2", + "answer_a37bdf22_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "27016adc", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_a37bdf22_2", + "answer_a37bdf22_1", + "9f554b46_2", + "fa858208_2", + "28ea20bf", + "ultrachat_495725", + "ff5e0db0_1", + "sharegpt_3b5ojYa_39", + "ultrachat_131737", + "ultrachat_284316" + ] + }, + { + "answer_session_ids": [ + "answer_c9dfeaea_1", + "answer_c9dfeaea_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "720133ac", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c9dfeaea_2", + "answer_c9dfeaea_1", + "4c128d97_2", + "e200d96c_3", + "ff2de758", + "ultrachat_479833", + "sharegpt_bZg2XwX_69", + "0cf86cbe_2", + "ultrachat_112896", + "ultrachat_512422" + ] + }, + { + "answer_session_ids": [ + "answer_33c251f0_2", + "answer_33c251f0_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "77eafa52", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_33c251f0_1", + "answer_33c251f0_2", + "bc1f1051_4", + "sharegpt_qEukY6n_14", + "sharegpt_FlvsHKg_4", + "sharegpt_C9GjDsF_0", + "sharegpt_bpxJpAU_0", + "ultrachat_309057", + "9c4358a4", + "7b13b4d6_2" + ] + }, + { + "answer_session_ids": [ + "answer_35e36341_1", + "answer_35e36341_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "8979f9ec", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_35e36341_1", + "answer_35e36341_2", + "91c71874_1", + "ultrachat_466891", + "bcc9382a_2", + "ultrachat_562471", + "4c967baa_1", + "a6b4c4c2_1", + "f18ebe36_7", + "1064ed80_2" + ] + }, + { + "answer_session_ids": [ + "answer_35c9798c_2", + "answer_35c9798c_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "0100672e", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_35c9798c_2", + "answer_35c9798c_1", + "a4b223a7_1", + "091aa510_2", + "326e86e0_3", + "8237de6a_2", + "ultrachat_372868", + "07313722_2", + "b167a4a8", + "f60c9de5" + ] + }, + { + "answer_session_ids": [ + "answer_ef84b994_1", + "answer_ef84b994_2" + ], + "ndcg_at_10": 0.8503, + "ndcg_at_5": 0.8503, + "question_id": "a96c20ee", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ef84b994_1", + "1b51fb71_1", + "dff89d9d_1", + "9ca3534f_1", + "answer_ef84b994_2", + "2aa70c9c_2", + "sharegpt_opQbgPL_7", + "32e292f2", + "sharegpt_lWLBUhQ_539", + "sharegpt_00qHEQ6_0" + ] + }, + { + "answer_session_ids": [ + "answer_6cb8f792_1", + "answer_6cb8f792_2" + ], + "ndcg_at_10": 0.525, + "ndcg_at_5": 0.5, + "question_id": "92a0aa75", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "ultrachat_178564", + "sharegpt_YwmoLVl_0", + "answer_6cb8f792_1", + "a5854a8d_2", + "2bdd78cb_1", + "answer_6cb8f792_2", + "bd00a24d_2", + "sharegpt_xkWMnwQ_0", + "sharegpt_9l7gjYP_17", + "sharegpt_hPax8pw_11" + ] + }, + { + "answer_session_ids": [ + "answer_1bb63ea5_2", + "answer_1bb63ea5_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "3fe836c9", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_1bb63ea5_1", + "answer_1bb63ea5_2", + "e41b78c7_3", + "0a42783a", + "c81897cd_2", + "sharegpt_2xK3jjf_0", + "8129a7a8", + "f0d7d364", + "ba184a34", + "ultrachat_258020" + ] + }, + { + "answer_session_ids": [ + "answer_4cb841a8_1", + "answer_4cb841a8_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "1c549ce4", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_4cb841a8_1", + "answer_4cb841a8_2", + "f1e4daa8_2", + "f999b05c_1", + "d743153b_2", + "ultrachat_441412", + "162ff451", + "sharegpt_miOKGt1_13", + "35f052f9_2", + "sharegpt_yLBm8Vr_23" + ] + }, + { + "answer_session_ids": [ + "answer_cc1ced21_2", + "answer_cc1ced21_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "6c49646a", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_cc1ced21_2", + "answer_cc1ced21_1", + "d4380d3f", + "8acb29f9_2", + "ultrachat_565061", + "fffa6e59", + "9f220c66_1", + "6a2f306a", + "sharegpt_1X61wv2_7", + "edf2a486" + ] + }, + { + "answer_session_ids": [ + "answer_e184e4c3_2", + "answer_e184e4c3_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "1192316e", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e184e4c3_2", + "answer_e184e4c3_1", + "574df152_3", + "c5db849c_1", + "6cd7234e_2", + "ee82ed2c_2", + "21ef2d05_1", + "sharegpt_pZ3I356_11", + "1ba79ea8_1", + "ultrachat_457634" + ] + }, + { + "answer_session_ids": [ + "answer_dc5e537d_1", + "answer_dc5e537d_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "0ea62687", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_dc5e537d_1", + "answer_dc5e537d_2", + "fba28a35_3", + "26eedae1_1", + "0835368c", + "c5e9cbd8_1", + "d3ee5958_1", + "ultrachat_108112", + "sharegpt_HYI3HgX_0", + "sharegpt_Ktz3pBy_0" + ] + }, + { + "answer_session_ids": [ + "answer_3a5010af_2", + "answer_3a5010af_1" + ], + "ndcg_at_10": 0.8772, + "ndcg_at_5": 0.8772, + "question_id": "67e0d0f2", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_3a5010af_2", + "6868c94f", + "98cd882c_1", + "answer_3a5010af_1", + "ultrachat_217333", + "0cde5602_2", + "8c568d75_1", + "2252d3dd", + "sharegpt_tJkPZbz_2", + "sharegpt_GJpf1ou_0" + ] + }, + { + "answer_session_ids": [ + "answer_de64539a_1", + "answer_de64539a_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "bb7c3b45", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_de64539a_2", + "answer_de64539a_1", + "60ff3a8c", + "3891baea_4", + "a51dc9ca", + "64b7d9cc_2", + "ultrachat_305663", + "f3d43a12_3", + "4f2d6be4_1", + "752b10e6_1" + ] + }, + { + "answer_session_ids": [ + "answer_cbd08e3c_1", + "answer_cbd08e3c_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "ba358f49", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_cbd08e3c_1", + "2a2ea005_1", + "20b5caab_2", + "9cc709cb", + "8db0ab95_1", + "4ea6f550", + "385683f0_2", + "648d9cda", + "d964ed3b", + "4f838497_2" + ] + }, + { + "answer_session_ids": [ + "answer_872e8da2_2", + "answer_872e8da2_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "61f8c8f8", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_872e8da2_1", + "answer_872e8da2_2", + "a3107e2a_1", + "6e110a53_1", + "7b88c38b_2", + "2154c810_1", + "7af38385_2", + "e10846c4", + "sharegpt_iL51bzd_0", + "sharegpt_7jGo66i_99" + ] + }, + { + "answer_session_ids": [ + "answer_75eca223_2", + "answer_75eca223_1" + ], + "ndcg_at_10": 0.4825, + "ndcg_at_5": 0.4307, + "question_id": "60159905", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "21f8f481_3", + "d5f4d9fa_2", + "02eae87c_5", + "answer_75eca223_1", + "sharegpt_V7ePzVN_1", + "answer_75eca223_2", + "6ade84f0", + "c351fa3b_1", + "c92b27e2_1", + "c36f4196" + ] + }, + { + "answer_session_ids": [ + "answer_87e3a1cb_1", + "answer_87e3a1cb_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "ef9cf60a", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_87e3a1cb_2", + "answer_87e3a1cb_1", + "2357de36_2", + "b330d6b1", + "7b88c38b_1", + "sharegpt_FNyKOSt_0", + "7a34bc76_3", + "12dd7681_4", + "0babf5b6_2", + "de47d59b_1" + ] + }, + { + "answer_session_ids": [ + "answer_1881e7db_1", + "answer_1881e7db_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "73d42213", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_1881e7db_2", + "answer_1881e7db_1", + "sharegpt_ab6IEma_0", + "9614d065_1", + "sharegpt_G8j6c8J_87", + "sharegpt_ndynIbS_0", + "sharegpt_GKjb2be_0", + "ultrachat_275858", + "a13abca5_1", + "sharegpt_2Na4ASq_0" + ] + }, + { + "answer_session_ids": [ + "answer_92147866_1", + "answer_92147866_2" + ], + "ndcg_at_10": 0.8503, + "ndcg_at_5": 0.8503, + "question_id": "bc149d6b", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_92147866_1", + "7211c69e", + "86935301_2", + "451ecb23_3", + "answer_92147866_2", + "4f435d20_2", + "sharegpt_bmEgv3O_12", + "05be3ff0_3", + "eb5c65e2_1", + "db73b7e4_3" + ] + }, + { + "answer_session_ids": [ + "answer_80d6d664_2", + "answer_80d6d664_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "099778bb", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_80d6d664_2", + "answer_80d6d664_1", + "sharegpt_s1I3UkG_0", + "47103d08", + "0070840d_2", + "b1793bba", + "6bb8af95_1", + "33877350_2", + "95ce0e21_1", + "sharegpt_kshUqZh_6" + ] + }, + { + "answer_session_ids": [ + "answer_96c743d0_2", + "answer_96c743d0_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "09ba9854", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_96c743d0_1", + "answer_96c743d0_2", + "ultrachat_177260", + "32e6c308", + "7a2bdd15_1", + "ae32e467_2", + "8c63238e", + "864a563d_3", + "69a42467_2", + "8682e696_2" + ] + }, + { + "answer_session_ids": [ + "answer_23f3a657_2", + "answer_23f3a657_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "d6062bb9", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_23f3a657_1", + "answer_23f3a657_2", + "sharegpt_xuhzAgG_19", + "b35610f4_4", + "4f8caea3_1", + "5b6aedac", + "59c704ad_2", + "86298245_2", + "sharegpt_afCogMg_13", + "ultrachat_496826" + ] + }, + { + "answer_session_ids": [ + "answer_8de18468_2", + "answer_8de18468_1" + ], + "ndcg_at_10": 0.301, + "ndcg_at_5": 0.0, + "question_id": "157a136e", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 0.0, + "top_session_ids": [ + "ultrachat_255132", + "01049599_1", + "ultrachat_3770", + "sharegpt_qqo6qRj_0", + "a2e2cb72_5", + "sharegpt_6ZXBs1w_0", + "ultrachat_506827", + "298bf7a3_1", + "answer_8de18468_2", + "ultrachat_446084" + ] + }, + { + "answer_session_ids": [ + "answer_2e2085fa_2", + "answer_2e2085fa_1" + ], + "ndcg_at_10": 0.4307, + "ndcg_at_5": 0.4307, + "question_id": "c18a7dc8", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "9d5a389d", + "ultrachat_353377", + "5022812c", + "answer_2e2085fa_1", + "fc6c3985", + "f716720c_3", + "ultrachat_415226", + "37141183", + "ultrachat_463016", + "sharegpt_YPC6Oll_18" + ] + }, + { + "answer_session_ids": [ + "answer_16ece55f_2", + "answer_16ece55f_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "a3332713", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_16ece55f_2", + "answer_16ece55f_1", + "62be624f", + "67388dfc_2", + "3e59ee68_1", + "fe291ebc", + "117098d9_3", + "33994682_3", + "sharegpt_bYymtlW_0", + "sharegpt_8zrM3l4_0" + ] + }, + { + "answer_session_ids": [ + "answer_fa08bf49_1", + "answer_fa08bf49_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "55241a1f", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_fa08bf49_1", + "answer_fa08bf49_2", + "d117e2da", + "2aeb1268", + "76491ee4_1", + "9c9a22d8", + "60153a02_4", + "sharegpt_TzwvePR_0", + "46e4a8eb_2", + "1d105d8d" + ] + }, + { + "answer_session_ids": [ + "answer_47152166_2", + "answer_47152166_1" + ], + "ndcg_at_10": 0.6934, + "ndcg_at_5": 0.6934, + "question_id": "a08a253f", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "6e5130ed_2", + "answer_47152166_2", + "answer_47152166_1", + "798fa5f2", + "ultrachat_242484", + "33bc1102", + "c7fc2bda_1", + "ultrachat_329610", + "a79f1a04_1", + "ultrachat_403700" + ] + }, + { + "answer_session_ids": [ + "answer_cfcf5340_1", + "answer_cfcf5340_2" + ], + "ndcg_at_10": 0.8503, + "ndcg_at_5": 0.8503, + "question_id": "f0e564bc", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_cfcf5340_2", + "65ca99fb_2", + "798e4ba3_2", + "bc1f1051_1", + "answer_cfcf5340_1", + "sharegpt_W3YFpVK_7", + "sharegpt_3hCiKrk_0", + "06fe8cfb_1", + "ultrachat_197576", + "932a97c6" + ] + }, + { + "answer_session_ids": [ + "answer_254d8b09_1", + "answer_254d8b09_2" + ], + "ndcg_at_10": 0.8316, + "ndcg_at_5": 1.0, + "question_id": "078150f1", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_254d8b09_1", + "61b27d67_3", + "203a48bb_1", + "46e4a8eb_1", + "ultrachat_202549", + "answer_254d8b09_2", + "0e218647_2", + "57bf2079", + "ultrachat_276164", + "b27b0097_1" + ] + }, + { + "answer_session_ids": [ + "answer_e2278b24_1", + "answer_e2278b24_2" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "8cf4d046", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e2278b24_2", + "a30f52a4", + "answer_e2278b24_1", + "sharegpt_P1Okivz_9", + "sharegpt_6DlPq48_0", + "7ef465f3_1", + "sharegpt_WjpyexI_15", + "7a5c32b0_3", + "c8b1178f_1", + "c36f4196" + ] + }, + { + "answer_session_ids": [ + "answer_4934b2d7_2", + "answer_4934b2d7_1" + ], + "ndcg_at_10": 0.8316, + "ndcg_at_5": 1.0, + "question_id": "a346bb18", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_4934b2d7_2", + "771ba1d5", + "5726dc37_2", + "fd5448f0", + "84fb50bb_1", + "answer_4934b2d7_1", + "sharegpt_VeexfGM_0", + "sharegpt_cYNuFCK_50", + "sharegpt_ooEuxDg_0", + "e5713180_1" + ] + }, + { + "answer_session_ids": [ + "answer_6b9b2b1e_2", + "answer_6b9b2b1e_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "37f165cf", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_6b9b2b1e_2", + "answer_6b9b2b1e_1", + "20b1b65f_2", + "a41d46df", + "c85d1682_1", + "c41e97c9_2", + "sharegpt_2yaKUmC_0", + "395a2f92", + "c0f6571a_2", + "7e8e790a" + ] + }, + { + "answer_session_ids": [ + "answer_477ae455_1", + "answer_477ae455_2" + ], + "ndcg_at_10": 0.0, + "ndcg_at_5": 0.0, + "question_id": "8e91e7d9", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 0.0, + "r_at_5": 0.0, + "top_session_ids": [ + "sharegpt_PZrBCF2_0", + "ultrachat_147574", + "4e3da326_1", + "bc790dfb", + "2f980ae0_2", + "c3023a45_4", + "06170195_1", + "sharegpt_8lpX5R7_19", + "c2381e1a_2", + "sharegpt_Z35PJ8x_2" + ] + }, + { + "answer_session_ids": [ + "answer_f56e6152_2", + "answer_f56e6152_1" + ], + "ndcg_at_10": 0.4306, + "ndcg_at_5": 0.3869, + "question_id": "87f22b4a", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "02b99ec3_1", + "ultrachat_350147", + "28741edc_1", + "ultrachat_182225", + "answer_f56e6152_2", + "889076d6", + "c4c504d2_6", + "answer_f56e6152_1", + "a8ac3d1d_3", + "ea28bb75" + ] + }, + { + "answer_session_ids": [ + "answer_430d0a87_1", + "answer_430d0a87_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e56a43b9", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_430d0a87_1", + "answer_430d0a87_2", + "83e14506_2", + "0eef411a_2", + "38e81260_7", + "7596ba6a", + "ce21f0e4", + "sharegpt_XaM55h6_28", + "ultrachat_245457", + "ultrachat_12201" + ] + }, + { + "answer_session_ids": [ + "answer_fc81d1a2_1", + "answer_fc81d1a2_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "efc3f7c2", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_fc81d1a2_1", + "answer_fc81d1a2_2", + "ultrachat_38938", + "263f2395", + "ultrachat_166366", + "bbdc7b0a", + "ultrachat_102676", + "ultrachat_552063", + "48cb014c", + "fc005760" + ] + }, + { + "answer_session_ids": [ + "answer_2c637141_2", + "answer_2c637141_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "21d02d0d", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_2c637141_2", + "answer_2c637141_1", + "eb8195a6_1", + "ultrachat_71299", + "sharegpt_adeEi7y_7", + "06a39bb5_1", + "42d9467c_2", + "a15fa6eb", + "sharegpt_B9jlITg_0", + "414b8ddb" + ] + }, + { + "answer_session_ids": [ + "answer_d00ba6d0_1", + "answer_d00ba6d0_2" + ], + "ndcg_at_10": 0.8772, + "ndcg_at_5": 0.8772, + "question_id": "gpt4_59149c77", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_d00ba6d0_2", + "9e2c2a6c_2", + "9242fbe3_1", + "answer_d00ba6d0_1", + "6e672b84_1", + "sharegpt_XlNs8Lz_199", + "148f87a0_2", + "54a05e72_2", + "ultrachat_318178", + "ultrachat_280434" + ] + }, + { + "answer_session_ids": [ + "answer_3e9fce53_1", + "answer_3e9fce53_2", + "answer_3e9fce53_3" + ], + "ndcg_at_10": 0.9469, + "ndcg_at_5": 0.9469, + "question_id": "gpt4_f49edff3", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_3e9fce53_3", + "answer_3e9fce53_2", + "7d33fcb4_1", + "1a29bf1f", + "answer_3e9fce53_1", + "dc7c53fa_4", + "9f8fa5e4_2", + "acfdbdcb", + "4376945f", + "ultrachat_257206" + ] + }, + { + "answer_session_ids": [ + "answer_0b4a8adc_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "71017276", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_0b4a8adc_1", + "683c8358_1", + "58ec258f_2", + "sharegpt_tCLMKZY_0", + "10889cf0_1", + "4d6927c2_2", + "sharegpt_11WxPMZ_0", + "44e4ae7e", + "398f23e7", + "sharegpt_tmFunkX_20" + ] + }, + { + "answer_session_ids": [ + "answer_4bfcc250_4", + "answer_4bfcc250_3", + "answer_4bfcc250_2", + "answer_4bfcc250_1" + ], + "ndcg_at_10": 0.8529, + "ndcg_at_5": 0.8529, + "question_id": "b46e15ed", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_4bfcc250_3", + "aeab3296", + "fa34119a_1", + "answer_4bfcc250_2", + "answer_4bfcc250_1", + "4bf66c38_2", + "a410bab2_3", + "47311a4c_1", + "ultrachat_520419", + "f598d30f_1" + ] + }, + { + "answer_session_ids": [ + "answer_ff201786_1", + "answer_ff201786_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_fa19884c", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ff201786_2", + "answer_ff201786_1", + "a7b1b27c", + "f3745025_1", + "1b066639", + "f4f4b504_2", + "800cfd94", + "ultrachat_423544", + "cbf48918_1", + "bb2180e9_3" + ] + }, + { + "answer_session_ids": [ + "answer_f4ea84fb_3", + "answer_f4ea84fb_2", + "answer_f4ea84fb_1" + ], + "ndcg_at_10": 0.6462, + "ndcg_at_5": 0.6509, + "question_id": "0bc8ad92", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "0868982f_1", + "answer_f4ea84fb_3", + "f58c36b8_1", + "answer_f4ea84fb_2", + "ultrachat_349938", + "fccc9400", + "d3cc5bdc_1", + "answer_f4ea84fb_1", + "eca613fc", + "sharegpt_scMKXPh_9" + ] + }, + { + "answer_session_ids": [ + "answer_b51b6115_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "af082822", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_b51b6115_1", + "bc5827b5", + "facef6e0_1", + "b28990c8_7", + "d048ae4b_2", + "1dfef591_2", + "cae65795_2", + "cdf068b1_2", + "1fcb4134_2", + "54f0d6f9_1" + ] + }, + { + "answer_session_ids": [ + "answer_add9b012_2", + "answer_add9b012_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_4929293a", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_add9b012_2", + "answer_add9b012_1", + "623ea729_2", + "ultrachat_329640", + "sharegpt_oQacgfY_0", + "ultrachat_512282", + "b8bc1464_4", + "73353aee_1", + "c1e598cd_1", + "sharegpt_mWFmQxk_36" + ] + }, + { + "answer_session_ids": [ + "answer_a17423e7_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_b5700ca9", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_a17423e7_1", + "44c62f26", + "5053474c_2", + "a53c7542_1", + "33bdd89a_1", + "sharegpt_FqOC5e9_169", + "d743153b_1", + "ultrachat_366301", + "47382adf_1", + "ultrachat_383774" + ] + }, + { + "answer_session_ids": [ + "answer_dba89487_2", + "answer_dba89487_1" + ], + "ndcg_at_10": 0.8316, + "ndcg_at_5": 1.0, + "question_id": "9a707b81", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_dba89487_2", + "4abbeb2a_2", + "5a13d047_1", + "ultrachat_233519", + "6862e478_2", + "answer_dba89487_1", + "990f3ef9_6", + "3fcbb58c_4", + "e1d11615_1", + "59510ab6" + ] + }, + { + "answer_session_ids": [ + "answer_febde667_1", + "answer_febde667_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_1d4ab0c9", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_febde667_2", + "answer_febde667_1", + "ab855b34_2", + "3f1d1224_3", + "7257bd15_1", + "082b7e52_1", + "ultrachat_201007", + "2dbfff2f", + "ultrachat_310433", + "sharegpt_MIYBRJz_27" + ] + }, + { + "answer_session_ids": [ + "answer_c19bd2bf_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_e072b769", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c19bd2bf_1", + "5558a42e_1", + "4f7b5dc9_1", + "b9e32ff8_1", + "2fc9ce79_2", + "sharegpt_1VhiETr_37", + "66d3fdb7_2", + "ultrachat_371691", + "sharegpt_9Ln5Ood_0", + "sharegpt_VbIxRWJ_33" + ] + }, + { + "answer_session_ids": [ + "answer_b9e32ff8_1", + "answer_b9e32ff8_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "0db4c65d", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_b9e32ff8_2", + "answer_b9e32ff8_1", + "71dc2037_2", + "953d5a4b_1", + "667465d6_1", + "sharegpt_5NoXULS_0", + "e9e19917_1", + "b22a2540", + "d36d11b9_2", + "e6c3a50a" + ] + }, + { + "answer_session_ids": [ + "answer_661b711f_1", + "answer_661b711f_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_1d80365e", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_661b711f_1", + "answer_661b711f_2", + "f7a61595_2", + "ultrachat_293442", + "805e571c_1", + "554b054e_2", + "ultrachat_561552", + "ultrachat_271411", + "3be19aed_1", + "sharegpt_5uZ7IdY_0" + ] + }, + { + "answer_session_ids": [ + "answer_5d8c99d3_1", + "answer_5d8c99d3_2", + "answer_5d8c99d3_3" + ], + "ndcg_at_10": 0.4825, + "ndcg_at_5": 0.4307, + "question_id": "gpt4_7f6b06db", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "39c1bf6a", + "928890e4_3", + "ultrachat_31232", + "answer_5d8c99d3_2", + "631e4016", + "answer_5d8c99d3_3", + "d9ca4aca", + "4bf66c38_2", + "da42526b_1", + "sharegpt_3sKfamn_27" + ] + }, + { + "answer_session_ids": [ + "answer_c4df007f_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_6dc9b45b", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c4df007f_1", + "d8f48b0a", + "ultrachat_384387", + "9adc81d7_2", + "ultrachat_355155", + "ultrachat_35147", + "afa0c2e7_5", + "94ff80d3", + "sharegpt_jyynvvk_0", + "sharegpt_0SDNlJ3_63" + ] + }, + { + "answer_session_ids": [ + "answer_56521e65_1" + ], + "ndcg_at_10": 0.5, + "ndcg_at_5": 0.5, + "question_id": "gpt4_8279ba02", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "b909542d_1", + "85f19a30_1", + "answer_56521e65_1", + "b6018747_2", + "86c5d31d", + "e1c807f9", + "62b40a05_2", + "3a735bfa_1", + "ultrachat_78027", + "1a037873_2" + ] + }, + { + "answer_session_ids": [ + "answer_c862f65a_2", + "answer_c862f65a_3", + "answer_c862f65a_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_18c2b244", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c862f65a_3", + "answer_c862f65a_2", + "answer_c862f65a_1", + "dfbc46bd_1", + "7e2c6065", + "d9e1d5fd", + "aeb58d86_4", + "ultrachat_503659", + "84896d13", + "746ae854_2" + ] + }, + { + "answer_session_ids": [ + "answer_e9ad5914_1", + "answer_e9ad5914_2", + "answer_e9ad5914_3", + "answer_e9ad5914_4", + "answer_e9ad5914_5", + "answer_e9ad5914_6" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_a1b77f9c", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e9ad5914_3", + "answer_e9ad5914_4", + "answer_e9ad5914_6", + "answer_e9ad5914_1", + "answer_e9ad5914_5", + "answer_e9ad5914_2", + "199a5225_1", + "47ec1674_2", + "d52d478b", + "f4a4a28a" + ] + }, + { + "answer_session_ids": [ + "answer_447052a5_2", + "answer_447052a5_1" + ], + "ndcg_at_10": 0.8175, + "ndcg_at_5": 1.0, + "question_id": "gpt4_1916e0ea", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_447052a5_2", + "d3fe25ff", + "280352e9", + "4b72fe4c_1", + "fa1ce406_2", + "987b5d00", + "answer_447052a5_1", + "7a36e820_2", + "dffa3157", + "sharegpt_tPA8329_0" + ] + }, + { + "answer_session_ids": [ + "answer_4d5490f1_1", + "answer_4d5490f1_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_7a0daae1", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_4d5490f1_1", + "answer_4d5490f1_2", + "8c68df0d_1", + "014a5d36", + "819dfad7_2", + "2ff297a8_1", + "bf3aebdb_2", + "1c8832b4_1", + "sharegpt_ZWqMvoL_607", + "99883f38_3" + ] + }, + { + "answer_session_ids": [ + "answer_9b09d95b_1" + ], + "ndcg_at_10": 0.3333, + "ndcg_at_5": 0.0, + "question_id": "gpt4_468eb063", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 0.0, + "top_session_ids": [ + "e60a93ff_2", + "ultrachat_108278", + "sharegpt_Sf51OjE_8", + "d97db962_2", + "sharegpt_gfhUw40_17", + "sharegpt_1rIk8tS_0", + "answer_9b09d95b_1", + "ultrachat_91161", + "sharegpt_8M7HcM3_11", + "sharegpt_ipLglky_42" + ] + }, + { + "answer_session_ids": [ + "answer_7093d898_1", + "answer_7093d898_2", + "answer_7093d898_3", + "answer_7093d898_4", + "answer_7093d898_5", + "answer_7093d898_6" + ], + "ndcg_at_10": 0.6199, + "ndcg_at_5": 0.5706, + "question_id": "gpt4_7abb270c", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "5808284e", + "ultrachat_253954", + "answer_7093d898_2", + "answer_7093d898_1", + "39948bcf_4", + "answer_7093d898_6", + "087d2b0a_2", + "sharegpt_W6t3Ck0_0", + "answer_7093d898_5", + "ultrachat_41971" + ] + }, + { + "answer_session_ids": [ + "answer_16bd5ea5_1", + "answer_16bd5ea5_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_1e4a8aeb", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_16bd5ea5_1", + "answer_16bd5ea5_2", + "28621d6a_2", + "f7838a91_1", + "faad7d7a_1", + "bc5827b5", + "15564b61_2", + "fd18e0dd_1", + "ultrachat_413946", + "ultrachat_292933" + ] + }, + { + "answer_session_ids": [ + "answer_be07688f_1", + "answer_be07688f_2" + ], + "ndcg_at_10": 0.8503, + "ndcg_at_5": 0.8503, + "question_id": "gpt4_4fc4f797", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_be07688f_1", + "39566615_1", + "814d3b22", + "47c5482a", + "answer_be07688f_2", + "660cea1d", + "bdcee74e_3", + "7e02e59b_4", + "86298245_2", + "0c783e4a_1" + ] + }, + { + "answer_session_ids": [ + "answer_4bebc782_1", + "answer_4bebc782_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "4dfccbf7", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_4bebc782_1", + "answer_4bebc782_2", + "a5e69b52_1", + "8fa624b2_6", + "491ebc5d_2", + "36f8b380_1", + "ultrachat_327454", + "sharegpt_e1DTWDa_23", + "sharegpt_cXkL3cR_69", + "ultrachat_373047" + ] + }, + { + "answer_session_ids": [ + "answer_e831a29f_1", + "answer_e831a29f_2" + ], + "ndcg_at_10": 0.8503, + "ndcg_at_5": 0.8503, + "question_id": "gpt4_61e13b3c", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e831a29f_1", + "f3fcf4fd_2", + "a6f2320f_2", + "c0d099e6_2", + "answer_e831a29f_2", + "cc380119_3", + "8ca0085a_2", + "2b5c911e_3", + "8e10bf6b_1", + "b4923852" + ] + }, + { + "answer_session_ids": [ + "answer_e6c20e52_3", + "answer_e6c20e52_2", + "answer_e6c20e52_1" + ], + "ndcg_at_10": 0.5641, + "ndcg_at_5": 0.6309, + "question_id": "gpt4_45189cb4", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "5c16fe0b_1", + "answer_e6c20e52_3", + "ultrachat_307270", + "c9d35c00_2", + "dc3ee1d1_1", + "sharegpt_hgGAUvu_0", + "ebf5b3bc_2", + "6593cb8b_1", + "66ab6260", + "answer_e6c20e52_1" + ] + }, + { + "answer_session_ids": [ + "answer_c9d35c00_1", + "answer_c9d35c00_2" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "2ebe6c90", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c9d35c00_2", + "0f73eee7_2", + "answer_c9d35c00_1", + "sharegpt_jyMHY1J_31", + "8fa64936_1", + "9ccea2f0_2", + "c3c1f84a_1", + "4d1b9b0d_4", + "453d354e_1", + "ultrachat_68168" + ] + }, + { + "answer_session_ids": [ + "answer_8c64ce25_2", + "answer_8c64ce25_1", + "answer_8c64ce25_3" + ], + "ndcg_at_10": 0.3155, + "ndcg_at_5": 0.0, + "question_id": "gpt4_e061b84f", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 0.0, + "top_session_ids": [ + "0a6bf5e4_1", + "3275acf9", + "4cc9270f", + "bab3f409", + "73bc3176_2", + "539c2346_3", + "77974d02", + "answer_8c64ce25_3", + "ultrachat_129606", + "9345f7dc_4" + ] + }, + { + "answer_session_ids": [ + "answer_61d1be50_1", + "answer_61d1be50_2" + ], + "ndcg_at_10": 0.8772, + "ndcg_at_5": 0.8772, + "question_id": "370a8ff4", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_61d1be50_1", + "11c0b1c5", + "cfde748a", + "answer_61d1be50_2", + "sharegpt_UbMVpdp_93", + "2b4eaee0_1", + "sharegpt_gVwWKqg_0", + "sharegpt_JRQtEcd_115", + "31d5ad91_1", + "8c64ce25_1" + ] + }, + { + "answer_session_ids": [ + "answer_f999b05b_1", + "answer_f999b05b_4", + "answer_f999b05b_2", + "answer_f999b05b_5", + "answer_f999b05b_3" + ], + "ndcg_at_10": 0.8401, + "ndcg_at_5": 0.8855, + "question_id": "gpt4_d6585ce8", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_f999b05b_3", + "51c82c8e_2", + "answer_f999b05b_5", + "ultrachat_225491", + "answer_f999b05b_2", + "dc0802a9_1", + "c1e598cd_3", + "6ec9bed9", + "answer_f999b05b_4", + "answer_f999b05b_1" + ] + }, + { + "answer_session_ids": [ + "answer_f964cea3_1", + "answer_f964cea3_2" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "gpt4_4ef30696", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_f964cea3_2", + "80b3f093", + "answer_f964cea3_1", + "d9868305_2", + "sharegpt_ELYlA30_25", + "4b29957c_1", + "4067bede", + "1ccb08a0", + "750b437b_1", + "sharegpt_K8mXaIT_0" + ] + }, + { + "answer_session_ids": [ + "answer_e3aa84c4_2", + "answer_e3aa84c4_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_ec93e27f", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e3aa84c4_1", + "answer_e3aa84c4_2", + "5a994ed3_2", + "sharegpt_Sg0ctDe_14", + "6b63bdc5_2", + "5de5c1ed_1", + "85846900_2", + "ultrachat_357838", + "dd049be8", + "sharegpt_1ZQntAk_0" + ] + }, + { + "answer_session_ids": [ + "answer_88841f26_1", + "answer_88841f26_2" + ], + "ndcg_at_10": 0.5, + "ndcg_at_5": 0.5, + "question_id": "6e984301", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "b38766c1_2", + "535ce50e_2", + "answer_88841f26_1", + "b500d9be", + "55c23a23_2", + "ultrachat_402356", + "954c790c_2", + "answer_88841f26_2", + "2bc4ae7d", + "018426a5" + ] + }, + { + "answer_session_ids": [ + "answer_0dd54b7c_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "8077ef71", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_0dd54b7c_1", + "f7ffddbc", + "fd88019d", + "5365840b_1", + "ultrachat_468198", + "sharegpt_Wi7Op1u_39", + "sharegpt_11WxPMZ_0", + "d87a6ef8", + "a993bf57_2", + "sharegpt_hoQsVGz_51" + ] + }, + { + "answer_session_ids": [ + "answer_d8a1af6b_1", + "answer_d8a1af6b_2", + "answer_d8a1af6b_3", + "answer_d8a1af6b_4", + "answer_d8a1af6b_5" + ], + "ndcg_at_10": 0.6181, + "ndcg_at_5": 0.5013, + "question_id": "gpt4_f420262c", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "sharegpt_jFDrHZ3_0", + "sharegpt_B0riiep_0", + "116bae09", + "answer_d8a1af6b_4", + "answer_d8a1af6b_3", + "answer_d8a1af6b_5", + "answer_d8a1af6b_1", + "answer_d8a1af6b_2", + "ultrachat_446175", + "ultrachat_548517" + ] + }, + { + "answer_session_ids": [ + "answer_d97db962_1", + "answer_d97db962_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_8e165409", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_d97db962_2", + "answer_d97db962_1", + "sharegpt_JRQtEcd_115", + "4c9f6bf5_2", + "09599d45_1", + "ultrachat_572191", + "ultrachat_106695", + "d5b3cf54_4", + "f1fe6512_1", + "ultrachat_577133" + ] + }, + { + "answer_session_ids": [ + "answer_aed8cf17_1", + "answer_aed8cf17_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_74aed68e", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_aed8cf17_2", + "answer_aed8cf17_1", + "318048af_2", + "9523d17c", + "69a9211c_1", + "25b3e36c_2", + "470f29c0_1", + "50d66391_6", + "sharegpt_O3FeoHQ_153", + "sharegpt_DXFHJtN_0" + ] + }, + { + "answer_session_ids": [ + "answer_43ba3965_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "bcbe585f", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_43ba3965_1", + "a158b7e7", + "7fa8099b_1", + "a98ff421_4", + "44f94343_1", + "5dbe11b9", + "6bed7862", + "8166673a", + "896162a2", + "8953c8c8_2" + ] + }, + { + "answer_session_ids": [ + "answer_1e2369c9_1", + "answer_1e2369c9_2" + ], + "ndcg_at_10": 0.8772, + "ndcg_at_5": 0.8772, + "question_id": "gpt4_21adecb5", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_1e2369c9_1", + "ultrachat_555122", + "ultrachat_444790", + "answer_1e2369c9_2", + "e1f37e24", + "sharegpt_ipLglky_42", + "6c574737_2", + "128082f8_2", + "ultrachat_333888", + "19f2e2c5_2" + ] + }, + { + "answer_session_ids": [ + "answer_c18d480b_1" + ], + "ndcg_at_10": 0.4307, + "ndcg_at_5": 0.4307, + "question_id": "5e1b23de", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "1a5344e9_3", + "1380576d_1", + "ultrachat_153798", + "answer_c18d480b_1", + "809021de", + "c3f945f3_1", + "a13abca5_2", + "c28aed33", + "d8e33f5c_2", + "8278312e_1" + ] + }, + { + "answer_session_ids": [ + "answer_5850de18_2", + "answer_5850de18_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_98f46fc6", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_5850de18_2", + "answer_5850de18_1", + "sharegpt_rnL5VQO_0", + "6172cc72_3", + "8e3b479f_1", + "sharegpt_hPTUZia_0", + "ultrachat_238480", + "28897044", + "ultrachat_45511", + "32e292f2" + ] + }, + { + "answer_session_ids": [ + "answer_184c8f56_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_af6db32f", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_184c8f56_1", + "aee13015_6", + "ultrachat_552923", + "sharegpt_1jjEIai_173", + "e0b8c937_1", + "ea834a17_2", + "f26846e1_3", + "b7c70f28_3", + "8930493f_2", + "sharegpt_hDjMGr7_70" + ] + }, + { + "answer_session_ids": [ + "answer_0d4d0347_1", + "answer_0d4d0347_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "eac54adc", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_0d4d0347_2", + "answer_0d4d0347_1", + "b40f0a7a", + "ae051325_3", + "bf234051_2", + "32fc393b_2", + "ultrachat_387180", + "74133043", + "75a306dd_3", + "sharegpt_PgmAp0p_0" + ] + }, + { + "answer_session_ids": [ + "answer_ad8a76cb_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_7ddcf75f", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ad8a76cb_1", + "sharegpt_ZUxBndS_26", + "8c652eb0_2", + "sharegpt_mMdyu3t_0", + "79d122f0", + "7ba84ed3_2", + "9283c152", + "ultrachat_281183", + "07e6a3bd_1", + "sharegpt_5RciDTC_0" + ] + }, + { + "answer_session_ids": [ + "answer_f6d6e33f_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_a2d1d1f6", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_f6d6e33f_1", + "ultrachat_476519", + "f5cf8815_4", + "ultrachat_575969", + "894dd0c2_3", + "2c31d8be_3", + "sharegpt_gI1U1eK_4", + "90f7041a_2", + "a1bfb382_1", + "ultrachat_68856" + ] + }, + { + "answer_session_ids": [ + "answer_581ab834_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_85da3956", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_581ab834_1", + "fcc7fe77_2", + "ultrachat_373754", + "sharegpt_h1UXagU_1", + "14b000d9_1", + "sharegpt_i59HUGf_0", + "sharegpt_ewad5Sr_0", + "8da8c7e0", + "6c184bde_2", + "97f8a545_2" + ] + }, + { + "answer_session_ids": [ + "answer_550bb2d1_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_b0863698", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_550bb2d1_1", + "91e581ad_3", + "4a79d078_2", + "63bc9fa3", + "5b014f9b_1", + "554b054e_2", + "65f6f130_1", + "833750a1", + "bdcee74e_2", + "sharegpt_NITaMP6_0" + ] + }, + { + "answer_session_ids": [ + "answer_9793daa3_2", + "answer_9793daa3_1" + ], + "ndcg_at_10": 0.8316, + "ndcg_at_5": 1.0, + "question_id": "gpt4_68e94287", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_9793daa3_2", + "ultrachat_209526", + "60876def_1", + "50f136f2_3", + "74bcc8f4_1", + "answer_9793daa3_1", + "sharegpt_nh3eDLi_9", + "sharegpt_XzAEFL4_0", + "dadd6379_1", + "afc64a5c_1" + ] + }, + { + "answer_session_ids": [ + "answer_e28c1f0d_1", + "answer_e28c1f0d_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_e414231e", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e28c1f0d_1", + "answer_e28c1f0d_2", + "9f8fc173_3", + "d3065d85", + "1bab001f", + "1980bbfa_3", + "d4439279_2", + "a490e4b8_1", + "7e3fa056_2", + "ultrachat_524205" + ] + }, + { + "answer_session_ids": [ + "answer_cf021b36_1", + "answer_cf021b36_2", + "answer_cf021b36_3" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_7ca326fa", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_cf021b36_2", + "answer_cf021b36_3", + "answer_cf021b36_1", + "8d0ca228", + "b4025421", + "5429e768_1", + "sharegpt_7SCdbX2_0", + "1301fbd1", + "sharegpt_CYVtC0w_0", + "604de0a3_2" + ] + }, + { + "answer_session_ids": [ + "answer_e22d6aef_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_7bc6cf22", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e22d6aef_1", + "e6cc6157_1", + "09d021e7_1", + "ultrachat_215356", + "c82c18dd_1", + "ultrachat_330542", + "ultrachat_465755", + "3e43bf49_1", + "20a1ca39", + "eca3b71c_2" + ] + }, + { + "answer_session_ids": [ + "answer_c9d35c00_1", + "answer_c9d35c00_2" + ], + "ndcg_at_10": 0.3562, + "ndcg_at_5": 0.0, + "question_id": "2ebe6c92", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 0.0, + "top_session_ids": [ + "9d4312f6_3", + "fb303dd2_2", + "ca3a4e4f_1", + "54c5a89c_1", + "16756728_1", + "answer_c9d35c00_2", + "84048eba_5", + "sharegpt_9dKNTQo_0", + "b30a47be_2", + "ultrachat_575902" + ] + }, + { + "answer_session_ids": [ + "answer_8c64ce26_2", + "answer_8c64ce26_1", + "answer_8c64ce26_3" + ], + "ndcg_at_10": 0.3869, + "ndcg_at_5": 0.3869, + "question_id": "gpt4_e061b84g", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "ae15e8b6_1", + "9dac9e37_1", + "68705467_1", + "3d6a2b24_1", + "answer_8c64ce26_3", + "51221cac", + "ce584ba0", + "sharegpt_rB6IP4G_0", + "ultrachat_231231", + "ultrachat_391029" + ] + }, + { + "answer_session_ids": [ + "answer_0b4a8adc_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "71017277", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_0b4a8adc_1", + "85a1be56_2", + "976ca0d9", + "f8de4e92_4", + "5854eebc_2", + "e64f9553_1", + "40a1e681_1", + "ultrachat_557308", + "ultrachat_542251", + "e5b86922_1" + ] + }, + { + "answer_session_ids": [ + "answer_4bfcc251_4", + "answer_4bfcc251_3", + "answer_4bfcc251_2", + "answer_4bfcc251_1" + ], + "ndcg_at_10": 0.8667, + "ndcg_at_5": 0.8855, + "question_id": "b46e15ee", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_4bfcc251_1", + "918c4788_1", + "answer_4bfcc251_2", + "69664c72", + "answer_4bfcc251_3", + "c92239bc", + "answer_4bfcc251_4", + "495cf518_2", + "sharegpt_RhSlNyz_0", + "aa31c397_2" + ] + }, + { + "answer_session_ids": [ + "answer_f999b05c_1", + "answer_f999b05c_4", + "answer_f999b05c_2", + "answer_f999b05c_5", + "answer_f999b05c_3" + ], + "ndcg_at_10": 0.6154, + "ndcg_at_5": 0.5438, + "question_id": "gpt4_d6585ce9", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "a554ed79_4", + "58dd3f35", + "answer_f999b05c_3", + "87f5cb44", + "answer_f999b05c_2", + "answer_f999b05c_4", + "answer_f999b05c_5", + "a7aed4cc_2", + "17d91635_2", + "d8e33f5c_abs_1" + ] + }, + { + "answer_session_ids": [ + "answer_16bd5ea6_1", + "answer_16bd5ea6_2" + ], + "ndcg_at_10": 0.3333, + "ndcg_at_5": 0.0, + "question_id": "gpt4_1e4a8aec", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 0.0, + "top_session_ids": [ + "9f1f3bf4", + "b34a52aa_3", + "a8086137", + "4c8455cb_2", + "b1265fd7_1", + "sharegpt_9JZCfee_0", + "answer_16bd5ea6_2", + "e675c69a_1", + "1c662b7b_1", + "e41b78c7_2" + ] + }, + { + "answer_session_ids": [ + "answer_d8a1af6c_1", + "answer_d8a1af6c_2", + "answer_d8a1af6c_5", + "answer_d8a1af6c_4", + "answer_d8a1af6c_3" + ], + "ndcg_at_10": 0.967, + "ndcg_at_5": 0.9829, + "question_id": "gpt4_f420262d", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_d8a1af6c_5", + "answer_d8a1af6c_2", + "answer_d8a1af6c_1", + "27639dd8_3", + "answer_d8a1af6c_3", + "sharegpt_UUFCRmF_13", + "answer_d8a1af6c_4", + "09e6f061", + "e1023f28", + "00c7b769_1" + ] + }, + { + "answer_session_ids": [ + "answer_d00ba6d1_1", + "answer_d00ba6d1_2" + ], + "ndcg_at_10": 0.4486, + "ndcg_at_5": 0.4307, + "question_id": "gpt4_59149c78", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "sharegpt_GI6737T_2", + "765ce8a7_2", + "23754665", + "answer_d00ba6d1_2", + "a8ac3d1d_1", + "84762061_1", + "sharegpt_YENXhFv_0", + "ultrachat_131385", + "answer_d00ba6d1_1", + "b2341a22" + ] + }, + { + "answer_session_ids": [ + "answer_e28c1f0e_1", + "answer_e28c1f0e_2" + ], + "ndcg_at_10": 0.7904, + "ndcg_at_5": 1.0, + "question_id": "gpt4_e414231f", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e28c1f0e_1", + "ultrachat_150948", + "07a26dfa_5", + "deb5bb60", + "135f761b_1", + "78ef59d0_1", + "sharegpt_K8mXaIT_0", + "752c438c_3", + "fe82a033_2", + "answer_e28c1f0e_2" + ] + }, + { + "answer_session_ids": [ + "answer_add9b013_2", + "answer_add9b013_1" + ], + "ndcg_at_10": 0.0, + "ndcg_at_5": 0.0, + "question_id": "gpt4_4929293b", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 0.0, + "r_at_5": 0.0, + "top_session_ids": [ + "sharegpt_KFhIUCO_0", + "b4ed7223_3", + "cae65795_1", + "ultrachat_511609", + "bda611f6_3", + "ea8c38f8", + "sharegpt_MB6e5CL_23", + "sharegpt_15lfOiQ_39", + "78bab597_2", + "ultrachat_156176" + ] + }, + { + "answer_session_ids": [ + "answer_9b09d95b_1" + ], + "ndcg_at_10": 0.4307, + "ndcg_at_5": 0.4307, + "question_id": "gpt4_468eb064", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "a2f0054f", + "db1aefb6_3", + "4d4df0e0_1", + "answer_9b09d95b_1", + "1e5bd28d_2", + "sharegpt_0PLBHdX_15", + "ultrachat_109577", + "sharegpt_P54kbvt_0", + "ultrachat_337251", + "cc3c5fa9_1" + ] + }, + { + "answer_session_ids": [ + "answer_ff201787_1", + "answer_ff201787_2" + ], + "ndcg_at_10": 0.6509, + "ndcg_at_5": 0.6509, + "question_id": "gpt4_fa19884d", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "ccc87da9_1", + "answer_ff201787_1", + "5f9dd782", + "answer_ff201787_2", + "eb4d1c82_1", + "55da3cbf_2", + "f7a61595_3", + "f910dc31_4", + "ultrachat_525919", + "ultrachat_483034" + ] + }, + { + "answer_session_ids": [ + "answer_dba89488_2", + "answer_dba89488_1" + ], + "ndcg_at_10": 0.3333, + "ndcg_at_5": 0.0, + "question_id": "9a707b82", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 0.0, + "top_session_ids": [ + "8e78fa70_1", + "fab41c07", + "79d122f0", + "sharegpt_bUaSM2v_0", + "990f3ef9_2", + "f8e9d4fa", + "answer_dba89488_1", + "7a4d00b3_2", + "ultrachat_466884", + "09e97c6f_2" + ] + }, + { + "answer_session_ids": [ + "answer_0d4d0348_1", + "answer_0d4d0348_2" + ], + "ndcg_at_10": 0.301, + "ndcg_at_5": 0.0, + "question_id": "eac54add", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 0.0, + "top_session_ids": [ + "2ea6fda4_1", + "7bb01253", + "7966888b_2", + "577da76d_2", + "2946bf12", + "c3023a45_5", + "773aebbd_3", + "369695b4_2", + "answer_0d4d0348_2", + "28269633_2" + ] + }, + { + "answer_session_ids": [ + "answer_4bebc783_1", + "answer_4bebc783_2" + ], + "ndcg_at_10": 0.7977, + "ndcg_at_5": 1.0, + "question_id": "4dfccbf8", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_4bebc783_1", + "sharegpt_3D3oQC0_213", + "a63ad8e3_3", + "1c177942_4", + "b4f63a70_3", + "7db28e68_1", + "1bc87711_1", + "edd89480_1", + "answer_4bebc783_2", + "ultrachat_444490" + ] + }, + { + "answer_session_ids": [ + "answer_f4ea84fc_3", + "answer_f4ea84fc_2", + "answer_f4ea84fc_1" + ], + "ndcg_at_10": 0.9469, + "ndcg_at_5": 0.9469, + "question_id": "0bc8ad93", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_f4ea84fc_1", + "answer_f4ea84fc_2", + "97338ddd_1", + "2c185a2b_1", + "answer_f4ea84fc_3", + "ultrachat_260376", + "dd25eeb5_2", + "7285299a_2", + "d59d1a8d_1", + "0d8f8cc0_2" + ] + }, + { + "answer_session_ids": [ + "answer_88841f27_1", + "answer_88841f27_2" + ], + "ndcg_at_10": 0.3562, + "ndcg_at_5": 0.0, + "question_id": "6e984302", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 0.0, + "top_session_ids": [ + "7285299a_8", + "de1f4aec_1", + "3535dbf0_4", + "7cd7c296_1", + "c9dc9158_1", + "answer_88841f27_1", + "6efebca9", + "96c743d0_2", + "c03fc56c_1", + "sharegpt_4Wjx3pH_0" + ] + }, + { + "answer_session_ids": [ + "answer_56521e66_1" + ], + "ndcg_at_10": 0.0, + "ndcg_at_5": 0.0, + "question_id": "gpt4_8279ba03", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 0.0, + "r_at_5": 0.0, + "top_session_ids": [ + "b357fb8b_2", + "ultrachat_526748", + "2ef55f49_3", + "864a563d_5", + "49b78a55_1", + "ba9f938b_1", + "518d26d3_4", + "bb107057_2", + "570fe405", + "50d66391_4" + ] + }, + { + "answer_session_ids": [ + "answer_a17423e8_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_b5700ca0", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_a17423e8_1", + "93395e5f_3", + "534be93d_1", + "109e07e8_2", + "sharegpt_FpTfRvR_0", + "ultrachat_23298", + "b403d6f0_4", + "e789afdb_2", + "ultrachat_101752", + "ultrachat_160178" + ] + }, + { + "answer_session_ids": [ + "answer_9793daa4_2", + "answer_9793daa4_1" + ], + "ndcg_at_10": 0.4307, + "ndcg_at_5": 0.4307, + "question_id": "gpt4_68e94288", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "4c49e37f", + "128f4e4d_2", + "9d2fdc71", + "answer_9793daa4_1", + "c99dcd81", + "0a7c4f1c_1", + "767b1b89", + "cff08ec1_1", + "ultrachat_396489", + "d2606fe9" + ] + }, + { + "answer_session_ids": [ + "answer_4be1b6b4_1", + "answer_4be1b6b4_2", + "answer_4be1b6b4_3" + ], + "ndcg_at_10": 0.5391, + "ndcg_at_5": 0.5, + "question_id": "gpt4_2655b836", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "d8a1af6b_4", + "9e05d9a0", + "answer_4be1b6b4_2", + "ultrachat_268202", + "e3e66b50", + "0f2434e8", + "answer_4be1b6b4_3", + "answer_4be1b6b4_1", + "b876911c", + "ea8cc87b" + ] + }, + { + "answer_session_ids": [ + "answer_1c6b85ea_2", + "answer_1c6b85ea_1" + ], + "ndcg_at_10": 0.6934, + "ndcg_at_5": 0.6934, + "question_id": "gpt4_2487a7cb", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "4f5880c6_6", + "answer_1c6b85ea_1", + "answer_1c6b85ea_2", + "sharegpt_QN26oUg_0", + "67a8ef5f", + "f87fea58_4", + "sharegpt_CHaU4ax_10", + "sharegpt_XNcyasy_17", + "4c1982d8_2", + "sharegpt_EJC4YOh_0" + ] + }, + { + "answer_session_ids": [ + "answer_b535969f_1", + "answer_b535969f_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_76048e76", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_b535969f_1", + "answer_b535969f_2", + "6a78e959_2", + "33a39aa7_2", + "88e01130_1", + "33630c9e_2", + "ultrachat_273271", + "sharegpt_okVqBzp_13", + "sharegpt_inuIr9J_119", + "sharegpt_vx14orQ_11" + ] + }, + { + "answer_session_ids": [ + "answer_5328c3c2_2", + "answer_5328c3c2_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_2312f94c", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_5328c3c2_2", + "answer_5328c3c2_1", + "a764b677_2", + "869fe9ba_2", + "sharegpt_jE25Ibo_0", + "3bbf4800_1", + "e831a29f_2", + "a1c1739c_3", + "ultrachat_44021", + "sharegpt_XYtuANl_9" + ] + }, + { + "answer_session_ids": [ + "answer_e936197f_1", + "answer_e936197f_2" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "0bb5a684", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e936197f_1", + "1a14d6cb_1", + "answer_e936197f_2", + "3c8e7c0e_3", + "ultrachat_62919", + "565929a2_2", + "b10d0907_2", + "826d51da_3", + "d794dec9_2", + "sharegpt_d3UV0eu_0" + ] + }, + { + "answer_session_ids": [ + "answer_6ea1541e_2", + "answer_6ea1541e_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "08f4fc43", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_6ea1541e_1", + "answer_6ea1541e_2", + "f3bd00ec_2", + "ultrachat_292416", + "sharegpt_JJu53iW_0", + "50303fd0_1", + "9216c4d3_4", + "sharegpt_zqQa6sO_0", + "sharegpt_wXqtiUe_0", + "cdbd6862_1" + ] + }, + { + "answer_session_ids": [ + "answer_d39b7977_1", + "answer_d39b7977_2" + ], + "ndcg_at_10": 0.8772, + "ndcg_at_5": 0.8772, + "question_id": "2c63a862", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_d39b7977_2", + "33baaaab_1", + "93a5cfe6", + "answer_d39b7977_1", + "1854abc7_2", + "ad175dc4_2", + "0dde26d4_3", + "ultrachat_179062", + "ultrachat_149914", + "918c4788_2" + ] + }, + { + "answer_session_ids": [ + "answer_7a4a93f1_2", + "answer_7a4a93f1_1" + ], + "ndcg_at_10": 0.8772, + "ndcg_at_5": 0.8772, + "question_id": "gpt4_385a5000", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_7a4a93f1_2", + "8062f4ab_2", + "80aefc49_1", + "answer_7a4a93f1_1", + "29d27eaa_2", + "sharegpt_hcaZN94_507", + "28f74b09_1", + "ultrachat_480527", + "ultrachat_528011", + "ultrachat_286149" + ] + }, + { + "answer_session_ids": [ + "answer_1cc3cd0c_1", + "answer_1cc3cd0c_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "2a1811e2", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_1cc3cd0c_2", + "answer_1cc3cd0c_1", + "b0e040c1", + "901a6763_2", + "ecfd2047_1", + "e9fb10e7_1", + "ultrachat_187883", + "sharegpt_c9R1uSD_0", + "89e05490", + "f5a4d792_1" + ] + }, + { + "answer_session_ids": [ + "answer_b3763b6b_1", + "answer_b3763b6b_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "bbf86515", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_b3763b6b_1", + "answer_b3763b6b_2", + "df003c93_2", + "592cb8d2", + "75b98d2a_1", + "0c260a71_2", + "e9e19917_1", + "33d0e4b3_2", + "sharegpt_d1Rh99l_63", + "sharegpt_G8j6c8J_38" + ] + }, + { + "answer_session_ids": [ + "answer_099c1b6c_5", + "answer_099c1b6c_2", + "answer_099c1b6c_4", + "answer_099c1b6c_3", + "answer_099c1b6c_1" + ], + "ndcg_at_10": 0.9896, + "ndcg_at_5": 1.0, + "question_id": "gpt4_5dcc0aab", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_099c1b6c_3", + "answer_099c1b6c_1", + "answer_099c1b6c_5", + "answer_099c1b6c_2", + "353d3c6d_1", + "answer_099c1b6c_4", + "3a192d16_1", + "3891baea_3", + "b10c041b_1", + "b31f923a_2" + ] + }, + { + "answer_session_ids": [ + "answer_c4e5d969_2", + "answer_c4e5d969_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_0b2f1d21", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c4e5d969_2", + "answer_c4e5d969_1", + "1cafd864_3", + "f31a0422_1", + "sharegpt_rhWNt54_19", + "ultrachat_455430", + "ultrachat_49268", + "sharegpt_4pDZgCL_14", + "5260c678_2", + "ultrachat_1415" + ] + }, + { + "answer_session_ids": [ + "answer_932ea3bf_2", + "answer_932ea3bf_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "f0853d11", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_932ea3bf_2", + "answer_932ea3bf_1", + "bd8708e2_2", + "sharegpt_9upOSlE_27", + "10c0752c", + "b16a8656_2", + "6e436d54", + "ultrachat_325202", + "ultrachat_114660", + "4c967baa_2" + ] + }, + { + "answer_session_ids": [ + "answer_d50a8a33_1", + "answer_d50a8a33_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_6ed717ea", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_d50a8a33_2", + "answer_d50a8a33_1", + "637bc32a_2", + "15ed03f0_1", + "7e7ecab6", + "d4b6ec0b", + "sharegpt_HQMYIkt_0", + "sharegpt_PuRfIep_7", + "8e657ff7_1", + "e0585cb5_1" + ] + }, + { + "answer_session_ids": [ + "answer_b3070ec4_2", + "answer_b3070ec4_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_70e84552", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_b3070ec4_1", + "answer_b3070ec4_2", + "ultrachat_352927", + "2b9b6224_2", + "30491f6f_1", + "45b155ec_2", + "3f787a78_1", + "ultrachat_573894", + "6ebd9e18_1", + "d5527e62" + ] + }, + { + "answer_session_ids": [ + "answer_4ffa04a2_1", + "answer_4ffa04a2_6", + "answer_4ffa04a2_4", + "answer_4ffa04a2_3", + "answer_4ffa04a2_5", + "answer_4ffa04a2_2" + ], + "ndcg_at_10": 0.9797, + "ndcg_at_5": 1.0, + "question_id": "a3838d2b", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_4ffa04a2_1", + "answer_4ffa04a2_6", + "answer_4ffa04a2_4", + "answer_4ffa04a2_2", + "answer_4ffa04a2_5", + "06317e04", + "8cf0d2a9", + "10305ca9_2", + "638e7094_1", + "answer_4ffa04a2_3" + ] + }, + { + "answer_session_ids": [ + "answer_e5131a1b_2", + "answer_e5131a1b_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_93159ced", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e5131a1b_2", + "ultrachat_387032", + "460be54a", + "ultrachat_365033", + "sharegpt_A83arI9_383", + "sharegpt_9srz3L6_0", + "ultrachat_162445", + "ultrachat_475376", + "ultrachat_304047", + "ultrachat_244924" + ] + }, + { + "answer_session_ids": [ + "answer_3e11e0ae_1", + "answer_3e11e0ae_2" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "gpt4_2d58bcd6", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_3e11e0ae_1", + "0bd0a01c", + "answer_3e11e0ae_2", + "be32ad25_1", + "1bf8b212", + "sharegpt_4tLDGmp_0", + "sharegpt_Xl3kKYf_15", + "3c53154d_1", + "1b205d70_1", + "47e1f8dd_1" + ] + }, + { + "answer_session_ids": [ + "answer_30dfe889_1", + "answer_30dfe889_2" + ], + "ndcg_at_10": 0.6934, + "ndcg_at_5": 0.6934, + "question_id": "gpt4_65aabe59", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "6a7e3aec", + "answer_30dfe889_2", + "answer_30dfe889_1", + "77b8fcd9_1", + "10771408_4", + "31cde680", + "05a37787", + "ultrachat_190669", + "78bcce6a", + "ultrachat_169466" + ] + }, + { + "answer_session_ids": [ + "answer_ab603dd5_1", + "answer_ab603dd5_2" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "982b5123", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ab603dd5_1", + "07ba9acd_2", + "answer_ab603dd5_2", + "e36b0031", + "1bfd5a8b_3", + "359a2a0d", + "f89f184e_1", + "bf8ab267", + "91168bd2_1", + "a200b713_4" + ] + }, + { + "answer_session_ids": [ + "answer_5e3bb940_3", + "answer_5e3bb940_2", + "answer_5e3bb940_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "b9cfe692", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_5e3bb940_1", + "answer_5e3bb940_2", + "answer_5e3bb940_3", + "ed88aded_2", + "sharegpt_oXgiN7q_53", + "sharegpt_JRQtEcd_115", + "2714885f_3", + "da650f3b", + "91d847c1_6", + "ultrachat_265626" + ] + }, + { + "answer_session_ids": [ + "answer_0a00c163_1", + "answer_0a00c163_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_4edbafa2", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_0a00c163_2", + "answer_0a00c163_1", + "22be78fc_2", + "18c0af9c", + "4a006b77", + "725f95e1", + "sharegpt_TQNHiIB_12", + "sharegpt_jZOf9E5_21", + "sharegpt_6MPnc5F_9", + "7533d546_5" + ] + }, + { + "answer_session_ids": [ + "answer_70dc7d08_2", + "answer_70dc7d08_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "c8090214", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_70dc7d08_1", + "answer_70dc7d08_2", + "a4c2a6f7_2", + "sharegpt_xOzzaof_0", + "005ec7cf_1", + "d6c6dadf", + "162ad3bf_1", + "c21c9d97", + "sharegpt_BQ3jrV3_0", + "sharegpt_3cVjZ2b_8" + ] + }, + { + "answer_session_ids": [ + "answer_fb793c87_1", + "answer_fb793c87_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_483dd43c", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_fb793c87_1", + "answer_fb793c87_2", + "3fd56ebf", + "52161c9e_1", + "19c24c11", + "29695e1c_3", + "ultrachat_48790", + "sharegpt_Sg0ctDe_0", + "4243ce01_1", + "ultrachat_374858" + ] + }, + { + "answer_session_ids": [ + "answer_cf425855_1", + "answer_cf425855_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e4e14d04", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_cf425855_2", + "answer_cf425855_1", + "a92165a5", + "ffc627f2", + "5944b36a_1", + "ultrachat_542420", + "f8de4e92_2", + "1c1f5ccc_3", + "ultrachat_325291", + "ultrachat_200098" + ] + }, + { + "answer_session_ids": [ + "answer_cdba3d9f_1", + "answer_cdba3d9f_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "c9f37c46", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_cdba3d9f_1", + "answer_cdba3d9f_2", + "b38766c1_3", + "0cf86cbe_1", + "4ffa04a2_1", + "2ba1710e_1", + "02df1225_1", + "7c9b09ac", + "211bb1c2_1", + "9d4312f6_3" + ] + }, + { + "answer_session_ids": [ + "answer_9af4e346_1", + "answer_9af4e346_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_2c50253f", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_9af4e346_2", + "answer_9af4e346_1", + "ultrachat_120586", + "sharegpt_a6pZtrg_0", + "0138fdec", + "dd81b163_2", + "b6f35f3a_1", + "65bc93cf_2", + "sharegpt_ueZYbCA_15", + "ultrachat_267493" + ] + }, + { + "answer_session_ids": [ + "answer_5e3eeb12_2", + "answer_5e3eeb12_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "dcfa8644", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_5e3eeb12_2", + "answer_5e3eeb12_1", + "b2243528_1", + "0d129c99", + "ultrachat_431004", + "7de51ffe_1", + "sharegpt_NCuLVp2_0", + "b906870f_1", + "4eedac88_1", + "sharegpt_hUAh1Tu_0" + ] + }, + { + "answer_session_ids": [ + "answer_b9d9150e_2", + "answer_b9d9150e_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_b4a80587", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_b9d9150e_2", + "answer_b9d9150e_1", + "ultrachat_389307", + "3cb41ab8_1", + "sharegpt_4tqCoYJ_115", + "9071ba70_2", + "55bcfb77", + "sharegpt_IT52MRw_37", + "sharegpt_HxOhjmq_29", + "9b4f5c73" + ] + }, + { + "answer_session_ids": [ + "answer_8a42fedf_1", + "answer_8a42fedf_3", + "answer_8a42fedf_2" + ], + "ndcg_at_10": 0.9218, + "ndcg_at_5": 1.0, + "question_id": "gpt4_9a159967", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_8a42fedf_2", + "answer_8a42fedf_3", + "ultrachat_534927", + "253742b4_3", + "300cfc4a", + "4ea94f85", + "answer_8a42fedf_1", + "7169e342_2", + "sharegpt_MJrfKtS_0", + "75671e3f_1" + ] + }, + { + "answer_session_ids": [ + "answer_be73098b_2", + "answer_be73098b_1" + ], + "ndcg_at_10": 0.8316, + "ndcg_at_5": 1.0, + "question_id": "cc6d1ec1", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_be73098b_2", + "22db3cc3_4", + "aabf201b", + "9a297400_4", + "ultrachat_316268", + "answer_be73098b_1", + "e549b6b2_2", + "33baaaab_3", + "ultrachat_436945", + "sharegpt_iNlPmuC_9" + ] + }, + { + "answer_session_ids": [ + "answer_72d9aa58_1", + "answer_72d9aa58_2" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "gpt4_8c8961ae", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_72d9aa58_1", + "3b22d17b", + "answer_72d9aa58_2", + "1d14730c_1", + "1cde7ee4_2", + "02f9bedb_2", + "b77d0b58", + "37f0ce6b_1", + "ultrachat_265707", + "d7281662_2" + ] + }, + { + "answer_session_ids": [ + "answer_da704e79_2", + "answer_da704e79_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_d9af6064", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_da704e79_1", + "answer_da704e79_2", + "sharegpt_w9YUs4c_0", + "0d9a6f01", + "sharegpt_VbIxRWJ_33", + "98c83fe4", + "ultrachat_519452", + "sharegpt_brc2wJS_109", + "6cff6108", + "3e3b6d77_2" + ] + }, + { + "answer_session_ids": [ + "answer_6a78e959_2", + "answer_6a78e959_1" + ], + "ndcg_at_10": 0.8772, + "ndcg_at_5": 0.8772, + "question_id": "gpt4_7de946e7", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_6a78e959_2", + "sharegpt_6Yfj9A3_5", + "6155addd_2", + "answer_6a78e959_1", + "caa00337_2", + "sharegpt_mjN4DdI_0", + "f1cf6f00_1", + "c4d370d3_1", + "sharegpt_BacsrkR_0", + "ultrachat_560313" + ] + }, + { + "answer_session_ids": [ + "answer_991d55e5_1", + "answer_991d55e5_2" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "d01c6aa8", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_991d55e5_2", + "2adf224b_2", + "answer_991d55e5_1", + "64460e9d_1", + "f2fa97f5_1", + "fc9a62c0_1", + "7d1f5395", + "ultrachat_333097", + "51fe163e_2", + "sharegpt_YLhXcw8_79" + ] + }, + { + "answer_session_ids": [ + "answer_54f0d6f9_1", + "answer_54f0d6f9_2" + ], + "ndcg_at_10": 0.6509, + "ndcg_at_5": 0.6509, + "question_id": "993da5e2", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "b16a8656_2", + "answer_54f0d6f9_1", + "bcf96d28_3", + "answer_54f0d6f9_2", + "8953c8c8_3", + "a6c0f032_3", + "c69a28f0", + "ultrachat_266485", + "sharegpt_wpbTVG0_0", + "sharegpt_v11Tpg9_0" + ] + }, + { + "answer_session_ids": [ + "answer_016f6bd4_2", + "answer_016f6bd4_1" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "a3045048", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_016f6bd4_1", + "536a334d", + "answer_016f6bd4_2", + "aaf71ce2_2", + "d837b5fa", + "5cb58cc6", + "sharegpt_asoSnhq_0", + "d12d5a98_1", + "ultrachat_406386", + "sharegpt_LC9Pzkr_0" + ] + }, + { + "answer_session_ids": [ + "answer_8464304d_1", + "answer_8464304d_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_d31cdae3", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_8464304d_2", + "answer_8464304d_1", + "41743aae_2", + "ultrachat_74113", + "de695a4e", + "55033f6f", + "809cabca_2", + "ultrachat_324099", + "5ace29d7_1", + "ultrachat_169591" + ] + }, + { + "answer_session_ids": [ + "answer_aa930b56_1", + "answer_aa930b56_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_cd90e484", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_aa930b56_2", + "answer_aa930b56_1", + "9aad36bb_1", + "08c306e2", + "ultrachat_347068", + "4d452cdb_3", + "835e1b84_2", + "4bfcc251_1", + "6043dcf6_1", + "018eac29" + ] + }, + { + "answer_session_ids": [ + "answer_e60a93ff_1", + "answer_e60a93ff_2" + ], + "ndcg_at_10": 0.6509, + "ndcg_at_5": 0.6509, + "question_id": "gpt4_88806d6e", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "ultrachat_562852", + "answer_e60a93ff_1", + "sharegpt_bJh9LPd_0", + "answer_e60a93ff_2", + "20b1b65f_1", + "sharegpt_XYtuANl_11", + "ultrachat_258611", + "c11fb634_1", + "3c11dc79", + "24bfc009_5" + ] + }, + { + "answer_session_ids": [ + "answer_5fcca8bc_2", + "answer_5fcca8bc_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_4cd9eba1", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_5fcca8bc_2", + "answer_5fcca8bc_1", + "f54d04cc", + "456860e7_2", + "6b9b2b1e_1", + "d61f3bb5", + "8242fa3f_1", + "2ef53f61_4", + "0e193841_6", + "sharegpt_2nvkR60_0" + ] + }, + { + "answer_session_ids": [ + "answer_544fe66c_2", + "answer_544fe66c_1", + "answer_544fe66c_3" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_93f6379c", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_544fe66c_2", + "answer_544fe66c_3", + "1b9b2960_3", + "b8a688f6", + "sharegpt_cfESFSo_0", + "e06c6d99_5", + "c07b8034", + "ultrachat_443550", + "d3a34811", + "82e27408_1" + ] + }, + { + "answer_session_ids": [ + "answer_436d4309_2", + "answer_436d4309_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "b29f3365", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_436d4309_2", + "answer_436d4309_1", + "6b7605d1_1", + "6a2f9452_1", + "06ee3665", + "ultrachat_243663", + "95c9666f_5", + "sharegpt_K0i5Zon_0", + "ultrachat_58020", + "4ccc2061" + ] + }, + { + "answer_session_ids": [ + "answer_7a36e820_2", + "answer_7a36e820_3", + "answer_7a36e820_1" + ], + "ndcg_at_10": 0.6871, + "ndcg_at_5": 0.6934, + "question_id": "gpt4_2f56ae70", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "3aac691d_2", + "answer_7a36e820_2", + "answer_7a36e820_1", + "f8e8d445_4", + "647f5851", + "ultrachat_159439", + "answer_7a36e820_3", + "83cd701b_3", + "e61d966a_1", + "sharegpt_KzDwaf1_165" + ] + }, + { + "answer_session_ids": [ + "answer_aaf71ce2_3", + "answer_aaf71ce2_2", + "answer_aaf71ce2_1" + ], + "ndcg_at_10": 0.8772, + "ndcg_at_5": 0.8772, + "question_id": "6613b389", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_aaf71ce2_2", + "e3d8da67", + "5eef2132", + "answer_aaf71ce2_3", + "55bc2e1e", + "2780323d_3", + "f1cc1ddb_1", + "1a65880d", + "2366adbc", + "86c8f348_1" + ] + }, + { + "answer_session_ids": [ + "answer_5a78688d_1", + "answer_5a78688d_2" + ], + "ndcg_at_10": 0.8772, + "ndcg_at_5": 0.8772, + "question_id": "gpt4_78cf46a3", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_5a78688d_1", + "b89a3ec3", + "sharegpt_rx39ipj_0", + "answer_5a78688d_2", + "sharegpt_89M8Aiz_0", + "52b19cba_1", + "ultrachat_305042", + "e8e65a1f", + "a8b4bfb3_1", + "27099532_2" + ] + }, + { + "answer_session_ids": [ + "answer_a68db5db_2", + "answer_a68db5db_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_0a05b494", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_a68db5db_2", + "answer_a68db5db_1", + "0abe3e51", + "sharegpt_24ey7mz_0", + "ultrachat_135150", + "48b68664", + "30677a32", + "ultrachat_273213", + "18687689", + "ultrachat_105014" + ] + }, + { + "answer_session_ids": [ + "answer_faad7d7a_2", + "answer_faad7d7a_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_1a1dc16d", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_faad7d7a_2", + "answer_faad7d7a_1", + "87f1f950_2", + "c12d5181_1", + "14940349_2", + "ultrachat_428346", + "ultrachat_63236", + "ultrachat_92302", + "8bd3814a_2", + "661ee6d9_1" + ] + }, + { + "answer_session_ids": [ + "answer_11a8f823_2", + "answer_11a8f823_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_2f584639", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_11a8f823_2", + "answer_11a8f823_1", + "55ca8b20_3", + "ca555919_3", + "sharegpt_bq9G6bT_2", + "4b3a0ee6_1", + "c6eb17f0_1", + "ultrachat_101401", + "68131948", + "d441f4c5" + ] + }, + { + "answer_session_ids": [ + "answer_53582e7e_2", + "answer_53582e7e_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_213fd887", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_53582e7e_2", + "answer_53582e7e_1", + "9bdbfc62_2", + "10771408_1", + "ultrachat_165091", + "sharegpt_EXJAjmw_7", + "8aa17385", + "dc1f9713_1", + "368276ab_3", + "d10f0307_3" + ] + }, + { + "answer_session_ids": [ + "answer_b10f3828_1", + "answer_b10f3828_2" + ], + "ndcg_at_10": 0.6509, + "ndcg_at_5": 0.6509, + "question_id": "gpt4_5438fa52", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "ultrachat_69599", + "answer_b10f3828_2", + "a8e24e22", + "answer_b10f3828_1", + "290fe09a", + "sharegpt_LVUZee8_35", + "b4f07fef_2", + "ab8dce45", + "d68bab3c_1", + "528f481d_6" + ] + }, + { + "answer_session_ids": [ + "answer_d8e33f5c_1", + "answer_d8e33f5c_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_c27434e8", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_d8e33f5c_2", + "answer_d8e33f5c_1", + "sharegpt_9XlbWvL_0", + "16933961", + "6e2cca63_3", + "4d578deb", + "fd56c5bd_4", + "5d1c7bf7", + "ultrachat_212645", + "sharegpt_DVswHqE_0" + ] + }, + { + "answer_session_ids": [ + "answer_65600ff6_2", + "answer_65600ff6_1" + ], + "ndcg_at_10": 0.6241, + "ndcg_at_5": 0.6241, + "question_id": "gpt4_fe651585", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "e30c4a7b_2", + "answer_65600ff6_1", + "ff6b8624", + "9f220c66_1", + "answer_65600ff6_2", + "ultrachat_576354", + "da828711", + "sharegpt_izuEpAx_20", + "sharegpt_mJ7MnQS_219", + "cee0b8d5" + ] + }, + { + "answer_session_ids": [ + "answer_124f5dc3_2", + "answer_124f5dc3_1" + ], + "ndcg_at_10": 0.6053, + "ndcg_at_5": 0.6309, + "question_id": "8c18457d", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "98cd882c_2", + "answer_124f5dc3_1", + "deb12028_5", + "3c8e7c0e_2", + "2dfc1443_1", + "answer_124f5dc3_2", + "b53f447a_1", + "dc0802a9_3", + "sharegpt_2wY0N2C_0", + "1a5369b9" + ] + }, + { + "answer_session_ids": [ + "answer_a25d4a91_1", + "answer_a25d4a91_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "6a1eabeb", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_a25d4a91_2", + "answer_a25d4a91_1", + "d03995c6", + "4aaf678f_3", + "59a6150d_1", + "6da0bba2_1", + "9f80607d", + "6387e969", + "sharegpt_ZWqMvoL_178", + "sharegpt_9ApEWXK_0" + ] + }, + { + "answer_session_ids": [ + "answer_3f9693b7_1", + "answer_3f9693b7_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "6aeb4375", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_3f9693b7_2", + "answer_3f9693b7_1", + "402e0082_5", + "sharegpt_MWlvrvZ_0", + "1bc87711_1", + "2ed7c45e_2", + "sharegpt_wbbH6cZ_0", + "93bed3d8_1", + "e47d38f4_2", + "b65a30db" + ] + }, + { + "answer_session_ids": [ + "answer_0b1a0942_1", + "answer_0b1a0942_2" + ], + "ndcg_at_10": 0.8772, + "ndcg_at_5": 0.8772, + "question_id": "830ce83f", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_0b1a0942_1", + "8d77be9a_2", + "47c5482a", + "answer_0b1a0942_2", + "639cec58", + "dfbda5e9", + "020149a5_1", + "8bc305e4_2", + "d1e7d11c_2", + "e19c8fd9_1" + ] + }, + { + "answer_session_ids": [ + "answer_3a6f1e82_1", + "answer_3a6f1e82_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "852ce960", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_3a6f1e82_1", + "answer_3a6f1e82_2", + "b0fd1357", + "aaf22693", + "d36d11b9_2", + "a110c4dc_1", + "a76e7e3c_1", + "sharegpt_FYhsZ0Q_0", + "010a28ab_3", + "sharegpt_uiECyCD_66" + ] + }, + { + "answer_session_ids": [ + "answer_6a4f8626_1", + "answer_6a4f8626_2" + ], + "ndcg_at_10": 0.8316, + "ndcg_at_5": 1.0, + "question_id": "945e3d21", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_6a4f8626_2", + "sharegpt_E7VUcfj_0", + "5fcca8bc_1", + "9418c255", + "ultrachat_518638", + "answer_6a4f8626_1", + "dc880372_3", + "sharegpt_zn81iL6_9", + "87252d80_2", + "9bbd38a6" + ] + }, + { + "answer_session_ids": [ + "answer_eecb10d9_1", + "answer_eecb10d9_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "d7c942c3", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_eecb10d9_2", + "answer_eecb10d9_1", + "sharegpt_ezPxUrC_0", + "sharegpt_LOF3smB_25", + "ddf0f116_3", + "sharegpt_vYQdLPe_2", + "sharegpt_OTILwRq_0", + "e5b72a5d_3", + "b2243528_1", + "d15c9567_2" + ] + }, + { + "answer_session_ids": [ + "answer_c44b9df4_1", + "answer_c44b9df4_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "71315a70", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c44b9df4_2", + "answer_c44b9df4_1", + "ultrachat_399278", + "c86a22a2_2", + "b5e94516_2", + "8cfa91c3_1", + "ultrachat_42515", + "sharegpt_ZUxBndS_26", + "ultrachat_119182", + "a08fbe88_1" + ] + }, + { + "answer_session_ids": [ + "answer_e1403127_1", + "answer_e1403127_2" + ], + "ndcg_at_10": 0.8772, + "ndcg_at_5": 0.8772, + "question_id": "89941a93", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e1403127_1", + "8e72316f", + "ultrachat_400703", + "answer_e1403127_2", + "ultrachat_349321", + "sharegpt_bn0XI2B_0", + "ultrachat_352196", + "ultrachat_214088", + "ultrachat_150565", + "ultrachat_151269" + ] + }, + { + "answer_session_ids": [ + "answer_73540165_1", + "answer_73540165_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "ce6d2d27", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_73540165_2", + "answer_73540165_1", + "7a4d00b3_2", + "d8d7eddf_2", + "48ad2a93", + "a46d10a2_1", + "9f6bec4f", + "cd92ac85_1", + "4da24fe5_2", + "sharegpt_jZOf9E5_61" + ] + }, + { + "answer_session_ids": [ + "answer_02e66dec_1", + "answer_02e66dec_2" + ], + "ndcg_at_10": 0.6934, + "ndcg_at_5": 0.6934, + "question_id": "9ea5eabc", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "418c27fa", + "answer_02e66dec_1", + "answer_02e66dec_2", + "62e6593e_1", + "ultrachat_243996", + "d1f30ac6_5", + "6df6fff4", + "8c8b180e", + "7c44bc2d_1", + "6e2cca63_1" + ] + }, + { + "answer_session_ids": [ + "answer_7e9ad7b4_1", + "answer_7e9ad7b4_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "07741c44", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_7e9ad7b4_2", + "answer_7e9ad7b4_1", + "sharegpt_UGg8d44_4", + "af4d4ecd_4", + "sharegpt_CEWG2XW_0", + "d1d6b55b_1", + "3b4c63ba_3", + "aa71db11_1", + "1e0c4b50", + "sharegpt_fH7KHW1_119" + ] + }, + { + "answer_session_ids": [ + "answer_0eb23770_1", + "answer_0eb23770_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "a1eacc2a", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_0eb23770_1", + "answer_0eb23770_2", + "1f035408", + "7daa121d_2", + "521255f7_2", + "fde0ca18", + "93f23301_1", + "80082060_4", + "1ccb1920", + "sharegpt_ePBzYD0_57" + ] + }, + { + "answer_session_ids": [ + "answer_e2f4f947_1", + "answer_e2f4f947_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "184da446", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e2f4f947_2", + "answer_e2f4f947_1", + "bf633415_2", + "82a89303", + "85a49fe0_2", + "341a737e", + "ultrachat_195877", + "sharegpt_w7bjHzU_103", + "30a3c930", + "61ad6074" + ] + }, + { + "answer_session_ids": [ + "answer_8748f791_1", + "answer_8748f791_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "031748ae", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_8748f791_1", + "answer_8748f791_2", + "ultrachat_406392", + "5ac7f9c5_3", + "e0f71050_2", + "39a056ae", + "ultrachat_556006", + "ultrachat_69185", + "ultrachat_219083", + "7a085b95_1" + ] + }, + { + "answer_session_ids": [ + "answer_766ab8da_1", + "answer_766ab8da_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "4d6b87c8", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_766ab8da_2", + "answer_766ab8da_1", + "5c85023a_1", + "cffde796", + "ultrachat_40253", + "60d4332d", + "sharegpt_NUyiNZE_0", + "9af4e346_1", + "sharegpt_mmYyki5_0", + "a0201607_3" + ] + }, + { + "answer_session_ids": [ + "answer_d6d2eba8_1", + "answer_d6d2eba8_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "0f05491a", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_d6d2eba8_1", + "answer_d6d2eba8_2", + "27aba903_2", + "0c260a71_3", + "9760f4ca", + "1b205d70_2", + "e2efa128", + "13905cf6", + "9260a72d", + "sharegpt_DM7idTU_0" + ] + }, + { + "answer_session_ids": [ + "answer_cdbe2250_1", + "answer_cdbe2250_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "08e075c7", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_cdbe2250_2", + "answer_cdbe2250_1", + "b1d9d555_1", + "f7344cf4", + "ae77c245", + "25b3e36c_1", + "77827bdf", + "ultrachat_359377", + "ultrachat_556776", + "ultrachat_269248" + ] + }, + { + "answer_session_ids": [ + "answer_b191df5b_1", + "answer_b191df5b_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "f9e8c073", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_b191df5b_1", + "answer_b191df5b_2", + "0a042aef_2", + "ultrachat_88473", + "ultrachat_219619", + "445e6a7a_1", + "sharegpt_bLaSZdD_0", + "22db3cc3_1", + "50b32eed_2", + "d5283595_1" + ] + }, + { + "answer_session_ids": [ + "answer_c7ddc051_1", + "answer_c7ddc051_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "41698283", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c7ddc051_2", + "answer_c7ddc051_1", + "631e4016", + "47a2bd5a_1", + "431ae25c", + "0e8ba03b_1", + "6554fb34_1", + "28e948f5", + "ultrachat_135408", + "sharegpt_7mlXHfH_41" + ] + }, + { + "answer_session_ids": [ + "answer_9282283d_1", + "answer_9282283d_2" + ], + "ndcg_at_10": 0.5912, + "ndcg_at_5": 0.6309, + "question_id": "2698e78f", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "d76030af_1", + "answer_9282283d_2", + "862218f3_2", + "ebd61d29_2", + "338281a4_1", + "d99bab55_2", + "answer_9282283d_1", + "89c1cdfa_2", + "1cf6c966_2", + "619e96bc" + ] + }, + { + "answer_session_ids": [ + "answer_67074b4b_1", + "answer_67074b4b_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "b6019101", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_67074b4b_1", + "answer_67074b4b_2", + "de43030f_2", + "ultrachat_72569", + "sharegpt_CvVLg2J_18", + "ultrachat_146343", + "9bd36386", + "sharegpt_aVExml0_235", + "ac089c83_4", + "59c6685a_2" + ] + }, + { + "answer_session_ids": [ + "answer_07664d43_1", + "answer_07664d43_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "45dc21b6", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_07664d43_1", + "answer_07664d43_2", + "2f57c190_1", + "5934b91f", + "sharegpt_mMdyu3t_0", + "eb34144a_2", + "0504a710_4", + "ultrachat_318979", + "sharegpt_NSacJOD_0", + "8acb29f9_1" + ] + }, + { + "answer_session_ids": [ + "answer_b0f3dfff_1", + "answer_b0f3dfff_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "5a4f22c0", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_b0f3dfff_2", + "answer_b0f3dfff_1", + "007e7d81_2", + "af778ece_1", + "dba97bb1_3", + "sharegpt_1ZrnYv8_23", + "57a6a404_2", + "sharegpt_cXkL3cR_77", + "sharegpt_IMtsj65_0", + "195f2929" + ] + }, + { + "answer_session_ids": [ + "answer_4dac77cb_1", + "answer_4dac77cb_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "6071bd76", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_4dac77cb_2", + "answer_4dac77cb_1", + "a58eeaf8", + "04616813_2", + "a8899aee_1", + "8d0b4363", + "9de53d68_1", + "ultrachat_551846", + "ultrachat_56513", + "34a230b3" + ] + }, + { + "answer_session_ids": [ + "answer_1a374afa_1", + "answer_1a374afa_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e493bb7c", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_1a374afa_2", + "answer_1a374afa_1", + "9161500f_2", + "7aaa23d7_2", + "sharegpt_jHGL67f_0", + "ultrachat_185447", + "ultrachat_151926", + "ultrachat_575969", + "ultrachat_415114", + "0aacd15f_1" + ] + }, + { + "answer_session_ids": [ + "answer_caf5b52e_1", + "answer_caf5b52e_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "618f13b2", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_caf5b52e_1", + "answer_caf5b52e_2", + "86b56135_2", + "748eb152_2", + "01a120b5", + "dda70510_2", + "a20bc58f", + "ultrachat_47868", + "ultrachat_270398", + "sharegpt_aF6djes_0" + ] + }, + { + "answer_session_ids": [ + "answer_d7de9a6a_1", + "answer_d7de9a6a_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "72e3ee87", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_d7de9a6a_2", + "answer_d7de9a6a_1", + "1c1f5ccc_4", + "ultrachat_523550", + "080a4000_2", + "ultrachat_195333", + "ultrachat_129606", + "sharegpt_KdbXhTW_9", + "ultrachat_233144", + "6f2c01fc_1" + ] + }, + { + "answer_session_ids": [ + "answer_d3bf812b_1", + "answer_d3bf812b_2" + ], + "ndcg_at_10": 0.4218, + "ndcg_at_5": 0.3869, + "question_id": "c4ea545c", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "sharegpt_JJu53iW_0", + "sharegpt_I0uNTjm_16", + "9f9b402a_5", + "ee7f5084_1", + "answer_d3bf812b_1", + "sharegpt_CPzL3ju_0", + "a1f43f8d_2", + "b2d284b1", + "answer_d3bf812b_2", + "ultrachat_369982" + ] + }, + { + "answer_session_ids": [ + "answer_a7b44747_1", + "answer_a7b44747_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "01493427", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_a7b44747_1", + "answer_a7b44747_2", + "55e6ff53", + "077f7bcb_3", + "780b6c7c_1", + "5a840928_1", + "29332a29_2", + "88e73f02", + "c2204106_3", + "sharegpt_kjeGJvK_13" + ] + }, + { + "answer_session_ids": [ + "answer_77f32504_1", + "answer_77f32504_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "6a27ffc2", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_77f32504_1", + "answer_77f32504_2", + "sharegpt_kkqlRlo_13", + "ab483540", + "08effb2b_2", + "sharegpt_KSCqghj_0", + "ultrachat_20656", + "sharegpt_5clkQjb_12", + "0651f7bd_2", + "9ded9e01_2" + ] + }, + { + "answer_session_ids": [ + "answer_52382508_1", + "answer_52382508_2" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "2133c1b5", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_52382508_1", + "ultrachat_354956", + "answer_52382508_2", + "ultrachat_502724", + "25b1afbc", + "8ca0085a_1", + "sharegpt_Jm8wWJN_0", + "cf021b36_1", + "09d14ddf_2", + "1126be1e_2" + ] + }, + { + "answer_session_ids": [ + "answer_fff743f5_1", + "answer_fff743f5_2" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "18bc8abd", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_fff743f5_2", + "ba6b67af_3", + "answer_fff743f5_1", + "ultrachat_379579", + "ultrachat_131871", + "sharegpt_afCogMg_34", + "a35679d5_2", + "sharegpt_vyHqfrX_0", + "sharegpt_hH4KVIN_10" + ] + }, + { + "answer_session_ids": [ + "answer_611b6e83_1", + "answer_611b6e83_2" + ], + "ndcg_at_10": 0.8175, + "ndcg_at_5": 1.0, + "question_id": "db467c8c", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_611b6e83_2", + "6cc835bd", + "1a9f5723_1", + "16ebc8f8_1", + "ecfd2047_1", + "77cf4551_1", + "answer_611b6e83_1", + "ultrachat_137520", + "sharegpt_HBmf6q0_0", + "76967d22" + ] + }, + { + "answer_session_ids": [ + "answer_d08a934d_1", + "answer_d08a934d_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "7a87bd0c", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_d08a934d_1", + "answer_d08a934d_2", + "62369c56_1", + "b98914c6_2", + "2c543c87", + "381227c4_1", + "e250791e_3", + "55a59bc9_4", + "60f9246d", + "15519944_1" + ] + }, + { + "answer_session_ids": [ + "answer_f25c32f5_1", + "answer_f25c32f5_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e61a7584", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_f25c32f5_2", + "answer_f25c32f5_1", + "ultrachat_183954", + "82159da4_1", + "1fd16470_2", + "d02bcc9e_2", + "51add7da_1", + "ultrachat_367590", + "sharegpt_o0MporM_0", + "e1416816_2" + ] + }, + { + "answer_session_ids": [ + "answer_79c395a9_1", + "answer_79c395a9_2" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "1cea1afa", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_79c395a9_2", + "7af38385_1", + "answer_79c395a9_1", + "602ad002_1", + "52d46b4a_1", + "ultrachat_51727", + "ultrachat_67100", + "e9ad5914_5", + "ultrachat_255553", + "sharegpt_X1NXUxZ_0" + ] + }, + { + "answer_session_ids": [ + "answer_babbaccb_1", + "answer_babbaccb_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "ed4ddc30", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_babbaccb_2", + "answer_babbaccb_1", + "d2f4a309", + "8b726b64_1", + "ultrachat_238480", + "4ffa04a2_4", + "499652a0_3", + "ultrachat_342966", + "5d2dc022_1", + "a0c40934_1" + ] + }, + { + "answer_session_ids": [ + "answer_966cecbb_1", + "answer_966cecbb_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "8fb83627", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_966cecbb_1", + "answer_966cecbb_2", + "9315f268_1", + "c15319aa_1", + "02b81ad9", + "0638f6f6_1", + "ultrachat_172298", + "88e01130_1", + "be050ab3_2", + "ultrachat_307166" + ] + }, + { + "answer_session_ids": [ + "answer_8c0712af_1", + "answer_8c0712af_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "b01defab", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_8c0712af_2", + "answer_8c0712af_1", + "b1d9eb66_2", + "sharegpt_VbIxRWJ_33", + "1dcfb9a0_1", + "fc69fd44_7", + "9aa07e25", + "ultrachat_479632", + "sharegpt_fB2ri01_0", + "3891baea_2" + ] + }, + { + "answer_session_ids": [ + "answer_bcce0b73_1", + "answer_bcce0b73_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "22d2cb42", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_bcce0b73_1", + "answer_bcce0b73_2", + "74932466_1", + "060aeced", + "eec72a82_4", + "5a76fadb_2", + "ff898925", + "ultrachat_18212", + "sharegpt_hGQUdgE_10", + "ultrachat_227551" + ] + }, + { + "answer_session_ids": [ + "answer_f2f998c7_1", + "answer_f2f998c7_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "0e4e4c46", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_f2f998c7_1", + "answer_f2f998c7_2", + "b42cbaf0_1", + "60876def_2", + "0773943b", + "71d1e644", + "sharegpt_1YtwybC_15", + "ultrachat_467820", + "ultrachat_180729", + "debc34d7_1" + ] + }, + { + "answer_session_ids": [ + "answer_2cec623b_1", + "answer_2cec623b_2" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "4b24c848", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_2cec623b_2", + "d9e0c03e", + "answer_2cec623b_1", + "1815d1b7", + "sharegpt_iqJr8nh_0", + "d38e5b38", + "ultrachat_198571", + "16e094b3_1", + "d0c1453b_1", + "b0855671_2" + ] + }, + { + "answer_session_ids": [ + "answer_c9f5693c_1", + "answer_c9f5693c_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "7e974930", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c9f5693c_1", + "answer_c9f5693c_2", + "705b5399_3", + "a1e169b1", + "898ce7a5_1", + "bf139883", + "07b2420e_1", + "ultrachat_380252", + "8bd5b3c4_1", + "2cc24b7c_2" + ] + }, + { + "answer_session_ids": [ + "answer_8afdebac_1", + "answer_8afdebac_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "603deb26", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_8afdebac_2", + "answer_8afdebac_1", + "c77250d2", + "433ce12c", + "58e5adf2", + "f379d356_3", + "88611fff", + "7b0dea50_1", + "sharegpt_z6pSZ19_0", + "sharegpt_etklNfN_11" + ] + }, + { + "answer_session_ids": [ + "answer_b28f2c7a_1", + "answer_b28f2c7a_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "59524333", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_b28f2c7a_2", + "answer_b28f2c7a_1", + "ultrachat_458180", + "e4882e09_5", + "9b9ff448_1", + "4507b423", + "sharegpt_KhxXaow_0", + "sharegpt_5FHWj0a_0", + "ultrachat_493944", + "sharegpt_7gIKsqO_104" + ] + }, + { + "answer_session_ids": [ + "answer_8d63a897_1", + "answer_8d63a897_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "5831f84d", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_8d63a897_1", + "answer_8d63a897_2", + "6efce493_2", + "ef84b994_abs_2", + "41c90f4c_2", + "ff67236f_1", + "83dae08b", + "sharegpt_NHidbra_0", + "b818e2a1_1", + "sharegpt_3Rk1i3g_0" + ] + }, + { + "answer_session_ids": [ + "answer_8a791264_1", + "answer_8a791264_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "eace081b", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_8a791264_2", + "answer_8a791264_1", + "4dae77d3", + "4d04b866", + "413b57cb_3", + "9316aae3_1", + "ultrachat_455723", + "164bd5a4", + "29a18eab_3", + "sharegpt_HOO59Ue_137" + ] + }, + { + "answer_session_ids": [ + "answer_90de9b4d_1", + "answer_90de9b4d_2" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "affe2881", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_90de9b4d_1", + "fb5dd87f_7", + "answer_90de9b4d_2", + "ultrachat_443550", + "6c4a5aee", + "f334878c", + "1b71c896_2", + "sharegpt_eM7toF0_0", + "ultrachat_156501", + "ultrachat_541196" + ] + }, + { + "answer_session_ids": [ + "answer_dcd74827_1", + "answer_dcd74827_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "50635ada", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_dcd74827_1", + "answer_dcd74827_2", + "9f9b402a_5", + "42bcee92_2", + "2e53911b_1", + "cb88f886_1", + "add7cc1e", + "1de8082c", + "35bc3132_2", + "b6f35f3a_1" + ] + }, + { + "answer_session_ids": [ + "answer_ac0140ce_1", + "answer_ac0140ce_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e66b632c", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ac0140ce_2", + "answer_ac0140ce_1", + "84098c1b", + "259d58da_2", + "sharegpt_nspc8Nf_199", + "75621e39_1", + "4f51038b", + "805528d6_1", + "487fd03c", + "b3070ec4_1" + ] + }, + { + "answer_session_ids": [ + "answer_a22b654d_1", + "answer_a22b654d_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "0ddfec37", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_a22b654d_2", + "answer_a22b654d_1", + "a8a0db2b_1", + "147ab7e9_7", + "0a699029_2", + "sharegpt_UyBTuTv_0", + "ultrachat_139747", + "sharegpt_cXkL3cR_45", + "f2199726_2", + "60f9246d" + ] + }, + { + "answer_session_ids": [ + "answer_25df025b_1", + "answer_25df025b_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "f685340e", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_25df025b_1", + "answer_25df025b_2", + "a4815e8e_2", + "e862f726_1", + "5035d8d6", + "e7dcd5df", + "4a9eb139_2", + "142e3203_1", + "sharegpt_RuTDId0_29", + "992eb1f3_2" + ] + }, + { + "answer_session_ids": [ + "answer_a5b68517_1", + "answer_a5b68517_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "cc5ded98", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_a5b68517_1", + "answer_a5b68517_2", + "f14a4532_1", + "c63b2c8f", + "ultrachat_323437", + "bd8708e2_2", + "b13d5e4e_1", + "sharegpt_IEsU45Z_23", + "sharegpt_BJmgxmf_0", + "753b7c1f_1" + ] + }, + { + "answer_session_ids": [ + "answer_35d6c0be_1", + "answer_35d6c0be_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "dfde3500", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_35d6c0be_1", + "answer_35d6c0be_2", + "ad1c7ac6", + "86e218e3_2", + "fa68c9d6_2", + "5c18cb32_2", + "59c704ad_1", + "134127c2_2", + "ultrachat_368201", + "7de51ffe_1" + ] + }, + { + "answer_session_ids": [ + "answer_d6028d6e_1", + "answer_d6028d6e_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "69fee5aa", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_d6028d6e_1", + "answer_d6028d6e_2", + "a554ed79_2", + "a0b8b0ad_7", + "8464304d_2", + "ultrachat_152449", + "6685e6cd", + "2af27403_2", + "42adb80d_2", + "5d284d7d_2" + ] + }, + { + "answer_session_ids": [ + "answer_94650bfa_1", + "answer_94650bfa_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "7401057b", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_94650bfa_1", + "answer_94650bfa_2", + "c38a915a_1", + "30f20967_2", + "8de5b7cf_2", + "c3757c5b", + "1450ffad", + "sharegpt_Ah0tRO5_0", + "ultrachat_402967", + "ultrachat_385162" + ] + }, + { + "answer_session_ids": [ + "answer_ae3a122b_1", + "answer_ae3a122b_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "cf22b7bf", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ae3a122b_2", + "answer_ae3a122b_1", + "36d5bbde_1", + "457e26bf", + "8ab0292c_1", + "c8fcdf5a_1", + "8064b6ca", + "22fe1c93", + "e201572a", + "ultrachat_345143" + ] + }, + { + "answer_session_ids": [ + "answer_5126c02d_1", + "answer_5126c02d_2" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "a2f3aa27", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_5126c02d_1", + "b65f51fb_5", + "answer_5126c02d_2", + "57bf4122_1", + "2fa50779_2", + "a8028ce7_2", + "104055de", + "ultrachat_475462", + "a135a911", + "4cf2ea34" + ] + }, + { + "answer_session_ids": [ + "answer_0cdbca92_1", + "answer_0cdbca92_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "c7dc5443", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_0cdbca92_1", + "answer_0cdbca92_2", + "064e1984_1", + "sharegpt_6cz1Sq6_264", + "ultrachat_435372", + "debc34d7_2", + "8a355b36_1", + "728fc45a", + "aefbe381", + "ultrachat_308370" + ] + }, + { + "answer_session_ids": [ + "answer_da72b1b4_1", + "answer_da72b1b4_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "06db6396", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_da72b1b4_1", + "answer_da72b1b4_2", + "6ebdc1fe_2", + "sharegpt_2pwdGxJ_16", + "ultrachat_456942", + "5bd9f1e6_1", + "35038174_4", + "8235b87d", + "824634d9_2", + "sharegpt_C26CwPQ_0" + ] + }, + { + "answer_session_ids": [ + "answer_cd345582_1", + "answer_cd345582_2" + ], + "ndcg_at_10": 0.5912, + "ndcg_at_5": 0.6309, + "question_id": "3ba21379", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "49576fd6", + "answer_cd345582_1", + "67620dbc_4", + "d557e57a", + "ultrachat_224436", + "sharegpt_YGmoq5F_7", + "answer_cd345582_2", + "ultrachat_74850", + "d68838d0_2", + "f64c4793_1" + ] + }, + { + "answer_session_ids": [ + "answer_c6a0c6c2_1", + "answer_c6a0c6c2_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "9bbe84a2", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c6a0c6c2_1", + "answer_c6a0c6c2_2", + "114c1fb7_6", + "ef69c258", + "sharegpt_hcaZN94_546", + "362aa011", + "acad845c_1", + "1fc31387", + "a0f8468c_2", + "sharegpt_hLkmbBL_0" + ] + }, + { + "answer_session_ids": [ + "answer_67be2c38_1", + "answer_67be2c38_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "10e09553", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_67be2c38_2", + "answer_67be2c38_1", + "7d1f5395", + "e5f785de", + "d3da4592_2", + "918c4788_1", + "ff67236f_2", + "ultrachat_196381", + "f4e2933b", + "sharegpt_YPM1YqM_0" + ] + }, + { + "answer_session_ids": [ + "answer_4a97ae40_1", + "answer_4a97ae40_2" + ], + "ndcg_at_10": 0.6934, + "ndcg_at_5": 0.6934, + "question_id": "dad224aa", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "6de8645d_1", + "answer_4a97ae40_2", + "answer_4a97ae40_1", + "c86a22a2_1", + "sharegpt_opQbgPL_13", + "56b5b760", + "dd345e24_2", + "ultrachat_106352", + "ultrachat_468289", + "5c690047_2" + ] + }, + { + "answer_session_ids": [ + "answer_f377cda7_1", + "answer_f377cda7_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "ba61f0b9", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_f377cda7_1", + "answer_f377cda7_2", + "sharegpt_iqJr8nh_0", + "17a1e5c1", + "c9213a03_1", + "85846900_6", + "495cf518_1", + "6b7605d1_2", + "sharegpt_s8Opwwu_0", + "ad578e11_3" + ] + }, + { + "answer_session_ids": [ + "answer_e3892371_1", + "answer_e3892371_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "42ec0761", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e3892371_2", + "answer_e3892371_1", + "27d0d3e7", + "4fb01417_2", + "sharegpt_vWO8y7i_35", + "f849d84c", + "493bd421_2", + "53acd215", + "b37a44e4_2", + "37c3726a_4" + ] + }, + { + "answer_session_ids": [ + "answer_1cb52d0a_1", + "answer_1cb52d0a_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "5c40ec5b", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_1cb52d0a_2", + "answer_1cb52d0a_1", + "ultrachat_333871", + "be050ab3_2", + "a200b713_3", + "9eb7bb35", + "5cc9e1ed_3", + "ultrachat_122152", + "ultrachat_91084", + "6446f6e6" + ] + }, + { + "answer_session_ids": [ + "answer_626e93c4_1", + "answer_626e93c4_2" + ], + "ndcg_at_10": 0.8503, + "ndcg_at_5": 0.8503, + "question_id": "c6853660", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_626e93c4_2", + "6ed1c85e", + "ultrachat_223464", + "33994682_3", + "answer_626e93c4_1", + "71dc2037_2", + "a4863bcb_2", + "ultrachat_544305", + "f8ab60d7", + "bed5fbab" + ] + }, + { + "answer_session_ids": [ + "answer_f762ad8d_1", + "answer_f762ad8d_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "26bdc477", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_f762ad8d_1", + "answer_f762ad8d_2", + "27ce9f13_2", + "sharegpt_EeXTDjc_23", + "ultrachat_438698", + "37b3c75d", + "551eae69_1", + "sharegpt_lWLBUhQ_517", + "sharegpt_pRBsKdE_5", + "sharegpt_bq9G6bT_2" + ] + }, + { + "answer_session_ids": [ + "answer_3bf5b73b_1", + "answer_3bf5b73b_2" + ], + "ndcg_at_10": 0.6309, + "ndcg_at_5": 0.6309, + "question_id": "0977f2af", + "question_type": "knowledge-update", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "b1d9eb66_4", + "answer_3bf5b73b_2", + "b24eddd9_1", + "f115a3db_1", + "ultrachat_554962", + "809cbce9_3", + "357e33b5", + "1692563c_1", + "9b083158_1", + "sharegpt_FqOC5e9_45" + ] + }, + { + "answer_session_ids": [ + "answer_e1403127_1", + "answer_e1403127_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "89941a94", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e1403127_2", + "answer_e1403127_1", + "6e5130ed_1", + "de275215_3", + "74b0227e_2", + "6a7d7055_2", + "f334878c", + "2ec60f51_3", + "522dd987_1", + "sharegpt_CQ3HZO9_71" + ] + }, + { + "answer_session_ids": [ + "answer_7e9ad7b4_1", + "answer_7e9ad7b4_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "07741c45", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_7e9ad7b4_2", + "answer_7e9ad7b4_1", + "37d65d40_3", + "af4a2fd1", + "66c23110_2", + "ce21f0e4", + "ultrachat_162268", + "ultrachat_267176", + "e06c6d99_3", + "ultrachat_27772" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_5Lzox6N_0" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "7161e7e2", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_5Lzox6N_0", + "3c3fee41", + "5850de18_2", + "ultrachat_68050", + "sharegpt_yguSguz_0", + "6b17c99b_2", + "dba97bb1_4", + "5aed819f", + "sharegpt_PjFSCFK_9", + "sharegpt_6kDZTZ2_0" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_234453" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "c4f10528", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_234453", + "107d40f0", + "51c89181_1", + "29116b50_1", + "689fec3d_1", + "11248f50", + "6cf1848e_2", + "sharegpt_yMwEvl7_557", + "21ff9566_2", + "e12cfbcd" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_YkWn1Ne_0" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "89527b6b", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_YkWn1Ne_0", + "ultrachat_117600", + "48d385f0_1", + "644e1d00", + "51c262b6_2", + "4b637e89_1", + "ultrachat_84388", + "sharegpt_htSCqmh_171", + "9c722816", + "07e363d9_2" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_480665" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e9327a54", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_480665", + "ba944cdc", + "11decc61", + "c58fdf1e_2", + "c0d099e6_2", + "28e0ea23_2", + "7613d5ec_2", + "b65f51fb_5", + "sharegpt_NP5eyFN_5", + "sharegpt_Q7EQHIJ_0" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_448704" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "4c36ccef", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_448704", + "3dc8ba5f_2", + "07e6a3bd_1", + "5fc83435_2", + "1ba83815_4", + "89749c78_1", + "7b1d1f43_2", + "sharegpt_m8F4NxL_0", + "940629b2_1", + "sharegpt_1PRjwYK_11" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_IUWQYGQ_0" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "6ae235be", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_IUWQYGQ_0", + "0a1e90d1_2", + "99188307", + "809cabca_1", + "31cde680", + "8e608aad", + "39358a85_1", + "61a91bc9_2", + "cb692a80_3", + "sharegpt_KzDwaf1_211" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_370515" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "7e00a6cb", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_370515", + "4dd1ba8d_2", + "b7dca8f1_1", + "24f78a31_2", + "2fa50779_5", + "956ce238", + "39358a85_3", + "9998814e_2", + "ultrachat_188131", + "a9981dc6_2" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_hA7AkP3_0" + ], + "ndcg_at_10": 0.4307, + "ndcg_at_5": 0.4307, + "question_id": "1903aded", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "66369fde_1", + "184efb91_3", + "9b4c070c", + "answer_sharegpt_hA7AkP3_0", + "ultrachat_110523", + "sharegpt_64onasJ_0", + "201df030", + "8adb232f_1", + "f8246a66_1", + "sharegpt_emE20LI_0" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_cGdjmYo_0" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "ceb54acb", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_cGdjmYo_0", + "511bdbe3_2", + "197d99cc", + "7aa2a9af_2", + "de293134_1", + "f6fd00cf", + "0bbd7094_1", + "ultrachat_150417", + "63b72857", + "c60d5c8a_1" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_m2xJfjo_0" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "f523d9fe", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_m2xJfjo_0", + "193c23bd_1", + "22aa7cca_4", + "debc34d7_1", + "9ccea2f0_1", + "73f75ab4", + "a07f9d5c_1", + "ultrachat_391251", + "661b711f_1", + "fbfda981" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_113156" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "0e5e2d1a", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_113156", + "f84b4266", + "ultrachat_201585", + "c756c34e", + "5d79fd65_2", + "22f9f163_1", + "sharegpt_DcVIUDF_0", + "sharegpt_GPluPmJ_7", + "sharegpt_btv4SkQ_0", + "c4e5d969_2" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_187684" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "fea54f57", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_187684", + "34590cab_1", + "0aa99e2e_1", + "c8854b28", + "712e923c", + "ee659010_2", + "ultrachat_435757", + "ultrachat_129340", + "ultrachat_562162", + "a1e90def_1" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_374124" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "cc539528", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_374124", + "18807892_4", + "8bd3814a_1", + "15a8ffb7_2", + "faa194ea", + "20507d15_1", + "fe1faff9_1", + "sharegpt_0z277Yx_0", + "2a394047", + "sharegpt_1SSsoox_0" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_459954" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "dc439ea3", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_459954", + "1bfd5a8b_2", + "eb7366f2_1", + "6ef99651_1", + "5a81277c", + "ultrachat_186512", + "49d8a382", + "cb5637b3", + "sharegpt_pttddrt_5", + "ultrachat_201433" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_hn3IS1q_0" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "18dcd5a5", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_hn3IS1q_0", + "6ea1541e_1", + "ultrachat_425586", + "959e6cb0_2", + "33d16079_2", + "d9fb1588_1", + "c00077f5_2", + "sharegpt_d1tE544_5", + "sharegpt_MW0whoh_79", + "ultrachat_39471" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_275993" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "488d3006", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_275993", + "ce19df54_3", + "a68090b0_2", + "9a57d65b", + "82b82901", + "a394f6b5_1", + "d53b6bfe_1", + "1af2c0fb_1", + "sharegpt_d9F8HCC_0", + "7e4dab66_1" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_U4oCSfU_7" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "58470ed2", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_U4oCSfU_7", + "sharegpt_oPOTyid_141", + "ff49b5a5_3", + "4050ebff_6", + "a49a1438_2", + "2f2884ad_3", + "b6018747_4", + "ultrachat_458590", + "e45db473_2", + "sharegpt_e8sgAJX_0" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_HFMn2ZX_0" + ], + "ndcg_at_10": 0.6309, + "ndcg_at_5": 0.6309, + "question_id": "8cf51dda", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "daa09295", + "answer_sharegpt_HFMn2ZX_0", + "sharegpt_ftoCcOn_0", + "507514d9_1", + "27639dd8_2", + "ultrachat_365938", + "3358b5ca_2", + "9f0f59d1_1", + "sharegpt_jpiaPoJ_738", + "sharegpt_yoSUjfa_0" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_348449" + ], + "ndcg_at_10": 0.5, + "ndcg_at_5": 0.5, + "question_id": "1d4da289", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "09f1c793", + "ultrachat_213759", + "answer_ultrachat_348449", + "4cd929f8_1", + "517ad0f0", + "sharegpt_eVmxjQZ_0", + "bf85791d", + "a05c7d7d", + "ultrachat_322093", + "ultrachat_329521" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_467053" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "8464fc84", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_467053", + "21d86ebb_2", + "6ac43c5b_4", + "bdc804df_2", + "602b4074_3", + "1d5998ca_1", + "d43b63a6", + "7a0abbe2_2", + "98d39a6d", + "9582aa59_2" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_563222" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "8aef76bc", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_563222", + "146dfabe", + "5209e813_2", + "27d378b2_2", + "08e084f1_1", + "sharegpt_n3Hy63g_0", + "ultrachat_351310", + "ultrachat_368669", + "44b1595a", + "c7f6643e_2" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_417348" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "71a3fd6b", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_417348", + "8419b385_3", + "7f7e26d2_2", + "878271ae_1", + "36743359_3", + "sharegpt_1L5GXZJ_0", + "sharegpt_n3kgf14_17", + "sharegpt_asilfXk_321", + "ultrachat_168420", + "b7146673_1" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_2kpncbX_13" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "2bf43736", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_2kpncbX_13", + "16c3be68_1", + "8897a2e5", + "661ee6d9_1", + "2e89bd03_1", + "7e4cd916_2", + "d79173aa_3", + "445e6a7a_1", + "1a5fb2cd", + "a93750ef_2" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_334948" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "70b3e69b", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_334948", + "09e1a929_2", + "ultrachat_47234", + "sharegpt_oTNStu2_51", + "dd92723f_1", + "ultrachat_478639", + "1388b204_2", + "sharegpt_O8qJclh_0", + "sharegpt_s0wsteT_0", + "c2204106_2" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_6pWK9yx_0" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "8752c811", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_6pWK9yx_0", + "d0a41222_1", + "93f23301_2", + "06b5b860", + "9db30f1c_3", + "5c6a5b7c_1", + "3e48ed93", + "02b63d04_1", + "ultrachat_160178", + "9272a6a2_2" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_CaxTGYP_0" + ], + "ndcg_at_10": 0.6309, + "ndcg_at_5": 0.6309, + "question_id": "3249768e", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "ed1d68f1_1", + "answer_sharegpt_CaxTGYP_0", + "0868982f_1", + "71f2b8ba", + "ultrachat_77515", + "56cf2973", + "172b47cb_2", + "e0bd5f14_2", + "23450b93_3", + "sharegpt_M5zaN8c_0" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_115151" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "1b9b7252", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_115151", + "3829d412_2", + "d3cc5bdc_4", + "53ab4734", + "af4d4ecd_6", + "0e726047", + "330c05e7_1", + "b2e5a019", + "4f6e1b77_1", + "sharegpt_1AhFMpw_5" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_d6JJiqH_76" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "1568498a", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_d6JJiqH_76", + "sharegpt_kRkQrRx_0", + "38de4c2a_1", + "6ad46850_2", + "70fed904", + "4022d960_2", + "sharegpt_trljpZ5_27", + "sharegpt_69agvDU_27", + "983b1954", + "f8e8d445_2" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_H9PiM5G_0" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "6222b6eb", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_H9PiM5G_0", + "caf7480d_1", + "sharegpt_cyc2xbb_0", + "40a90d51", + "2fd5e208", + "544c1e7b", + "b35fadaa", + "facb94a8_3", + "2a500dce_1", + "5209e813_2" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_13075" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e8a79c70", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_13075", + "sharegpt_ha8kUEr_0", + "c0d099e6_2", + "sharegpt_vKSMO9J_69", + "742ea86b_1", + "89aed484_2", + "c2b2b2ea_1", + "87252d80_2", + "ultrachat_225061", + "sharegpt_tfIfm00_14" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_252214" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "d596882b", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_252214", + "dc9bf721_1", + "71888aff_2", + "456807b7_1", + "bef3abcd", + "0dc2efcc_2", + "ultrachat_299672", + "087d2b0a_1", + "5fb3b5ac", + "ultrachat_306390" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_5m7gg5F_0" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e3fc4d6e", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_5m7gg5F_0", + "4d261642", + "sharegpt_TVK4gjw_24", + "d31c3ec8_1", + "ultrachat_161238", + "ultrachat_325941", + "sharegpt_sXKNzPE_0", + "ultrachat_495326", + "ultrachat_443354", + "sharegpt_nAacldL_21" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_427265" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "51b23612", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_427265", + "255c753b_1", + "8672f398_2", + "409015ef_1", + "17f444f3", + "ultrachat_343", + "ultrachat_522170", + "ultrachat_555062", + "ultrachat_221034", + "848f9a52" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_94624" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "3e321797", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_94624", + "9252145e_2", + "11302335_4", + "57144028_2", + "00305dd2_1", + "193c23bd_1", + "385a60c8_1", + "sharegpt_OzK1xD9_0", + "ultrachat_557903", + "sharegpt_EZmhz8p_0" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_195444" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e982271f", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_195444", + "eb0d8dc1_4", + "8a7c65c1_1", + "f72c924e", + "db65997e", + "ultrachat_247528", + "ultrachat_383774", + "sharegpt_Xlpa70t_0", + "934419a6_1", + "3e8f07c9_1" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_NoDZzot_7" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "352ab8bd", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_NoDZzot_7", + "6593cb8b_6", + "337cbfc8_2", + "d600c646", + "55a59bc9_2", + "4a42c62b", + "ultrachat_194309", + "27971ce6_5", + "ultrachat_101751", + "sharegpt_SM8NnLU_13" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_39395" + ], + "ndcg_at_10": 0.3869, + "ndcg_at_5": 0.3869, + "question_id": "fca762bc", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "06a34d82_1", + "d0cdfddb_2", + "8fcc3543_3", + "2bc4ae7d", + "answer_ultrachat_39395", + "42d9467c_1", + "sharegpt_wrN9uUo_77", + "3b7a6ecd_1", + "136dfd61_2", + "sharegpt_ZgtoeGU_0" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_i0tMT9q_9" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "7a8d0b71", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_i0tMT9q_9", + "f07752be_1", + "f8169a6d", + "7a8f5003", + "d5eab084_2", + "c14e3f09_2", + "147ab7e9_5", + "f20e72e4_2", + "sharegpt_P0Qe2Gx_0", + "881d8e9b_1" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_269020" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "a40e080f", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_269020", + "4d32ba1b_1", + "85736a07_2", + "sharegpt_fyAkcGP_81", + "89c7a48a_2", + "4fd6129f_1", + "cd38b6ee", + "ed109bc3", + "sharegpt_oXgiN7q_327", + "c9489af0_1" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_289157" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "8b9d4367", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_289157", + "sharegpt_WZzri9J_0", + "7babf05b", + "031061e6_4", + "sharegpt_iHP3XDi_39", + "5534efcb_1", + "ultrachat_79945", + "36743359_1", + "007e7d81_3", + "sharegpt_MHiC4J8_0" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_4aJsGCH_0" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "5809eb10", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_4aJsGCH_0", + "fc3d6c80", + "sharegpt_GDFyxeT_0", + "9b820206", + "sharegpt_obCC1BP_103", + "d3069305_2", + "22ce6625_1", + "ultrachat_536416", + "ultrachat_275993", + "sharegpt_9M4WbfX_0" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_81riySf_0" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "41275add", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_81riySf_0", + "5b83c26e_3", + "33a39aa7_1", + "b3619c2c", + "a68db5db_1", + "4d84cbfa", + "ultrachat_36085", + "ultrachat_556585", + "sharegpt_dUoAhvI_71", + "sharegpt_fV5wNsl_0" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_qTi81nS_0" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "4388e9dd", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_qTi81nS_0", + "cfd21744_2", + "3197d998_7", + "1ebd753f_2", + "ultrachat_412706", + "1b8f0ba5_4", + "fc2d70fe_1", + "0826def6_1", + "cf9eda76", + "9ff53e00_1" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_i9adwQn_0" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "4baee567", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_i9adwQn_0", + "sharegpt_wduYbsX_0", + "sharegpt_NxG2nGm_91", + "385683f0_1", + "sharegpt_kt5ZCxz_10", + "ec830058_5", + "sharegpt_hRVKLa2_0", + "ultrachat_427430", + "d5415748", + "ultrachat_210474" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_hChsWOp_97" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "561fabcd", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_hChsWOp_97", + "864a563d_4", + "34b38398_2", + "sharegpt_QaiMtpK_0", + "aee13015_3", + "3be19aed_1", + "83fb74bf_1", + "d75f4fb5_1", + "33da50d0_5", + "ultrachat_194831" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_2BSXlAr_0" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "b759caee", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_2BSXlAr_0", + "sharegpt_6gBVFdg_1", + "b41aae7f_1", + "bab41bb6_5", + "d3575920", + "ultrachat_554750", + "f55fee5b_4", + "sharegpt_I9veIBa_1", + "ultrachat_281823", + "0bad887e_2" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_GYqnAhC_190" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "ac031881", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_GYqnAhC_190", + "eb95f93c", + "6608c1da", + "sharegpt_LgJ688S_0", + "e7989082_1", + "c986f83a", + "ultrachat_58138", + "2dbe975c", + "sharegpt_cIj2dFl_4", + "19c24c11" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_446979" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "28bcfaac", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_446979", + "8d410160", + "be05680b_2", + "17cb8d2f_5", + "135a62d7_2", + "a27a2811", + "044e200f_2", + "ultrachat_433665", + "989ad9e6_3", + "sharegpt_MO9Xxet_71" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_294807" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "16c90bf4", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_294807", + "8464304d_2", + "5053474c_2", + "b8770374_2", + "sharegpt_Ou9rQ6U_8", + "b0da5097_2", + "ad72ce15", + "d576152e_4", + "d4230511_1", + "32449a42" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_519486" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "c8f1aeed", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_519486", + "sharegpt_ebYKddM_87", + "aa7a2ec0_1", + "a53fdd02_2", + "af631aa3_3", + "48a72204_3", + "f7cd21e5", + "sharegpt_fH5kcER_0", + "12cd736c", + "c3023a45_5" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_SS141vi_0" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "eaca4986", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_SS141vi_0", + "3477c37c_1", + "931c521e_1", + "ultrachat_311235", + "7bc2c7a3_1", + "1c1a2b7f_2", + "564761aa", + "24ca2090_2", + "sharegpt_NelPQLT_0", + "0e543e9e_2" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_456407" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "c7cf7dfd", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_456407", + "de5a59bf", + "95cec590_1", + "285a29e7_1", + "sharegpt_f28uI6i_31", + "780b6c7c_1", + "c1e681e7_2", + "e27fe923_5", + "1382d6fb_1", + "9db6d988_2" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_174360" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e48988bc", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_174360", + "6e110a53_1", + "ultrachat_494373", + "e388463b_1", + "3fd56ebf", + "ff6bccaa_3", + "476b1cbb", + "sharegpt_TzwvePR_21", + "ultrachat_172828", + "ultrachat_307163" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_440262" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "1de5cff2", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_440262", + "4c6775ed_1", + "848d721f_1", + "8fcfbe43_3", + "ultrachat_58031", + "64048e35", + "28031c43_2", + "c14dbe17", + "42d9467c_1", + "4cb33063" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_403752" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "65240037", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_403752", + "37141183", + "46f07bd0_2", + "f8dc1273", + "fce669f5_3", + "0984a772", + "7df4c735_1", + "3c3a9042_1", + "dba5a924", + "sharegpt_Caxpnd3_6" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_399000" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "778164c6", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_399000", + "sharegpt_ki9IVDq_6", + "ultrachat_346016", + "483ddd90", + "490bb46d_1", + "05b551b6", + "b605741a_1", + "ultrachat_144598", + "c15dadce_4", + "35201d43" + ] + } + ], + "runtime_seconds": 403.852, + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_new_brainctl_cmd.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_new_brainctl_cmd.json new file mode 100644 index 0000000..50ef218 --- /dev/null +++ b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_new_brainctl_cmd.json @@ -0,0 +1,12197 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\longmemeval_new_brainctl_cmd.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "measured": true, + "metadata": {}, + "metrics": { + "ndcg_at_10": 0.8753, + "ndcg_at_5": 0.8715, + "r_at_10": 0.9894, + "r_at_5": 0.9596 + }, + "mode": "cmd", + "notes": [ + "top_k=10", + "Legacy 470-question session-level slice." + ], + "primary_metric": "r_at_5", + "primary_metric_value": 0.9596, + "reference_kind": "measured", + "rows": [ + { + "answer_session_ids": [ + "answer_280352e9" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e47becba", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_280352e9", + "02bd2b90_3", + "ultrachat_214101", + "5dac7cc2_1", + "sharegpt_Cr2tc1f_0", + "f6859b48_2" + ] + }, + { + "answer_session_ids": [ + "answer_40a90d51" + ], + "ndcg_at_10": 0.4307, + "ndcg_at_5": 0.4307, + "question_id": "118b2229", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "ultrachat_392094", + "db73b7e4_4", + "afe04238_1", + "answer_40a90d51", + "5dce60dd", + "a0b8b0ad_6", + "db50c0f6", + "3ea9f765", + "d600c646", + "sharegpt_i1iuP70_0" + ] + }, + { + "answer_session_ids": [ + "answer_d61669c7" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "51a45a95", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_d61669c7", + "98c198fb", + "217debf7", + "6e0b1800_2", + "5c44d9fe_1", + "07ba9acd_2", + "sharegpt_CyJ3dal_0", + "sharegpt_hChsWOp_128", + "sharegpt_3vxz2Zr_0", + "ultrachat_282235" + ] + }, + { + "answer_session_ids": [ + "answer_355c48bb" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "58bf7951", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_355c48bb", + "ultrachat_304942", + "ultrachat_268434", + "ultrachat_155546", + "sharegpt_gmLF3PE_17", + "7e17ae56_5", + "sharegpt_M84blrA_53", + "sharegpt_RkFlOeC_19", + "sharegpt_OFiElWf_37", + "sharegpt_CvVLg2J_18" + ] + }, + { + "answer_session_ids": [ + "answer_3e012175" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "1e043500", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_3e012175", + "01bd9def", + "ultrachat_227993", + "7b1d1f43_1", + "8020d945_2", + "e132259f", + "sharegpt_bph7DTk_7", + "ultrachat_271928", + "4d0d7984", + "sharegpt_6QUDIXG_24" + ] + }, + { + "answer_session_ids": [ + "answer_f6168136" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "c5e8278d", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_f6168136", + "89b1028d_1", + "64767b7f", + "aaf71ce2_2", + "d777e7da_1", + "fc6549b2", + "sharegpt_WPGrlAK_0", + "sharegpt_EXJAjmw_0", + "ultrachat_474903", + "3fa35683_7" + ] + }, + { + "answer_session_ids": [ + "answer_9398da02" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "6ade9755", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_9398da02", + "7c5ab02a_2", + "ec8fa65d_2", + "ultrachat_84599", + "ultrachat_393203", + "910ed479_3", + "ultrachat_270925", + "ultrachat_462", + "28621d6a_2", + "sharegpt_YsgacQi_0" + ] + }, + { + "answer_session_ids": [ + "answer_feb5200f" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "6f9b354f", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_feb5200f", + "a79f1a04_1", + "0c0f2449_2", + "ultrachat_549317", + "60207b41_1", + "54d06eeb", + "sharegpt_IuLT3VI_9", + "sharegpt_VAELSB3_0", + "sharegpt_kRjBUsA_79", + "ultrachat_224845" + ] + }, + { + "answer_session_ids": [ + "answer_59547700" + ], + "ndcg_at_10": 0.6309, + "ndcg_at_5": 0.6309, + "question_id": "58ef2f1c", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "13f3da23_3", + "answer_59547700", + "e6ab6a7b_2", + "ultrachat_5212", + "e82ce7cd_1", + "ultrachat_246324", + "ultrachat_477344", + "dbcfab21", + "b546cb39_2", + "403ee298_1" + ] + }, + { + "answer_session_ids": [ + "answer_c3567066" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "f8c5f88b", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c3567066", + "68ace657_1", + "sharegpt_TKj3yO8_0", + "91880423", + "15232887_1", + "ultrachat_360085", + "ultrachat_412973", + "7ff89315_3", + "58de8f9b", + "3b5205b6_1" + ] + }, + { + "answer_session_ids": [ + "answer_235eb6fb" + ], + "ndcg_at_10": 0.6309, + "ndcg_at_5": 0.6309, + "question_id": "5d3d2817", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "ultrachat_384575", + "answer_235eb6fb", + "670ca40e", + "sharegpt_ipLglky_48", + "sharegpt_8zsFRKh_0", + "e93efd9a_1", + "cef33b28_2", + "ultrachat_106114", + "sharegpt_bTDMt3m_64", + "25d1830a_1" + ] + }, + { + "answer_session_ids": [ + "answer_7cb94507" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "7527f7e2", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_7cb94507", + "864a563d_3", + "sharegpt_cRrfdsc_0", + "sharegpt_rraH1Q7_0", + "44bbd8cd_2", + "d81d9846", + "sharegpt_bTDMt3m_0", + "bee872da_2", + "ultrachat_62919", + "18a06652_3" + ] + }, + { + "answer_session_ids": [ + "answer_e05e4612" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "c960da58", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e05e4612", + "89d3053f", + "ultrachat_228940", + "ultrachat_36718", + "ultrachat_109577", + "ultrachat_129096", + "sharegpt_9iUZNsL_0", + "47db3b56_1", + "b76006cb_3", + "f01feb31" + ] + }, + { + "answer_session_ids": [ + "answer_94030872" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "3b6f954b", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_94030872", + "c271c071", + "6d3f5c68_4", + "ultrachat_475284", + "c566edbb_3", + "90b1c6d4_2", + "sharegpt_kpnwrGe_12", + "ultrachat_298929", + "4ed9cd1b_1", + "f999b05c_2" + ] + }, + { + "answer_session_ids": [ + "answer_f38f679b" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "726462e0", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_f38f679b", + "9c05acfe_2", + "741c6781_1", + "082889ba_1", + "5b83c26e_1", + "6ef99651_2", + "ce144f08", + "cdd9704d_1", + "7e4dab66_1", + "322a96e8" + ] + }, + { + "answer_session_ids": [ + "answer_c63c0458" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "94f70d80", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c63c0458", + "67045503", + "58de8f9b", + "sharegpt_6fTnCnw_76", + "ultrachat_380138", + "ultrachat_172828", + "sharegpt_mJ7MnQS_31", + "ec628e8d_2", + "sharegpt_ZvjRGRN_0", + "9a59023e_1" + ] + }, + { + "answer_session_ids": [ + "answer_fea2e4d3" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "66f24dbb", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_fea2e4d3", + "89452526_2", + "8d46b8fa_1", + "798e4ba3_2", + "21c6c41d", + "ca3a4e4f_1", + "sharegpt_83fkMsQ_0", + "ultrachat_445260", + "sharegpt_ZcfatzD_0", + "76299a49_1" + ] + }, + { + "answer_session_ids": [ + "answer_679840f8" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "ad7109d1", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_679840f8", + "22aa7cca_3", + "3dc02d26_2", + "d92cc997", + "ultrachat_74974", + "495cf518_1", + "sharegpt_Ck9R9HX_0", + "1d6744b5_1", + "fb721812_2", + "ultrachat_365256" + ] + }, + { + "answer_session_ids": [ + "answer_82a04f59" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "af8d2e46", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_82a04f59", + "33c3b457", + "73d5f10c", + "ultrachat_560731", + "266d604c", + "32a67b35", + "ultrachat_284576", + "13047a02", + "d5a6f455_2", + "ultrachat_210554" + ] + }, + { + "answer_session_ids": [ + "answer_8f276838" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "dccbc061", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_8f276838", + "0f417163", + "sharegpt_nc62Spr_7", + "ultrachat_278202", + "959e6cb0_3", + "b2a885b9", + "3c614701", + "8d74c5f6", + "sharegpt_MKMWjX0_25", + "ultrachat_519037" + ] + }, + { + "answer_session_ids": [ + "answer_761acef8" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "c8c3f81d", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_761acef8", + "eb5d5a93_1", + "66e94928", + "19ec83c5_1", + "sharegpt_pRqHb1o_0", + "eb403c80_2", + "9c5f1314_1", + "2b16f3d6_1", + "998616b4_4", + "25432e10" + ] + }, + { + "answer_session_ids": [ + "answer_8ad8a34f" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "8ebdbe50", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_8ad8a34f", + "b53f447a_1", + "90f16f14_2", + "01e806f0_1", + "27d378b2_3", + "39911f94", + "90a5e1f3_1", + "sharegpt_QxC97cL_0", + "d4f7a065_2", + "326f41d8" + ] + }, + { + "answer_session_ids": [ + "answer_e623ae87" + ], + "ndcg_at_10": 0.301, + "ndcg_at_5": 0.0, + "question_id": "6b168ec8", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 0.0, + "top_session_ids": [ + "ultrachat_360618", + "ultrachat_377741", + "33dff20c_3", + "ultrachat_249928", + "ultrachat_499691", + "dded1725_3", + "sharegpt_D9kTAIU_0", + "02f0738e_1", + "answer_e623ae87", + "f5b33470_abs" + ] + }, + { + "answer_session_ids": [ + "answer_723bf11f" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "75499fd8", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_723bf11f", + "5328c3c2_2", + "990b60df_1", + "18b0d357_1", + "eb0d8dc1_7", + "3e7df334_1", + "ddf0f116_3", + "sharegpt_1kE6haW_276", + "sharegpt_yPCVYhX_0", + "168feef7_1" + ] + }, + { + "answer_session_ids": [ + "answer_1e6d4567" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "21436231", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_1e6d4567", + "0d0a89dc_1", + "7a7a4bf0_4", + "ultrachat_172276", + "dc378711_1", + "sharegpt_6byuqlt_0", + "sharegpt_DGiqvVv_169", + "d0785ee9_1", + "3197d998_6", + "6bd0b406_1" + ] + }, + { + "answer_session_ids": [ + "answer_cb742a61" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "95bcc1c8", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_cb742a61", + "3919227f", + "ultrachat_320154", + "c9763bff", + "396238f9", + "abeda028", + "56266c38_1", + "99e5e380", + "sharegpt_uATA5m5_41", + "ultrachat_203404" + ] + }, + { + "answer_session_ids": [ + "answer_c6fd8ebd" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "0862e8bf", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c6fd8ebd", + "0566c21b", + "2442d5b9", + "sharegpt_3BkB8g5_0", + "sharegpt_2I1TxKW_0", + "ultrachat_47999", + "sharegpt_A7KJApk_21", + "sharegpt_KOaA4jf_0", + "sharegpt_isQJkCz_17", + "sharegpt_sCFErnY_22" + ] + }, + { + "answer_session_ids": [ + "answer_69811d4a" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "853b0a1d", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_69811d4a", + "sharegpt_n3MgsgL_0", + "c70f9f9c", + "3fa35683_3", + "a8b4bfb3_2", + "f71bf532_1", + "66a6a097_2", + "ebb5bc7c_2", + "451120d3_2", + "sharegpt_B7mSlfQ_55" + ] + }, + { + "answer_session_ids": [ + "answer_6fe9fb49" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "a06e4cfe", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_6fe9fb49", + "ee43db4a", + "3c2e6725", + "84fb50bb_3", + "de64539a_2", + "8dd2d3cc", + "af257b0b_2", + "0c891c0b_1", + "ultrachat_442201", + "ultrachat_231287" + ] + }, + { + "answer_session_ids": [ + "answer_55161935" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "37d43f65", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_55161935", + "3dcf1e0b_2", + "63587769", + "ultrachat_370801", + "ultrachat_525148", + "sharegpt_MOpCbrr_0", + "sharegpt_brc2wJS_264", + "4050ebff_5", + "sharegpt_ErOTMZ3_35", + "sharegpt_afbMhMS_45" + ] + }, + { + "answer_session_ids": [ + "answer_645b0329" + ], + "ndcg_at_10": 0.5, + "ndcg_at_5": 0.5, + "question_id": "b86304ba", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "ea8bb4f8_2", + "sharegpt_xGoJZ6Z_0", + "answer_645b0329", + "ultrachat_328696", + "ultrachat_10525", + "3b73120a_1", + "e831a29f_1", + "fe1e4351_1", + "91074ce7", + "091aa510_1" + ] + }, + { + "answer_session_ids": [ + "answer_0df6aa4b" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "d52b4f67", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_0df6aa4b", + "d31c3ec8_3", + "da1797c4_1", + "ultrachat_258008", + "18a06652_6", + "62a6d083", + "974fe383_2", + "sharegpt_IqVsDp7_89", + "sharegpt_MM7pEKH_0", + "sharegpt_cn3iUbp_7" + ] + }, + { + "answer_session_ids": [ + "answer_986de8c3" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "25e5aa4f", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_986de8c3", + "18807892_4", + "48f804a3", + "ultrachat_381447", + "a459d477", + "ca5ff3b4", + "13ad258b", + "sharegpt_8cFNcVG_0", + "ultrachat_225537", + "087d2b0a_2" + ] + }, + { + "answer_session_ids": [ + "answer_0714183a" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "caf9ead2", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_0714183a", + "f684ac4c_2", + "fc167d00", + "sharegpt_3D3oQC0_213", + "44713827_2", + "9f8bdd23_1", + "48a72204_1", + "ultrachat_30245", + "fcc6d66d_1", + "ultrachat_532107" + ] + }, + { + "answer_session_ids": [ + "answer_c8354ae9" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "8550ddae", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c8354ae9", + "a7189b75", + "71cf5aeb_3", + "e10dfffb_1", + "4f234e2c", + "9deebbc2_2", + "8ec23b2c", + "295dc1ab_1", + "c7a15bdc", + "sharegpt_ipLglky_42" + ] + }, + { + "answer_session_ids": [ + "answer_9cddca88" + ], + "ndcg_at_10": 0.4307, + "ndcg_at_5": 0.4307, + "question_id": "60d45044", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "fcc6d66d_1", + "4ebf9095_1", + "af4d4ecd_2", + "answer_9cddca88", + "26bc645b_5", + "a76e7e3c_2", + "fec059c5", + "bda4a421", + "0c0f2449_1", + "446173bb_1" + ] + }, + { + "answer_session_ids": [ + "answer_57fc1954" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "3f1e9474", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_57fc1954", + "sharegpt_wWUI4SO_7", + "ultrachat_177477", + "e7497c07_2", + "ultrachat_462827", + "3aac691d_2", + "cdf068b1_1", + "4f23a396_1", + "205f348b_2", + "954d4451_2" + ] + }, + { + "answer_session_ids": [ + "answer_dc11c1eb" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "86b68151", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_dc11c1eb", + "sharegpt_beV15ZO_0", + "d8454317_4", + "3d72c0c0", + "ultrachat_182718", + "0abaca24_1", + "sharegpt_rnL5VQO_0", + "4e671700", + "5ace87df_1", + "eed7b3ac_2" + ] + }, + { + "answer_session_ids": [ + "answer_0dd4d99a" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "577d4d32", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_0dd4d99a", + "ultrachat_125084", + "88925167_1", + "6e2cca63_2", + "449b918c_1", + "2c0d04d6_1", + "2ef53f61_2", + "sharegpt_pcoHTby_85", + "b960cbd3", + "sharegpt_101i5OR_0" + ] + }, + { + "answer_session_ids": [ + "answer_ed1982fc" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "ec81a493", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ed1982fc", + "6f55019d", + "ultrachat_447961", + "d2fc150a_1", + "sharegpt_ST22P5t_0", + "ultrachat_27488", + "ab4643a2_4", + "ultrachat_413665", + "ultrachat_492664", + "616fdd27" + ] + }, + { + "answer_session_ids": [ + "answer_586de428" + ], + "ndcg_at_10": 0.6309, + "ndcg_at_5": 0.6309, + "question_id": "15745da0", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "07d33eaa", + "answer_586de428", + "64a97d9b", + "sharegpt_IzpzBot_0", + "1f5370f7_4", + "1c662b7b_1", + "0d73afc8", + "ultrachat_383425", + "sharegpt_iF4hlNs_5", + "3cec6e2b_2" + ] + }, + { + "answer_session_ids": [ + "answer_5ca6cd28" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e01b8e2f", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_5ca6cd28", + "97338ddd_1", + "8acfa731", + "sharegpt_jyMHY1J_31", + "f9d0bc67", + "ultrachat_433104", + "d31c3ec8_1", + "e74be4c6_4", + "8be2c3f1", + "1aaf8057_3" + ] + }, + { + "answer_session_ids": [ + "answer_e6143162" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "bc8a6e93", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e6143162", + "f62bfe2b_1", + "8c1da8f9_1", + "b869d7ed_2", + "a1b3dd27_1", + "sharegpt_qyWsgp8_13", + "552c7fd0", + "c6bed037_1", + "sharegpt_jl0GWjL_0", + "ultrachat_453950" + ] + }, + { + "answer_session_ids": [ + "answer_f1fbb330" + ], + "ndcg_at_10": 0.5, + "ndcg_at_5": 0.5, + "question_id": "ccb36322", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "bdcee74e_3", + "6d52ee93_1", + "answer_f1fbb330", + "c38b33ae", + "d99bab55_1", + "04a0b385", + "8b1019b8_1", + "0e349efa", + "06d1a1a0", + "f79a48dd_1" + ] + }, + { + "answer_session_ids": [ + "answer_530960c1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "001be529", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_530960c1", + "0f71db75_1", + "9becef17_3", + "sharegpt_YQ4UlQM_83", + "sharegpt_PKXxlMF_0", + "ultrachat_351532", + "sharegpt_ETJwE0s_0", + "sharegpt_o3cfukb_0", + "ultrachat_95472", + "ultrachat_39884" + ] + }, + { + "answer_session_ids": [ + "answer_5cc9b056" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "b320f3f8", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_5cc9b056", + "d3a3f421_2", + "514a7856_2", + "b89f68c4_4", + "8bb8bc0d", + "aae4411b_2", + "40628271_1", + "5e4bb245_2", + "sharegpt_4FCMpJR_0", + "1b71c896_2" + ] + }, + { + "answer_session_ids": [ + "answer_5ff494b9" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "19b5f2b3", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_5ff494b9", + "f379d356_1", + "ultrachat_440262", + "ultrachat_463145", + "ultrachat_302261", + "ultrachat_8325", + "ultrachat_268389", + "ultrachat_68139", + "fbe6fa2c_1", + "ultrachat_138083" + ] + }, + { + "answer_session_ids": [ + "answer_2952aee4" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "4fd1909e", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_2952aee4", + "2479cbae", + "94c582bb_2", + "ultrachat_186237", + "sharegpt_hO4FoVt_0", + "fef185b1", + "ultrachat_472809", + "sharegpt_bTDMt3m_0", + "sharegpt_FNyKOSt_0", + "sharegpt_xX1L0kg_29" + ] + }, + { + "answer_session_ids": [ + "answer_47ffab4c" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "545bd2b5", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_47ffab4c", + "b7f69a7d_1", + "5fb3b5ac", + "a79f1a04_1", + "700c16e2_1", + "cc021f81_2", + "6e672b84_2", + "d1a1b9ea_1", + "4404a52a_1", + "sharegpt_nuYnG53_0" + ] + }, + { + "answer_session_ids": [ + "answer_15d63a22" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "8a137a7f", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_15d63a22", + "77908e43_1", + "fcff2dc4_3", + "20507d15_2", + "ultrachat_90882", + "ultrachat_186116", + "ultrachat_382344", + "sharegpt_sj5ZvVB_0", + "ultrachat_151889", + "ultrachat_192894" + ] + }, + { + "answer_session_ids": [ + "answer_bbdc7b0a" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "76d63226", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_bbdc7b0a", + "6a2f9452_2", + "adc47df1_1", + "a5ab6ba7_3", + "f27e27f9_2", + "cb289226_1", + "sharegpt_sRhcMiu_13", + "ultrachat_328229", + "ultrachat_180710", + "50136b31" + ] + }, + { + "answer_session_ids": [ + "answer_96b8c9ee" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "86f00804", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_96b8c9ee", + "sharegpt_ErOTMZ3_277", + "137d8c6a_2", + "0424a40a_2", + "35c5419d_abs_3", + "6e672b84_3", + "sharegpt_PQ71V9I_7", + "60153a02_1", + "fc41f919", + "sharegpt_sCFErnY_22" + ] + }, + { + "answer_session_ids": [ + "answer_7bdcbd23" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "8e9d538c", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_7bdcbd23", + "ultrachat_549592", + "3cdad629_1", + "0bd928bf_2", + "ultrachat_26072", + "a200b713_4", + "2deed26f", + "ultrachat_232141", + "sharegpt_KtWBRv1_4", + "sharegpt_cADPA4R_0" + ] + }, + { + "answer_session_ids": [ + "answer_e40b054e" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "311778f1", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e40b054e", + "5db56d24_2", + "8e6e058b_3", + "sharegpt_vXNQZ2I_0", + "ultrachat_267100", + "d09360d3_1", + "31903f54", + "95cec590_1", + "27fb983c_2", + "ultrachat_316997" + ] + }, + { + "answer_session_ids": [ + "answer_f442ccbe" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "c19f7a0b", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_f442ccbe", + "58ab5fc5", + "eb336de0_3", + "sharegpt_6ZeW9Vb_0", + "c9292210_2", + "1b0d77b0", + "0e829b8f", + "195c8fe6_3", + "8bb8bc0d", + "81ba81ab" + ] + }, + { + "answer_session_ids": [ + "answer_83c13ff9" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "4100d0a0", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_83c13ff9", + "8d068598", + "ea3db78e_1", + "sharegpt_eV0krsR_0", + "b9890c15_1", + "ultrachat_130655", + "f8ab60d7", + "0f6b9698_1", + "665c09b1_1", + "7fc9c056" + ] + }, + { + "answer_session_ids": [ + "answer_7cc5362f" + ], + "ndcg_at_10": 0.5, + "ndcg_at_5": 0.5, + "question_id": "29f2956b", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "6d6ffac5", + "88361fdd", + "answer_7cc5362f", + "ultrachat_238255", + "c4ed2287_3", + "d50a8a33_1", + "sharegpt_jZOf9E5_0", + "715b1b28", + "eb0d8dc1_5", + "sharegpt_UCVg3jO_0" + ] + }, + { + "answer_session_ids": [ + "answer_d01949bf" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "1faac195", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_d01949bf", + "b2543a58_3", + "8064b6ca", + "9d3d54f7", + "ultrachat_283922", + "f2ccf83b", + "94f460be_1", + "ultrachat_12436", + "sharegpt_NhF2Neg_0", + "48d385f0_1" + ] + }, + { + "answer_session_ids": [ + "answer_39b12014" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "faba32e5", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_39b12014", + "67045503", + "69f3fc12_1", + "c6c5bb8b_4", + "770d11f5", + "sharegpt_FNyKOSt_0", + "sharegpt_benxw0S_9", + "ultrachat_205840", + "d3da4592_2", + "ultrachat_250588" + ] + }, + { + "answer_session_ids": [ + "answer_f5b33470" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "f4f1d8a4", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_f5b33470", + "2bfea7af", + "3aac691d_2", + "9614d065_1", + "eb682e52_2", + "6f051087_2", + "ecb24dd8_1", + "76ea88ba_2", + "ultrachat_67942", + "ultrachat_576354" + ] + }, + { + "answer_session_ids": [ + "answer_304511ce" + ], + "ndcg_at_10": 0.5, + "ndcg_at_5": 0.5, + "question_id": "c14c00dd", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "c77250d2", + "7cd7c296_3", + "answer_304511ce", + "ffd66315_2", + "cd1f5147", + "ultrachat_272210", + "183b6085", + "sharegpt_gU0qvZu_0", + "sharegpt_FfqJ2xI_0", + "sharegpt_vXWOUqx_11" + ] + }, + { + "answer_session_ids": [ + "answer_93e1bd22" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "36580ce8", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_93e1bd22", + "ultrachat_53784", + "9b38bda6_2", + "ultrachat_272552", + "ultrachat_270984", + "ultrachat_32917", + "sharegpt_FpTfRvR_0", + "ultrachat_299779", + "ultrachat_149170", + "sharegpt_ZbUcxUa_0" + ] + }, + { + "answer_session_ids": [ + "answer_19c24c11" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "3d86fd0a", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_19c24c11", + "cf8e352f_1", + "sharegpt_d7bP5fb_0", + "69674f25", + "d1d3fd12_2", + "1c1a2b7f_2", + "06d1a1a0", + "091ba7f5_1", + "5a13d047_1", + "7e4dab66_2" + ] + }, + { + "answer_session_ids": [ + "answer_787e6a6d" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "a82c026e", + "question_type": "single-session-user", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_787e6a6d", + "d9727262_4", + "f18ebe36_4", + "3009ba66_2", + "ccaabf0b_1", + "516b0aae_1", + "a6c41826_3", + "daa32134", + "c71e1c7a_2", + "bdf735de_1" + ] + }, + { + "answer_session_ids": [ + "answer_afa9873b_2", + "answer_afa9873b_3", + "answer_afa9873b_1" + ], + "ndcg_at_10": 0.9675, + "ndcg_at_5": 0.9675, + "question_id": "0a995998", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_afa9873b_1", + "answer_afa9873b_2", + "85846900_2", + "answer_afa9873b_3", + "439f9ed8_3", + "3f05c474", + "e6790684_1", + "fa46a10e", + "sharegpt_ZzArUrr_0", + "ultrachat_331531" + ] + }, + { + "answer_session_ids": [ + "answer_ec904b3c_1", + "answer_ec904b3c_4", + "answer_ec904b3c_3", + "answer_ec904b3c_2" + ], + "ndcg_at_10": 0.8772, + "ndcg_at_5": 0.8772, + "question_id": "6d550036", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ec904b3c_1", + "2e4430d8_2", + "0bab76de", + "answer_ec904b3c_4", + "ultrachat_184799", + "a9981dc6_3", + "sharegpt_UbMVpdp_93", + "f6246b5f", + "dae3906e_2", + "sharegpt_J7ZAFLd_0" + ] + }, + { + "answer_session_ids": [ + "answer_593bdffd_4", + "answer_593bdffd_1", + "answer_593bdffd_3", + "answer_593bdffd_2" + ], + "ndcg_at_10": 0.5508, + "ndcg_at_5": 0.5013, + "question_id": "gpt4_59c863d7", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "9ef698bc_2", + "de5f2701_1", + "a7d014e4_1", + "answer_593bdffd_1", + "answer_593bdffd_4", + "answer_593bdffd_2", + "2f09d4c8", + "ultrachat_126967", + "eb47739f_2", + "ultrachat_59708" + ] + }, + { + "answer_session_ids": [ + "answer_a8b4290f_3", + "answer_a8b4290f_1", + "answer_a8b4290f_2" + ], + "ndcg_at_10": 0.6395, + "ndcg_at_5": 0.6509, + "question_id": "b5ef892d", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "ultrachat_565056", + "answer_a8b4290f_3", + "sharegpt_XWqXdom_0", + "answer_a8b4290f_1", + "ultrachat_53385", + "35dcacdc_2", + "ultrachat_367439", + "ultrachat_375427", + "answer_a8b4290f_2", + "e552e1f9_2" + ] + }, + { + "answer_session_ids": [ + "answer_86c505e7_1", + "answer_86c505e7_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e831120c", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_86c505e7_2", + "answer_86c505e7_1", + "sharegpt_sDfu38Q_0", + "8fb3fe3a_1", + "a63ad8e3_2", + "ultrachat_310037", + "7c82a6c3", + "6387e969", + "sharegpt_hvdBqMM_0", + "sharegpt_d8EB6Ze_0" + ] + }, + { + "answer_session_ids": [ + "answer_c2204106_2", + "answer_c2204106_3", + "answer_c2204106_1" + ], + "ndcg_at_10": 0.8529, + "ndcg_at_5": 0.8529, + "question_id": "3a704032", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c2204106_1", + "7712aded", + "86e830d2_3", + "answer_c2204106_2", + "answer_c2204106_3", + "78764b10_2", + "ultrachat_501733", + "sharegpt_MRCLhIj_0", + "ultrachat_357807", + "sharegpt_HrxBbBK_0" + ] + }, + { + "answer_session_ids": [ + "answer_2880eb6c_2", + "answer_2880eb6c_4", + "answer_2880eb6c_1", + "answer_2880eb6c_3" + ], + "ndcg_at_10": 0.7606, + "ndcg_at_5": 0.7606, + "question_id": "gpt4_d84a3211", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "18d68208_1", + "answer_2880eb6c_4", + "answer_2880eb6c_2", + "answer_2880eb6c_1", + "answer_2880eb6c_3", + "4889cd54_1", + "4a32d58c_2", + "sharegpt_OHd1RQ7_37", + "ultrachat_83526", + "13c55c69_1" + ] + }, + { + "answer_session_ids": [ + "answer_526354c8_1", + "answer_526354c8_3", + "answer_526354c8_2" + ], + "ndcg_at_10": 0.9218, + "ndcg_at_5": 1.0, + "question_id": "aae3761f", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_526354c8_2", + "answer_526354c8_1", + "b192ae00_2", + "ultrachat_266173", + "sharegpt_8I9vH7n_7", + "3d85d91e_2", + "answer_526354c8_3", + "sharegpt_ynLoh9N_0", + "2a86d0da_2", + "ultrachat_92484" + ] + }, + { + "answer_session_ids": [ + "answer_55a6940c_3", + "answer_55a6940c_1", + "answer_55a6940c_2" + ], + "ndcg_at_10": 0.4716, + "ndcg_at_5": 0.0, + "question_id": "gpt4_f2262a51", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 0.0, + "top_session_ids": [ + "c0d6fa6f_2", + "dd81b163_1", + "sharegpt_eoEbthf_0", + "ultrachat_268434", + "sharegpt_BWMyoNr_0", + "answer_55a6940c_3", + "answer_55a6940c_2", + "answer_55a6940c_1", + "sharegpt_hChsWOp_128", + "sharegpt_ab6IEma_0" + ] + }, + { + "answer_session_ids": [ + "answer_f9de4602_2", + "answer_f9de4602_1" + ], + "ndcg_at_10": 0.8316, + "ndcg_at_5": 1.0, + "question_id": "dd2973ad", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_f9de4602_2", + "sharegpt_O6NO7Vo_9", + "e5f08e40", + "41743aae_1", + "0f73eee7_2", + "answer_f9de4602_1", + "ultrachat_26627", + "e720dbe2", + "32c0dae1_3", + "b192ae00_2" + ] + }, + { + "answer_session_ids": [ + "answer_56d02cab_3", + "answer_56d02cab_4", + "answer_56d02cab_2", + "answer_56d02cab_1" + ], + "ndcg_at_10": 0.9047, + "ndcg_at_5": 0.9047, + "question_id": "c4a1ceb8", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_56d02cab_3", + "40c77045", + "answer_56d02cab_1", + "answer_56d02cab_4", + "answer_56d02cab_2", + "ba9f938b_2", + "cef33b28_1", + "sharegpt_1sxNkyQ_67", + "0b64c6cb_2", + "sharegpt_WanXxRQ_0" + ] + }, + { + "answer_session_ids": [ + "answer_cf9e3940_2", + "answer_cf9e3940_1", + "answer_cf9e3940_3" + ], + "ndcg_at_10": 0.6462, + "ndcg_at_5": 0.6509, + "question_id": "gpt4_a56e767c", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "88c8df0e_3", + "answer_cf9e3940_3", + "f56e6152_1", + "answer_cf9e3940_2", + "ultrachat_257906", + "ultrachat_104129", + "d75245ea", + "answer_cf9e3940_1", + "dac54d1e_1", + "71621be0" + ] + }, + { + "answer_session_ids": [ + "answer_a4204937_2", + "answer_a4204937_1" + ], + "ndcg_at_10": 0.8772, + "ndcg_at_5": 0.8772, + "question_id": "6cb6f249", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_a4204937_1", + "9b6db1a9_2", + "ultrachat_404221", + "answer_a4204937_2", + "5edfe253_1", + "f584ba36_2", + "67620dbc_2", + "ultrachat_374192", + "cbe70f21", + "sharegpt_IkYYvSd_79" + ] + }, + { + "answer_session_ids": [ + "answer_c65042d7_3", + "answer_c65042d7_2", + "answer_c65042d7_1" + ], + "ndcg_at_10": 0.9066, + "ndcg_at_5": 1.0, + "question_id": "46a3abf7", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c65042d7_3", + "answer_c65042d7_2", + "09f2c6c5", + "28eb86ab", + "a95bedca", + "ultrachat_406179", + "c1e598cd_1", + "4fe91719_2", + "answer_c65042d7_1", + "e224317f_2" + ] + }, + { + "answer_session_ids": [ + "answer_ef74281f_2", + "answer_ef74281f_1", + "answer_ef74281f_3" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "36b9f61e", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ef74281f_2", + "answer_ef74281f_1", + "answer_ef74281f_3", + "e05baf83_2", + "3159942d_2", + "fc6f9c59", + "5829ab38", + "3ceb6783_2", + "fd6f60f0_3", + "85cd56c7_1" + ] + }, + { + "answer_session_ids": [ + "answer_8d015d9d_3", + "answer_8d015d9d_5", + "answer_8d015d9d_2", + "answer_8d015d9d_4", + "answer_8d015d9d_1" + ], + "ndcg_at_10": 0.9609, + "ndcg_at_5": 0.9829, + "question_id": "28dc39ac", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_8d015d9d_2", + "answer_8d015d9d_3", + "answer_8d015d9d_1", + "a7cd1c0f_1", + "answer_8d015d9d_4", + "bfa499f7", + "ultrachat_452868", + "answer_8d015d9d_5", + "5854eebc_1", + "95578ea8_2" + ] + }, + { + "answer_session_ids": [ + "answer_e7b0637e_2", + "answer_e7b0637e_1", + "answer_e7b0637e_3" + ], + "ndcg_at_10": 0.5574, + "ndcg_at_5": 0.5438, + "question_id": "gpt4_2f8be40d", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "ultrachat_217382", + "1d6e01e8", + "answer_e7b0637e_2", + "ultrachat_304474", + "answer_e7b0637e_1", + "sharegpt_L64rHOA_0", + "81b971b8_2", + "cb1f962f_4", + "answer_e7b0637e_3", + "dfe646a7_2" + ] + }, + { + "answer_session_ids": [ + "answer_fa526fc0_4", + "answer_fa526fc0_3", + "answer_fa526fc0_1", + "answer_fa526fc0_2" + ], + "ndcg_at_10": 0.8118, + "ndcg_at_5": 0.8772, + "question_id": "2e6d26dc", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_fa526fc0_4", + "sharegpt_vXNQZ2I_0", + "ultrachat_541982", + "answer_fa526fc0_3", + "7a3cbde3", + "14a520c8_2", + "answer_fa526fc0_1", + "answer_fa526fc0_2", + "sharegpt_IigLRfw_0", + "sharegpt_Wt2YDZs_35" + ] + }, + { + "answer_session_ids": [ + "answer_8858d9dc_3", + "answer_8858d9dc_1", + "answer_8858d9dc_4", + "answer_8858d9dc_2" + ], + "ndcg_at_10": 0.6509, + "ndcg_at_5": 0.6509, + "question_id": "gpt4_15e38248", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "sharegpt_mm9tc5g_0", + "answer_8858d9dc_2", + "1e7f3172_5", + "answer_8858d9dc_4", + "530742d1", + "d4ab49f1", + "ec616e7e_5", + "sharegpt_01VDd0u_0", + "15acfb99", + "8c9bc932_2" + ] + }, + { + "answer_session_ids": [ + "answer_733e443a_3", + "answer_733e443a_4", + "answer_733e443a_2", + "answer_733e443a_1" + ], + "ndcg_at_10": 0.4306, + "ndcg_at_5": 0.3869, + "question_id": "88432d0a", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "f777f641", + "1c8832b4_2", + "c9d35c00_2", + "d1f6080f_1", + "answer_733e443a_2", + "ultrachat_122782", + "ultrachat_351103", + "answer_733e443a_3", + "sharegpt_HpkiAOb_9", + "sharegpt_tESxx1y_0" + ] + }, + { + "answer_session_ids": [ + "answer_990c8992_2", + "answer_990c8992_1", + "answer_990c8992_3" + ], + "ndcg_at_10": 0.8278, + "ndcg_at_5": 0.8772, + "question_id": "80ec1f4f", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_990c8992_2", + "f6ce25ef", + "6884f11d", + "answer_990c8992_1", + "b909542d_1", + "d22c92b2", + "answer_990c8992_3", + "ultrachat_318441", + "ultrachat_497101", + "sharegpt_1kE6haW_52" + ] + }, + { + "answer_session_ids": [ + "answer_5a0d28f8_4", + "answer_5a0d28f8_2", + "answer_5a0d28f8_3", + "answer_5a0d28f8_1" + ], + "ndcg_at_10": 0.9439, + "ndcg_at_5": 0.9675, + "question_id": "d23cf73b", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_5a0d28f8_4", + "answer_5a0d28f8_2", + "66369fde_2", + "answer_5a0d28f8_3", + "b124e4a1_1", + "answer_5a0d28f8_1", + "ultrachat_224488", + "ultrachat_283390", + "ultrachat_456819", + "ultrachat_375734" + ] + }, + { + "answer_session_ids": [ + "answer_a679a86a_5", + "answer_a679a86a_4", + "answer_a679a86a_2", + "answer_a679a86a_3", + "answer_a679a86a_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_7fce9456", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_a679a86a_1", + "answer_a679a86a_5", + "answer_a679a86a_4", + "answer_a679a86a_3", + "541ecc45_3", + "1da409cd_1", + "sharegpt_OjaM9hX_225", + "191e1832", + "67154fb2_1", + "288e7d30" + ] + }, + { + "answer_session_ids": [ + "answer_c008e5df_1", + "answer_c008e5df_2", + "answer_c008e5df_3" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "d682f1a2", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c008e5df_3", + "answer_c008e5df_1", + "sharegpt_E7VUcfj_0", + "7332a779_2", + "9de29219_1", + "d7f9742a_2", + "sharegpt_LFPdgVp_22", + "e30307d8_2", + "58335f60_2", + "ba0ae49c_2" + ] + }, + { + "answer_session_ids": [ + "answer_a21f3697_1", + "answer_a21f3697_2", + "answer_a21f3697_3" + ], + "ndcg_at_10": 0.8711, + "ndcg_at_5": 0.9197, + "question_id": "7024f17c", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_a21f3697_1", + "sharegpt_rshCiS5_1", + "answer_a21f3697_2", + "4f838497_3", + "8c652eb0_1", + "answer_a21f3697_3", + "3c12ff0c", + "f8476198", + "d7281662_2", + "445e06ce_2" + ] + }, + { + "answer_session_ids": [ + "answer_203bf3fa_1", + "answer_203bf3fa_3", + "answer_203bf3fa_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_5501fe77", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_203bf3fa_1", + "answer_203bf3fa_3", + "bc5827b5", + "ultrachat_163385", + "021c0d09_2", + "7a2c618c_1", + "6dfb33f1_2", + "sharegpt_x7apNZ0_0", + "631e4016", + "13b79e39_3" + ] + }, + { + "answer_session_ids": [ + "answer_6a3b5c13_3", + "answer_6a3b5c13_1", + "answer_6a3b5c13_2", + "answer_6a3b5c13_4" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_2ba83207", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_6a3b5c13_1", + "answer_6a3b5c13_2", + "answer_6a3b5c13_4", + "answer_6a3b5c13_3", + "ultrachat_568233", + "cc3c5fa9_2", + "ultrachat_385963", + "ultrachat_224880", + "ea336da0", + "bc6de190_1" + ] + }, + { + "answer_session_ids": [ + "answer_eaa8e3ef_1", + "answer_eaa8e3ef_2" + ], + "ndcg_at_10": 0.8066, + "ndcg_at_5": 1.0, + "question_id": "2318644b", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_eaa8e3ef_2", + "920f808b", + "c4ea8219_1", + "e491f2d4_2", + "7d02ef5c_5", + "6ec23dc5_1", + "fc733b20_2", + "answer_eaa8e3ef_1", + "fcae5c39_2", + "7ff89315_3" + ] + }, + { + "answer_session_ids": [ + "answer_901a6763_2", + "answer_901a6763_4", + "answer_901a6763_1", + "answer_901a6763_3" + ], + "ndcg_at_10": 0.4914, + "ndcg_at_5": 0.4307, + "question_id": "2ce6a0f2", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "a1714d2c_2", + "c1634164_2", + "ultrachat_355740", + "answer_901a6763_4", + "23ddce03_1", + "600077b0_3", + "ultrachat_36844", + "answer_901a6763_2", + "answer_901a6763_1", + "3e90815f" + ] + }, + { + "answer_session_ids": [ + "answer_2504635e_3", + "answer_2504635e_2", + "answer_2504635e_1" + ], + "ndcg_at_10": 0.7929, + "ndcg_at_5": 1.0, + "question_id": "gpt4_d12ceb0e", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_2504635e_2", + "73f4798f", + "sharegpt_dxirwR4_25", + "2fe5510e_3", + "2f23dd1a", + "answer_2504635e_1", + "answer_2504635e_3", + "ultrachat_55863", + "d8e33f5c_abs_2", + "sharegpt_dIgdYPv_13" + ] + }, + { + "answer_session_ids": [ + "answer_39900a0a_3", + "answer_39900a0a_2", + "answer_39900a0a_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "00ca467f", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_39900a0a_3", + "answer_39900a0a_1", + "answer_39900a0a_2", + "e4cb6c56", + "ultrachat_145200", + "ultrachat_239705", + "07942c06", + "4951c0b9_1", + "36828c66_1", + "sharegpt_3cVjZ2b_16" + ] + }, + { + "answer_session_ids": [ + "answer_05d808e6_1", + "answer_05d808e6_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "b3c15d39", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_05d808e6_1", + "answer_05d808e6_2", + "5d4ff8db_1", + "a16a4158", + "sharegpt_H4fpIsr_9", + "sharegpt_inuIr9J_119", + "8d25b813_1", + "d298714d_1", + "sharegpt_V2j1zkI_0", + "sharegpt_5n9NBC8_0" + ] + }, + { + "answer_session_ids": [ + "answer_02b63d04_1", + "answer_02b63d04_5", + "answer_02b63d04_2", + "answer_02b63d04_3", + "answer_02b63d04_4" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_31ff4165", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_02b63d04_1", + "answer_02b63d04_5", + "answer_02b63d04_3", + "35201d43", + "sharegpt_tKmxch5_0", + "e2393b63_2", + "c625cff4_3", + "2d0b80f1", + "6aa87dc3", + "ultrachat_555273" + ] + }, + { + "answer_session_ids": [ + "answer_3e5fea0e_1", + "answer_3e5fea0e_2" + ], + "ndcg_at_10": 0.6053, + "ndcg_at_5": 0.6309, + "question_id": "eeda8a6d", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "sharegpt_PTZ2ime_0", + "answer_3e5fea0e_1", + "ea67b29e_1", + "cf7aad73_3", + "f3745025_1", + "answer_3e5fea0e_2", + "91c71874_1", + "sharegpt_JsCcnH4_0", + "sharegpt_wrXNAYQ_0", + "sharegpt_khNFc2W_0" + ] + }, + { + "answer_session_ids": [ + "answer_8f6b938d_1", + "answer_8f6b938d_3", + "answer_8f6b938d_4", + "answer_8f6b938d_2" + ], + "ndcg_at_10": 0.7397, + "ndcg_at_5": 0.7328, + "question_id": "2788b940", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "544fe66c_2", + "answer_8f6b938d_4", + "answer_8f6b938d_2", + "answer_8f6b938d_1", + "ff6e824b_1", + "sharegpt_vgBnRfE_0", + "answer_8f6b938d_3", + "7033540c_2", + "3f863722", + "sharegpt_xFriTfz_35" + ] + }, + { + "answer_session_ids": [ + "answer_e0956e0a_1", + "answer_e0956e0a_2" + ], + "ndcg_at_10": 0.525, + "ndcg_at_5": 0.5, + "question_id": "60bf93ed", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "a200b713_2", + "388c5534_3", + "answer_e0956e0a_1", + "4fe27c66_4", + "0e297c49", + "answer_e0956e0a_2", + "6414f676_4", + "ultrachat_7648", + "e9590b7d", + "sharegpt_f40Yvlz_0" + ] + }, + { + "answer_session_ids": [ + "answer_fcff2dc4_2", + "answer_fcff2dc4_1", + "answer_fcff2dc4_3" + ], + "ndcg_at_10": 0.8194, + "ndcg_at_5": 0.8772, + "question_id": "9d25d4e0", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_fcff2dc4_3", + "464f3821", + "a4b774c4", + "answer_fcff2dc4_1", + "37beb8da", + "ca555919_3", + "46d9d476", + "answer_fcff2dc4_2", + "39911f94", + "41abc171_1" + ] + }, + { + "answer_session_ids": [ + "answer_1de862d6_1", + "answer_1de862d6_3", + "answer_1de862d6_2" + ], + "ndcg_at_10": 0.906, + "ndcg_at_5": 0.906, + "question_id": "129d1232", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_1de862d6_2", + "69a9211c_2", + "answer_1de862d6_1", + "answer_1de862d6_3", + "5e4bb245_2", + "1ab255ba_1", + "sharegpt_De0lanR_0", + "sharegpt_jBwwg7B_0", + "bf3aebdb_1", + "sharegpt_YfIEYo1_0" + ] + }, + { + "answer_session_ids": [ + "answer_e7fe8c8b_1", + "answer_e7fe8c8b_2", + "answer_e7fe8c8b_3" + ], + "ndcg_at_10": 0.8711, + "ndcg_at_5": 0.9197, + "question_id": "60472f9c", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e7fe8c8b_1", + "b1815b81", + "answer_e7fe8c8b_2", + "7b74d32d_3", + "sharegpt_kF8hHMt_45", + "answer_e7fe8c8b_3", + "ultrachat_334324", + "ultrachat_126217", + "ultrachat_470951", + "sharegpt_5BPO9KJ_0" + ] + }, + { + "answer_session_ids": [ + "answer_3826dc55_1", + "answer_3826dc55_3", + "answer_3826dc55_5", + "answer_3826dc55_2", + "answer_3826dc55_4" + ], + "ndcg_at_10": 0.7157, + "ndcg_at_5": 0.7123, + "question_id": "gpt4_194be4b3", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "ultrachat_58180", + "answer_3826dc55_3", + "answer_3826dc55_5", + "31403c51", + "answer_3826dc55_4", + "ultrachat_385467", + "ultrachat_474217", + "answer_3826dc55_1", + "sharegpt_KwbJJ66_12", + "sharegpt_VO7dTG3_0" + ] + }, + { + "answer_session_ids": [ + "answer_cc021f81_2", + "answer_cc021f81_3", + "answer_cc021f81_1" + ], + "ndcg_at_10": 0.906, + "ndcg_at_5": 0.906, + "question_id": "a9f6b44c", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_cc021f81_3", + "837f258b", + "answer_cc021f81_1", + "answer_cc021f81_2", + "d4912cd9", + "ca896bdc_1", + "sharegpt_AqgYTct_0", + "ec830058_1", + "ultrachat_279504", + "b7e36cc5_2" + ] + }, + { + "answer_session_ids": [ + "answer_5cdf9bd2_2", + "answer_5cdf9bd2_1", + "answer_5cdf9bd2_3", + "answer_5cdf9bd2_4" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "d851d5ba", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_5cdf9bd2_2", + "answer_5cdf9bd2_1", + "answer_5cdf9bd2_4", + "answer_5cdf9bd2_3", + "sharegpt_0gZVK1A_15", + "68ace657_2", + "sharegpt_wduYbsX_0", + "sharegpt_dxirwR4_25", + "sharegpt_x7apNZ0_0", + "d77d4ac9_1" + ] + }, + { + "answer_session_ids": [ + "answer_4cef8a3c_3", + "answer_4cef8a3c_1", + "answer_4cef8a3c_2" + ], + "ndcg_at_10": 0.9675, + "ndcg_at_5": 0.9675, + "question_id": "5a7937c8", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_4cef8a3c_3", + "answer_4cef8a3c_2", + "3b38be11_2", + "answer_4cef8a3c_1", + "ultrachat_288747", + "ultrachat_377263", + "b6bc5b09_2", + "a76e7e3c_4", + "b70ac29f_2", + "07417b21" + ] + }, + { + "answer_session_ids": [ + "answer_728deb4d_5", + "answer_728deb4d_2", + "answer_728deb4d_3", + "answer_728deb4d_1", + "answer_728deb4d_4" + ], + "ndcg_at_10": 0.9709, + "ndcg_at_5": 1.0, + "question_id": "gpt4_ab202e7f", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_728deb4d_4", + "answer_728deb4d_5", + "answer_728deb4d_3", + "answer_728deb4d_2", + "ultrachat_433876", + "sharegpt_q3qJzui_0", + "29bc69b3", + "cdcbdf13", + "answer_728deb4d_1", + "sharegpt_xD2q9ER_14" + ] + }, + { + "answer_session_ids": [ + "answer_6350aa4f_1", + "answer_6350aa4f_2", + "answer_6350aa4f_3", + "answer_6350aa4f_4" + ], + "ndcg_at_10": 0.9709, + "ndcg_at_5": 1.0, + "question_id": "gpt4_e05b82a6", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_6350aa4f_2", + "answer_6350aa4f_4", + "answer_6350aa4f_3", + "88e01130_1", + "b405d3d8_1", + "answer_6350aa4f_1", + "aed6b1b6_1", + "386f6df9", + "ultrachat_43272", + "ultrachat_256330" + ] + }, + { + "answer_session_ids": [ + "answer_826d51da_3", + "answer_826d51da_4", + "answer_826d51da_2", + "answer_826d51da_1" + ], + "ndcg_at_10": 0.7614, + "ndcg_at_5": 1.0, + "question_id": "gpt4_731e37d7", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_826d51da_1", + "07820595_1", + "5365840b_2", + "267a912c", + "c566edbb_3", + "sharegpt_Rwql31f_62", + "answer_826d51da_2", + "7e4aa7c2_1", + "baa5b468_3", + "answer_826d51da_3" + ] + }, + { + "answer_session_ids": [ + "answer_60e8941a_2", + "answer_60e8941a_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "edced276", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_60e8941a_2", + "answer_60e8941a_1", + "1da409cd_1", + "f584ba36_2", + "59a6150d_3", + "726fa34a_1", + "5726dc37_2", + "f5e28561_4", + "93f23301_1", + "e4bbcd5f_4" + ] + }, + { + "answer_session_ids": [ + "answer_e0585cb5_2", + "answer_e0585cb5_1" + ], + "ndcg_at_10": 0.5, + "ndcg_at_5": 0.5, + "question_id": "10d9b85a", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "c51583cd_3", + "cbd1fe79_2", + "answer_e0585cb5_2", + "8dd2fca2", + "84889496_1", + "aa6afba8", + "d576152e_1", + "answer_e0585cb5_1", + "faac3c60", + "5cbfaf3e_4" + ] + }, + { + "answer_session_ids": [ + "answer_b6018747_2", + "answer_b6018747_4", + "answer_b6018747_1", + "answer_b6018747_3" + ], + "ndcg_at_10": 0.7107, + "ndcg_at_5": 0.6934, + "question_id": "e3038f8c", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "sharegpt_hgGAUvu_0", + "answer_b6018747_2", + "answer_b6018747_4", + "85846900_2", + "a3d8e134_2", + "answer_b6018747_3", + "answer_b6018747_1", + "sharegpt_s72sGRc_0", + "4629287b", + "sharegpt_WxCMnMO_5" + ] + }, + { + "answer_session_ids": [ + "answer_23759615_2", + "answer_23759615_3", + "answer_23759615_1" + ], + "ndcg_at_10": 0.9066, + "ndcg_at_5": 1.0, + "question_id": "2b8f3739", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_23759615_3", + "answer_23759615_1", + "dd25eeb5_6", + "c075835c", + "67d6f18f_1", + "285525b2_1", + "ec8691f5", + "sharegpt_ZH6IAa2_11", + "answer_23759615_2", + "ce6cae1b_3" + ] + }, + { + "answer_session_ids": [ + "answer_2bd23659_3", + "answer_2bd23659_2", + "answer_2bd23659_4", + "answer_2bd23659_1" + ], + "ndcg_at_10": 0.906, + "ndcg_at_5": 0.906, + "question_id": "1a8a66a6", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_2bd23659_3", + "54aa4875", + "answer_2bd23659_1", + "answer_2bd23659_2", + "1e91a236_1", + "dc71023e", + "sharegpt_lsiJM0Y_0", + "d6792df8", + "ebb5d262", + "ultrachat_287097" + ] + }, + { + "answer_session_ids": [ + "answer_923c0221_1", + "answer_923c0221_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "c2ac3c61", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_923c0221_2", + "answer_923c0221_1", + "0c483a3c", + "8e4e215b", + "sharegpt_2BSHRQc_25", + "6009ba4e_5", + "9f803df5_1", + "ultrachat_412857", + "ultrachat_38171", + "0b64c6cb_1" + ] + }, + { + "answer_session_ids": [ + "answer_7726e7e9_1", + "answer_7726e7e9_3", + "answer_7726e7e9_2" + ], + "ndcg_at_10": 0.8385, + "ndcg_at_5": 0.8772, + "question_id": "bf659f65", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_7726e7e9_1", + "f2835879_2", + "9d480413_5", + "answer_7726e7e9_2", + "ultrachat_465222", + "answer_7726e7e9_3", + "258645ba_3", + "71619051_1", + "ultrachat_377765", + "sharegpt_SBJezEx_4" + ] + }, + { + "answer_session_ids": [ + "answer_35c5419d_3", + "answer_35c5419d_2", + "answer_35c5419d_1" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "gpt4_372c3eed", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_35c5419d_3", + "4d04b866", + "answer_35c5419d_1", + "ultrachat_400797", + "ultrachat_290513", + "ultrachat_424492", + "0d396995_1", + "bc1f1051_5", + "c95f88fd_1", + "sharegpt_d1pTYei_39" + ] + }, + { + "answer_session_ids": [ + "answer_669318cf_2", + "answer_669318cf_1", + "answer_669318cf_3" + ], + "ndcg_at_10": 0.9469, + "ndcg_at_5": 0.9469, + "question_id": "gpt4_2f91af09", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_669318cf_2", + "answer_669318cf_1", + "22b3af37_1", + "ultrachat_42946", + "answer_669318cf_3", + "51c82c8e_2", + "d75245ea", + "541ecc45_1", + "fd6f60f0_2", + "e7e416f5_2" + ] + }, + { + "answer_session_ids": [ + "answer_da3c1266_3", + "answer_da3c1266_4", + "answer_da3c1266_1", + "answer_da3c1266_5", + "answer_da3c1266_2" + ], + "ndcg_at_10": 0.9829, + "ndcg_at_5": 0.9829, + "question_id": "81507db6", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_da3c1266_2", + "answer_da3c1266_4", + "answer_da3c1266_5", + "5c2f41a0_1", + "answer_da3c1266_1", + "9071ba70_3", + "ultrachat_162129", + "88c09142_2", + "42fa05d3", + "058f17e5_1" + ] + }, + { + "answer_session_ids": [ + "answer_edb03329" + ], + "ndcg_at_10": 0.6309, + "ndcg_at_5": 0.6309, + "question_id": "8a2466db", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "6a5b5a78", + "answer_edb03329", + "6dcf5fa0_1", + "3392c0c7", + "6be54739_3", + "d8b3e1c8_2", + "3d0c9f89_1", + "76967d22", + "sharegpt_MKMWjX0_25", + "sharegpt_osTHjYi_0" + ] + }, + { + "answer_session_ids": [ + "answer_555dfb94" + ], + "ndcg_at_10": 0.3869, + "ndcg_at_5": 0.3869, + "question_id": "06878be2", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "a6074da9_1", + "ca929779_1", + "3e9fce53_1", + "465e7ef9", + "answer_555dfb94", + "8fcfbe43_2", + "sharegpt_GSC090N_6", + "ultrachat_448784", + "79d53fd0_2", + "sharegpt_InRLwN7_0" + ] + }, + { + "answer_session_ids": [ + "answer_d87a6ef8" + ], + "ndcg_at_10": 0.0, + "ndcg_at_5": 0.0, + "question_id": "75832dbd", + "question_type": "single-session-preference", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 0.0, + "r_at_5": 0.0, + "top_session_ids": [ + "d49b1a24_1", + "1ef63c66", + "c884b67a_1", + "sharegpt_xni9yYO_6", + "ultrachat_165893", + "sharegpt_FYqt26U_0", + "47ec1674_2", + "11e9d591_1", + "sharegpt_fK7pfNf_0", + "ultrachat_93325" + ] + }, + { + "answer_session_ids": [ + "answer_d586e9cd" + ], + "ndcg_at_10": 0.5, + "ndcg_at_5": 0.5, + "question_id": "0edc2aef", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "f20e72e4_1", + "6a747f2e", + "answer_d586e9cd", + "08ca1f31_2", + "sharegpt_AsgRx0A_0", + "53dc1394", + "3c3a9042_3", + "8464304d_2", + "c6c3a982_1", + "52939d20" + ] + }, + { + "answer_session_ids": [ + "answer_9b182436" + ], + "ndcg_at_10": 0.5, + "ndcg_at_5": 0.5, + "question_id": "35a27287", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "ultrachat_267380", + "465f0d1b_1", + "answer_9b182436", + "58ae7c4f_3", + "880ee13e", + "e4a1b565_2", + "203bf3fa_2", + "c9b1c309_4", + "ultrachat_422353", + "7128c070_2" + ] + }, + { + "answer_session_ids": [ + "answer_0250ae1c" + ], + "ndcg_at_10": 0.5, + "ndcg_at_5": 0.5, + "question_id": "32260d93", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "c4d370d3_2", + "04bef53b", + "answer_0250ae1c", + "573dc24b_1", + "3f787a78_4", + "ultrachat_26485", + "ultrachat_171173", + "sharegpt_H4jw5s7_39", + "sharegpt_8jxRz9U_0", + "sharegpt_cMHMPNy_29" + ] + }, + { + "answer_session_ids": [ + "answer_6dc4305e" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "195a1a1b", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_6dc4305e", + "ultrachat_396124", + "9f12daa1_1", + "1b71c896_2", + "d03b6a05_2", + "ultrachat_576932", + "9b90460f_3", + "29695e1c_1", + "87fff4b4_1", + "ultrachat_199480" + ] + }, + { + "answer_session_ids": [ + "answer_8549e5e0" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "afdc33df", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_8549e5e0", + "0b0bc8ae_2", + "22f9f163_3", + "b6558bfb", + "afa0c2e7_2", + "d03b6a05_3", + "43b53ac4_1", + "ultrachat_381452", + "ultrachat_512490", + "e58109ed_1" + ] + }, + { + "answer_session_ids": [ + "answer_2fc6aabb" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "caf03d32", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_2fc6aabb", + "ec806228", + "6b137bce_4", + "eb0d8dc1_6", + "4374f9a6_2", + "ultrachat_501938", + "sharegpt_3D7Qpuq_14", + "af631aa3_1", + "bd6d0687", + "147e93c4" + ] + }, + { + "answer_session_ids": [ + "answer_f7b22c66" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "54026fce", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_f7b22c66", + "ba4db050_1", + "fab585c2_1", + "670c49e6", + "ultrachat_166270", + "ef5d4aa7_1", + "1c95e152_1", + "sharegpt_jyHBcl1_0", + "sharegpt_DnGdKtp_0", + "sharegpt_8YTuqXO_0" + ] + }, + { + "answer_session_ids": [ + "answer_92d5f7cd" + ], + "ndcg_at_10": 0.0, + "ndcg_at_5": 0.0, + "question_id": "06f04340", + "question_type": "single-session-preference", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 0.0, + "r_at_5": 0.0, + "top_session_ids": [ + "6e6fbb6b", + "91223fd5_1", + "sharegpt_h4ZC1fl_203", + "66bfa1db", + "0844dea6", + "42924d15", + "8b156015_2", + "728deb4d_4", + "sharegpt_MkLNumZ_0", + "sharegpt_tb5H6IH_3" + ] + }, + { + "answer_session_ids": [ + "answer_f6502d0f" + ], + "ndcg_at_10": 0.3869, + "ndcg_at_5": 0.3869, + "question_id": "6b7dfb22", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "574df152_1", + "9ee69ca6_3", + "377f8fd9_1", + "56859d37", + "answer_f6502d0f", + "be581797", + "ultrachat_443160", + "fc69fd44_3", + "603a8580_2", + "ultrachat_336051" + ] + }, + { + "answer_session_ids": [ + "answer_719502eb" + ], + "ndcg_at_10": 0.6309, + "ndcg_at_5": 0.6309, + "question_id": "1a1907b4", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "9585e0c6_1", + "answer_719502eb", + "4f3b36d4_5", + "3dced3e9_1", + "e42e7876_5", + "4e3da326_1", + "ultrachat_183428", + "ultrachat_490921", + "097c93d8", + "sharegpt_buGcwO5_0" + ] + }, + { + "answer_session_ids": [ + "answer_b10dce5e" + ], + "ndcg_at_10": 0.3562, + "ndcg_at_5": 0.0, + "question_id": "09d032c9", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 0.0, + "top_session_ids": [ + "b21bd3e2", + "3fc2244f", + "af631aa3_2", + "d3971322_1", + "612e23f1", + "answer_b10dce5e", + "26d9aaaf", + "9d1999d7_1", + "e8bfacec_2", + "sharegpt_e9sAtcZ_63" + ] + }, + { + "answer_session_ids": [ + "answer_772472c8" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "38146c39", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_772472c8", + "8064b6ca", + "909d57ca_2", + "5263736d_1", + "87aaee77", + "381b80a3", + "5806662d_1", + "a86b30e4", + "sharegpt_yBVQLaX_15", + "ultrachat_233519" + ] + }, + { + "answer_session_ids": [ + "answer_7c0ade93" + ], + "ndcg_at_10": 0.6309, + "ndcg_at_5": 0.6309, + "question_id": "d24813b1", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "3124bb28_1", + "answer_7c0ade93", + "084802f9_3", + "61a46ff7_3", + "c2a34674_1", + "ultrachat_494027", + "446173bb_1", + "1bc08e44", + "128f4e4d_3", + "sharegpt_tntUB4J_0" + ] + }, + { + "answer_session_ids": [ + "answer_1bde8d3b" + ], + "ndcg_at_10": 0.3869, + "ndcg_at_5": 0.3869, + "question_id": "57f827a0", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "d0f42e3f", + "1bff1675", + "e85e0eaf_1", + "2ef55f49_1", + "answer_1bde8d3b", + "1fc5074c_2", + "5150a4e9_2", + "d05d6391_1", + "6dd592c3", + "963e896f_1" + ] + }, + { + "answer_session_ids": [ + "answer_5e613445" + ], + "ndcg_at_10": 0.3155, + "ndcg_at_5": 0.0, + "question_id": "95228167", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 0.0, + "top_session_ids": [ + "b916da64_2", + "3d2e66f2_2", + "385a60c8_3", + "084802f9_2", + "99e0fd9f_3", + "5d8c99d3_3", + "7e17ae56_3", + "answer_5e613445", + "e2691068", + "c34b6a1c_1" + ] + }, + { + "answer_session_ids": [ + "answer_f3164f2c" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "505af2f5", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_f3164f2c", + "7d52ca48_2", + "4f2d6be4_2", + "5e4bb245_1", + "af631aa3_1", + "0e193841_8", + "sharegpt_cjulkGS_15", + "sharegpt_kjeGJvK_11", + "ultrachat_208471", + "9f8fa5e4_1" + ] + }, + { + "answer_session_ids": [ + "answer_8ee04a2e" + ], + "ndcg_at_10": 0.6309, + "ndcg_at_5": 0.6309, + "question_id": "75f70248", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "2d74df23_4", + "answer_8ee04a2e", + "53dc1394", + "75671e3f_1", + "204862e4_2", + "ff67236f_3", + "6f689aee", + "ultrachat_27339", + "sharegpt_ijZ2Bps_0", + "ultrachat_309088" + ] + }, + { + "answer_session_ids": [ + "answer_b0fac439" + ], + "ndcg_at_10": 0.4307, + "ndcg_at_5": 0.4307, + "question_id": "d6233ab6", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "e419b7c3_4", + "94bc18df_3", + "87cfeb28_2", + "answer_b0fac439", + "911f859b_5", + "f916c63a_2", + "ultrachat_457634", + "sharegpt_xni9yYO_6", + "b45e05f6", + "ultrachat_329160" + ] + }, + { + "answer_session_ids": [ + "answer_4d3be2ab" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "1da05512", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_4d3be2ab", + "f2565b3b_2", + "sharegpt_eSTYwMj_14", + "ce919391_2", + "6b93018e_2", + "6dfa3dc6_2", + "ce3ad11f_2", + "5aa88956_2", + "ultrachat_159846", + "6de70f46_1" + ] + }, + { + "answer_session_ids": [ + "answer_a1e169b1" + ], + "ndcg_at_10": 0.3869, + "ndcg_at_5": 0.3869, + "question_id": "fca70973", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "fc005760", + "f504b269", + "5d5e80c5_1", + "99b061c5_1", + "answer_a1e169b1", + "2f2884ad_1", + "ultrachat_3370", + "312bf939", + "sharegpt_lSeiu11_0", + "sharegpt_fPvtTbw_0" + ] + }, + { + "answer_session_ids": [ + "answer_8414cc57" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "b6025781", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_8414cc57", + "f20c73f5_3", + "1a693999_4", + "3a1ea336_1", + "d6956a2e_1", + "1d6744b5_1", + "612c8368_2", + "6446f6e6", + "ebe60348_1", + "2ef53f61_1" + ] + }, + { + "answer_session_ids": [ + "answer_8f15ac24" + ], + "ndcg_at_10": 0.5, + "ndcg_at_5": 0.5, + "question_id": "a89d7624", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "99f2f5b1_1", + "02f70006_1", + "answer_8f15ac24", + "a479a7d6_4", + "fd778d8e", + "39a056ae", + "082fc07f_1", + "168776ed_2", + "bef3247f_1", + "sharegpt_G3RxeFJ_94" + ] + }, + { + "answer_session_ids": [ + "answer_30f63ddb" + ], + "ndcg_at_10": 0.4307, + "ndcg_at_5": 0.4307, + "question_id": "b0479f84", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "6af947bf_1", + "0cf86cbe_2", + "fbfda981", + "answer_30f63ddb", + "457b1adb_1", + "94cffbd2_2", + "9090b1f0_2", + "ddf41a24", + "d1bb5323_1", + "1cde7ee4_1" + ] + }, + { + "answer_session_ids": [ + "answer_e6b6353d" + ], + "ndcg_at_10": 0.3155, + "ndcg_at_5": 0.0, + "question_id": "1d4e3b97", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 0.0, + "top_session_ids": [ + "ecb24dd8_2", + "809cabca_1", + "c51583cd_3", + "03633a51", + "fb2368b8_3", + "1b320137_1", + "12b6f0f1_3", + "answer_e6b6353d", + "5cd6ab1b", + "ultrachat_8041" + ] + }, + { + "answer_session_ids": [ + "answer_d03098f9" + ], + "ndcg_at_10": 0.3562, + "ndcg_at_5": 0.0, + "question_id": "07b6f563", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 0.0, + "top_session_ids": [ + "dcc7f430_1", + "66ab6260", + "5e7b1c98_1", + "13b79e39_2", + "0ac1a6f9_2", + "answer_d03098f9", + "bbccfe79", + "6af947bf_1", + "55c267aa", + "39358a85_2" + ] + }, + { + "answer_session_ids": [ + "answer_8da8c7e0" + ], + "ndcg_at_10": 0.4307, + "ndcg_at_5": 0.4307, + "question_id": "1c0ddc50", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "2aa70c9c_1", + "5439cbf0_2", + "b33e89b5_1", + "answer_8da8c7e0", + "c7ca6dff", + "a0aa5035", + "ultrachat_339938", + "2566382f_2", + "6cd203f7_1", + "0a7d5bf6_2" + ] + }, + { + "answer_session_ids": [ + "answer_cebb7159" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "0a34ad58", + "question_type": "single-session-preference", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_cebb7159", + "fb048b5d_1", + "9ca2a195_1", + "f849dd04", + "121efb3b_2", + "4b495a04_1", + "f2b8e6a8_1", + "58ad78d6_2", + "ultrachat_122225", + "f7e682c3_5" + ] + }, + { + "answer_session_ids": [ + "answer_5237bb0b_2", + "answer_5237bb0b_1" + ], + "ndcg_at_10": 0.8503, + "ndcg_at_5": 0.8503, + "question_id": "d3ab962e", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_5237bb0b_2", + "ultrachat_564044", + "3329e5e8_1", + "sharegpt_6fTnCnw_49", + "answer_5237bb0b_1", + "aa235649_1", + "ultrachat_164834", + "ultrachat_155769", + "sharegpt_CDacKfJ_0", + "ultrachat_248741" + ] + }, + { + "answer_session_ids": [ + "answer_bf633415_2", + "answer_bf633415_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "2311e44b", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_bf633415_2", + "answer_bf633415_1", + "b5b8f8f9_3", + "5854eebc_2", + "ultrachat_360918", + "c824c711_1", + "84fedd9f_2", + "c6bed037_1", + "sharegpt_YAiYuAD_7", + "2714885f_1" + ] + }, + { + "answer_session_ids": [ + "answer_4fb01417_2", + "answer_4fb01417_1" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "cc06de0d", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_4fb01417_1", + "01551a39_2", + "answer_4fb01417_2", + "55033f6f", + "d5eab084_2", + "1dbe77da_1", + "ultrachat_104440", + "66ffbb8b_1", + "670c49e6", + "92d74e73" + ] + }, + { + "answer_session_ids": [ + "answer_c69ee1f9_2", + "answer_c69ee1f9_1" + ], + "ndcg_at_10": 0.8503, + "ndcg_at_5": 0.8503, + "question_id": "a11281a2", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c69ee1f9_2", + "ultrachat_55839", + "bb2180e9_3", + "c4c4d85a", + "answer_c69ee1f9_1", + "b8adda7f_1", + "28209b6a_5", + "d95e64a7_4", + "9126833b", + "60f10fd3" + ] + }, + { + "answer_session_ids": [ + "answer_50940cb7_2", + "answer_50940cb7_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "4f54b7c9", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_50940cb7_1", + "answer_50940cb7_2", + "bcfd4ab7", + "dde142b2_1", + "ultrachat_518898", + "8a834aee_1", + "b89f68c4_1", + "49a91ed8", + "663337ce_1", + "ultrachat_473718" + ] + }, + { + "answer_session_ids": [ + "answer_dcb18b9b_2", + "answer_dcb18b9b_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "85fa3a3f", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_dcb18b9b_2", + "answer_dcb18b9b_1", + "8f707521_1", + "71110dbd_1", + "0d66562c_1", + "c6c06e6e", + "93ff2f73", + "d31c3ec8_1", + "b7fd3b48", + "sharegpt_3WzI4bW_0" + ] + }, + { + "answer_session_ids": [ + "answer_353d3c6d_2", + "answer_353d3c6d_1" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "9aaed6a3", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_353d3c6d_2", + "c846422a_1", + "answer_353d3c6d_1", + "819dfad7_2", + "3f9f66f1_1", + "2e89bd03_1", + "11e942da_2", + "sharegpt_HpdNUlz_9", + "1bf761b8_2", + "73726c40_1" + ] + }, + { + "answer_session_ids": [ + "answer_d8454588_1", + "answer_d8454588_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "1f2b8d4f", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_d8454588_2", + "answer_d8454588_1", + "d3ee5958_2", + "c9b1c309_4", + "c8c64252", + "b5c147c7_2", + "9e96d5b9_1", + "0bd59f15_3", + "23759615_1", + "sharegpt_OHIxTQm_22" + ] + }, + { + "answer_session_ids": [ + "answer_4eb6d671_2", + "answer_4eb6d671_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e6041065", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_4eb6d671_2", + "answer_4eb6d671_1", + "d79173aa_1", + "b73a4d04_1", + "49f2e396_2", + "4a006b77", + "ec616e7e_4", + "2a86d0da_2", + "bdcee74e_3", + "b7dca8f1_1" + ] + }, + { + "answer_session_ids": [ + "answer_58820c75_1", + "answer_58820c75_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "51c32626", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_58820c75_1", + "answer_58820c75_2", + "c051e740_2", + "sharegpt_hzwc0HI_13", + "84d7650f_1", + "809cbce9_1", + "sharegpt_ErOTMZ3_59", + "932ea3bf_1", + "ultrachat_273746", + "sharegpt_jBafAKm_0" + ] + }, + { + "answer_session_ids": [ + "answer_85a77c48_2", + "answer_85a77c48_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "d905b33f", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_85a77c48_1", + "answer_85a77c48_2", + "8cfa91c3_2", + "2d364fb0", + "a9b7f49e_1", + "368276ab_3", + "dd4a0e15", + "6423d6a4_2", + "7613d5ec_2", + "sharegpt_ektieAO_13" + ] + }, + { + "answer_session_ids": [ + "answer_80323f3f_1", + "answer_80323f3f_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "7405e8b1", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_80323f3f_1", + "answer_80323f3f_2", + "f2f2a606_3", + "a6b00510_4", + "6c1c5bc3_2", + "c578da1f_1", + "6e983235_1", + "3a4012a5_1", + "8f8f2ced_2", + "643d285a_3" + ] + }, + { + "answer_session_ids": [ + "answer_e9bb9500_2", + "answer_e9bb9500_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "f35224e0", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e9bb9500_2", + "answer_e9bb9500_1", + "96da07f9_4", + "e184e4c3_1", + "9393001b_3", + "ultrachat_215582", + "4648f214_2", + "42234f98", + "ultrachat_558221", + "sharegpt_Pqhvz5n_0" + ] + }, + { + "answer_session_ids": [ + "answer_743f03a1_2", + "answer_743f03a1_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "6456829e", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_743f03a1_1", + "answer_743f03a1_2", + "c44d1533_2", + "ultrachat_490336", + "ultrachat_518760", + "a8c73552", + "898cfd46_2", + "ultrachat_555576", + "3304f57b_3", + "ultrachat_247641" + ] + }, + { + "answer_session_ids": [ + "answer_feb5f98a_1", + "answer_feb5f98a_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "a4996e51", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_feb5f98a_2", + "answer_feb5f98a_1", + "252715f8_3", + "e9fba4f8", + "c1dac02f", + "082b7e52_2", + "d1a61aa1_3", + "ultrachat_506025", + "sharegpt_KjF70Iy_0", + "eea3e4be_2" + ] + }, + { + "answer_session_ids": [ + "answer_c8cc60d6_2", + "answer_c8cc60d6_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "3c1045c8", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c8cc60d6_2", + "ultrachat_561782", + "ultrachat_20893", + "b52f03a1_1", + "62541047", + "sharegpt_oIB2JG9_0", + "4f3b36d4_3", + "ultrachat_299649", + "sharegpt_GYqnAhC_190", + "cf422c07" + ] + }, + { + "answer_session_ids": [ + "answer_e552e1f9_1", + "answer_e552e1f9_2" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "60036106", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e552e1f9_2", + "sharegpt_moARQa7_0", + "answer_e552e1f9_1", + "sharegpt_ckvrgF1_99", + "c575291e_2", + "e9cb7125", + "ultrachat_466668", + "23a7f0ec", + "763f028a", + "ultrachat_134557" + ] + }, + { + "answer_session_ids": [ + "answer_3be95d43_1", + "answer_3be95d43_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "681a1674", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_3be95d43_2", + "answer_3be95d43_1", + "ultrachat_64650", + "fdd25ea1", + "50e8f2b5_1", + "d81e853b", + "sharegpt_8Kzmn95_0", + "7d33fcb4_2", + "ultrachat_436833", + "9e411500_1" + ] + }, + { + "answer_session_ids": [ + "answer_6702277b_1", + "answer_6702277b_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e25c3b8d", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_6702277b_2", + "answer_6702277b_1", + "sharegpt_40qcQLS_17", + "sharegpt_WYQSZ3n_0", + "90fea364_1", + "sharegpt_Nk10TjE_29", + "824634d9_1", + "ff201786_1", + "4256f981_2", + "ultrachat_363088" + ] + }, + { + "answer_session_ids": [ + "answer_6efce493_1", + "answer_6efce493_2" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "4adc0475", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_6efce493_1", + "bb27df5e_2", + "answer_6efce493_2", + "4fc2cf8c_1", + "fc6f9c59", + "2880eb6c_3", + "0bde5dac", + "sharegpt_wcTU09Q_31", + "debc34d7_2", + "ee659010_2" + ] + }, + { + "answer_session_ids": [ + "answer_9ef115d4_1", + "answer_9ef115d4_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "4bc144e2", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_9ef115d4_1", + "answer_9ef115d4_2", + "4f23a396_2", + "d75245ea", + "ultrachat_380790", + "42bcee92_2", + "099c1b6c_3", + "60207b41_3", + "26d9aaaf", + "2e2ae792_1" + ] + }, + { + "answer_session_ids": [ + "answer_f7fd1029_2", + "answer_f7fd1029_1" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "ef66a6e5", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_f7fd1029_1", + "48941fec_4", + "answer_f7fd1029_2", + "74c2d3a1_1", + "3f05a03e_1", + "41e778e4_2", + "afb58af0", + "295645f5_1", + "67045503", + "5273d4b8" + ] + }, + { + "answer_session_ids": [ + "answer_688f9a3f_1", + "answer_688f9a3f_2" + ], + "ndcg_at_10": 0.6934, + "ndcg_at_5": 0.6934, + "question_id": "5025383b", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "22f9af5f", + "answer_688f9a3f_2", + "answer_688f9a3f_1", + "f40fa3ed", + "2d21f921", + "ultrachat_50563", + "d10f0307_1", + "ultrachat_150500", + "sharegpt_BmS3AX0_10", + "ultrachat_137351" + ] + }, + { + "answer_session_ids": [ + "answer_17dc2f5b_2", + "answer_17dc2f5b_1" + ], + "ndcg_at_10": 0.6309, + "ndcg_at_5": 0.6309, + "question_id": "a1cc6108", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "298a07de_5", + "answer_17dc2f5b_2", + "fce56dfb_4", + "7285299a_4", + "6dfc99bf_1", + "001cefa7_2", + "fb328ace_1" + ] + }, + { + "answer_session_ids": [ + "answer_66c23110_1", + "answer_66c23110_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "9ee3ecd6", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_66c23110_2", + "answer_66c23110_1", + "fd56c5bd_2", + "a9eda378", + "6d9d5941", + "ultrachat_282862", + "fa0fa74d_2", + "ultrachat_96128", + "sharegpt_6cz1Sq6_264", + "42b45f6e_1" + ] + }, + { + "answer_session_ids": [ + "answer_419d21d5_2", + "answer_419d21d5_1" + ], + "ndcg_at_10": 0.8316, + "ndcg_at_5": 1.0, + "question_id": "3fdac837", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_419d21d5_2", + "92b06116_1", + "a98ff421_7", + "ultrachat_526748", + "c85ef24e_1", + "answer_419d21d5_1", + "ac8a2cd0", + "ultrachat_293546", + "88c8df0e_3", + "22210456_1" + ] + }, + { + "answer_session_ids": [ + "answer_5404a208_1", + "answer_5404a208_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "91b15a6e", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_5404a208_1", + "answer_5404a208_2", + "ultrachat_15125", + "f9a304e9_2", + "6169ef55_1", + "ultrachat_217922", + "7cfeb89c_2", + "ultrachat_338803", + "ultrachat_326780", + "ultrachat_222016" + ] + }, + { + "answer_session_ids": [ + "answer_a37bdf22_2", + "answer_a37bdf22_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "27016adc", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_a37bdf22_2", + "answer_a37bdf22_1", + "9f554b46_2", + "fa858208_2", + "28ea20bf", + "ultrachat_495725", + "ff5e0db0_1", + "sharegpt_3b5ojYa_39", + "ultrachat_131737", + "ultrachat_284316" + ] + }, + { + "answer_session_ids": [ + "answer_c9dfeaea_1", + "answer_c9dfeaea_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "720133ac", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c9dfeaea_2", + "answer_c9dfeaea_1", + "4c128d97_2", + "e200d96c_3", + "ff2de758", + "ultrachat_479833", + "sharegpt_bZg2XwX_69", + "0cf86cbe_2", + "ultrachat_112896", + "ultrachat_512422" + ] + }, + { + "answer_session_ids": [ + "answer_33c251f0_2", + "answer_33c251f0_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "77eafa52", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_33c251f0_1", + "answer_33c251f0_2", + "bc1f1051_4", + "sharegpt_qEukY6n_14", + "sharegpt_FlvsHKg_4", + "sharegpt_C9GjDsF_0", + "sharegpt_bpxJpAU_0", + "ultrachat_309057", + "9c4358a4", + "7b13b4d6_2" + ] + }, + { + "answer_session_ids": [ + "answer_35e36341_1", + "answer_35e36341_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "8979f9ec", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_35e36341_1", + "answer_35e36341_2", + "91c71874_1", + "ultrachat_466891", + "bcc9382a_2", + "ultrachat_562471", + "4c967baa_1", + "a6b4c4c2_1", + "f18ebe36_7", + "1064ed80_2" + ] + }, + { + "answer_session_ids": [ + "answer_35c9798c_2", + "answer_35c9798c_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "0100672e", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_35c9798c_2", + "answer_35c9798c_1", + "a4b223a7_1", + "091aa510_2", + "326e86e0_3", + "8237de6a_2", + "ultrachat_372868", + "07313722_2", + "b167a4a8", + "f60c9de5" + ] + }, + { + "answer_session_ids": [ + "answer_ef84b994_1", + "answer_ef84b994_2" + ], + "ndcg_at_10": 0.8503, + "ndcg_at_5": 0.8503, + "question_id": "a96c20ee", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ef84b994_1", + "1b51fb71_1", + "dff89d9d_1", + "9ca3534f_1", + "answer_ef84b994_2", + "2aa70c9c_2", + "sharegpt_opQbgPL_7", + "32e292f2", + "sharegpt_lWLBUhQ_539", + "sharegpt_00qHEQ6_0" + ] + }, + { + "answer_session_ids": [ + "answer_6cb8f792_1", + "answer_6cb8f792_2" + ], + "ndcg_at_10": 0.525, + "ndcg_at_5": 0.5, + "question_id": "92a0aa75", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "ultrachat_178564", + "sharegpt_YwmoLVl_0", + "answer_6cb8f792_1", + "a5854a8d_2", + "2bdd78cb_1", + "answer_6cb8f792_2", + "bd00a24d_2", + "sharegpt_xkWMnwQ_0", + "sharegpt_9l7gjYP_17", + "sharegpt_hPax8pw_11" + ] + }, + { + "answer_session_ids": [ + "answer_1bb63ea5_2", + "answer_1bb63ea5_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "3fe836c9", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_1bb63ea5_1", + "answer_1bb63ea5_2", + "e41b78c7_3", + "0a42783a", + "c81897cd_2", + "sharegpt_2xK3jjf_0", + "8129a7a8", + "f0d7d364", + "ba184a34", + "ultrachat_258020" + ] + }, + { + "answer_session_ids": [ + "answer_4cb841a8_1", + "answer_4cb841a8_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "1c549ce4", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_4cb841a8_1", + "answer_4cb841a8_2", + "f1e4daa8_2", + "f999b05c_1", + "d743153b_2", + "ultrachat_441412", + "162ff451", + "sharegpt_miOKGt1_13", + "35f052f9_2", + "sharegpt_yLBm8Vr_23" + ] + }, + { + "answer_session_ids": [ + "answer_cc1ced21_2", + "answer_cc1ced21_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "6c49646a", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_cc1ced21_2", + "answer_cc1ced21_1", + "d4380d3f", + "8acb29f9_2", + "ultrachat_565061", + "fffa6e59", + "9f220c66_1", + "6a2f306a", + "sharegpt_1X61wv2_7", + "edf2a486" + ] + }, + { + "answer_session_ids": [ + "answer_e184e4c3_2", + "answer_e184e4c3_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "1192316e", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e184e4c3_2", + "answer_e184e4c3_1", + "574df152_3", + "c5db849c_1", + "6cd7234e_2", + "ee82ed2c_2", + "21ef2d05_1", + "sharegpt_pZ3I356_11", + "1ba79ea8_1", + "ultrachat_457634" + ] + }, + { + "answer_session_ids": [ + "answer_dc5e537d_1", + "answer_dc5e537d_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "0ea62687", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_dc5e537d_1", + "answer_dc5e537d_2", + "fba28a35_3", + "26eedae1_1", + "0835368c", + "c5e9cbd8_1", + "d3ee5958_1", + "ultrachat_108112", + "sharegpt_HYI3HgX_0", + "sharegpt_Ktz3pBy_0" + ] + }, + { + "answer_session_ids": [ + "answer_3a5010af_2", + "answer_3a5010af_1" + ], + "ndcg_at_10": 0.8772, + "ndcg_at_5": 0.8772, + "question_id": "67e0d0f2", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_3a5010af_2", + "6868c94f", + "98cd882c_1", + "answer_3a5010af_1", + "ultrachat_217333", + "0cde5602_2", + "8c568d75_1", + "2252d3dd", + "sharegpt_tJkPZbz_2", + "sharegpt_GJpf1ou_0" + ] + }, + { + "answer_session_ids": [ + "answer_de64539a_1", + "answer_de64539a_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "bb7c3b45", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_de64539a_2", + "answer_de64539a_1", + "60ff3a8c", + "3891baea_4", + "a51dc9ca", + "64b7d9cc_2", + "ultrachat_305663", + "f3d43a12_3", + "4f2d6be4_1", + "752b10e6_1" + ] + }, + { + "answer_session_ids": [ + "answer_cbd08e3c_1", + "answer_cbd08e3c_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "ba358f49", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_cbd08e3c_1", + "2a2ea005_1", + "20b5caab_2", + "9cc709cb", + "8db0ab95_1", + "4ea6f550", + "385683f0_2", + "648d9cda", + "d964ed3b", + "4f838497_2" + ] + }, + { + "answer_session_ids": [ + "answer_872e8da2_2", + "answer_872e8da2_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "61f8c8f8", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_872e8da2_1", + "answer_872e8da2_2", + "a3107e2a_1", + "6e110a53_1", + "7b88c38b_2", + "2154c810_1", + "7af38385_2", + "e10846c4", + "sharegpt_iL51bzd_0", + "sharegpt_7jGo66i_99" + ] + }, + { + "answer_session_ids": [ + "answer_75eca223_2", + "answer_75eca223_1" + ], + "ndcg_at_10": 0.4575, + "ndcg_at_5": 0.4307, + "question_id": "60159905", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "21f8f481_3", + "d5f4d9fa_2", + "02eae87c_5", + "answer_75eca223_1", + "sharegpt_V7ePzVN_1", + "c351fa3b_1", + "c92b27e2_1", + "answer_75eca223_2", + "6ade84f0", + "c36f4196" + ] + }, + { + "answer_session_ids": [ + "answer_87e3a1cb_1", + "answer_87e3a1cb_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "ef9cf60a", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_87e3a1cb_2", + "answer_87e3a1cb_1", + "2357de36_2", + "7b88c38b_1", + "b330d6b1", + "sharegpt_FNyKOSt_0", + "12dd7681_4", + "7a34bc76_3", + "0babf5b6_2", + "de47d59b_1" + ] + }, + { + "answer_session_ids": [ + "answer_1881e7db_1", + "answer_1881e7db_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "73d42213", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_1881e7db_2", + "answer_1881e7db_1", + "sharegpt_ab6IEma_0", + "9614d065_1", + "sharegpt_G8j6c8J_87", + "sharegpt_ndynIbS_0", + "sharegpt_GKjb2be_0", + "ultrachat_275858", + "a13abca5_1", + "sharegpt_2Na4ASq_0" + ] + }, + { + "answer_session_ids": [ + "answer_92147866_1", + "answer_92147866_2" + ], + "ndcg_at_10": 0.8503, + "ndcg_at_5": 0.8503, + "question_id": "bc149d6b", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_92147866_1", + "7211c69e", + "86935301_2", + "451ecb23_3", + "answer_92147866_2", + "4f435d20_2", + "sharegpt_bmEgv3O_12", + "05be3ff0_3", + "eb5c65e2_1", + "db73b7e4_3" + ] + }, + { + "answer_session_ids": [ + "answer_80d6d664_2", + "answer_80d6d664_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "099778bb", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_80d6d664_2", + "answer_80d6d664_1", + "sharegpt_s1I3UkG_0", + "47103d08", + "0070840d_2", + "b1793bba", + "6bb8af95_1", + "33877350_2", + "95ce0e21_1", + "sharegpt_kshUqZh_6" + ] + }, + { + "answer_session_ids": [ + "answer_96c743d0_2", + "answer_96c743d0_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "09ba9854", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_96c743d0_1", + "answer_96c743d0_2", + "ultrachat_177260", + "32e6c308", + "7a2bdd15_1", + "ae32e467_2", + "8c63238e", + "864a563d_3", + "69a42467_2", + "8682e696_2" + ] + }, + { + "answer_session_ids": [ + "answer_23f3a657_2", + "answer_23f3a657_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "d6062bb9", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_23f3a657_1", + "answer_23f3a657_2", + "sharegpt_xuhzAgG_19", + "b35610f4_4", + "4f8caea3_1", + "5b6aedac", + "59c704ad_2", + "86298245_2", + "sharegpt_afCogMg_13", + "ultrachat_496826" + ] + }, + { + "answer_session_ids": [ + "answer_8de18468_2", + "answer_8de18468_1" + ], + "ndcg_at_10": 0.301, + "ndcg_at_5": 0.0, + "question_id": "157a136e", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 0.0, + "top_session_ids": [ + "ultrachat_255132", + "01049599_1", + "ultrachat_3770", + "sharegpt_qqo6qRj_0", + "a2e2cb72_5", + "sharegpt_6ZXBs1w_0", + "ultrachat_506827", + "298bf7a3_1", + "answer_8de18468_2", + "ultrachat_446084" + ] + }, + { + "answer_session_ids": [ + "answer_2e2085fa_2", + "answer_2e2085fa_1" + ], + "ndcg_at_10": 0.4307, + "ndcg_at_5": 0.4307, + "question_id": "c18a7dc8", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "9d5a389d", + "ultrachat_353377", + "5022812c", + "answer_2e2085fa_1", + "fc6c3985", + "f716720c_3", + "ultrachat_415226", + "37141183", + "ultrachat_463016", + "sharegpt_YPC6Oll_18" + ] + }, + { + "answer_session_ids": [ + "answer_16ece55f_2", + "answer_16ece55f_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "a3332713", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_16ece55f_2", + "answer_16ece55f_1", + "62be624f", + "67388dfc_2", + "3e59ee68_1", + "fe291ebc", + "117098d9_3", + "33994682_3", + "sharegpt_bYymtlW_0", + "sharegpt_8zrM3l4_0" + ] + }, + { + "answer_session_ids": [ + "answer_fa08bf49_1", + "answer_fa08bf49_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "55241a1f", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_fa08bf49_1", + "answer_fa08bf49_2", + "d117e2da", + "2aeb1268", + "76491ee4_1", + "9c9a22d8", + "60153a02_4", + "sharegpt_TzwvePR_0", + "46e4a8eb_2", + "1d105d8d" + ] + }, + { + "answer_session_ids": [ + "answer_47152166_2", + "answer_47152166_1" + ], + "ndcg_at_10": 0.6934, + "ndcg_at_5": 0.6934, + "question_id": "a08a253f", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "6e5130ed_2", + "answer_47152166_2", + "answer_47152166_1", + "798fa5f2", + "ultrachat_242484", + "33bc1102", + "c7fc2bda_1", + "ultrachat_329610", + "a79f1a04_1", + "ultrachat_403700" + ] + }, + { + "answer_session_ids": [ + "answer_cfcf5340_1", + "answer_cfcf5340_2" + ], + "ndcg_at_10": 0.8503, + "ndcg_at_5": 0.8503, + "question_id": "f0e564bc", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_cfcf5340_2", + "65ca99fb_2", + "798e4ba3_2", + "bc1f1051_1", + "answer_cfcf5340_1", + "sharegpt_W3YFpVK_7", + "sharegpt_3hCiKrk_0", + "06fe8cfb_1", + "ultrachat_197576", + "932a97c6" + ] + }, + { + "answer_session_ids": [ + "answer_254d8b09_1", + "answer_254d8b09_2" + ], + "ndcg_at_10": 0.8316, + "ndcg_at_5": 1.0, + "question_id": "078150f1", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_254d8b09_1", + "61b27d67_3", + "203a48bb_1", + "46e4a8eb_1", + "ultrachat_202549", + "answer_254d8b09_2", + "0e218647_2", + "57bf2079", + "ultrachat_276164", + "b27b0097_1" + ] + }, + { + "answer_session_ids": [ + "answer_e2278b24_1", + "answer_e2278b24_2" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "8cf4d046", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e2278b24_2", + "a30f52a4", + "answer_e2278b24_1", + "sharegpt_P1Okivz_9", + "sharegpt_6DlPq48_0", + "7ef465f3_1", + "sharegpt_WjpyexI_15", + "7a5c32b0_3", + "c8b1178f_1", + "c36f4196" + ] + }, + { + "answer_session_ids": [ + "answer_4934b2d7_2", + "answer_4934b2d7_1" + ], + "ndcg_at_10": 0.8316, + "ndcg_at_5": 1.0, + "question_id": "a346bb18", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_4934b2d7_2", + "771ba1d5", + "5726dc37_2", + "fd5448f0", + "84fb50bb_1", + "answer_4934b2d7_1", + "sharegpt_VeexfGM_0", + "sharegpt_cYNuFCK_50", + "sharegpt_ooEuxDg_0", + "e5713180_1" + ] + }, + { + "answer_session_ids": [ + "answer_6b9b2b1e_2", + "answer_6b9b2b1e_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "37f165cf", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_6b9b2b1e_2", + "answer_6b9b2b1e_1", + "20b1b65f_2", + "a41d46df", + "c85d1682_1", + "c41e97c9_2", + "sharegpt_2yaKUmC_0", + "395a2f92", + "c0f6571a_2", + "7e8e790a" + ] + }, + { + "answer_session_ids": [ + "answer_477ae455_1", + "answer_477ae455_2" + ], + "ndcg_at_10": 0.0, + "ndcg_at_5": 0.0, + "question_id": "8e91e7d9", + "question_type": "multi-session", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 0.0, + "r_at_5": 0.0, + "top_session_ids": [ + "sharegpt_PZrBCF2_0", + "ultrachat_147574", + "4e3da326_1", + "bc790dfb", + "2f980ae0_2", + "c3023a45_4", + "06170195_1", + "sharegpt_8lpX5R7_19", + "c2381e1a_2", + "sharegpt_Z35PJ8x_2" + ] + }, + { + "answer_session_ids": [ + "answer_f56e6152_2", + "answer_f56e6152_1" + ], + "ndcg_at_10": 0.4306, + "ndcg_at_5": 0.3869, + "question_id": "87f22b4a", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "02b99ec3_1", + "ultrachat_350147", + "28741edc_1", + "ultrachat_182225", + "answer_f56e6152_2", + "889076d6", + "c4c504d2_6", + "answer_f56e6152_1", + "a8ac3d1d_3", + "ea28bb75" + ] + }, + { + "answer_session_ids": [ + "answer_430d0a87_1", + "answer_430d0a87_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e56a43b9", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_430d0a87_1", + "answer_430d0a87_2", + "83e14506_2", + "0eef411a_2", + "38e81260_7", + "7596ba6a", + "ce21f0e4", + "sharegpt_XaM55h6_28", + "ultrachat_245457", + "ultrachat_12201" + ] + }, + { + "answer_session_ids": [ + "answer_fc81d1a2_1", + "answer_fc81d1a2_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "efc3f7c2", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_fc81d1a2_1", + "answer_fc81d1a2_2", + "ultrachat_38938", + "263f2395", + "ultrachat_166366", + "bbdc7b0a", + "ultrachat_102676", + "ultrachat_552063", + "48cb014c", + "fc005760" + ] + }, + { + "answer_session_ids": [ + "answer_2c637141_2", + "answer_2c637141_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "21d02d0d", + "question_type": "multi-session", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_2c637141_2", + "answer_2c637141_1", + "eb8195a6_1", + "ultrachat_71299", + "sharegpt_adeEi7y_7", + "06a39bb5_1", + "42d9467c_2", + "a15fa6eb", + "sharegpt_B9jlITg_0", + "414b8ddb" + ] + }, + { + "answer_session_ids": [ + "answer_d00ba6d0_1", + "answer_d00ba6d0_2" + ], + "ndcg_at_10": 0.8772, + "ndcg_at_5": 0.8772, + "question_id": "gpt4_59149c77", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_d00ba6d0_2", + "9e2c2a6c_2", + "9242fbe3_1", + "answer_d00ba6d0_1", + "6e672b84_1", + "sharegpt_XlNs8Lz_199", + "148f87a0_2", + "54a05e72_2", + "ultrachat_318178", + "ultrachat_280434" + ] + }, + { + "answer_session_ids": [ + "answer_3e9fce53_1", + "answer_3e9fce53_2", + "answer_3e9fce53_3" + ], + "ndcg_at_10": 0.9469, + "ndcg_at_5": 0.9469, + "question_id": "gpt4_f49edff3", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_3e9fce53_3", + "answer_3e9fce53_2", + "7d33fcb4_1", + "1a29bf1f", + "answer_3e9fce53_1", + "dc7c53fa_4", + "9f8fa5e4_2", + "acfdbdcb", + "4376945f", + "ultrachat_257206" + ] + }, + { + "answer_session_ids": [ + "answer_0b4a8adc_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "71017276", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_0b4a8adc_1", + "683c8358_1", + "58ec258f_2", + "sharegpt_tCLMKZY_0", + "10889cf0_1", + "4d6927c2_2", + "sharegpt_11WxPMZ_0", + "44e4ae7e", + "398f23e7", + "sharegpt_tmFunkX_20" + ] + }, + { + "answer_session_ids": [ + "answer_4bfcc250_4", + "answer_4bfcc250_3", + "answer_4bfcc250_2", + "answer_4bfcc250_1" + ], + "ndcg_at_10": 0.8529, + "ndcg_at_5": 0.8529, + "question_id": "b46e15ed", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_4bfcc250_3", + "aeab3296", + "fa34119a_1", + "answer_4bfcc250_2", + "answer_4bfcc250_1", + "4bf66c38_2", + "a410bab2_3", + "47311a4c_1", + "ultrachat_520419", + "f598d30f_1" + ] + }, + { + "answer_session_ids": [ + "answer_ff201786_1", + "answer_ff201786_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_fa19884c", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ff201786_2", + "answer_ff201786_1", + "a7b1b27c", + "f3745025_1", + "1b066639", + "f4f4b504_2", + "800cfd94", + "ultrachat_423544", + "cbf48918_1", + "bb2180e9_3" + ] + }, + { + "answer_session_ids": [ + "answer_f4ea84fb_3", + "answer_f4ea84fb_2", + "answer_f4ea84fb_1" + ], + "ndcg_at_10": 0.6462, + "ndcg_at_5": 0.6509, + "question_id": "0bc8ad92", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "0868982f_1", + "answer_f4ea84fb_3", + "f58c36b8_1", + "answer_f4ea84fb_2", + "ultrachat_349938", + "fccc9400", + "d3cc5bdc_1", + "answer_f4ea84fb_1", + "eca613fc", + "sharegpt_scMKXPh_9" + ] + }, + { + "answer_session_ids": [ + "answer_b51b6115_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "af082822", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_b51b6115_1", + "bc5827b5", + "facef6e0_1", + "b28990c8_7", + "d048ae4b_2", + "1dfef591_2", + "cae65795_2", + "cdf068b1_2", + "1fcb4134_2", + "54f0d6f9_1" + ] + }, + { + "answer_session_ids": [ + "answer_add9b012_2", + "answer_add9b012_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_4929293a", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_add9b012_2", + "answer_add9b012_1", + "623ea729_2", + "ultrachat_329640", + "sharegpt_oQacgfY_0", + "ultrachat_512282", + "b8bc1464_4", + "73353aee_1", + "c1e598cd_1", + "sharegpt_mWFmQxk_36" + ] + }, + { + "answer_session_ids": [ + "answer_a17423e7_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_b5700ca9", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_a17423e7_1", + "44c62f26", + "5053474c_2", + "a53c7542_1", + "33bdd89a_1", + "sharegpt_FqOC5e9_169", + "d743153b_1", + "ultrachat_366301", + "47382adf_1", + "ultrachat_383774" + ] + }, + { + "answer_session_ids": [ + "answer_dba89487_2", + "answer_dba89487_1" + ], + "ndcg_at_10": 0.8316, + "ndcg_at_5": 1.0, + "question_id": "9a707b81", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_dba89487_2", + "4abbeb2a_2", + "5a13d047_1", + "ultrachat_233519", + "6862e478_2", + "answer_dba89487_1", + "990f3ef9_6", + "3fcbb58c_4", + "e1d11615_1", + "59510ab6" + ] + }, + { + "answer_session_ids": [ + "answer_febde667_1", + "answer_febde667_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_1d4ab0c9", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_febde667_2", + "answer_febde667_1", + "ab855b34_2", + "3f1d1224_3", + "7257bd15_1", + "082b7e52_1", + "ultrachat_201007", + "2dbfff2f", + "ultrachat_310433", + "sharegpt_MIYBRJz_27" + ] + }, + { + "answer_session_ids": [ + "answer_c19bd2bf_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_e072b769", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c19bd2bf_1", + "5558a42e_1", + "4f7b5dc9_1", + "b9e32ff8_1", + "2fc9ce79_2", + "sharegpt_1VhiETr_37", + "66d3fdb7_2", + "ultrachat_371691", + "sharegpt_9Ln5Ood_0", + "sharegpt_VbIxRWJ_33" + ] + }, + { + "answer_session_ids": [ + "answer_b9e32ff8_1", + "answer_b9e32ff8_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "0db4c65d", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_b9e32ff8_2", + "answer_b9e32ff8_1", + "71dc2037_2", + "953d5a4b_1", + "667465d6_1", + "sharegpt_5NoXULS_0", + "e9e19917_1", + "b22a2540", + "d36d11b9_2", + "e6c3a50a" + ] + }, + { + "answer_session_ids": [ + "answer_661b711f_1", + "answer_661b711f_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_1d80365e", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_661b711f_1", + "answer_661b711f_2", + "f7a61595_2", + "ultrachat_293442", + "805e571c_1", + "554b054e_2", + "ultrachat_561552", + "ultrachat_271411", + "3be19aed_1", + "sharegpt_5uZ7IdY_0" + ] + }, + { + "answer_session_ids": [ + "answer_5d8c99d3_1", + "answer_5d8c99d3_2", + "answer_5d8c99d3_3" + ], + "ndcg_at_10": 0.4825, + "ndcg_at_5": 0.4307, + "question_id": "gpt4_7f6b06db", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "39c1bf6a", + "928890e4_3", + "ultrachat_31232", + "answer_5d8c99d3_2", + "631e4016", + "answer_5d8c99d3_3", + "d9ca4aca", + "4bf66c38_2", + "da42526b_1", + "sharegpt_3sKfamn_27" + ] + }, + { + "answer_session_ids": [ + "answer_c4df007f_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_6dc9b45b", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c4df007f_1", + "d8f48b0a", + "ultrachat_384387", + "9adc81d7_2", + "ultrachat_355155", + "ultrachat_35147", + "afa0c2e7_5", + "94ff80d3", + "sharegpt_jyynvvk_0", + "sharegpt_0SDNlJ3_63" + ] + }, + { + "answer_session_ids": [ + "answer_56521e65_1" + ], + "ndcg_at_10": 0.5, + "ndcg_at_5": 0.5, + "question_id": "gpt4_8279ba02", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "b909542d_1", + "85f19a30_1", + "answer_56521e65_1", + "b6018747_2", + "86c5d31d", + "e1c807f9", + "62b40a05_2", + "3a735bfa_1", + "ultrachat_78027", + "1a037873_2" + ] + }, + { + "answer_session_ids": [ + "answer_c862f65a_2", + "answer_c862f65a_3", + "answer_c862f65a_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_18c2b244", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c862f65a_3", + "answer_c862f65a_2", + "answer_c862f65a_1", + "dfbc46bd_1", + "7e2c6065", + "d9e1d5fd", + "aeb58d86_4", + "ultrachat_503659", + "84896d13", + "746ae854_2" + ] + }, + { + "answer_session_ids": [ + "answer_e9ad5914_1", + "answer_e9ad5914_2", + "answer_e9ad5914_3", + "answer_e9ad5914_4", + "answer_e9ad5914_5", + "answer_e9ad5914_6" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_a1b77f9c", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e9ad5914_3", + "answer_e9ad5914_4", + "answer_e9ad5914_6", + "answer_e9ad5914_1", + "answer_e9ad5914_5", + "answer_e9ad5914_2", + "199a5225_1", + "47ec1674_2", + "d52d478b", + "f4a4a28a" + ] + }, + { + "answer_session_ids": [ + "answer_447052a5_2", + "answer_447052a5_1" + ], + "ndcg_at_10": 0.8175, + "ndcg_at_5": 1.0, + "question_id": "gpt4_1916e0ea", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_447052a5_2", + "d3fe25ff", + "280352e9", + "4b72fe4c_1", + "fa1ce406_2", + "987b5d00", + "answer_447052a5_1", + "7a36e820_2", + "dffa3157", + "sharegpt_tPA8329_0" + ] + }, + { + "answer_session_ids": [ + "answer_4d5490f1_1", + "answer_4d5490f1_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_7a0daae1", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_4d5490f1_1", + "answer_4d5490f1_2", + "8c68df0d_1", + "014a5d36", + "819dfad7_2", + "2ff297a8_1", + "bf3aebdb_2", + "1c8832b4_1", + "sharegpt_ZWqMvoL_607", + "99883f38_3" + ] + }, + { + "answer_session_ids": [ + "answer_9b09d95b_1" + ], + "ndcg_at_10": 0.3333, + "ndcg_at_5": 0.0, + "question_id": "gpt4_468eb063", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 0.0, + "top_session_ids": [ + "e60a93ff_2", + "ultrachat_108278", + "sharegpt_Sf51OjE_8", + "d97db962_2", + "sharegpt_gfhUw40_17", + "sharegpt_1rIk8tS_0", + "answer_9b09d95b_1", + "ultrachat_91161", + "sharegpt_8M7HcM3_11", + "sharegpt_ipLglky_42" + ] + }, + { + "answer_session_ids": [ + "answer_7093d898_1", + "answer_7093d898_2", + "answer_7093d898_3", + "answer_7093d898_4", + "answer_7093d898_5", + "answer_7093d898_6" + ], + "ndcg_at_10": 0.6199, + "ndcg_at_5": 0.5706, + "question_id": "gpt4_7abb270c", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "5808284e", + "ultrachat_253954", + "answer_7093d898_2", + "answer_7093d898_1", + "39948bcf_4", + "answer_7093d898_6", + "087d2b0a_2", + "sharegpt_W6t3Ck0_0", + "answer_7093d898_5", + "ultrachat_41971" + ] + }, + { + "answer_session_ids": [ + "answer_16bd5ea5_1", + "answer_16bd5ea5_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_1e4a8aeb", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_16bd5ea5_1", + "answer_16bd5ea5_2", + "28621d6a_2", + "f7838a91_1", + "faad7d7a_1", + "bc5827b5", + "15564b61_2", + "fd18e0dd_1", + "ultrachat_413946", + "ultrachat_292933" + ] + }, + { + "answer_session_ids": [ + "answer_be07688f_1", + "answer_be07688f_2" + ], + "ndcg_at_10": 0.8503, + "ndcg_at_5": 0.8503, + "question_id": "gpt4_4fc4f797", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_be07688f_1", + "39566615_1", + "814d3b22", + "47c5482a", + "answer_be07688f_2", + "660cea1d", + "bdcee74e_3", + "7e02e59b_4", + "86298245_2", + "0c783e4a_1" + ] + }, + { + "answer_session_ids": [ + "answer_4bebc782_1", + "answer_4bebc782_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "4dfccbf7", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_4bebc782_1", + "answer_4bebc782_2", + "a5e69b52_1", + "8fa624b2_6", + "491ebc5d_2", + "36f8b380_1", + "ultrachat_327454", + "sharegpt_e1DTWDa_23", + "sharegpt_cXkL3cR_69", + "ultrachat_373047" + ] + }, + { + "answer_session_ids": [ + "answer_e831a29f_1", + "answer_e831a29f_2" + ], + "ndcg_at_10": 0.8503, + "ndcg_at_5": 0.8503, + "question_id": "gpt4_61e13b3c", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e831a29f_1", + "f3fcf4fd_2", + "a6f2320f_2", + "c0d099e6_2", + "answer_e831a29f_2", + "cc380119_3", + "8ca0085a_2", + "2b5c911e_3", + "8e10bf6b_1", + "b4923852" + ] + }, + { + "answer_session_ids": [ + "answer_e6c20e52_3", + "answer_e6c20e52_2", + "answer_e6c20e52_1" + ], + "ndcg_at_10": 0.5641, + "ndcg_at_5": 0.6309, + "question_id": "gpt4_45189cb4", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "5c16fe0b_1", + "answer_e6c20e52_3", + "ultrachat_307270", + "c9d35c00_2", + "dc3ee1d1_1", + "sharegpt_hgGAUvu_0", + "ebf5b3bc_2", + "6593cb8b_1", + "66ab6260", + "answer_e6c20e52_1" + ] + }, + { + "answer_session_ids": [ + "answer_c9d35c00_1", + "answer_c9d35c00_2" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "2ebe6c90", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c9d35c00_2", + "0f73eee7_2", + "answer_c9d35c00_1", + "sharegpt_jyMHY1J_31", + "8fa64936_1", + "9ccea2f0_2", + "c3c1f84a_1", + "4d1b9b0d_4", + "453d354e_1", + "ultrachat_68168" + ] + }, + { + "answer_session_ids": [ + "answer_8c64ce25_2", + "answer_8c64ce25_1", + "answer_8c64ce25_3" + ], + "ndcg_at_10": 0.3155, + "ndcg_at_5": 0.0, + "question_id": "gpt4_e061b84f", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 0.0, + "top_session_ids": [ + "0a6bf5e4_1", + "3275acf9", + "4cc9270f", + "bab3f409", + "73bc3176_2", + "539c2346_3", + "77974d02", + "answer_8c64ce25_3", + "ultrachat_129606", + "9345f7dc_4" + ] + }, + { + "answer_session_ids": [ + "answer_61d1be50_1", + "answer_61d1be50_2" + ], + "ndcg_at_10": 0.8772, + "ndcg_at_5": 0.8772, + "question_id": "370a8ff4", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_61d1be50_1", + "11c0b1c5", + "cfde748a", + "answer_61d1be50_2", + "sharegpt_UbMVpdp_93", + "2b4eaee0_1", + "sharegpt_gVwWKqg_0", + "sharegpt_JRQtEcd_115", + "31d5ad91_1", + "8c64ce25_1" + ] + }, + { + "answer_session_ids": [ + "answer_f999b05b_1", + "answer_f999b05b_4", + "answer_f999b05b_2", + "answer_f999b05b_5", + "answer_f999b05b_3" + ], + "ndcg_at_10": 0.8401, + "ndcg_at_5": 0.8855, + "question_id": "gpt4_d6585ce8", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_f999b05b_3", + "51c82c8e_2", + "answer_f999b05b_5", + "ultrachat_225491", + "answer_f999b05b_2", + "dc0802a9_1", + "c1e598cd_3", + "6ec9bed9", + "answer_f999b05b_4", + "answer_f999b05b_1" + ] + }, + { + "answer_session_ids": [ + "answer_f964cea3_1", + "answer_f964cea3_2" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "gpt4_4ef30696", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_f964cea3_2", + "80b3f093", + "answer_f964cea3_1", + "d9868305_2", + "sharegpt_ELYlA30_25", + "4b29957c_1", + "4067bede", + "1ccb08a0", + "750b437b_1", + "sharegpt_K8mXaIT_0" + ] + }, + { + "answer_session_ids": [ + "answer_e3aa84c4_2", + "answer_e3aa84c4_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_ec93e27f", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e3aa84c4_1", + "answer_e3aa84c4_2", + "5a994ed3_2", + "sharegpt_Sg0ctDe_14", + "6b63bdc5_2", + "5de5c1ed_1", + "85846900_2", + "ultrachat_357838", + "dd049be8", + "sharegpt_1ZQntAk_0" + ] + }, + { + "answer_session_ids": [ + "answer_88841f26_1", + "answer_88841f26_2" + ], + "ndcg_at_10": 0.5, + "ndcg_at_5": 0.5, + "question_id": "6e984301", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "b38766c1_2", + "535ce50e_2", + "answer_88841f26_1", + "b500d9be", + "55c23a23_2", + "ultrachat_402356", + "954c790c_2", + "answer_88841f26_2", + "2bc4ae7d", + "018426a5" + ] + }, + { + "answer_session_ids": [ + "answer_0dd54b7c_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "8077ef71", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_0dd54b7c_1", + "f7ffddbc", + "fd88019d", + "5365840b_1", + "ultrachat_468198", + "sharegpt_Wi7Op1u_39", + "sharegpt_11WxPMZ_0", + "d87a6ef8", + "a993bf57_2", + "sharegpt_hoQsVGz_51" + ] + }, + { + "answer_session_ids": [ + "answer_d8a1af6b_1", + "answer_d8a1af6b_2", + "answer_d8a1af6b_3", + "answer_d8a1af6b_4", + "answer_d8a1af6b_5" + ], + "ndcg_at_10": 0.6181, + "ndcg_at_5": 0.5013, + "question_id": "gpt4_f420262c", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "sharegpt_jFDrHZ3_0", + "sharegpt_B0riiep_0", + "116bae09", + "answer_d8a1af6b_4", + "answer_d8a1af6b_3", + "answer_d8a1af6b_5", + "answer_d8a1af6b_1", + "answer_d8a1af6b_2", + "ultrachat_446175", + "ultrachat_548517" + ] + }, + { + "answer_session_ids": [ + "answer_d97db962_1", + "answer_d97db962_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_8e165409", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_d97db962_2", + "answer_d97db962_1", + "sharegpt_JRQtEcd_115", + "4c9f6bf5_2", + "09599d45_1", + "ultrachat_572191", + "ultrachat_106695", + "d5b3cf54_4", + "f1fe6512_1", + "ultrachat_577133" + ] + }, + { + "answer_session_ids": [ + "answer_aed8cf17_1", + "answer_aed8cf17_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_74aed68e", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_aed8cf17_2", + "answer_aed8cf17_1", + "318048af_2", + "9523d17c", + "69a9211c_1", + "25b3e36c_2", + "470f29c0_1", + "50d66391_6", + "sharegpt_O3FeoHQ_153", + "sharegpt_DXFHJtN_0" + ] + }, + { + "answer_session_ids": [ + "answer_43ba3965_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "bcbe585f", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_43ba3965_1", + "a158b7e7", + "7fa8099b_1", + "a98ff421_4", + "44f94343_1", + "5dbe11b9", + "6bed7862", + "8166673a", + "896162a2", + "8953c8c8_2" + ] + }, + { + "answer_session_ids": [ + "answer_1e2369c9_1", + "answer_1e2369c9_2" + ], + "ndcg_at_10": 0.8772, + "ndcg_at_5": 0.8772, + "question_id": "gpt4_21adecb5", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_1e2369c9_1", + "ultrachat_555122", + "ultrachat_444790", + "answer_1e2369c9_2", + "e1f37e24", + "sharegpt_ipLglky_42", + "6c574737_2", + "128082f8_2", + "ultrachat_333888", + "19f2e2c5_2" + ] + }, + { + "answer_session_ids": [ + "answer_c18d480b_1" + ], + "ndcg_at_10": 0.4307, + "ndcg_at_5": 0.4307, + "question_id": "5e1b23de", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "1a5344e9_3", + "1380576d_1", + "ultrachat_153798", + "answer_c18d480b_1", + "809021de", + "c3f945f3_1", + "a13abca5_2", + "c28aed33", + "d8e33f5c_2", + "8278312e_1" + ] + }, + { + "answer_session_ids": [ + "answer_5850de18_2", + "answer_5850de18_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_98f46fc6", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_5850de18_2", + "answer_5850de18_1", + "sharegpt_rnL5VQO_0", + "6172cc72_3", + "sharegpt_hPTUZia_0", + "ultrachat_238480", + "8e3b479f_1", + "28897044", + "ultrachat_45511", + "32e292f2" + ] + }, + { + "answer_session_ids": [ + "answer_184c8f56_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_af6db32f", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_184c8f56_1", + "aee13015_6", + "ultrachat_552923", + "sharegpt_1jjEIai_173", + "e0b8c937_1", + "ea834a17_2", + "f26846e1_3", + "b7c70f28_3", + "8930493f_2", + "sharegpt_hDjMGr7_70" + ] + }, + { + "answer_session_ids": [ + "answer_0d4d0347_1", + "answer_0d4d0347_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "eac54adc", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_0d4d0347_2", + "answer_0d4d0347_1", + "b40f0a7a", + "ae051325_3", + "bf234051_2", + "32fc393b_2", + "ultrachat_387180", + "74133043", + "75a306dd_3", + "sharegpt_PgmAp0p_0" + ] + }, + { + "answer_session_ids": [ + "answer_ad8a76cb_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_7ddcf75f", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ad8a76cb_1", + "sharegpt_ZUxBndS_26", + "8c652eb0_2", + "sharegpt_mMdyu3t_0", + "79d122f0", + "7ba84ed3_2", + "9283c152", + "ultrachat_281183", + "07e6a3bd_1", + "sharegpt_5RciDTC_0" + ] + }, + { + "answer_session_ids": [ + "answer_f6d6e33f_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_a2d1d1f6", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_f6d6e33f_1", + "ultrachat_476519", + "f5cf8815_4", + "ultrachat_575969", + "894dd0c2_3", + "2c31d8be_3", + "sharegpt_gI1U1eK_4", + "90f7041a_2", + "a1bfb382_1", + "ultrachat_68856" + ] + }, + { + "answer_session_ids": [ + "answer_581ab834_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_85da3956", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_581ab834_1", + "fcc7fe77_2", + "ultrachat_373754", + "sharegpt_h1UXagU_1", + "14b000d9_1", + "sharegpt_i59HUGf_0", + "sharegpt_ewad5Sr_0", + "8da8c7e0", + "6c184bde_2", + "97f8a545_2" + ] + }, + { + "answer_session_ids": [ + "answer_550bb2d1_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_b0863698", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_550bb2d1_1", + "91e581ad_3", + "4a79d078_2", + "63bc9fa3", + "5b014f9b_1", + "554b054e_2", + "65f6f130_1", + "833750a1", + "bdcee74e_2", + "sharegpt_NITaMP6_0" + ] + }, + { + "answer_session_ids": [ + "answer_9793daa3_2", + "answer_9793daa3_1" + ], + "ndcg_at_10": 0.8316, + "ndcg_at_5": 1.0, + "question_id": "gpt4_68e94287", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_9793daa3_2", + "60876def_1", + "50f136f2_3", + "ultrachat_209526", + "74bcc8f4_1", + "answer_9793daa3_1", + "sharegpt_nh3eDLi_9", + "sharegpt_XzAEFL4_0", + "dadd6379_1", + "afc64a5c_1" + ] + }, + { + "answer_session_ids": [ + "answer_e28c1f0d_1", + "answer_e28c1f0d_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_e414231e", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e28c1f0d_1", + "answer_e28c1f0d_2", + "9f8fc173_3", + "d3065d85", + "1bab001f", + "1980bbfa_3", + "d4439279_2", + "a490e4b8_1", + "7e3fa056_2", + "ultrachat_524205" + ] + }, + { + "answer_session_ids": [ + "answer_cf021b36_1", + "answer_cf021b36_2", + "answer_cf021b36_3" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_7ca326fa", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_cf021b36_2", + "answer_cf021b36_3", + "answer_cf021b36_1", + "8d0ca228", + "b4025421", + "5429e768_1", + "sharegpt_7SCdbX2_0", + "1301fbd1", + "sharegpt_CYVtC0w_0", + "604de0a3_2" + ] + }, + { + "answer_session_ids": [ + "answer_e22d6aef_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_7bc6cf22", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e22d6aef_1", + "e6cc6157_1", + "09d021e7_1", + "ultrachat_215356", + "c82c18dd_1", + "ultrachat_330542", + "ultrachat_465755", + "20a1ca39", + "eca3b71c_2", + "3e43bf49_1" + ] + }, + { + "answer_session_ids": [ + "answer_c9d35c00_1", + "answer_c9d35c00_2" + ], + "ndcg_at_10": 0.3562, + "ndcg_at_5": 0.0, + "question_id": "2ebe6c92", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 0.0, + "top_session_ids": [ + "9d4312f6_3", + "fb303dd2_2", + "ca3a4e4f_1", + "54c5a89c_1", + "16756728_1", + "answer_c9d35c00_2", + "84048eba_5", + "sharegpt_9dKNTQo_0", + "b30a47be_2", + "ultrachat_575902" + ] + }, + { + "answer_session_ids": [ + "answer_8c64ce26_2", + "answer_8c64ce26_1", + "answer_8c64ce26_3" + ], + "ndcg_at_10": 0.3869, + "ndcg_at_5": 0.3869, + "question_id": "gpt4_e061b84g", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "ae15e8b6_1", + "9dac9e37_1", + "68705467_1", + "3d6a2b24_1", + "answer_8c64ce26_3", + "51221cac", + "ce584ba0", + "sharegpt_rB6IP4G_0", + "ultrachat_231231", + "ultrachat_391029" + ] + }, + { + "answer_session_ids": [ + "answer_0b4a8adc_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "71017277", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_0b4a8adc_1", + "85a1be56_2", + "976ca0d9", + "f8de4e92_4", + "5854eebc_2", + "e64f9553_1", + "40a1e681_1", + "ultrachat_557308", + "ultrachat_542251", + "e5b86922_1" + ] + }, + { + "answer_session_ids": [ + "answer_4bfcc251_4", + "answer_4bfcc251_3", + "answer_4bfcc251_2", + "answer_4bfcc251_1" + ], + "ndcg_at_10": 0.8667, + "ndcg_at_5": 0.8855, + "question_id": "b46e15ee", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_4bfcc251_1", + "918c4788_1", + "answer_4bfcc251_2", + "69664c72", + "answer_4bfcc251_3", + "c92239bc", + "answer_4bfcc251_4", + "495cf518_2", + "sharegpt_RhSlNyz_0", + "aa31c397_2" + ] + }, + { + "answer_session_ids": [ + "answer_f999b05c_1", + "answer_f999b05c_4", + "answer_f999b05c_2", + "answer_f999b05c_5", + "answer_f999b05c_3" + ], + "ndcg_at_10": 0.6154, + "ndcg_at_5": 0.5438, + "question_id": "gpt4_d6585ce9", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "a554ed79_4", + "58dd3f35", + "answer_f999b05c_3", + "87f5cb44", + "answer_f999b05c_2", + "answer_f999b05c_4", + "answer_f999b05c_5", + "a7aed4cc_2", + "17d91635_2", + "d8e33f5c_abs_1" + ] + }, + { + "answer_session_ids": [ + "answer_16bd5ea6_1", + "answer_16bd5ea6_2" + ], + "ndcg_at_10": 0.3333, + "ndcg_at_5": 0.0, + "question_id": "gpt4_1e4a8aec", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 0.0, + "top_session_ids": [ + "9f1f3bf4", + "b34a52aa_3", + "a8086137", + "4c8455cb_2", + "b1265fd7_1", + "sharegpt_9JZCfee_0", + "answer_16bd5ea6_2", + "e675c69a_1", + "1c662b7b_1", + "e41b78c7_2" + ] + }, + { + "answer_session_ids": [ + "answer_d8a1af6c_1", + "answer_d8a1af6c_2", + "answer_d8a1af6c_5", + "answer_d8a1af6c_4", + "answer_d8a1af6c_3" + ], + "ndcg_at_10": 0.967, + "ndcg_at_5": 0.9829, + "question_id": "gpt4_f420262d", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_d8a1af6c_5", + "answer_d8a1af6c_2", + "answer_d8a1af6c_1", + "27639dd8_3", + "answer_d8a1af6c_3", + "sharegpt_UUFCRmF_13", + "answer_d8a1af6c_4", + "09e6f061", + "e1023f28", + "00c7b769_1" + ] + }, + { + "answer_session_ids": [ + "answer_d00ba6d1_1", + "answer_d00ba6d1_2" + ], + "ndcg_at_10": 0.4486, + "ndcg_at_5": 0.4307, + "question_id": "gpt4_59149c78", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "sharegpt_GI6737T_2", + "765ce8a7_2", + "23754665", + "answer_d00ba6d1_2", + "a8ac3d1d_1", + "84762061_1", + "sharegpt_YENXhFv_0", + "ultrachat_131385", + "answer_d00ba6d1_1", + "b2341a22" + ] + }, + { + "answer_session_ids": [ + "answer_e28c1f0e_1", + "answer_e28c1f0e_2" + ], + "ndcg_at_10": 0.7904, + "ndcg_at_5": 1.0, + "question_id": "gpt4_e414231f", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e28c1f0e_1", + "ultrachat_150948", + "07a26dfa_5", + "deb5bb60", + "135f761b_1", + "78ef59d0_1", + "sharegpt_K8mXaIT_0", + "752c438c_3", + "fe82a033_2", + "answer_e28c1f0e_2" + ] + }, + { + "answer_session_ids": [ + "answer_add9b013_2", + "answer_add9b013_1" + ], + "ndcg_at_10": 0.0, + "ndcg_at_5": 0.0, + "question_id": "gpt4_4929293b", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 0.0, + "r_at_5": 0.0, + "top_session_ids": [ + "sharegpt_KFhIUCO_0", + "b4ed7223_3", + "cae65795_1", + "ultrachat_511609", + "bda611f6_3", + "ea8c38f8", + "sharegpt_MB6e5CL_23", + "sharegpt_15lfOiQ_39", + "78bab597_2", + "ultrachat_156176" + ] + }, + { + "answer_session_ids": [ + "answer_9b09d95b_1" + ], + "ndcg_at_10": 0.4307, + "ndcg_at_5": 0.4307, + "question_id": "gpt4_468eb064", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "a2f0054f", + "db1aefb6_3", + "4d4df0e0_1", + "answer_9b09d95b_1", + "1e5bd28d_2", + "sharegpt_0PLBHdX_15", + "ultrachat_109577", + "sharegpt_P54kbvt_0", + "ultrachat_337251", + "cc3c5fa9_1" + ] + }, + { + "answer_session_ids": [ + "answer_ff201787_1", + "answer_ff201787_2" + ], + "ndcg_at_10": 0.6509, + "ndcg_at_5": 0.6509, + "question_id": "gpt4_fa19884d", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "ccc87da9_1", + "answer_ff201787_1", + "5f9dd782", + "answer_ff201787_2", + "eb4d1c82_1", + "55da3cbf_2", + "f7a61595_3", + "f910dc31_4", + "ultrachat_525919", + "ultrachat_483034" + ] + }, + { + "answer_session_ids": [ + "answer_dba89488_2", + "answer_dba89488_1" + ], + "ndcg_at_10": 0.3333, + "ndcg_at_5": 0.0, + "question_id": "9a707b82", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 0.0, + "top_session_ids": [ + "8e78fa70_1", + "fab41c07", + "79d122f0", + "sharegpt_bUaSM2v_0", + "990f3ef9_2", + "f8e9d4fa", + "answer_dba89488_1", + "7a4d00b3_2", + "ultrachat_466884", + "09e97c6f_2" + ] + }, + { + "answer_session_ids": [ + "answer_0d4d0348_1", + "answer_0d4d0348_2" + ], + "ndcg_at_10": 0.301, + "ndcg_at_5": 0.0, + "question_id": "eac54add", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 0.0, + "top_session_ids": [ + "2ea6fda4_1", + "7bb01253", + "7966888b_2", + "577da76d_2", + "2946bf12", + "c3023a45_5", + "773aebbd_3", + "369695b4_2", + "answer_0d4d0348_2", + "28269633_2" + ] + }, + { + "answer_session_ids": [ + "answer_4bebc783_1", + "answer_4bebc783_2" + ], + "ndcg_at_10": 0.7977, + "ndcg_at_5": 1.0, + "question_id": "4dfccbf8", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_4bebc783_1", + "sharegpt_3D3oQC0_213", + "a63ad8e3_3", + "1c177942_4", + "b4f63a70_3", + "7db28e68_1", + "1bc87711_1", + "edd89480_1", + "answer_4bebc783_2", + "ultrachat_444490" + ] + }, + { + "answer_session_ids": [ + "answer_f4ea84fc_3", + "answer_f4ea84fc_2", + "answer_f4ea84fc_1" + ], + "ndcg_at_10": 0.9469, + "ndcg_at_5": 0.9469, + "question_id": "0bc8ad93", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_f4ea84fc_1", + "answer_f4ea84fc_2", + "97338ddd_1", + "2c185a2b_1", + "answer_f4ea84fc_3", + "ultrachat_260376", + "dd25eeb5_2", + "7285299a_2", + "d59d1a8d_1", + "0d8f8cc0_2" + ] + }, + { + "answer_session_ids": [ + "answer_88841f27_1", + "answer_88841f27_2" + ], + "ndcg_at_10": 0.3562, + "ndcg_at_5": 0.0, + "question_id": "6e984302", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 0.0, + "top_session_ids": [ + "7285299a_8", + "de1f4aec_1", + "3535dbf0_4", + "7cd7c296_1", + "c9dc9158_1", + "answer_88841f27_1", + "6efebca9", + "96c743d0_2", + "c03fc56c_1", + "sharegpt_4Wjx3pH_0" + ] + }, + { + "answer_session_ids": [ + "answer_56521e66_1" + ], + "ndcg_at_10": 0.0, + "ndcg_at_5": 0.0, + "question_id": "gpt4_8279ba03", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 0.0, + "r_at_5": 0.0, + "top_session_ids": [ + "b357fb8b_2", + "ultrachat_526748", + "2ef55f49_3", + "864a563d_5", + "49b78a55_1", + "518d26d3_4", + "bb107057_2", + "ba9f938b_1", + "570fe405", + "50d66391_4" + ] + }, + { + "answer_session_ids": [ + "answer_a17423e8_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_b5700ca0", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_a17423e8_1", + "534be93d_1", + "93395e5f_3", + "109e07e8_2", + "sharegpt_FpTfRvR_0", + "ultrachat_23298", + "b403d6f0_4", + "e789afdb_2", + "ultrachat_101752", + "ultrachat_160178" + ] + }, + { + "answer_session_ids": [ + "answer_9793daa4_2", + "answer_9793daa4_1" + ], + "ndcg_at_10": 0.6309, + "ndcg_at_5": 0.6309, + "question_id": "gpt4_68e94288", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "4c49e37f", + "answer_9793daa4_1", + "128f4e4d_2", + "9d2fdc71", + "c99dcd81", + "0a7c4f1c_1", + "767b1b89", + "cff08ec1_1", + "ultrachat_396489", + "d2606fe9" + ] + }, + { + "answer_session_ids": [ + "answer_4be1b6b4_1", + "answer_4be1b6b4_2", + "answer_4be1b6b4_3" + ], + "ndcg_at_10": 0.5391, + "ndcg_at_5": 0.5, + "question_id": "gpt4_2655b836", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "d8a1af6b_4", + "9e05d9a0", + "answer_4be1b6b4_2", + "ultrachat_268202", + "e3e66b50", + "0f2434e8", + "answer_4be1b6b4_3", + "answer_4be1b6b4_1", + "b876911c", + "ea8cc87b" + ] + }, + { + "answer_session_ids": [ + "answer_1c6b85ea_2", + "answer_1c6b85ea_1" + ], + "ndcg_at_10": 0.6934, + "ndcg_at_5": 0.6934, + "question_id": "gpt4_2487a7cb", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "4f5880c6_6", + "answer_1c6b85ea_1", + "answer_1c6b85ea_2", + "sharegpt_QN26oUg_0", + "67a8ef5f", + "f87fea58_4", + "sharegpt_CHaU4ax_10", + "sharegpt_XNcyasy_17", + "4c1982d8_2", + "sharegpt_EJC4YOh_0" + ] + }, + { + "answer_session_ids": [ + "answer_b535969f_1", + "answer_b535969f_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_76048e76", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_b535969f_1", + "answer_b535969f_2", + "6a78e959_2", + "33a39aa7_2", + "88e01130_1", + "33630c9e_2", + "sharegpt_okVqBzp_13", + "ultrachat_273271", + "sharegpt_inuIr9J_119", + "sharegpt_vx14orQ_11" + ] + }, + { + "answer_session_ids": [ + "answer_5328c3c2_2", + "answer_5328c3c2_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_2312f94c", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_5328c3c2_2", + "answer_5328c3c2_1", + "a764b677_2", + "869fe9ba_2", + "sharegpt_jE25Ibo_0", + "3bbf4800_1", + "a1c1739c_3", + "e831a29f_2", + "ultrachat_44021", + "sharegpt_XYtuANl_9" + ] + }, + { + "answer_session_ids": [ + "answer_e936197f_1", + "answer_e936197f_2" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "0bb5a684", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e936197f_1", + "1a14d6cb_1", + "answer_e936197f_2", + "3c8e7c0e_3", + "ultrachat_62919", + "565929a2_2", + "b10d0907_2", + "826d51da_3", + "d794dec9_2", + "sharegpt_d3UV0eu_0" + ] + }, + { + "answer_session_ids": [ + "answer_6ea1541e_2", + "answer_6ea1541e_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "08f4fc43", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_6ea1541e_1", + "answer_6ea1541e_2", + "f3bd00ec_2", + "ultrachat_292416", + "sharegpt_JJu53iW_0", + "50303fd0_1", + "9216c4d3_4", + "sharegpt_zqQa6sO_0", + "sharegpt_wXqtiUe_0", + "cdbd6862_1" + ] + }, + { + "answer_session_ids": [ + "answer_d39b7977_1", + "answer_d39b7977_2" + ], + "ndcg_at_10": 0.8772, + "ndcg_at_5": 0.8772, + "question_id": "2c63a862", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_d39b7977_2", + "93a5cfe6", + "33baaaab_1", + "answer_d39b7977_1", + "1854abc7_2", + "ad175dc4_2", + "0dde26d4_3", + "ultrachat_179062", + "ultrachat_149914", + "918c4788_2" + ] + }, + { + "answer_session_ids": [ + "answer_7a4a93f1_2", + "answer_7a4a93f1_1" + ], + "ndcg_at_10": 0.8772, + "ndcg_at_5": 0.8772, + "question_id": "gpt4_385a5000", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_7a4a93f1_2", + "8062f4ab_2", + "80aefc49_1", + "answer_7a4a93f1_1", + "29d27eaa_2", + "sharegpt_hcaZN94_507", + "28f74b09_1", + "ultrachat_480527", + "ultrachat_528011", + "ultrachat_286149" + ] + }, + { + "answer_session_ids": [ + "answer_1cc3cd0c_1", + "answer_1cc3cd0c_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "2a1811e2", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_1cc3cd0c_2", + "answer_1cc3cd0c_1", + "b0e040c1", + "901a6763_2", + "ecfd2047_1", + "e9fb10e7_1", + "ultrachat_187883", + "sharegpt_c9R1uSD_0", + "89e05490", + "f5a4d792_1" + ] + }, + { + "answer_session_ids": [ + "answer_b3763b6b_1", + "answer_b3763b6b_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "bbf86515", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_b3763b6b_1", + "answer_b3763b6b_2", + "df003c93_2", + "592cb8d2", + "75b98d2a_1", + "0c260a71_2", + "e9e19917_1", + "33d0e4b3_2", + "sharegpt_d1Rh99l_63", + "sharegpt_G8j6c8J_38" + ] + }, + { + "answer_session_ids": [ + "answer_099c1b6c_5", + "answer_099c1b6c_2", + "answer_099c1b6c_4", + "answer_099c1b6c_3", + "answer_099c1b6c_1" + ], + "ndcg_at_10": 0.9896, + "ndcg_at_5": 1.0, + "question_id": "gpt4_5dcc0aab", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_099c1b6c_3", + "answer_099c1b6c_1", + "answer_099c1b6c_5", + "answer_099c1b6c_2", + "353d3c6d_1", + "answer_099c1b6c_4", + "3a192d16_1", + "3891baea_3", + "b10c041b_1", + "b31f923a_2" + ] + }, + { + "answer_session_ids": [ + "answer_c4e5d969_2", + "answer_c4e5d969_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_0b2f1d21", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c4e5d969_2", + "answer_c4e5d969_1", + "1cafd864_3", + "f31a0422_1", + "sharegpt_rhWNt54_19", + "ultrachat_455430", + "ultrachat_49268", + "sharegpt_4pDZgCL_14", + "5260c678_2", + "ultrachat_1415" + ] + }, + { + "answer_session_ids": [ + "answer_932ea3bf_2", + "answer_932ea3bf_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "f0853d11", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_932ea3bf_2", + "answer_932ea3bf_1", + "bd8708e2_2", + "sharegpt_9upOSlE_27", + "10c0752c", + "b16a8656_2", + "6e436d54", + "ultrachat_325202", + "ultrachat_114660", + "4c967baa_2" + ] + }, + { + "answer_session_ids": [ + "answer_d50a8a33_1", + "answer_d50a8a33_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_6ed717ea", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_d50a8a33_2", + "answer_d50a8a33_1", + "637bc32a_2", + "15ed03f0_1", + "7e7ecab6", + "d4b6ec0b", + "sharegpt_HQMYIkt_0", + "sharegpt_PuRfIep_7", + "8e657ff7_1", + "e0585cb5_1" + ] + }, + { + "answer_session_ids": [ + "answer_b3070ec4_2", + "answer_b3070ec4_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_70e84552", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_b3070ec4_1", + "answer_b3070ec4_2", + "ultrachat_352927", + "2b9b6224_2", + "30491f6f_1", + "45b155ec_2", + "3f787a78_1", + "ultrachat_573894", + "6ebd9e18_1", + "d5527e62" + ] + }, + { + "answer_session_ids": [ + "answer_4ffa04a2_1", + "answer_4ffa04a2_6", + "answer_4ffa04a2_4", + "answer_4ffa04a2_3", + "answer_4ffa04a2_5", + "answer_4ffa04a2_2" + ], + "ndcg_at_10": 0.9797, + "ndcg_at_5": 1.0, + "question_id": "a3838d2b", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_4ffa04a2_1", + "answer_4ffa04a2_6", + "answer_4ffa04a2_4", + "answer_4ffa04a2_2", + "answer_4ffa04a2_5", + "06317e04", + "8cf0d2a9", + "10305ca9_2", + "638e7094_1", + "answer_4ffa04a2_3" + ] + }, + { + "answer_session_ids": [ + "answer_e5131a1b_2", + "answer_e5131a1b_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_93159ced", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e5131a1b_2", + "ultrachat_387032", + "460be54a", + "ultrachat_365033", + "sharegpt_A83arI9_383", + "sharegpt_9srz3L6_0", + "ultrachat_162445", + "ultrachat_475376", + "ultrachat_304047", + "ultrachat_244924" + ] + }, + { + "answer_session_ids": [ + "answer_3e11e0ae_1", + "answer_3e11e0ae_2" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "gpt4_2d58bcd6", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_3e11e0ae_1", + "0bd0a01c", + "answer_3e11e0ae_2", + "be32ad25_1", + "1bf8b212", + "sharegpt_4tLDGmp_0", + "sharegpt_Xl3kKYf_15", + "3c53154d_1", + "1b205d70_1", + "47e1f8dd_1" + ] + }, + { + "answer_session_ids": [ + "answer_30dfe889_1", + "answer_30dfe889_2" + ], + "ndcg_at_10": 0.6934, + "ndcg_at_5": 0.6934, + "question_id": "gpt4_65aabe59", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "6a7e3aec", + "answer_30dfe889_2", + "answer_30dfe889_1", + "77b8fcd9_1", + "10771408_4", + "31cde680", + "05a37787", + "ultrachat_190669", + "78bcce6a", + "ultrachat_169466" + ] + }, + { + "answer_session_ids": [ + "answer_ab603dd5_1", + "answer_ab603dd5_2" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "982b5123", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ab603dd5_1", + "07ba9acd_2", + "answer_ab603dd5_2", + "e36b0031", + "1bfd5a8b_3", + "359a2a0d", + "f89f184e_1", + "bf8ab267", + "91168bd2_1", + "a200b713_4" + ] + }, + { + "answer_session_ids": [ + "answer_5e3bb940_3", + "answer_5e3bb940_2", + "answer_5e3bb940_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "b9cfe692", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_5e3bb940_1", + "answer_5e3bb940_2", + "answer_5e3bb940_3", + "ed88aded_2", + "sharegpt_oXgiN7q_53", + "sharegpt_JRQtEcd_115", + "2714885f_3", + "da650f3b", + "91d847c1_6", + "ultrachat_265626" + ] + }, + { + "answer_session_ids": [ + "answer_0a00c163_1", + "answer_0a00c163_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_4edbafa2", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_0a00c163_2", + "answer_0a00c163_1", + "22be78fc_2", + "18c0af9c", + "4a006b77", + "725f95e1", + "sharegpt_TQNHiIB_12", + "sharegpt_jZOf9E5_21", + "sharegpt_6MPnc5F_9", + "7533d546_5" + ] + }, + { + "answer_session_ids": [ + "answer_70dc7d08_2", + "answer_70dc7d08_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "c8090214", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_70dc7d08_1", + "answer_70dc7d08_2", + "a4c2a6f7_2", + "sharegpt_xOzzaof_0", + "005ec7cf_1", + "d6c6dadf", + "162ad3bf_1", + "c21c9d97", + "sharegpt_BQ3jrV3_0", + "sharegpt_3cVjZ2b_8" + ] + }, + { + "answer_session_ids": [ + "answer_fb793c87_1", + "answer_fb793c87_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_483dd43c", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_fb793c87_1", + "answer_fb793c87_2", + "3fd56ebf", + "52161c9e_1", + "19c24c11", + "29695e1c_3", + "ultrachat_48790", + "sharegpt_Sg0ctDe_0", + "4243ce01_1", + "ultrachat_374858" + ] + }, + { + "answer_session_ids": [ + "answer_cf425855_1", + "answer_cf425855_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e4e14d04", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_cf425855_2", + "answer_cf425855_1", + "a92165a5", + "ffc627f2", + "5944b36a_1", + "ultrachat_542420", + "f8de4e92_2", + "1c1f5ccc_3", + "ultrachat_325291", + "ultrachat_200098" + ] + }, + { + "answer_session_ids": [ + "answer_cdba3d9f_1", + "answer_cdba3d9f_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "c9f37c46", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_cdba3d9f_1", + "answer_cdba3d9f_2", + "b38766c1_3", + "0cf86cbe_1", + "4ffa04a2_1", + "2ba1710e_1", + "02df1225_1", + "7c9b09ac", + "211bb1c2_1", + "9d4312f6_3" + ] + }, + { + "answer_session_ids": [ + "answer_9af4e346_1", + "answer_9af4e346_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_2c50253f", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_9af4e346_2", + "answer_9af4e346_1", + "ultrachat_120586", + "sharegpt_a6pZtrg_0", + "0138fdec", + "dd81b163_2", + "b6f35f3a_1", + "65bc93cf_2", + "sharegpt_ueZYbCA_15", + "ultrachat_267493" + ] + }, + { + "answer_session_ids": [ + "answer_5e3eeb12_2", + "answer_5e3eeb12_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "dcfa8644", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_5e3eeb12_2", + "answer_5e3eeb12_1", + "b2243528_1", + "0d129c99", + "ultrachat_431004", + "7de51ffe_1", + "sharegpt_NCuLVp2_0", + "b906870f_1", + "4eedac88_1", + "sharegpt_hUAh1Tu_0" + ] + }, + { + "answer_session_ids": [ + "answer_b9d9150e_2", + "answer_b9d9150e_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_b4a80587", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_b9d9150e_2", + "answer_b9d9150e_1", + "ultrachat_389307", + "3cb41ab8_1", + "sharegpt_4tqCoYJ_115", + "9071ba70_2", + "55bcfb77", + "sharegpt_IT52MRw_37", + "sharegpt_HxOhjmq_29", + "9b4f5c73" + ] + }, + { + "answer_session_ids": [ + "answer_8a42fedf_1", + "answer_8a42fedf_3", + "answer_8a42fedf_2" + ], + "ndcg_at_10": 0.9218, + "ndcg_at_5": 1.0, + "question_id": "gpt4_9a159967", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_8a42fedf_2", + "answer_8a42fedf_3", + "ultrachat_534927", + "253742b4_3", + "300cfc4a", + "4ea94f85", + "answer_8a42fedf_1", + "7169e342_2", + "sharegpt_MJrfKtS_0", + "75671e3f_1" + ] + }, + { + "answer_session_ids": [ + "answer_be73098b_2", + "answer_be73098b_1" + ], + "ndcg_at_10": 0.8316, + "ndcg_at_5": 1.0, + "question_id": "cc6d1ec1", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_be73098b_2", + "22db3cc3_4", + "9a297400_4", + "aabf201b", + "ultrachat_316268", + "answer_be73098b_1", + "e549b6b2_2", + "33baaaab_3", + "ultrachat_436945", + "sharegpt_iNlPmuC_9" + ] + }, + { + "answer_session_ids": [ + "answer_72d9aa58_1", + "answer_72d9aa58_2" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "gpt4_8c8961ae", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_72d9aa58_1", + "3b22d17b", + "answer_72d9aa58_2", + "1d14730c_1", + "1cde7ee4_2", + "02f9bedb_2", + "b77d0b58", + "37f0ce6b_1", + "ultrachat_265707", + "d7281662_2" + ] + }, + { + "answer_session_ids": [ + "answer_da704e79_2", + "answer_da704e79_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_d9af6064", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_da704e79_1", + "answer_da704e79_2", + "sharegpt_w9YUs4c_0", + "sharegpt_VbIxRWJ_33", + "0d9a6f01", + "98c83fe4", + "ultrachat_519452", + "sharegpt_brc2wJS_109", + "3e3b6d77_2", + "6cff6108" + ] + }, + { + "answer_session_ids": [ + "answer_6a78e959_2", + "answer_6a78e959_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_7de946e7", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_6a78e959_2", + "answer_6a78e959_1", + "caa00337_2", + "sharegpt_6Yfj9A3_5", + "sharegpt_mjN4DdI_0", + "6155addd_2", + "f1cf6f00_1", + "c4d370d3_1", + "sharegpt_BacsrkR_0", + "ultrachat_560313" + ] + }, + { + "answer_session_ids": [ + "answer_991d55e5_1", + "answer_991d55e5_2" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "d01c6aa8", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_991d55e5_2", + "2adf224b_2", + "answer_991d55e5_1", + "64460e9d_1", + "f2fa97f5_1", + "fc9a62c0_1", + "7d1f5395", + "ultrachat_333097", + "51fe163e_2", + "sharegpt_YLhXcw8_79" + ] + }, + { + "answer_session_ids": [ + "answer_54f0d6f9_1", + "answer_54f0d6f9_2" + ], + "ndcg_at_10": 0.8772, + "ndcg_at_5": 0.8772, + "question_id": "993da5e2", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_54f0d6f9_1", + "bcf96d28_3", + "b16a8656_2", + "answer_54f0d6f9_2", + "8953c8c8_3", + "a6c0f032_3", + "c69a28f0", + "ultrachat_266485", + "sharegpt_wpbTVG0_0", + "sharegpt_v11Tpg9_0" + ] + }, + { + "answer_session_ids": [ + "answer_016f6bd4_2", + "answer_016f6bd4_1" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "a3045048", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_016f6bd4_1", + "536a334d", + "answer_016f6bd4_2", + "aaf71ce2_2", + "d837b5fa", + "5cb58cc6", + "sharegpt_asoSnhq_0", + "d12d5a98_1", + "ultrachat_406386", + "sharegpt_LC9Pzkr_0" + ] + }, + { + "answer_session_ids": [ + "answer_8464304d_1", + "answer_8464304d_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_d31cdae3", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_8464304d_2", + "answer_8464304d_1", + "41743aae_2", + "ultrachat_74113", + "de695a4e", + "55033f6f", + "809cabca_2", + "ultrachat_324099", + "5ace29d7_1", + "ultrachat_169591" + ] + }, + { + "answer_session_ids": [ + "answer_aa930b56_1", + "answer_aa930b56_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_cd90e484", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_aa930b56_2", + "answer_aa930b56_1", + "9aad36bb_1", + "08c306e2", + "ultrachat_347068", + "4d452cdb_3", + "835e1b84_2", + "4bfcc251_1", + "6043dcf6_1", + "018eac29" + ] + }, + { + "answer_session_ids": [ + "answer_e60a93ff_1", + "answer_e60a93ff_2" + ], + "ndcg_at_10": 0.6509, + "ndcg_at_5": 0.6509, + "question_id": "gpt4_88806d6e", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "ultrachat_562852", + "answer_e60a93ff_1", + "sharegpt_bJh9LPd_0", + "answer_e60a93ff_2", + "20b1b65f_1", + "sharegpt_XYtuANl_11", + "ultrachat_258611", + "c11fb634_1", + "3c11dc79", + "24bfc009_5" + ] + }, + { + "answer_session_ids": [ + "answer_5fcca8bc_2", + "answer_5fcca8bc_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_4cd9eba1", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_5fcca8bc_2", + "answer_5fcca8bc_1", + "f54d04cc", + "456860e7_2", + "6b9b2b1e_1", + "d61f3bb5", + "8242fa3f_1", + "2ef53f61_4", + "0e193841_6", + "sharegpt_2nvkR60_0" + ] + }, + { + "answer_session_ids": [ + "answer_544fe66c_2", + "answer_544fe66c_1", + "answer_544fe66c_3" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_93f6379c", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_544fe66c_2", + "answer_544fe66c_3", + "1b9b2960_3", + "b8a688f6", + "sharegpt_cfESFSo_0", + "e06c6d99_5", + "c07b8034", + "ultrachat_443550", + "d3a34811", + "82e27408_1" + ] + }, + { + "answer_session_ids": [ + "answer_436d4309_2", + "answer_436d4309_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "b29f3365", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_436d4309_2", + "answer_436d4309_1", + "6b7605d1_1", + "6a2f9452_1", + "06ee3665", + "ultrachat_243663", + "95c9666f_5", + "sharegpt_K0i5Zon_0", + "ultrachat_58020", + "4ccc2061" + ] + }, + { + "answer_session_ids": [ + "answer_7a36e820_2", + "answer_7a36e820_3", + "answer_7a36e820_1" + ], + "ndcg_at_10": 0.6871, + "ndcg_at_5": 0.6934, + "question_id": "gpt4_2f56ae70", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "3aac691d_2", + "answer_7a36e820_2", + "answer_7a36e820_1", + "f8e8d445_4", + "647f5851", + "ultrachat_159439", + "answer_7a36e820_3", + "83cd701b_3", + "e61d966a_1", + "sharegpt_KzDwaf1_165" + ] + }, + { + "answer_session_ids": [ + "answer_aaf71ce2_3", + "answer_aaf71ce2_2", + "answer_aaf71ce2_1" + ], + "ndcg_at_10": 0.8772, + "ndcg_at_5": 0.8772, + "question_id": "6613b389", + "question_type": "temporal-reasoning", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_aaf71ce2_2", + "e3d8da67", + "5eef2132", + "answer_aaf71ce2_3", + "55bc2e1e", + "2780323d_3", + "f1cc1ddb_1", + "1a65880d", + "2366adbc", + "86c8f348_1" + ] + }, + { + "answer_session_ids": [ + "answer_5a78688d_1", + "answer_5a78688d_2" + ], + "ndcg_at_10": 0.8772, + "ndcg_at_5": 0.8772, + "question_id": "gpt4_78cf46a3", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_5a78688d_1", + "b89a3ec3", + "sharegpt_rx39ipj_0", + "answer_5a78688d_2", + "sharegpt_89M8Aiz_0", + "52b19cba_1", + "ultrachat_305042", + "e8e65a1f", + "a8b4bfb3_1", + "27099532_2" + ] + }, + { + "answer_session_ids": [ + "answer_a68db5db_2", + "answer_a68db5db_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_0a05b494", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_a68db5db_2", + "answer_a68db5db_1", + "0abe3e51", + "sharegpt_24ey7mz_0", + "ultrachat_135150", + "48b68664", + "30677a32", + "ultrachat_273213", + "18687689", + "ultrachat_105014" + ] + }, + { + "answer_session_ids": [ + "answer_faad7d7a_2", + "answer_faad7d7a_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_1a1dc16d", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_faad7d7a_2", + "answer_faad7d7a_1", + "87f1f950_2", + "c12d5181_1", + "14940349_2", + "ultrachat_428346", + "ultrachat_63236", + "ultrachat_92302", + "8bd3814a_2", + "661ee6d9_1" + ] + }, + { + "answer_session_ids": [ + "answer_11a8f823_2", + "answer_11a8f823_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_2f584639", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_11a8f823_2", + "answer_11a8f823_1", + "55ca8b20_3", + "ca555919_3", + "sharegpt_bq9G6bT_2", + "4b3a0ee6_1", + "c6eb17f0_1", + "ultrachat_101401", + "68131948", + "d441f4c5" + ] + }, + { + "answer_session_ids": [ + "answer_53582e7e_2", + "answer_53582e7e_1" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_213fd887", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_53582e7e_2", + "answer_53582e7e_1", + "9bdbfc62_2", + "10771408_1", + "ultrachat_165091", + "sharegpt_EXJAjmw_7", + "8aa17385", + "dc1f9713_1", + "368276ab_3", + "d10f0307_3" + ] + }, + { + "answer_session_ids": [ + "answer_b10f3828_1", + "answer_b10f3828_2" + ], + "ndcg_at_10": 0.6509, + "ndcg_at_5": 0.6509, + "question_id": "gpt4_5438fa52", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "ultrachat_69599", + "answer_b10f3828_2", + "a8e24e22", + "answer_b10f3828_1", + "290fe09a", + "sharegpt_LVUZee8_35", + "b4f07fef_2", + "ab8dce45", + "d68bab3c_1", + "528f481d_6" + ] + }, + { + "answer_session_ids": [ + "answer_d8e33f5c_1", + "answer_d8e33f5c_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "gpt4_c27434e8", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_d8e33f5c_2", + "answer_d8e33f5c_1", + "sharegpt_9XlbWvL_0", + "16933961", + "6e2cca63_3", + "4d578deb", + "fd56c5bd_4", + "5d1c7bf7", + "ultrachat_212645", + "sharegpt_DVswHqE_0" + ] + }, + { + "answer_session_ids": [ + "answer_65600ff6_2", + "answer_65600ff6_1" + ], + "ndcg_at_10": 0.6241, + "ndcg_at_5": 0.6241, + "question_id": "gpt4_fe651585", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "e30c4a7b_2", + "answer_65600ff6_1", + "ff6b8624", + "9f220c66_1", + "answer_65600ff6_2", + "ultrachat_576354", + "da828711", + "sharegpt_izuEpAx_20", + "sharegpt_mJ7MnQS_219", + "cee0b8d5" + ] + }, + { + "answer_session_ids": [ + "answer_124f5dc3_2", + "answer_124f5dc3_1" + ], + "ndcg_at_10": 0.6053, + "ndcg_at_5": 0.6309, + "question_id": "8c18457d", + "question_type": "temporal-reasoning", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "98cd882c_2", + "answer_124f5dc3_1", + "deb12028_5", + "3c8e7c0e_2", + "2dfc1443_1", + "answer_124f5dc3_2", + "b53f447a_1", + "dc0802a9_3", + "sharegpt_2wY0N2C_0", + "1a5369b9" + ] + }, + { + "answer_session_ids": [ + "answer_a25d4a91_1", + "answer_a25d4a91_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "6a1eabeb", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_a25d4a91_2", + "answer_a25d4a91_1", + "d03995c6", + "4aaf678f_3", + "59a6150d_1", + "6da0bba2_1", + "9f80607d", + "6387e969", + "sharegpt_ZWqMvoL_178", + "sharegpt_9ApEWXK_0" + ] + }, + { + "answer_session_ids": [ + "answer_3f9693b7_1", + "answer_3f9693b7_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "6aeb4375", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_3f9693b7_2", + "answer_3f9693b7_1", + "402e0082_5", + "sharegpt_MWlvrvZ_0", + "1bc87711_1", + "2ed7c45e_2", + "sharegpt_wbbH6cZ_0", + "93bed3d8_1", + "e47d38f4_2", + "b65a30db" + ] + }, + { + "answer_session_ids": [ + "answer_0b1a0942_1", + "answer_0b1a0942_2" + ], + "ndcg_at_10": 0.8772, + "ndcg_at_5": 0.8772, + "question_id": "830ce83f", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_0b1a0942_1", + "8d77be9a_2", + "47c5482a", + "answer_0b1a0942_2", + "dfbda5e9", + "020149a5_1", + "639cec58", + "8bc305e4_2", + "d1e7d11c_2", + "e19c8fd9_1" + ] + }, + { + "answer_session_ids": [ + "answer_3a6f1e82_1", + "answer_3a6f1e82_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "852ce960", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_3a6f1e82_1", + "answer_3a6f1e82_2", + "b0fd1357", + "aaf22693", + "d36d11b9_2", + "a110c4dc_1", + "a76e7e3c_1", + "sharegpt_FYhsZ0Q_0", + "010a28ab_3", + "sharegpt_uiECyCD_66" + ] + }, + { + "answer_session_ids": [ + "answer_6a4f8626_1", + "answer_6a4f8626_2" + ], + "ndcg_at_10": 0.8316, + "ndcg_at_5": 1.0, + "question_id": "945e3d21", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_6a4f8626_2", + "sharegpt_E7VUcfj_0", + "5fcca8bc_1", + "9418c255", + "ultrachat_518638", + "answer_6a4f8626_1", + "dc880372_3", + "sharegpt_zn81iL6_9", + "87252d80_2", + "9bbd38a6" + ] + }, + { + "answer_session_ids": [ + "answer_eecb10d9_1", + "answer_eecb10d9_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "d7c942c3", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_eecb10d9_2", + "answer_eecb10d9_1", + "sharegpt_ezPxUrC_0", + "sharegpt_LOF3smB_25", + "ddf0f116_3", + "sharegpt_vYQdLPe_2", + "sharegpt_OTILwRq_0", + "e5b72a5d_3", + "b2243528_1", + "d15c9567_2" + ] + }, + { + "answer_session_ids": [ + "answer_c44b9df4_1", + "answer_c44b9df4_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "71315a70", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c44b9df4_2", + "answer_c44b9df4_1", + "ultrachat_399278", + "c86a22a2_2", + "b5e94516_2", + "8cfa91c3_1", + "ultrachat_42515", + "sharegpt_ZUxBndS_26", + "ultrachat_119182", + "a08fbe88_1" + ] + }, + { + "answer_session_ids": [ + "answer_e1403127_1", + "answer_e1403127_2" + ], + "ndcg_at_10": 0.8772, + "ndcg_at_5": 0.8772, + "question_id": "89941a93", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e1403127_1", + "8e72316f", + "ultrachat_400703", + "answer_e1403127_2", + "ultrachat_349321", + "sharegpt_bn0XI2B_0", + "ultrachat_352196", + "ultrachat_214088", + "ultrachat_150565", + "ultrachat_151269" + ] + }, + { + "answer_session_ids": [ + "answer_73540165_1", + "answer_73540165_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "ce6d2d27", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_73540165_2", + "answer_73540165_1", + "7a4d00b3_2", + "d8d7eddf_2", + "48ad2a93", + "a46d10a2_1", + "9f6bec4f", + "cd92ac85_1", + "4da24fe5_2", + "sharegpt_jZOf9E5_61" + ] + }, + { + "answer_session_ids": [ + "answer_02e66dec_1", + "answer_02e66dec_2" + ], + "ndcg_at_10": 0.6934, + "ndcg_at_5": 0.6934, + "question_id": "9ea5eabc", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "418c27fa", + "answer_02e66dec_1", + "answer_02e66dec_2", + "62e6593e_1", + "ultrachat_243996", + "d1f30ac6_5", + "6df6fff4", + "8c8b180e", + "7c44bc2d_1", + "6e2cca63_1" + ] + }, + { + "answer_session_ids": [ + "answer_7e9ad7b4_1", + "answer_7e9ad7b4_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "07741c44", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_7e9ad7b4_2", + "answer_7e9ad7b4_1", + "sharegpt_UGg8d44_4", + "af4d4ecd_4", + "sharegpt_CEWG2XW_0", + "d1d6b55b_1", + "3b4c63ba_3", + "aa71db11_1", + "1e0c4b50", + "sharegpt_fH7KHW1_119" + ] + }, + { + "answer_session_ids": [ + "answer_0eb23770_1", + "answer_0eb23770_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "a1eacc2a", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_0eb23770_1", + "answer_0eb23770_2", + "1f035408", + "7daa121d_2", + "521255f7_2", + "fde0ca18", + "93f23301_1", + "80082060_4", + "1ccb1920", + "sharegpt_ePBzYD0_57" + ] + }, + { + "answer_session_ids": [ + "answer_e2f4f947_1", + "answer_e2f4f947_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "184da446", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e2f4f947_2", + "answer_e2f4f947_1", + "bf633415_2", + "82a89303", + "85a49fe0_2", + "341a737e", + "ultrachat_195877", + "sharegpt_w7bjHzU_103", + "30a3c930", + "61ad6074" + ] + }, + { + "answer_session_ids": [ + "answer_8748f791_1", + "answer_8748f791_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "031748ae", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_8748f791_1", + "answer_8748f791_2", + "ultrachat_406392", + "5ac7f9c5_3", + "e0f71050_2", + "39a056ae", + "ultrachat_556006", + "ultrachat_69185", + "ultrachat_219083", + "7a085b95_1" + ] + }, + { + "answer_session_ids": [ + "answer_766ab8da_1", + "answer_766ab8da_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "4d6b87c8", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_766ab8da_2", + "answer_766ab8da_1", + "5c85023a_1", + "cffde796", + "ultrachat_40253", + "60d4332d", + "sharegpt_NUyiNZE_0", + "9af4e346_1", + "sharegpt_mmYyki5_0", + "a0201607_3" + ] + }, + { + "answer_session_ids": [ + "answer_d6d2eba8_1", + "answer_d6d2eba8_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "0f05491a", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_d6d2eba8_1", + "answer_d6d2eba8_2", + "27aba903_2", + "0c260a71_3", + "9760f4ca", + "1b205d70_2", + "e2efa128", + "13905cf6", + "9260a72d", + "sharegpt_DM7idTU_0" + ] + }, + { + "answer_session_ids": [ + "answer_cdbe2250_1", + "answer_cdbe2250_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "08e075c7", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_cdbe2250_2", + "answer_cdbe2250_1", + "b1d9d555_1", + "f7344cf4", + "ae77c245", + "25b3e36c_1", + "77827bdf", + "ultrachat_359377", + "ultrachat_556776", + "ultrachat_269248" + ] + }, + { + "answer_session_ids": [ + "answer_b191df5b_1", + "answer_b191df5b_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "f9e8c073", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_b191df5b_1", + "answer_b191df5b_2", + "0a042aef_2", + "ultrachat_88473", + "ultrachat_219619", + "445e6a7a_1", + "sharegpt_bLaSZdD_0", + "22db3cc3_1", + "50b32eed_2", + "d5283595_1" + ] + }, + { + "answer_session_ids": [ + "answer_c7ddc051_1", + "answer_c7ddc051_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "41698283", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c7ddc051_2", + "answer_c7ddc051_1", + "631e4016", + "47a2bd5a_1", + "431ae25c", + "0e8ba03b_1", + "6554fb34_1", + "28e948f5", + "ultrachat_135408", + "sharegpt_7mlXHfH_41" + ] + }, + { + "answer_session_ids": [ + "answer_9282283d_1", + "answer_9282283d_2" + ], + "ndcg_at_10": 0.5912, + "ndcg_at_5": 0.6309, + "question_id": "2698e78f", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "d76030af_1", + "answer_9282283d_2", + "862218f3_2", + "ebd61d29_2", + "338281a4_1", + "d99bab55_2", + "answer_9282283d_1", + "89c1cdfa_2", + "1cf6c966_2", + "619e96bc" + ] + }, + { + "answer_session_ids": [ + "answer_67074b4b_1", + "answer_67074b4b_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "b6019101", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_67074b4b_1", + "answer_67074b4b_2", + "de43030f_2", + "ultrachat_72569", + "sharegpt_CvVLg2J_18", + "ultrachat_146343", + "9bd36386", + "sharegpt_aVExml0_235", + "ac089c83_4", + "59c6685a_2" + ] + }, + { + "answer_session_ids": [ + "answer_07664d43_1", + "answer_07664d43_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "45dc21b6", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_07664d43_1", + "answer_07664d43_2", + "2f57c190_1", + "5934b91f", + "sharegpt_mMdyu3t_0", + "eb34144a_2", + "0504a710_4", + "ultrachat_318979", + "sharegpt_NSacJOD_0", + "8acb29f9_1" + ] + }, + { + "answer_session_ids": [ + "answer_b0f3dfff_1", + "answer_b0f3dfff_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "5a4f22c0", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_b0f3dfff_2", + "answer_b0f3dfff_1", + "007e7d81_2", + "af778ece_1", + "dba97bb1_3", + "sharegpt_1ZrnYv8_23", + "57a6a404_2", + "sharegpt_cXkL3cR_77", + "sharegpt_IMtsj65_0", + "195f2929" + ] + }, + { + "answer_session_ids": [ + "answer_4dac77cb_1", + "answer_4dac77cb_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "6071bd76", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_4dac77cb_2", + "answer_4dac77cb_1", + "a58eeaf8", + "a8899aee_1", + "04616813_2", + "8d0b4363", + "9de53d68_1", + "ultrachat_551846", + "ultrachat_56513", + "34a230b3" + ] + }, + { + "answer_session_ids": [ + "answer_1a374afa_1", + "answer_1a374afa_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e493bb7c", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_1a374afa_2", + "answer_1a374afa_1", + "9161500f_2", + "7aaa23d7_2", + "sharegpt_jHGL67f_0", + "ultrachat_185447", + "ultrachat_151926", + "0aacd15f_1", + "ultrachat_415114", + "ultrachat_575969" + ] + }, + { + "answer_session_ids": [ + "answer_caf5b52e_1", + "answer_caf5b52e_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "618f13b2", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_caf5b52e_1", + "answer_caf5b52e_2", + "86b56135_2", + "748eb152_2", + "01a120b5", + "dda70510_2", + "a20bc58f", + "ultrachat_47868", + "ultrachat_270398", + "sharegpt_aF6djes_0" + ] + }, + { + "answer_session_ids": [ + "answer_d7de9a6a_1", + "answer_d7de9a6a_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "72e3ee87", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_d7de9a6a_2", + "answer_d7de9a6a_1", + "1c1f5ccc_4", + "ultrachat_523550", + "080a4000_2", + "ultrachat_195333", + "ultrachat_129606", + "sharegpt_KdbXhTW_9", + "ultrachat_233144", + "6f2c01fc_1" + ] + }, + { + "answer_session_ids": [ + "answer_d3bf812b_1", + "answer_d3bf812b_2" + ], + "ndcg_at_10": 0.4218, + "ndcg_at_5": 0.3869, + "question_id": "c4ea545c", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "sharegpt_JJu53iW_0", + "sharegpt_I0uNTjm_16", + "9f9b402a_5", + "ee7f5084_1", + "answer_d3bf812b_1", + "sharegpt_CPzL3ju_0", + "a1f43f8d_2", + "b2d284b1", + "answer_d3bf812b_2", + "ultrachat_369982" + ] + }, + { + "answer_session_ids": [ + "answer_a7b44747_1", + "answer_a7b44747_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "01493427", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_a7b44747_1", + "answer_a7b44747_2", + "55e6ff53", + "077f7bcb_3", + "780b6c7c_1", + "5a840928_1", + "29332a29_2", + "c2204106_3", + "88e73f02", + "sharegpt_kjeGJvK_13" + ] + }, + { + "answer_session_ids": [ + "answer_77f32504_1", + "answer_77f32504_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "6a27ffc2", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_77f32504_1", + "answer_77f32504_2", + "sharegpt_kkqlRlo_13", + "ab483540", + "08effb2b_2", + "sharegpt_KSCqghj_0", + "ultrachat_20656", + "sharegpt_5clkQjb_12", + "0651f7bd_2", + "9ded9e01_2" + ] + }, + { + "answer_session_ids": [ + "answer_52382508_1", + "answer_52382508_2" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "2133c1b5", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_52382508_1", + "ultrachat_354956", + "answer_52382508_2", + "ultrachat_502724", + "25b1afbc", + "8ca0085a_1", + "sharegpt_Jm8wWJN_0", + "cf021b36_1", + "09d14ddf_2", + "1126be1e_2" + ] + }, + { + "answer_session_ids": [ + "answer_fff743f5_1", + "answer_fff743f5_2" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "18bc8abd", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_fff743f5_2", + "ba6b67af_3", + "answer_fff743f5_1", + "ultrachat_379579", + "ultrachat_131871", + "sharegpt_afCogMg_34", + "a35679d5_2", + "sharegpt_vyHqfrX_0", + "sharegpt_hH4KVIN_10" + ] + }, + { + "answer_session_ids": [ + "answer_611b6e83_1", + "answer_611b6e83_2" + ], + "ndcg_at_10": 0.8772, + "ndcg_at_5": 0.8772, + "question_id": "db467c8c", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_611b6e83_2", + "6cc835bd", + "1a9f5723_1", + "answer_611b6e83_1", + "ultrachat_137520", + "16ebc8f8_1", + "ecfd2047_1", + "77cf4551_1", + "sharegpt_HBmf6q0_0", + "76967d22" + ] + }, + { + "answer_session_ids": [ + "answer_d08a934d_1", + "answer_d08a934d_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "7a87bd0c", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_d08a934d_1", + "answer_d08a934d_2", + "62369c56_1", + "b98914c6_2", + "2c543c87", + "381227c4_1", + "e250791e_3", + "55a59bc9_4", + "60f9246d", + "15519944_1" + ] + }, + { + "answer_session_ids": [ + "answer_f25c32f5_1", + "answer_f25c32f5_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e61a7584", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_f25c32f5_2", + "answer_f25c32f5_1", + "ultrachat_183954", + "82159da4_1", + "1fd16470_2", + "d02bcc9e_2", + "51add7da_1", + "ultrachat_367590", + "sharegpt_o0MporM_0", + "e1416816_2" + ] + }, + { + "answer_session_ids": [ + "answer_79c395a9_1", + "answer_79c395a9_2" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "1cea1afa", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_79c395a9_2", + "7af38385_1", + "answer_79c395a9_1", + "602ad002_1", + "52d46b4a_1", + "ultrachat_51727", + "ultrachat_67100", + "e9ad5914_5", + "ultrachat_255553", + "sharegpt_X1NXUxZ_0" + ] + }, + { + "answer_session_ids": [ + "answer_babbaccb_1", + "answer_babbaccb_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "ed4ddc30", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_babbaccb_2", + "answer_babbaccb_1", + "d2f4a309", + "8b726b64_1", + "ultrachat_238480", + "4ffa04a2_4", + "499652a0_3", + "ultrachat_342966", + "5d2dc022_1", + "a0c40934_1" + ] + }, + { + "answer_session_ids": [ + "answer_966cecbb_1", + "answer_966cecbb_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "8fb83627", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_966cecbb_1", + "answer_966cecbb_2", + "9315f268_1", + "c15319aa_1", + "02b81ad9", + "0638f6f6_1", + "ultrachat_172298", + "88e01130_1", + "be050ab3_2", + "ultrachat_307166" + ] + }, + { + "answer_session_ids": [ + "answer_8c0712af_1", + "answer_8c0712af_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "b01defab", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_8c0712af_2", + "answer_8c0712af_1", + "b1d9eb66_2", + "sharegpt_VbIxRWJ_33", + "1dcfb9a0_1", + "9aa07e25", + "fc69fd44_7", + "ultrachat_479632", + "sharegpt_fB2ri01_0", + "3891baea_2" + ] + }, + { + "answer_session_ids": [ + "answer_bcce0b73_1", + "answer_bcce0b73_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "22d2cb42", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_bcce0b73_1", + "answer_bcce0b73_2", + "74932466_1", + "060aeced", + "eec72a82_4", + "5a76fadb_2", + "ff898925", + "ultrachat_18212", + "sharegpt_hGQUdgE_10", + "ultrachat_227551" + ] + }, + { + "answer_session_ids": [ + "answer_f2f998c7_1", + "answer_f2f998c7_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "0e4e4c46", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_f2f998c7_1", + "answer_f2f998c7_2", + "b42cbaf0_1", + "60876def_2", + "0773943b", + "71d1e644", + "sharegpt_1YtwybC_15", + "ultrachat_467820", + "ultrachat_180729", + "debc34d7_1" + ] + }, + { + "answer_session_ids": [ + "answer_2cec623b_1", + "answer_2cec623b_2" + ], + "ndcg_at_10": 0.9197, + "ndcg_at_5": 0.9197, + "question_id": "4b24c848", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_2cec623b_2", + "d9e0c03e", + "answer_2cec623b_1", + "1815d1b7", + "sharegpt_iqJr8nh_0", + "d38e5b38", + "ultrachat_198571", + "16e094b3_1", + "d0c1453b_1", + "b0855671_2" + ] + }, + { + "answer_session_ids": [ + "answer_c9f5693c_1", + "answer_c9f5693c_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "7e974930", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c9f5693c_1", + "answer_c9f5693c_2", + "705b5399_3", + "a1e169b1", + "898ce7a5_1", + "bf139883", + "ultrachat_380252", + "07b2420e_1", + "8bd5b3c4_1", + "2cc24b7c_2" + ] + }, + { + "answer_session_ids": [ + "answer_8afdebac_1", + "answer_8afdebac_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "603deb26", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_8afdebac_2", + "answer_8afdebac_1", + "c77250d2", + "433ce12c", + "58e5adf2", + "f379d356_3", + "88611fff", + "7b0dea50_1", + "sharegpt_z6pSZ19_0", + "sharegpt_etklNfN_11" + ] + }, + { + "answer_session_ids": [ + "answer_b28f2c7a_1", + "answer_b28f2c7a_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "59524333", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_b28f2c7a_2", + "answer_b28f2c7a_1", + "ultrachat_458180", + "e4882e09_5", + "9b9ff448_1", + "4507b423", + "sharegpt_KhxXaow_0", + "sharegpt_5FHWj0a_0", + "ultrachat_493944", + "sharegpt_7gIKsqO_104" + ] + }, + { + "answer_session_ids": [ + "answer_8d63a897_1", + "answer_8d63a897_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "5831f84d", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_8d63a897_1", + "answer_8d63a897_2", + "6efce493_2", + "ef84b994_abs_2", + "41c90f4c_2", + "ff67236f_1", + "83dae08b", + "sharegpt_NHidbra_0", + "b818e2a1_1", + "sharegpt_3Rk1i3g_0" + ] + }, + { + "answer_session_ids": [ + "answer_8a791264_1", + "answer_8a791264_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "eace081b", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_8a791264_2", + "answer_8a791264_1", + "4dae77d3", + "413b57cb_3", + "9316aae3_1", + "4d04b866", + "ultrachat_455723", + "164bd5a4", + "29a18eab_3", + "sharegpt_HOO59Ue_137" + ] + }, + { + "answer_session_ids": [ + "answer_90de9b4d_1", + "answer_90de9b4d_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "affe2881", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_90de9b4d_1", + "answer_90de9b4d_2", + "fb5dd87f_7", + "ultrachat_443550", + "f334878c", + "6c4a5aee", + "1b71c896_2", + "sharegpt_eM7toF0_0", + "ultrachat_156501", + "ultrachat_541196" + ] + }, + { + "answer_session_ids": [ + "answer_dcd74827_1", + "answer_dcd74827_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "50635ada", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_dcd74827_1", + "answer_dcd74827_2", + "9f9b402a_5", + "2e53911b_1", + "42bcee92_2", + "add7cc1e", + "cb88f886_1", + "1de8082c", + "35bc3132_2", + "b6f35f3a_1" + ] + }, + { + "answer_session_ids": [ + "answer_ac0140ce_1", + "answer_ac0140ce_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e66b632c", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ac0140ce_2", + "answer_ac0140ce_1", + "84098c1b", + "259d58da_2", + "sharegpt_nspc8Nf_199", + "75621e39_1", + "4f51038b", + "805528d6_1", + "487fd03c", + "b3070ec4_1" + ] + }, + { + "answer_session_ids": [ + "answer_a22b654d_1", + "answer_a22b654d_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "0ddfec37", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_a22b654d_2", + "answer_a22b654d_1", + "a8a0db2b_1", + "147ab7e9_7", + "0a699029_2", + "sharegpt_UyBTuTv_0", + "ultrachat_139747", + "sharegpt_cXkL3cR_45", + "f2199726_2", + "60f9246d" + ] + }, + { + "answer_session_ids": [ + "answer_25df025b_1", + "answer_25df025b_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "f685340e", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_25df025b_1", + "answer_25df025b_2", + "a4815e8e_2", + "e862f726_1", + "5035d8d6", + "e7dcd5df", + "4a9eb139_2", + "142e3203_1", + "sharegpt_RuTDId0_29", + "992eb1f3_2" + ] + }, + { + "answer_session_ids": [ + "answer_a5b68517_1", + "answer_a5b68517_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "cc5ded98", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_a5b68517_1", + "answer_a5b68517_2", + "f14a4532_1", + "c63b2c8f", + "ultrachat_323437", + "bd8708e2_2", + "b13d5e4e_1", + "sharegpt_IEsU45Z_23", + "sharegpt_BJmgxmf_0", + "753b7c1f_1" + ] + }, + { + "answer_session_ids": [ + "answer_35d6c0be_1", + "answer_35d6c0be_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "dfde3500", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_35d6c0be_1", + "answer_35d6c0be_2", + "ad1c7ac6", + "86e218e3_2", + "fa68c9d6_2", + "5c18cb32_2", + "59c704ad_1", + "134127c2_2", + "ultrachat_368201", + "7de51ffe_1" + ] + }, + { + "answer_session_ids": [ + "answer_d6028d6e_1", + "answer_d6028d6e_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "69fee5aa", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_d6028d6e_1", + "answer_d6028d6e_2", + "a554ed79_2", + "a0b8b0ad_7", + "8464304d_2", + "ultrachat_152449", + "6685e6cd", + "42adb80d_2", + "2af27403_2", + "5d284d7d_2" + ] + }, + { + "answer_session_ids": [ + "answer_94650bfa_1", + "answer_94650bfa_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "7401057b", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_94650bfa_1", + "answer_94650bfa_2", + "c38a915a_1", + "30f20967_2", + "8de5b7cf_2", + "c3757c5b", + "1450ffad", + "sharegpt_Ah0tRO5_0", + "ultrachat_402967", + "ultrachat_385162" + ] + }, + { + "answer_session_ids": [ + "answer_ae3a122b_1", + "answer_ae3a122b_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "cf22b7bf", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ae3a122b_2", + "answer_ae3a122b_1", + "36d5bbde_1", + "457e26bf", + "8ab0292c_1", + "8064b6ca", + "22fe1c93", + "c8fcdf5a_1", + "e201572a", + "ultrachat_345143" + ] + }, + { + "answer_session_ids": [ + "answer_5126c02d_1", + "answer_5126c02d_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "a2f3aa27", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_5126c02d_1", + "answer_5126c02d_2", + "b65f51fb_5", + "57bf4122_1", + "2fa50779_2", + "a8028ce7_2", + "104055de", + "ultrachat_475462", + "a135a911", + "4cf2ea34" + ] + }, + { + "answer_session_ids": [ + "answer_0cdbca92_1", + "answer_0cdbca92_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "c7dc5443", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_0cdbca92_1", + "answer_0cdbca92_2", + "064e1984_1", + "sharegpt_6cz1Sq6_264", + "ultrachat_435372", + "debc34d7_2", + "ultrachat_308370", + "8a355b36_1", + "728fc45a", + "aefbe381" + ] + }, + { + "answer_session_ids": [ + "answer_da72b1b4_1", + "answer_da72b1b4_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "06db6396", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_da72b1b4_1", + "answer_da72b1b4_2", + "6ebdc1fe_2", + "sharegpt_2pwdGxJ_16", + "ultrachat_456942", + "5bd9f1e6_1", + "35038174_4", + "8235b87d", + "824634d9_2", + "sharegpt_C26CwPQ_0" + ] + }, + { + "answer_session_ids": [ + "answer_cd345582_1", + "answer_cd345582_2" + ], + "ndcg_at_10": 0.6053, + "ndcg_at_5": 0.6309, + "question_id": "3ba21379", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "49576fd6", + "answer_cd345582_1", + "67620dbc_4", + "d557e57a", + "sharegpt_YGmoq5F_7", + "answer_cd345582_2", + "ultrachat_224436", + "ultrachat_74850", + "d68838d0_2", + "f64c4793_1" + ] + }, + { + "answer_session_ids": [ + "answer_c6a0c6c2_1", + "answer_c6a0c6c2_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "9bbe84a2", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_c6a0c6c2_1", + "answer_c6a0c6c2_2", + "114c1fb7_6", + "ef69c258", + "sharegpt_hcaZN94_546", + "362aa011", + "acad845c_1", + "1fc31387", + "a0f8468c_2", + "sharegpt_hLkmbBL_0" + ] + }, + { + "answer_session_ids": [ + "answer_67be2c38_1", + "answer_67be2c38_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "10e09553", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_67be2c38_2", + "answer_67be2c38_1", + "7d1f5395", + "e5f785de", + "d3da4592_2", + "918c4788_1", + "ff67236f_2", + "ultrachat_196381", + "f4e2933b", + "sharegpt_YPM1YqM_0" + ] + }, + { + "answer_session_ids": [ + "answer_4a97ae40_1", + "answer_4a97ae40_2" + ], + "ndcg_at_10": 0.6934, + "ndcg_at_5": 0.6934, + "question_id": "dad224aa", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "6de8645d_1", + "answer_4a97ae40_2", + "answer_4a97ae40_1", + "c86a22a2_1", + "sharegpt_opQbgPL_13", + "56b5b760", + "dd345e24_2", + "ultrachat_106352", + "ultrachat_468289", + "5c690047_2" + ] + }, + { + "answer_session_ids": [ + "answer_f377cda7_1", + "answer_f377cda7_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "ba61f0b9", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_f377cda7_1", + "answer_f377cda7_2", + "sharegpt_iqJr8nh_0", + "17a1e5c1", + "c9213a03_1", + "85846900_6", + "495cf518_1", + "6b7605d1_2", + "sharegpt_s8Opwwu_0", + "ad578e11_3" + ] + }, + { + "answer_session_ids": [ + "answer_e3892371_1", + "answer_e3892371_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "42ec0761", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e3892371_2", + "answer_e3892371_1", + "27d0d3e7", + "4fb01417_2", + "sharegpt_vWO8y7i_35", + "f849d84c", + "493bd421_2", + "53acd215", + "b37a44e4_2", + "37c3726a_4" + ] + }, + { + "answer_session_ids": [ + "answer_1cb52d0a_1", + "answer_1cb52d0a_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "5c40ec5b", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_1cb52d0a_2", + "answer_1cb52d0a_1", + "ultrachat_333871", + "be050ab3_2", + "a200b713_3", + "9eb7bb35", + "5cc9e1ed_3", + "ultrachat_122152", + "ultrachat_91084", + "6446f6e6" + ] + }, + { + "answer_session_ids": [ + "answer_626e93c4_1", + "answer_626e93c4_2" + ], + "ndcg_at_10": 0.8503, + "ndcg_at_5": 0.8503, + "question_id": "c6853660", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_626e93c4_2", + "6ed1c85e", + "ultrachat_223464", + "33994682_3", + "answer_626e93c4_1", + "71dc2037_2", + "a4863bcb_2", + "ultrachat_544305", + "f8ab60d7", + "bed5fbab" + ] + }, + { + "answer_session_ids": [ + "answer_f762ad8d_1", + "answer_f762ad8d_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "26bdc477", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_f762ad8d_1", + "answer_f762ad8d_2", + "27ce9f13_2", + "sharegpt_EeXTDjc_23", + "ultrachat_438698", + "37b3c75d", + "551eae69_1", + "sharegpt_lWLBUhQ_517", + "sharegpt_pRBsKdE_5", + "sharegpt_bq9G6bT_2" + ] + }, + { + "answer_session_ids": [ + "answer_3bf5b73b_1", + "answer_3bf5b73b_2" + ], + "ndcg_at_10": 0.6309, + "ndcg_at_5": 0.6309, + "question_id": "0977f2af", + "question_type": "knowledge-update", + "r_all_at_10": 0.0, + "r_all_at_5": 0.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "b1d9eb66_4", + "answer_3bf5b73b_2", + "b24eddd9_1", + "f115a3db_1", + "ultrachat_554962", + "809cbce9_3", + "357e33b5", + "1692563c_1", + "9b083158_1", + "sharegpt_FqOC5e9_45" + ] + }, + { + "answer_session_ids": [ + "answer_e1403127_1", + "answer_e1403127_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "89941a94", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_e1403127_2", + "answer_e1403127_1", + "6e5130ed_1", + "de275215_3", + "74b0227e_2", + "6a7d7055_2", + "f334878c", + "2ec60f51_3", + "522dd987_1", + "sharegpt_CQ3HZO9_71" + ] + }, + { + "answer_session_ids": [ + "answer_7e9ad7b4_1", + "answer_7e9ad7b4_2" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "07741c45", + "question_type": "knowledge-update", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_7e9ad7b4_2", + "answer_7e9ad7b4_1", + "37d65d40_3", + "af4a2fd1", + "66c23110_2", + "ce21f0e4", + "ultrachat_162268", + "ultrachat_267176", + "e06c6d99_3", + "ultrachat_27772" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_5Lzox6N_0" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "7161e7e2", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_5Lzox6N_0", + "3c3fee41", + "5850de18_2", + "ultrachat_68050", + "sharegpt_yguSguz_0", + "6b17c99b_2", + "dba97bb1_4", + "5aed819f", + "sharegpt_PjFSCFK_9", + "sharegpt_6kDZTZ2_0" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_234453" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "c4f10528", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_234453", + "107d40f0", + "51c89181_1", + "29116b50_1", + "689fec3d_1", + "11248f50", + "6cf1848e_2", + "sharegpt_yMwEvl7_557", + "21ff9566_2", + "e12cfbcd" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_YkWn1Ne_0" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "89527b6b", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_YkWn1Ne_0", + "ultrachat_117600", + "48d385f0_1", + "644e1d00", + "51c262b6_2", + "4b637e89_1", + "ultrachat_84388", + "sharegpt_htSCqmh_171", + "9c722816", + "07e363d9_2" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_480665" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e9327a54", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_480665", + "ba944cdc", + "11decc61", + "c58fdf1e_2", + "c0d099e6_2", + "28e0ea23_2", + "7613d5ec_2", + "b65f51fb_5", + "sharegpt_NP5eyFN_5", + "sharegpt_Q7EQHIJ_0" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_448704" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "4c36ccef", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_448704", + "3dc8ba5f_2", + "07e6a3bd_1", + "5fc83435_2", + "1ba83815_4", + "89749c78_1", + "7b1d1f43_2", + "sharegpt_m8F4NxL_0", + "940629b2_1", + "sharegpt_1PRjwYK_11" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_IUWQYGQ_0" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "6ae235be", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_IUWQYGQ_0", + "0a1e90d1_2", + "99188307", + "809cabca_1", + "31cde680", + "8e608aad", + "39358a85_1", + "61a91bc9_2", + "cb692a80_3", + "sharegpt_KzDwaf1_211" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_370515" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "7e00a6cb", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_370515", + "4dd1ba8d_2", + "b7dca8f1_1", + "24f78a31_2", + "2fa50779_5", + "956ce238", + "39358a85_3", + "9998814e_2", + "ultrachat_188131", + "a9981dc6_2" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_hA7AkP3_0" + ], + "ndcg_at_10": 0.4307, + "ndcg_at_5": 0.4307, + "question_id": "1903aded", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "66369fde_1", + "184efb91_3", + "9b4c070c", + "answer_sharegpt_hA7AkP3_0", + "ultrachat_110523", + "sharegpt_64onasJ_0", + "201df030", + "8adb232f_1", + "f8246a66_1", + "sharegpt_emE20LI_0" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_cGdjmYo_0" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "ceb54acb", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_cGdjmYo_0", + "511bdbe3_2", + "197d99cc", + "7aa2a9af_2", + "de293134_1", + "f6fd00cf", + "0bbd7094_1", + "ultrachat_150417", + "63b72857", + "c60d5c8a_1" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_m2xJfjo_0" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "f523d9fe", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_m2xJfjo_0", + "193c23bd_1", + "22aa7cca_4", + "debc34d7_1", + "9ccea2f0_1", + "73f75ab4", + "a07f9d5c_1", + "ultrachat_391251", + "661b711f_1", + "fbfda981" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_113156" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "0e5e2d1a", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_113156", + "f84b4266", + "ultrachat_201585", + "c756c34e", + "5d79fd65_2", + "22f9f163_1", + "sharegpt_DcVIUDF_0", + "sharegpt_GPluPmJ_7", + "sharegpt_btv4SkQ_0", + "c4e5d969_2" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_187684" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "fea54f57", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_187684", + "34590cab_1", + "0aa99e2e_1", + "c8854b28", + "712e923c", + "ee659010_2", + "ultrachat_435757", + "ultrachat_129340", + "ultrachat_562162", + "a1e90def_1" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_374124" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "cc539528", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_374124", + "18807892_4", + "8bd3814a_1", + "15a8ffb7_2", + "faa194ea", + "20507d15_1", + "fe1faff9_1", + "sharegpt_0z277Yx_0", + "2a394047", + "sharegpt_1SSsoox_0" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_459954" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "dc439ea3", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_459954", + "1bfd5a8b_2", + "eb7366f2_1", + "6ef99651_1", + "5a81277c", + "ultrachat_186512", + "49d8a382", + "cb5637b3", + "sharegpt_pttddrt_5", + "ultrachat_201433" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_hn3IS1q_0" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "18dcd5a5", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_hn3IS1q_0", + "6ea1541e_1", + "ultrachat_425586", + "959e6cb0_2", + "33d16079_2", + "d9fb1588_1", + "c00077f5_2", + "sharegpt_d1tE544_5", + "sharegpt_MW0whoh_79", + "ultrachat_39471" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_275993" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "488d3006", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_275993", + "ce19df54_3", + "a68090b0_2", + "9a57d65b", + "82b82901", + "a394f6b5_1", + "d53b6bfe_1", + "sharegpt_d9F8HCC_0", + "1af2c0fb_1", + "7e4dab66_1" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_U4oCSfU_7" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "58470ed2", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_U4oCSfU_7", + "sharegpt_oPOTyid_141", + "ff49b5a5_3", + "4050ebff_6", + "a49a1438_2", + "2f2884ad_3", + "b6018747_4", + "ultrachat_458590", + "e45db473_2", + "sharegpt_e8sgAJX_0" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_HFMn2ZX_0" + ], + "ndcg_at_10": 0.6309, + "ndcg_at_5": 0.6309, + "question_id": "8cf51dda", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "daa09295", + "answer_sharegpt_HFMn2ZX_0", + "sharegpt_ftoCcOn_0", + "507514d9_1", + "27639dd8_2", + "ultrachat_365938", + "3358b5ca_2", + "9f0f59d1_1", + "sharegpt_jpiaPoJ_738", + "sharegpt_yoSUjfa_0" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_348449" + ], + "ndcg_at_10": 0.5, + "ndcg_at_5": 0.5, + "question_id": "1d4da289", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "09f1c793", + "ultrachat_213759", + "answer_ultrachat_348449", + "4cd929f8_1", + "517ad0f0", + "sharegpt_eVmxjQZ_0", + "bf85791d", + "a05c7d7d", + "ultrachat_322093", + "ultrachat_329521" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_467053" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "8464fc84", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_467053", + "21d86ebb_2", + "6ac43c5b_4", + "bdc804df_2", + "602b4074_3", + "1d5998ca_1", + "d43b63a6", + "7a0abbe2_2", + "98d39a6d", + "9582aa59_2" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_563222" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "8aef76bc", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_563222", + "146dfabe", + "5209e813_2", + "27d378b2_2", + "08e084f1_1", + "sharegpt_n3Hy63g_0", + "ultrachat_351310", + "ultrachat_368669", + "44b1595a", + "c7f6643e_2" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_417348" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "71a3fd6b", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_417348", + "8419b385_3", + "7f7e26d2_2", + "878271ae_1", + "36743359_3", + "sharegpt_1L5GXZJ_0", + "sharegpt_n3kgf14_17", + "ultrachat_168420", + "b7146673_1", + "sharegpt_asilfXk_321" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_2kpncbX_13" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "2bf43736", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_2kpncbX_13", + "16c3be68_1", + "8897a2e5", + "661ee6d9_1", + "2e89bd03_1", + "7e4cd916_2", + "d79173aa_3", + "445e6a7a_1", + "1a5fb2cd", + "a93750ef_2" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_334948" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "70b3e69b", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_334948", + "09e1a929_2", + "ultrachat_47234", + "sharegpt_oTNStu2_51", + "dd92723f_1", + "ultrachat_478639", + "1388b204_2", + "sharegpt_O8qJclh_0", + "sharegpt_s0wsteT_0", + "c2204106_2" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_6pWK9yx_0" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "8752c811", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_6pWK9yx_0", + "d0a41222_1", + "93f23301_2", + "06b5b860", + "9db30f1c_3", + "5c6a5b7c_1", + "3e48ed93", + "02b63d04_1", + "ultrachat_160178", + "9272a6a2_2" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_CaxTGYP_0" + ], + "ndcg_at_10": 0.6309, + "ndcg_at_5": 0.6309, + "question_id": "3249768e", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "ed1d68f1_1", + "answer_sharegpt_CaxTGYP_0", + "0868982f_1", + "71f2b8ba", + "ultrachat_77515", + "56cf2973", + "172b47cb_2", + "e0bd5f14_2", + "23450b93_3", + "sharegpt_M5zaN8c_0" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_115151" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "1b9b7252", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_115151", + "3829d412_2", + "d3cc5bdc_4", + "53ab4734", + "af4d4ecd_6", + "0e726047", + "330c05e7_1", + "b2e5a019", + "4f6e1b77_1", + "sharegpt_1AhFMpw_5" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_d6JJiqH_76" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "1568498a", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_d6JJiqH_76", + "sharegpt_kRkQrRx_0", + "38de4c2a_1", + "6ad46850_2", + "70fed904", + "4022d960_2", + "sharegpt_trljpZ5_27", + "sharegpt_69agvDU_27", + "983b1954", + "f8e8d445_2" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_H9PiM5G_0" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "6222b6eb", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_H9PiM5G_0", + "caf7480d_1", + "sharegpt_cyc2xbb_0", + "40a90d51", + "2fd5e208", + "544c1e7b", + "b35fadaa", + "facb94a8_3", + "2a500dce_1", + "5209e813_2" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_13075" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e8a79c70", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_13075", + "sharegpt_ha8kUEr_0", + "c0d099e6_2", + "sharegpt_vKSMO9J_69", + "742ea86b_1", + "89aed484_2", + "c2b2b2ea_1", + "87252d80_2", + "ultrachat_225061", + "sharegpt_tfIfm00_14" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_252214" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "d596882b", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_252214", + "dc9bf721_1", + "71888aff_2", + "456807b7_1", + "bef3abcd", + "0dc2efcc_2", + "ultrachat_299672", + "087d2b0a_1", + "5fb3b5ac", + "ultrachat_306390" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_5m7gg5F_0" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e3fc4d6e", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_5m7gg5F_0", + "4d261642", + "sharegpt_TVK4gjw_24", + "d31c3ec8_1", + "ultrachat_161238", + "ultrachat_325941", + "sharegpt_sXKNzPE_0", + "ultrachat_495326", + "ultrachat_443354", + "sharegpt_nAacldL_21" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_427265" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "51b23612", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_427265", + "255c753b_1", + "8672f398_2", + "409015ef_1", + "17f444f3", + "ultrachat_343", + "ultrachat_522170", + "ultrachat_555062", + "ultrachat_221034", + "848f9a52" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_94624" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "3e321797", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_94624", + "9252145e_2", + "11302335_4", + "57144028_2", + "00305dd2_1", + "193c23bd_1", + "385a60c8_1", + "sharegpt_OzK1xD9_0", + "ultrachat_557903", + "sharegpt_EZmhz8p_0" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_195444" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e982271f", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_195444", + "eb0d8dc1_4", + "8a7c65c1_1", + "f72c924e", + "db65997e", + "ultrachat_383774", + "ultrachat_247528", + "sharegpt_Xlpa70t_0", + "3e8f07c9_1", + "934419a6_1" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_NoDZzot_7" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "352ab8bd", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_NoDZzot_7", + "6593cb8b_6", + "337cbfc8_2", + "d600c646", + "55a59bc9_2", + "4a42c62b", + "ultrachat_194309", + "ultrachat_101751", + "27971ce6_5", + "sharegpt_SM8NnLU_13" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_39395" + ], + "ndcg_at_10": 0.3869, + "ndcg_at_5": 0.3869, + "question_id": "fca762bc", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "06a34d82_1", + "d0cdfddb_2", + "8fcc3543_3", + "2bc4ae7d", + "answer_ultrachat_39395", + "42d9467c_1", + "sharegpt_wrN9uUo_77", + "3b7a6ecd_1", + "136dfd61_2", + "sharegpt_ZgtoeGU_0" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_i0tMT9q_9" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "7a8d0b71", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_i0tMT9q_9", + "f07752be_1", + "f8169a6d", + "7a8f5003", + "d5eab084_2", + "c14e3f09_2", + "147ab7e9_5", + "f20e72e4_2", + "sharegpt_P0Qe2Gx_0", + "881d8e9b_1" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_269020" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "a40e080f", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_269020", + "4d32ba1b_1", + "85736a07_2", + "sharegpt_fyAkcGP_81", + "89c7a48a_2", + "4fd6129f_1", + "cd38b6ee", + "ed109bc3", + "sharegpt_oXgiN7q_327", + "c9489af0_1" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_289157" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "8b9d4367", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_289157", + "sharegpt_WZzri9J_0", + "7babf05b", + "031061e6_4", + "sharegpt_iHP3XDi_39", + "5534efcb_1", + "ultrachat_79945", + "36743359_1", + "007e7d81_3", + "sharegpt_MHiC4J8_0" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_4aJsGCH_0" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "5809eb10", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_4aJsGCH_0", + "fc3d6c80", + "sharegpt_GDFyxeT_0", + "9b820206", + "sharegpt_obCC1BP_103", + "d3069305_2", + "22ce6625_1", + "ultrachat_536416", + "ultrachat_275993", + "sharegpt_9M4WbfX_0" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_81riySf_0" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "41275add", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_81riySf_0", + "5b83c26e_3", + "33a39aa7_1", + "b3619c2c", + "a68db5db_1", + "4d84cbfa", + "ultrachat_36085", + "ultrachat_556585", + "sharegpt_dUoAhvI_71", + "sharegpt_fV5wNsl_0" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_qTi81nS_0" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "4388e9dd", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_qTi81nS_0", + "cfd21744_2", + "3197d998_7", + "1ebd753f_2", + "ultrachat_412706", + "1b8f0ba5_4", + "fc2d70fe_1", + "0826def6_1", + "cf9eda76", + "9ff53e00_1" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_i9adwQn_0" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "4baee567", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_i9adwQn_0", + "sharegpt_wduYbsX_0", + "sharegpt_NxG2nGm_91", + "385683f0_1", + "sharegpt_kt5ZCxz_10", + "ec830058_5", + "sharegpt_hRVKLa2_0", + "ultrachat_427430", + "d5415748", + "ultrachat_210474" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_hChsWOp_97" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "561fabcd", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_hChsWOp_97", + "864a563d_4", + "34b38398_2", + "sharegpt_QaiMtpK_0", + "aee13015_3", + "3be19aed_1", + "83fb74bf_1", + "d75f4fb5_1", + "33da50d0_5", + "ultrachat_194831" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_2BSXlAr_0" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "b759caee", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_2BSXlAr_0", + "sharegpt_6gBVFdg_1", + "b41aae7f_1", + "bab41bb6_5", + "d3575920", + "ultrachat_554750", + "f55fee5b_4", + "sharegpt_I9veIBa_1", + "ultrachat_281823", + "0bad887e_2" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_GYqnAhC_190" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "ac031881", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_GYqnAhC_190", + "eb95f93c", + "6608c1da", + "sharegpt_LgJ688S_0", + "e7989082_1", + "c986f83a", + "ultrachat_58138", + "2dbe975c", + "sharegpt_cIj2dFl_4", + "19c24c11" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_446979" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "28bcfaac", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_446979", + "8d410160", + "be05680b_2", + "17cb8d2f_5", + "135a62d7_2", + "a27a2811", + "044e200f_2", + "ultrachat_433665", + "989ad9e6_3", + "sharegpt_MO9Xxet_71" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_294807" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "16c90bf4", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_294807", + "8464304d_2", + "5053474c_2", + "b8770374_2", + "sharegpt_Ou9rQ6U_8", + "b0da5097_2", + "ad72ce15", + "d576152e_4", + "d4230511_1", + "32449a42" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_519486" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "c8f1aeed", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_519486", + "sharegpt_ebYKddM_87", + "aa7a2ec0_1", + "a53fdd02_2", + "af631aa3_3", + "48a72204_3", + "f7cd21e5", + "sharegpt_fH5kcER_0", + "12cd736c", + "c3023a45_5" + ] + }, + { + "answer_session_ids": [ + "answer_sharegpt_SS141vi_0" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "eaca4986", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_sharegpt_SS141vi_0", + "3477c37c_1", + "931c521e_1", + "ultrachat_311235", + "7bc2c7a3_1", + "1c1a2b7f_2", + "564761aa", + "24ca2090_2", + "sharegpt_NelPQLT_0", + "0e543e9e_2" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_456407" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "c7cf7dfd", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_456407", + "de5a59bf", + "95cec590_1", + "285a29e7_1", + "sharegpt_f28uI6i_31", + "780b6c7c_1", + "c1e681e7_2", + "e27fe923_5", + "1382d6fb_1", + "9db6d988_2" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_174360" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "e48988bc", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_174360", + "6e110a53_1", + "ultrachat_494373", + "e388463b_1", + "3fd56ebf", + "ff6bccaa_3", + "476b1cbb", + "sharegpt_TzwvePR_21", + "ultrachat_172828", + "ultrachat_307163" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_440262" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "1de5cff2", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_440262", + "4c6775ed_1", + "848d721f_1", + "8fcfbe43_3", + "ultrachat_58031", + "64048e35", + "28031c43_2", + "c14dbe17", + "42d9467c_1", + "4cb33063" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_403752" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "65240037", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_403752", + "37141183", + "46f07bd0_2", + "f8dc1273", + "fce669f5_3", + "0984a772", + "7df4c735_1", + "3c3a9042_1", + "dba5a924", + "sharegpt_Caxpnd3_6" + ] + }, + { + "answer_session_ids": [ + "answer_ultrachat_399000" + ], + "ndcg_at_10": 1.0, + "ndcg_at_5": 1.0, + "question_id": "778164c6", + "question_type": "single-session-assistant", + "r_all_at_10": 1.0, + "r_all_at_5": 1.0, + "r_at_10": 1.0, + "r_at_5": 1.0, + "top_session_ids": [ + "answer_ultrachat_399000", + "sharegpt_ki9IVDq_6", + "ultrachat_346016", + "483ddd90", + "490bb46d_1", + "05b551b6", + "b605741a_1", + "ultrachat_144598", + "c15dadce_4", + "35201d43" + ] + } + ], + "runtime_seconds": 251.044, + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_old_brainctl_brain.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_old_brainctl_brain.json new file mode 100644 index 0000000..c02f524 --- /dev/null +++ b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_old_brainctl_brain.json @@ -0,0 +1,32 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\longmemeval_old_brainctl_brain.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "measured": false, + "metadata": {}, + "metrics": { + "ndcg_at_10": 0.9253, + "ndcg_at_5": 0.9204, + "r_at_10": 0.9894, + "r_at_5": 0.9681 + }, + "mode": "brain", + "notes": [ + "top_k=10" + ], + "primary_metric": "r_at_5", + "primary_metric_value": 0.9681, + "reference_kind": "historical", + "runtime_seconds": 85.439, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_old_brainctl_cmd.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_old_brainctl_cmd.json new file mode 100644 index 0000000..d110c50 --- /dev/null +++ b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_old_brainctl_cmd.json @@ -0,0 +1,32 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\longmemeval_old_brainctl_cmd.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "measured": false, + "metadata": {}, + "metrics": { + "ndcg_at_10": 0.9247, + "ndcg_at_5": 0.9206, + "r_at_10": 0.9894, + "r_at_5": 0.9702 + }, + "mode": "cmd", + "notes": [ + "top_k=10" + ], + "primary_metric": "r_at_5", + "primary_metric_value": 0.9702, + "reference_kind": "historical", + "runtime_seconds": 130.863, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/membench_mempalace_raw_turn.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/membench_mempalace_raw_turn.json new file mode 100644 index 0000000..7654261 --- /dev/null +++ b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/membench_mempalace_raw_turn.json @@ -0,0 +1,35 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\membench_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\membench_mempalace_raw_turn.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" + }, + "benchmark": "membench", + "caveats": [ + "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 200, + "measured": false, + "metadata": {}, + "metrics": { + "hit_at_5": 0.885, + "simple_hit_at_5": 0.885, + "top_k": 5 + }, + "mode": "raw_turn", + "notes": [ + "FirstAgent slice only", + "turn-level retrieval", + "topic=all" + ], + "primary_metric": "hit_at_5", + "primary_metric_value": 0.885, + "reference_kind": "historical", + "runtime_seconds": 804.35, + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "partial", + "system_name": "mempalace" +} \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/membench_new_brainctl_cmd_turn.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/membench_new_brainctl_cmd_turn.json new file mode 100644 index 0000000..ff93a9f --- /dev/null +++ b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/membench_new_brainctl_cmd_turn.json @@ -0,0 +1,3369 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\membench_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\membench_new_brainctl_cmd_turn.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" + }, + "benchmark": "membench", + "caveats": [ + "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 200, + "measured": true, + "metadata": {}, + "metrics": { + "hit_at_5": 0.895, + "simple_hit_at_5": 0.895, + "top_k": 5 + }, + "mode": "cmd_turn", + "notes": [ + "FirstAgent slice only", + "turn-level retrieval", + "topic=all" + ], + "primary_metric": "hit_at_5", + "primary_metric_value": 0.895, + "reference_kind": "measured", + "rows": [ + { + "category": "simple", + "hit_at_k": false, + "question": "What is the name of my niece's company?", + "retrieved_ids": [], + "target_ids": [ + "119" + ], + "tid": 0, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the email address of my cousin, who is male?", + "retrieved_ids": [ + "simple_roles_1|sid=59|g=59|s=2|t=17", + "simple_roles_1|sid=143|g=143|s=6|t=18", + "simple_roles_1|sid=101|g=101|s=4|t=18", + "simple_roles_1|sid=33|g=33|s=1|t=12", + "simple_roles_1|sid=42|g=42|s=2|t=0" + ], + "target_ids": [ + "59" + ], + "tid": 1, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the position of my brother?", + "retrieved_ids": [ + "simple_roles_2|sid=85|g=85|s=4|t=1", + "simple_roles_2|sid=84|g=84|s=4|t=0", + "simple_roles_2|sid=28|g=28|s=1|t=7", + "simple_roles_2|sid=41|g=41|s=1|t=20", + "simple_roles_2|sid=123|g=123|s=5|t=18" + ], + "target_ids": [ + "84" + ], + "tid": 2, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the height of my aunt?", + "retrieved_ids": [ + "simple_roles_3|sid=13|g=13|s=0|t=13", + "simple_roles_3|sid=0|g=0|s=0|t=0", + "simple_roles_3|sid=125|g=125|s=6|t=1", + "simple_roles_3|sid=19|g=19|s=0|t=19", + "simple_roles_3|sid=98|g=98|s=4|t=16" + ], + "target_ids": [ + "13" + ], + "tid": 3, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the contact number for my subordinate?", + "retrieved_ids": [ + "simple_roles_4|sid=40|g=40|s=1|t=19", + "simple_roles_4|sid=135|g=135|s=6|t=9", + "simple_roles_4|sid=59|g=59|s=2|t=17", + "simple_roles_4|sid=98|g=98|s=4|t=14", + "simple_roles_4|sid=9|g=9|s=0|t=9" + ], + "target_ids": [ + "40" + ], + "tid": 4, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What hobby does my brother have?", + "retrieved_ids": [ + "simple_roles_5|sid=63|g=63|s=3|t=0", + "simple_roles_5|sid=81|g=81|s=3|t=18", + "simple_roles_5|sid=11|g=11|s=0|t=11", + "simple_roles_5|sid=77|g=77|s=3|t=14", + "simple_roles_5|sid=159|g=159|s=7|t=16" + ], + "target_ids": [ + "81" + ], + "tid": 5, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What hobby does my nephew have?", + "retrieved_ids": [ + "simple_roles_6|sid=103|g=103|s=5|t=0", + "simple_roles_6|sid=98|g=98|s=4|t=15", + "simple_roles_6|sid=52|g=52|s=2|t=10", + "simple_roles_6|sid=154|g=154|s=7|t=10", + "simple_roles_6|sid=105|g=105|s=5|t=2" + ], + "target_ids": [ + "105" + ], + "tid": 6, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the work location of my sister?", + "retrieved_ids": [ + "simple_roles_7|sid=144|g=144|s=7|t=0", + "simple_roles_7|sid=68|g=68|s=3|t=7", + "simple_roles_7|sid=145|g=145|s=7|t=1", + "simple_roles_7|sid=39|g=39|s=1|t=19", + "simple_roles_7|sid=127|g=127|s=6|t=3" + ], + "target_ids": [ + "144" + ], + "tid": 7, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the height of my male cousin?", + "retrieved_ids": [ + "simple_roles_8|sid=64|g=64|s=3|t=4", + "simple_roles_8|sid=82|g=82|s=4|t=1", + "simple_roles_8|sid=102|g=102|s=5|t=0", + "simple_roles_8|sid=60|g=60|s=3|t=0", + "simple_roles_8|sid=80|g=80|s=3|t=20" + ], + "target_ids": [ + "64" + ], + "tid": 8, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": false, + "question": "What is the educational background of that coworker?", + "retrieved_ids": [ + "simple_roles_9|sid=104|g=104|s=5|t=2", + "simple_roles_9|sid=48|g=48|s=2|t=7", + "simple_roles_9|sid=112|g=112|s=5|t=10", + "simple_roles_9|sid=138|g=138|s=6|t=16", + "simple_roles_9|sid=50|g=50|s=2|t=9" + ], + "target_ids": [ + "113" + ], + "tid": 9, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What does the coworker do for a living?", + "retrieved_ids": [ + "simple_roles_10|sid=123|g=123|s=6|t=0", + "simple_roles_10|sid=124|g=124|s=6|t=1", + "simple_roles_10|sid=140|g=140|s=6|t=17", + "simple_roles_10|sid=86|g=86|s=4|t=4", + "simple_roles_10|sid=134|g=134|s=6|t=11" + ], + "target_ids": [ + "124" + ], + "tid": 10, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": false, + "question": "What is the location of my father's workplace?", + "retrieved_ids": [ + "simple_roles_11|sid=81|g=81|s=4|t=0", + "simple_roles_11|sid=157|g=157|s=7|t=14", + "simple_roles_11|sid=94|g=94|s=4|t=13", + "simple_roles_11|sid=136|g=136|s=6|t=13", + "simple_roles_11|sid=153|g=153|s=7|t=10" + ], + "target_ids": [ + "92" + ], + "tid": 11, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the work location of the subordinate?", + "retrieved_ids": [ + "simple_roles_12|sid=32|g=32|s=1|t=11", + "simple_roles_12|sid=21|g=21|s=1|t=0", + "simple_roles_12|sid=46|g=46|s=2|t=4", + "simple_roles_12|sid=131|g=131|s=6|t=7", + "simple_roles_12|sid=47|g=47|s=2|t=5" + ], + "target_ids": [ + "32" + ], + "tid": 12, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What hobby does the coworker have?", + "retrieved_ids": [ + "simple_roles_13|sid=109|g=109|s=5|t=6", + "simple_roles_13|sid=158|g=158|s=7|t=15", + "simple_roles_13|sid=70|g=70|s=3|t=8", + "simple_roles_13|sid=148|g=148|s=7|t=5", + "simple_roles_13|sid=143|g=143|s=7|t=0" + ], + "target_ids": [ + "148" + ], + "tid": 13, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the position of the boss?", + "retrieved_ids": [ + "simple_roles_14|sid=42|g=42|s=2|t=0", + "simple_roles_14|sid=61|g=61|s=2|t=19", + "simple_roles_14|sid=56|g=56|s=2|t=14", + "simple_roles_14|sid=62|g=62|s=2|t=20", + "simple_roles_14|sid=11|g=11|s=0|t=11" + ], + "target_ids": [ + "56" + ], + "tid": 14, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the email address of the subordinate?", + "retrieved_ids": [ + "simple_roles_15|sid=13|g=13|s=0|t=13", + "simple_roles_15|sid=96|g=96|s=4|t=13", + "simple_roles_15|sid=35|g=35|s=1|t=13", + "simple_roles_15|sid=160|g=160|s=7|t=16", + "simple_roles_15|sid=73|g=73|s=3|t=11" + ], + "target_ids": [ + "13" + ], + "tid": 15, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the hometown of my aunt?", + "retrieved_ids": [ + "simple_roles_16|sid=123|g=123|s=6|t=0", + "simple_roles_16|sid=143|g=143|s=6|t=20", + "simple_roles_16|sid=85|g=85|s=4|t=3", + "simple_roles_16|sid=142|g=142|s=6|t=19", + "simple_roles_16|sid=32|g=32|s=1|t=11" + ], + "target_ids": [ + "143" + ], + "tid": 16, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the hometown of my sister?", + "retrieved_ids": [ + "simple_roles_17|sid=20|g=20|s=1|t=0", + "simple_roles_17|sid=24|g=24|s=1|t=4", + "simple_roles_17|sid=32|g=32|s=1|t=12", + "simple_roles_17|sid=5|g=5|s=0|t=5", + "simple_roles_17|sid=128|g=128|s=6|t=5" + ], + "target_ids": [ + "24" + ], + "tid": 17, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the education background of that coworker?", + "retrieved_ids": [ + "simple_roles_18|sid=145|g=145|s=7|t=1", + "simple_roles_18|sid=21|g=21|s=1|t=0", + "simple_roles_18|sid=24|g=24|s=1|t=3", + "simple_roles_18|sid=144|g=144|s=7|t=0", + "simple_roles_18|sid=165|g=165|s=7|t=21" + ], + "target_ids": [ + "144" + ], + "tid": 18, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the occupation of that subordinate?", + "retrieved_ids": [ + "simple_roles_19|sid=97|g=97|s=4|t=14" + ], + "target_ids": [ + "97" + ], + "tid": 19, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What hobby does my nephew have?", + "retrieved_ids": [ + "simple_roles_20|sid=0|g=0|s=0|t=0", + "simple_roles_20|sid=6|g=6|s=0|t=6", + "simple_roles_20|sid=115|g=115|s=5|t=9", + "simple_roles_20|sid=34|g=34|s=1|t=13", + "simple_roles_20|sid=72|g=72|s=3|t=8" + ], + "target_ids": [ + "6" + ], + "tid": 20, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What hobby does my coworker have?", + "retrieved_ids": [ + "simple_roles_21|sid=61|g=61|s=3|t=0", + "simple_roles_21|sid=72|g=72|s=3|t=11", + "simple_roles_21|sid=105|g=105|s=5|t=3", + "simple_roles_21|sid=70|g=70|s=3|t=9", + "simple_roles_21|sid=139|g=139|s=6|t=17" + ], + "target_ids": [ + "70" + ], + "tid": 21, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": false, + "question": "What is the position of my mother?", + "retrieved_ids": [ + "simple_roles_22|sid=21|g=21|s=1|t=0", + "simple_roles_22|sid=37|g=37|s=1|t=16", + "simple_roles_22|sid=36|g=36|s=1|t=15", + "simple_roles_22|sid=158|g=158|s=7|t=16", + "simple_roles_22|sid=115|g=115|s=5|t=13" + ], + "target_ids": [ + "38" + ], + "tid": 22, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the name of my sister?", + "retrieved_ids": [ + "simple_roles_23|sid=19|g=19|s=0|t=19", + "simple_roles_23|sid=0|g=0|s=0|t=0", + "simple_roles_23|sid=13|g=13|s=0|t=13", + "simple_roles_23|sid=40|g=40|s=2|t=0", + "simple_roles_23|sid=38|g=38|s=1|t=17" + ], + "target_ids": [ + "19" + ], + "tid": 23, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the name of my subordinate's company?", + "retrieved_ids": [ + "simple_roles_24|sid=61|g=61|s=2|t=19", + "simple_roles_24|sid=9|g=9|s=0|t=9", + "simple_roles_24|sid=42|g=42|s=2|t=0", + "simple_roles_24|sid=41|g=41|s=1|t=20", + "simple_roles_24|sid=114|g=114|s=5|t=10" + ], + "target_ids": [ + "61" + ], + "tid": 24, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the age of the subordinate?", + "retrieved_ids": [ + "simple_roles_25|sid=75|g=75|s=3|t=16" + ], + "target_ids": [ + "75" + ], + "tid": 25, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What hobby does the subordinate enjoy?", + "retrieved_ids": [ + "simple_roles_26|sid=135|g=135|s=6|t=12", + "simple_roles_26|sid=88|g=88|s=4|t=5", + "simple_roles_26|sid=32|g=32|s=1|t=11", + "simple_roles_26|sid=29|g=29|s=1|t=8", + "simple_roles_26|sid=11|g=11|s=0|t=11" + ], + "target_ids": [ + "135" + ], + "tid": 26, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": false, + "question": "What level of education has my mother completed?", + "retrieved_ids": [ + "simple_roles_27|sid=27|g=27|s=1|t=7", + "simple_roles_27|sid=6|g=6|s=0|t=6", + "simple_roles_27|sid=129|g=129|s=6|t=7", + "simple_roles_27|sid=139|g=139|s=6|t=17", + "simple_roles_27|sid=44|g=44|s=2|t=3" + ], + "target_ids": [ + "118" + ], + "tid": 27, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "When is my subordinate's birthday?", + "retrieved_ids": [ + "simple_roles_28|sid=15|g=15|s=0|t=15", + "simple_roles_28|sid=109|g=109|s=5|t=6", + "simple_roles_28|sid=44|g=44|s=2|t=2", + "simple_roles_28|sid=127|g=127|s=6|t=3", + "simple_roles_28|sid=0|g=0|s=0|t=0" + ], + "target_ids": [ + "15" + ], + "tid": 28, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the position of my sister?", + "retrieved_ids": [ + "simple_roles_29|sid=163|g=163|s=7|t=18", + "simple_roles_29|sid=93|g=93|s=4|t=10", + "simple_roles_29|sid=145|g=145|s=7|t=0", + "simple_roles_29|sid=17|g=17|s=0|t=17", + "simple_roles_29|sid=100|g=100|s=4|t=17" + ], + "target_ids": [ + "163" + ], + "tid": 29, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": false, + "question": "What is the hometown of my female cousin?", + "retrieved_ids": [], + "target_ids": [ + "24" + ], + "tid": 30, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the height of my uncle?", + "retrieved_ids": [ + "simple_roles_31|sid=67|g=67|s=3|t=7", + "simple_roles_31|sid=141|g=141|s=7|t=1", + "simple_roles_31|sid=121|g=121|s=6|t=1", + "simple_roles_31|sid=21|g=21|s=1|t=1", + "simple_roles_31|sid=60|g=60|s=3|t=0" + ], + "target_ids": [ + "67" + ], + "tid": 31, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the contact number for my male cousin?", + "retrieved_ids": [ + "simple_roles_32|sid=141|g=141|s=6|t=19", + "simple_roles_32|sid=109|g=109|s=5|t=8", + "simple_roles_32|sid=52|g=52|s=2|t=11", + "simple_roles_32|sid=74|g=74|s=3|t=13", + "simple_roles_32|sid=91|g=91|s=4|t=10" + ], + "target_ids": [ + "141" + ], + "tid": 32, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "How old is the coworker?", + "retrieved_ids": [ + "simple_roles_33|sid=151|g=151|s=7|t=7", + "simple_roles_33|sid=62|g=62|s=3|t=0", + "simple_roles_33|sid=162|g=162|s=7|t=18", + "simple_roles_33|sid=161|g=161|s=7|t=17", + "simple_roles_33|sid=144|g=144|s=7|t=0" + ], + "target_ids": [ + "151" + ], + "tid": 33, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the email address of that coworker?", + "retrieved_ids": [ + "simple_roles_34|sid=26|g=26|s=1|t=6", + "simple_roles_34|sid=139|g=139|s=6|t=15", + "simple_roles_34|sid=113|g=113|s=5|t=10", + "simple_roles_34|sid=96|g=96|s=4|t=13", + "simple_roles_34|sid=72|g=72|s=3|t=10" + ], + "target_ids": [ + "26" + ], + "tid": 34, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "Could anyone provide the contact number for my subordinate?", + "retrieved_ids": [ + "simple_roles_35|sid=16|g=16|s=0|t=16", + "simple_roles_35|sid=109|g=109|s=5|t=6", + "simple_roles_35|sid=77|g=77|s=3|t=17", + "simple_roles_35|sid=148|g=148|s=7|t=3", + "simple_roles_35|sid=55|g=55|s=2|t=15" + ], + "target_ids": [ + "109" + ], + "tid": 35, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the education background of the coworker?", + "retrieved_ids": [ + "simple_roles_36|sid=104|g=104|s=5|t=0", + "simple_roles_36|sid=61|g=61|s=2|t=19", + "simple_roles_36|sid=122|g=122|s=5|t=18", + "simple_roles_36|sid=112|g=112|s=5|t=8", + "simple_roles_36|sid=127|g=127|s=6|t=3" + ], + "target_ids": [ + "112" + ], + "tid": 36, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "When is my sister's birthday?", + "retrieved_ids": [ + "simple_roles_37|sid=139|g=139|s=6|t=18", + "simple_roles_37|sid=44|g=44|s=2|t=3", + "simple_roles_37|sid=118|g=118|s=5|t=17", + "simple_roles_37|sid=121|g=121|s=6|t=0", + "simple_roles_37|sid=20|g=20|s=1|t=0" + ], + "target_ids": [ + "139" + ], + "tid": 37, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What hobby does my nephew have?", + "retrieved_ids": [ + "simple_roles_38|sid=42|g=42|s=2|t=0", + "simple_roles_38|sid=48|g=48|s=2|t=6", + "simple_roles_38|sid=14|g=14|s=0|t=14", + "simple_roles_38|sid=10|g=10|s=0|t=10", + "simple_roles_38|sid=72|g=72|s=3|t=9" + ], + "target_ids": [ + "48" + ], + "tid": 38, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the height of my uncle?", + "retrieved_ids": [ + "simple_roles_39|sid=118|g=118|s=5|t=16", + "simple_roles_39|sid=102|g=102|s=5|t=0", + "simple_roles_39|sid=108|g=108|s=5|t=6", + "simple_roles_39|sid=82|g=82|s=4|t=1", + "simple_roles_39|sid=70|g=70|s=3|t=10" + ], + "target_ids": [ + "118" + ], + "tid": 39, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "When is my father's birthday?", + "retrieved_ids": [ + "simple_roles_40|sid=41|g=41|s=2|t=0", + "simple_roles_40|sid=84|g=84|s=4|t=3", + "simple_roles_40|sid=50|g=50|s=2|t=9", + "simple_roles_40|sid=59|g=59|s=2|t=18", + "simple_roles_40|sid=3|g=3|s=0|t=3" + ], + "target_ids": [ + "50" + ], + "tid": 40, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the hometown of my father?", + "retrieved_ids": [ + "simple_roles_41|sid=129|g=129|s=6|t=5", + "simple_roles_41|sid=46|g=46|s=2|t=4", + "simple_roles_41|sid=99|g=99|s=4|t=16", + "simple_roles_41|sid=21|g=21|s=1|t=0", + "simple_roles_41|sid=56|g=56|s=2|t=14" + ], + "target_ids": [ + "99" + ], + "tid": 41, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": false, + "question": "What position does the coworker hold?", + "retrieved_ids": [], + "target_ids": [ + "38" + ], + "tid": 42, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What position does that coworker hold?", + "retrieved_ids": [ + "simple_roles_43|sid=105|g=105|s=5|t=4", + "simple_roles_43|sid=110|g=110|s=5|t=9", + "simple_roles_43|sid=22|g=22|s=1|t=2", + "simple_roles_43|sid=34|g=34|s=1|t=14", + "simple_roles_43|sid=14|g=14|s=0|t=14" + ], + "target_ids": [ + "34" + ], + "tid": 43, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is my male cousin's email address?", + "retrieved_ids": [ + "simple_roles_44|sid=100|g=100|s=4|t=17", + "simple_roles_44|sid=51|g=51|s=2|t=10", + "simple_roles_44|sid=138|g=138|s=6|t=14", + "simple_roles_44|sid=156|g=156|s=7|t=12", + "simple_roles_44|sid=29|g=29|s=1|t=9" + ], + "target_ids": [ + "100" + ], + "tid": 44, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the highest level of education attained by my aunt?", + "retrieved_ids": [ + "simple_roles_45|sid=90|g=90|s=4|t=9", + "simple_roles_45|sid=42|g=42|s=2|t=0", + "simple_roles_45|sid=130|g=130|s=6|t=9", + "simple_roles_45|sid=51|g=51|s=2|t=9", + "simple_roles_45|sid=60|g=60|s=2|t=18" + ], + "target_ids": [ + "51" + ], + "tid": 45, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What hobby does my boss have?", + "retrieved_ids": [ + "simple_roles_46|sid=89|g=89|s=4|t=6", + "simple_roles_46|sid=9|g=9|s=0|t=9", + "simple_roles_46|sid=83|g=83|s=4|t=0", + "simple_roles_46|sid=112|g=112|s=5|t=8", + "simple_roles_46|sid=103|g=103|s=4|t=20" + ], + "target_ids": [ + "89" + ], + "tid": 46, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the age of my father?", + "retrieved_ids": [ + "simple_roles_47|sid=41|g=41|s=2|t=0", + "simple_roles_47|sid=101|g=101|s=5|t=0", + "simple_roles_47|sid=1|g=1|s=0|t=1", + "simple_roles_47|sid=108|g=108|s=5|t=7", + "simple_roles_47|sid=61|g=61|s=3|t=0" + ], + "target_ids": [ + "108" + ], + "tid": 47, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the hometown of my father?", + "retrieved_ids": [ + "simple_roles_48|sid=101|g=101|s=5|t=0", + "simple_roles_48|sid=64|g=64|s=3|t=3", + "simple_roles_48|sid=109|g=109|s=5|t=8", + "simple_roles_48|sid=54|g=54|s=2|t=12", + "simple_roles_48|sid=27|g=27|s=1|t=6" + ], + "target_ids": [ + "109" + ], + "tid": 48, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the email address of my father?", + "retrieved_ids": [ + "simple_roles_49|sid=6|g=6|s=0|t=6", + "simple_roles_49|sid=117|g=117|s=5|t=17", + "simple_roles_49|sid=91|g=91|s=4|t=12", + "simple_roles_49|sid=52|g=52|s=2|t=12", + "simple_roles_49|sid=31|g=31|s=1|t=11" + ], + "target_ids": [ + "52" + ], + "tid": 49, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": false, + "question": "What is the hobby of my mother?", + "retrieved_ids": [ + "simple_roles_50|sid=0|g=0|s=0|t=0", + "simple_roles_50|sid=71|g=71|s=3|t=8", + "simple_roles_50|sid=112|g=112|s=5|t=8", + "simple_roles_50|sid=30|g=30|s=1|t=9", + "simple_roles_50|sid=153|g=153|s=7|t=8" + ], + "target_ids": [ + "18" + ], + "tid": 50, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "When is my boss's birthday?", + "retrieved_ids": [ + "simple_roles_51|sid=79|g=79|s=3|t=18", + "simple_roles_51|sid=22|g=22|s=1|t=1", + "simple_roles_51|sid=49|g=49|s=2|t=8", + "simple_roles_51|sid=61|g=61|s=3|t=0", + "simple_roles_51|sid=81|g=81|s=3|t=20" + ], + "target_ids": [ + "79" + ], + "tid": 51, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the occupation of that subordinate?", + "retrieved_ids": [ + "simple_roles_52|sid=79|g=79|s=4|t=0", + "simple_roles_52|sid=7|g=7|s=0|t=7", + "simple_roles_52|sid=80|g=80|s=4|t=1" + ], + "target_ids": [ + "80" + ], + "tid": 52, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the level of education that my mother has completed?", + "retrieved_ids": [ + "simple_roles_53|sid=0|g=0|s=0|t=0", + "simple_roles_53|sid=93|g=93|s=4|t=11", + "simple_roles_53|sid=12|g=12|s=0|t=12", + "simple_roles_53|sid=111|g=111|s=5|t=8", + "simple_roles_53|sid=74|g=74|s=3|t=13" + ], + "target_ids": [ + "12" + ], + "tid": 53, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the work location of the coworker?", + "retrieved_ids": [ + "simple_roles_54|sid=117|g=117|s=5|t=16", + "simple_roles_54|sid=112|g=112|s=5|t=11", + "simple_roles_54|sid=46|g=46|s=2|t=6", + "simple_roles_54|sid=113|g=113|s=5|t=12", + "simple_roles_54|sid=118|g=118|s=5|t=17" + ], + "target_ids": [ + "118" + ], + "tid": 54, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "How old is my brother?", + "retrieved_ids": [ + "simple_roles_55|sid=70|g=70|s=3|t=7", + "simple_roles_55|sid=42|g=42|s=2|t=0", + "simple_roles_55|sid=1|g=1|s=0|t=1", + "simple_roles_55|sid=123|g=123|s=6|t=0", + "simple_roles_55|sid=63|g=63|s=3|t=0" + ], + "target_ids": [ + "70" + ], + "tid": 55, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What does my boss do for a living?", + "retrieved_ids": [ + "simple_roles_56|sid=59|g=59|s=2|t=18", + "simple_roles_56|sid=41|g=41|s=2|t=0", + "simple_roles_56|sid=61|g=61|s=2|t=20", + "simple_roles_56|sid=60|g=60|s=2|t=19", + "simple_roles_56|sid=162|g=162|s=7|t=18" + ], + "target_ids": [ + "59" + ], + "tid": 56, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the birthday of my subordinate?", + "retrieved_ids": [ + "simple_roles_57|sid=93|g=93|s=4|t=11", + "simple_roles_57|sid=145|g=145|s=7|t=2", + "simple_roles_57|sid=125|g=125|s=6|t=2", + "simple_roles_57|sid=23|g=23|s=1|t=2", + "simple_roles_57|sid=82|g=82|s=4|t=0" + ], + "target_ids": [ + "93" + ], + "tid": 57, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the hometown of my female cousin?", + "retrieved_ids": [ + "simple_roles_58|sid=124|g=124|s=6|t=0", + "simple_roles_58|sid=68|g=68|s=3|t=5", + "simple_roles_58|sid=116|g=116|s=5|t=13", + "simple_roles_58|sid=142|g=142|s=6|t=18", + "simple_roles_58|sid=139|g=139|s=6|t=15" + ], + "target_ids": [ + "139" + ], + "tid": 58, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What position does my aunt hold?", + "retrieved_ids": [ + "simple_roles_59|sid=124|g=124|s=6|t=0", + "simple_roles_59|sid=131|g=131|s=6|t=7", + "simple_roles_59|sid=22|g=22|s=1|t=2", + "simple_roles_59|sid=143|g=143|s=6|t=19", + "simple_roles_59|sid=47|g=47|s=2|t=6" + ], + "target_ids": [ + "131" + ], + "tid": 59, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the date of my niece's birthday?", + "retrieved_ids": [ + "simple_roles_60|sid=59|g=59|s=2|t=18", + "simple_roles_60|sid=64|g=64|s=3|t=3", + "simple_roles_60|sid=151|g=151|s=7|t=8", + "simple_roles_60|sid=60|g=60|s=2|t=19", + "simple_roles_60|sid=41|g=41|s=2|t=0" + ], + "target_ids": [ + "59" + ], + "tid": 60, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": false, + "question": "What hobby does my father enjoy?", + "retrieved_ids": [ + "simple_roles_61|sid=54|g=54|s=2|t=12", + "simple_roles_61|sid=157|g=157|s=7|t=12", + "simple_roles_61|sid=111|g=111|s=5|t=7", + "simple_roles_61|sid=0|g=0|s=0|t=0", + "simple_roles_61|sid=28|g=28|s=1|t=7" + ], + "target_ids": [ + "7" + ], + "tid": 61, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the contact number for my brother?", + "retrieved_ids": [ + "simple_roles_62|sid=61|g=61|s=2|t=19", + "simple_roles_62|sid=139|g=139|s=6|t=15", + "simple_roles_62|sid=84|g=84|s=4|t=2", + "simple_roles_62|sid=118|g=118|s=5|t=15", + "simple_roles_62|sid=162|g=162|s=7|t=18" + ], + "target_ids": [ + "61" + ], + "tid": 62, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "How old is the coworker?", + "retrieved_ids": [ + "simple_roles_63|sid=64|g=64|s=3|t=3", + "simple_roles_63|sid=103|g=103|s=5|t=0", + "simple_roles_63|sid=145|g=145|s=7|t=1", + "simple_roles_63|sid=61|g=61|s=3|t=0", + "simple_roles_63|sid=41|g=41|s=2|t=0" + ], + "target_ids": [ + "64" + ], + "tid": 63, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the height of my niece?", + "retrieved_ids": [ + "simple_roles_64|sid=115|g=115|s=5|t=14", + "simple_roles_64|sid=123|g=123|s=6|t=1", + "simple_roles_64|sid=101|g=101|s=5|t=0", + "simple_roles_64|sid=1|g=1|s=0|t=1", + "simple_roles_64|sid=146|g=146|s=7|t=4" + ], + "target_ids": [ + "115" + ], + "tid": 64, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "When is my subordinate's birthday?", + "retrieved_ids": [ + "simple_roles_65|sid=13|g=13|s=0|t=13", + "simple_roles_65|sid=102|g=102|s=5|t=2", + "simple_roles_65|sid=23|g=23|s=1|t=3", + "simple_roles_65|sid=64|g=64|s=3|t=3", + "simple_roles_65|sid=133|g=133|s=6|t=12" + ], + "target_ids": [ + "13" + ], + "tid": 65, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the name of my female cousin?", + "retrieved_ids": [ + "simple_roles_66|sid=65|g=65|s=3|t=4", + "simple_roles_66|sid=144|g=144|s=7|t=0", + "simple_roles_66|sid=155|g=155|s=7|t=11", + "simple_roles_66|sid=163|g=163|s=7|t=19", + "simple_roles_66|sid=99|g=99|s=4|t=18" + ], + "target_ids": [ + "65" + ], + "tid": 66, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What does my brother do for a living?", + "retrieved_ids": [ + "simple_roles_67|sid=61|g=61|s=3|t=0", + "simple_roles_67|sid=67|g=67|s=3|t=6", + "simple_roles_67|sid=72|g=72|s=3|t=11", + "simple_roles_67|sid=47|g=47|s=2|t=6", + "simple_roles_67|sid=98|g=98|s=4|t=16" + ], + "target_ids": [ + "67" + ], + "tid": 67, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the location where my sister works?", + "retrieved_ids": [ + "simple_roles_68|sid=73|g=73|s=3|t=13", + "simple_roles_68|sid=69|g=69|s=3|t=9", + "simple_roles_68|sid=149|g=149|s=7|t=8", + "simple_roles_68|sid=60|g=60|s=3|t=0", + "simple_roles_68|sid=66|g=66|s=3|t=6" + ], + "target_ids": [ + "69" + ], + "tid": 68, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the occupation of that coworker?", + "retrieved_ids": [ + "simple_roles_69|sid=0|g=0|s=0|t=0", + "simple_roles_69|sid=8|g=8|s=0|t=8", + "simple_roles_69|sid=15|g=15|s=0|t=15", + "simple_roles_69|sid=58|g=58|s=2|t=18" + ], + "target_ids": [ + "8" + ], + "tid": 69, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the position of my niece?", + "retrieved_ids": [ + "simple_roles_70|sid=115|g=115|s=5|t=16", + "simple_roles_70|sid=99|g=99|s=5|t=0", + "simple_roles_70|sid=117|g=117|s=5|t=18", + "simple_roles_70|sid=138|g=138|s=6|t=19", + "simple_roles_70|sid=19|g=19|s=0|t=19" + ], + "target_ids": [ + "115" + ], + "tid": 70, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the date of my coworker's birthday?", + "retrieved_ids": [ + "simple_roles_71|sid=159|g=159|s=7|t=17", + "simple_roles_71|sid=142|g=142|s=7|t=0", + "simple_roles_71|sid=151|g=151|s=7|t=9", + "simple_roles_71|sid=155|g=155|s=7|t=13", + "simple_roles_71|sid=81|g=81|s=4|t=0" + ], + "target_ids": [ + "159" + ], + "tid": 71, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the date of my brother's birthday?", + "retrieved_ids": [ + "simple_roles_72|sid=28|g=28|s=1|t=7", + "simple_roles_72|sid=21|g=21|s=1|t=0", + "simple_roles_72|sid=64|g=64|s=3|t=3", + "simple_roles_72|sid=44|g=44|s=2|t=3", + "simple_roles_72|sid=40|g=40|s=1|t=19" + ], + "target_ids": [ + "28" + ], + "tid": 72, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the name of the company where the coworker works?", + "retrieved_ids": [ + "simple_roles_73|sid=67|g=67|s=3|t=7", + "simple_roles_73|sid=153|g=153|s=7|t=11", + "simple_roles_73|sid=126|g=126|s=6|t=4", + "simple_roles_73|sid=77|g=77|s=3|t=17", + "simple_roles_73|sid=9|g=9|s=0|t=9" + ], + "target_ids": [ + "67" + ], + "tid": 73, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is my male cousin's email address?", + "retrieved_ids": [ + "simple_roles_74|sid=12|g=12|s=0|t=12", + "simple_roles_74|sid=57|g=57|s=2|t=15", + "simple_roles_74|sid=77|g=77|s=3|t=14", + "simple_roles_74|sid=146|g=146|s=6|t=20", + "simple_roles_74|sid=0|g=0|s=0|t=0" + ], + "target_ids": [ + "12" + ], + "tid": 74, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What position does my uncle hold?", + "retrieved_ids": [ + "simple_roles_75|sid=83|g=83|s=4|t=0", + "simple_roles_75|sid=41|g=41|s=2|t=0", + "simple_roles_75|sid=61|g=61|s=2|t=20", + "simple_roles_75|sid=52|g=52|s=2|t=11", + "simple_roles_75|sid=57|g=57|s=2|t=16" + ], + "target_ids": [ + "52" + ], + "tid": 75, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What hobby does my niece enjoy?", + "retrieved_ids": [ + "simple_roles_76|sid=112|g=112|s=5|t=7", + "simple_roles_76|sid=134|g=134|s=6|t=9", + "simple_roles_76|sid=105|g=105|s=5|t=0", + "simple_roles_76|sid=50|g=50|s=2|t=8", + "simple_roles_76|sid=124|g=124|s=5|t=19" + ], + "target_ids": [ + "112" + ], + "tid": 76, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": false, + "question": "What is the occupation of my mother?", + "retrieved_ids": [ + "simple_roles_77|sid=21|g=21|s=1|t=0", + "simple_roles_77|sid=147|g=147|s=7|t=4", + "simple_roles_77|sid=133|g=133|s=6|t=10", + "simple_roles_77|sid=161|g=161|s=7|t=18", + "simple_roles_77|sid=89|g=89|s=4|t=7" + ], + "target_ids": [ + "40" + ], + "tid": 77, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "How old is my sister?", + "retrieved_ids": [ + "simple_roles_78|sid=47|g=47|s=2|t=7", + "simple_roles_78|sid=80|g=80|s=4|t=0", + "simple_roles_78|sid=101|g=101|s=5|t=0", + "simple_roles_78|sid=0|g=0|s=0|t=0", + "simple_roles_78|sid=40|g=40|s=2|t=0" + ], + "target_ids": [ + "47" + ], + "tid": 78, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the name of the boss?", + "retrieved_ids": [ + "simple_roles_79|sid=119|g=119|s=5|t=19", + "simple_roles_79|sid=111|g=111|s=5|t=11", + "simple_roles_79|sid=125|g=125|s=6|t=4", + "simple_roles_79|sid=100|g=100|s=5|t=0" + ], + "target_ids": [ + "119" + ], + "tid": 79, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the work location of my niece?", + "retrieved_ids": [ + "simple_roles_80|sid=49|g=49|s=2|t=8", + "simple_roles_80|sid=41|g=41|s=2|t=0", + "simple_roles_80|sid=110|g=110|s=5|t=8", + "simple_roles_80|sid=58|g=58|s=2|t=17", + "simple_roles_80|sid=122|g=122|s=5|t=20" + ], + "target_ids": [ + "49" + ], + "tid": 80, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the date of my niece's birthday?", + "retrieved_ids": [ + "simple_roles_81|sid=101|g=101|s=4|t=19", + "simple_roles_81|sid=144|g=144|s=7|t=1", + "simple_roles_81|sid=30|g=30|s=1|t=9", + "simple_roles_81|sid=134|g=134|s=6|t=12", + "simple_roles_81|sid=82|g=82|s=4|t=0" + ], + "target_ids": [ + "101" + ], + "tid": 81, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "Where does my aunt work?", + "retrieved_ids": [ + "simple_roles_82|sid=2|g=2|s=0|t=2", + "simple_roles_82|sid=138|g=138|s=6|t=12", + "simple_roles_82|sid=0|g=0|s=0|t=0", + "simple_roles_82|sid=70|g=70|s=3|t=6", + "simple_roles_82|sid=19|g=19|s=0|t=19" + ], + "target_ids": [ + "2" + ], + "tid": 82, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the hometown of my niece?", + "retrieved_ids": [ + "simple_roles_83|sid=44|g=44|s=2|t=2", + "simple_roles_83|sid=107|g=107|s=5|t=3", + "simple_roles_83|sid=42|g=42|s=2|t=0", + "simple_roles_83|sid=89|g=89|s=4|t=6", + "simple_roles_83|sid=121|g=121|s=5|t=17" + ], + "target_ids": [ + "44" + ], + "tid": 83, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the contact number for my subordinate?", + "retrieved_ids": [ + "simple_roles_84|sid=98|g=98|s=4|t=17", + "simple_roles_84|sid=37|g=37|s=1|t=16", + "simple_roles_84|sid=12|g=12|s=0|t=12", + "simple_roles_84|sid=139|g=139|s=6|t=17", + "simple_roles_84|sid=73|g=73|s=3|t=12" + ], + "target_ids": [ + "98" + ], + "tid": 84, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the date of my sister's birthday?", + "retrieved_ids": [ + "simple_roles_85|sid=59|g=59|s=2|t=18", + "simple_roles_85|sid=41|g=41|s=2|t=0", + "simple_roles_85|sid=139|g=139|s=6|t=16", + "simple_roles_85|sid=156|g=156|s=7|t=13", + "simple_roles_85|sid=52|g=52|s=2|t=11" + ], + "target_ids": [ + "59" + ], + "tid": 85, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the hometown of my aunt?", + "retrieved_ids": [ + "simple_roles_86|sid=60|g=60|s=3|t=0", + "simple_roles_86|sid=30|g=30|s=1|t=12", + "simple_roles_86|sid=75|g=75|s=3|t=15", + "simple_roles_86|sid=72|g=72|s=3|t=12", + "simple_roles_86|sid=64|g=64|s=3|t=4" + ], + "target_ids": [ + "75" + ], + "tid": 86, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": false, + "question": "What does my female cousin do for a living?", + "retrieved_ids": [], + "target_ids": [ + "58" + ], + "tid": 87, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the name of my mother's company?", + "retrieved_ids": [ + "simple_roles_88|sid=164|g=164|s=7|t=18", + "simple_roles_88|sid=23|g=23|s=1|t=3", + "simple_roles_88|sid=146|g=146|s=7|t=0", + "simple_roles_88|sid=108|g=108|s=5|t=5", + "simple_roles_88|sid=124|g=124|s=5|t=21" + ], + "target_ids": [ + "164" + ], + "tid": 88, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the name of my subordinate?", + "retrieved_ids": [ + "simple_roles_89|sid=143|g=143|s=7|t=0", + "simple_roles_89|sid=149|g=149|s=7|t=6", + "simple_roles_89|sid=15|g=15|s=0|t=15", + "simple_roles_89|sid=144|g=144|s=7|t=1", + "simple_roles_89|sid=67|g=67|s=3|t=6" + ], + "target_ids": [ + "149" + ], + "tid": 89, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "How old is my female cousin?", + "retrieved_ids": [ + "simple_roles_90|sid=16|g=16|s=0|t=16", + "simple_roles_90|sid=40|g=40|s=2|t=0", + "simple_roles_90|sid=20|g=20|s=1|t=0", + "simple_roles_90|sid=44|g=44|s=2|t=4", + "simple_roles_90|sid=81|g=81|s=4|t=0" + ], + "target_ids": [ + "16" + ], + "tid": 90, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the name of my father?", + "retrieved_ids": [ + "simple_roles_91|sid=26|g=26|s=1|t=5", + "simple_roles_91|sid=117|g=117|s=5|t=13", + "simple_roles_91|sid=15|g=15|s=0|t=15", + "simple_roles_91|sid=148|g=148|s=7|t=4", + "simple_roles_91|sid=66|g=66|s=3|t=3" + ], + "target_ids": [ + "26" + ], + "tid": 91, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "When is the boss's birthday?", + "retrieved_ids": [ + "simple_roles_92|sid=69|g=69|s=3|t=8", + "simple_roles_92|sid=3|g=3|s=0|t=3", + "simple_roles_92|sid=124|g=124|s=6|t=3", + "simple_roles_92|sid=80|g=80|s=3|t=19", + "simple_roles_92|sid=61|g=61|s=3|t=0" + ], + "target_ids": [ + "69" + ], + "tid": 92, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the occupation of that subordinate?", + "retrieved_ids": [ + "simple_roles_93|sid=81|g=81|s=4|t=0", + "simple_roles_93|sid=98|g=98|s=4|t=17" + ], + "target_ids": [ + "98" + ], + "tid": 93, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the email address of my niece?", + "retrieved_ids": [ + "simple_roles_94|sid=138|g=138|s=6|t=15", + "simple_roles_94|sid=82|g=82|s=3|t=20", + "simple_roles_94|sid=155|g=155|s=7|t=12", + "simple_roles_94|sid=123|g=123|s=6|t=0", + "simple_roles_94|sid=58|g=58|s=2|t=17" + ], + "target_ids": [ + "138" + ], + "tid": 94, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What does my sister do for a living?", + "retrieved_ids": [ + "simple_roles_95|sid=148|g=148|s=7|t=4", + "simple_roles_95|sid=45|g=45|s=2|t=4", + "simple_roles_95|sid=4|g=4|s=0|t=4", + "simple_roles_95|sid=144|g=144|s=7|t=0", + "simple_roles_95|sid=89|g=89|s=4|t=6" + ], + "target_ids": [ + "148" + ], + "tid": 95, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the name of that uncle?", + "retrieved_ids": [ + "simple_roles_96|sid=152|g=152|s=7|t=11", + "simple_roles_96|sid=141|g=141|s=7|t=0" + ], + "target_ids": [ + "152" + ], + "tid": 96, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": false, + "question": "What does my father do for a living?", + "retrieved_ids": [ + "simple_roles_97|sid=102|g=102|s=5|t=0", + "simple_roles_97|sid=87|g=87|s=4|t=5", + "simple_roles_97|sid=48|g=48|s=2|t=7", + "simple_roles_97|sid=66|g=66|s=3|t=4", + "simple_roles_97|sid=74|g=74|s=3|t=12" + ], + "target_ids": [ + "114" + ], + "tid": 97, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the position of the boss?", + "retrieved_ids": [ + "simple_roles_98|sid=70|g=70|s=3|t=8", + "simple_roles_98|sid=79|g=79|s=3|t=17", + "simple_roles_98|sid=62|g=62|s=3|t=0", + "simple_roles_98|sid=89|g=89|s=4|t=6", + "simple_roles_98|sid=81|g=81|s=3|t=19" + ], + "target_ids": [ + "70" + ], + "tid": 98, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": false, + "question": "What is the occupation of my mother?", + "retrieved_ids": [ + "simple_roles_99|sid=21|g=21|s=1|t=0", + "simple_roles_99|sid=7|g=7|s=0|t=7", + "simple_roles_99|sid=52|g=52|s=2|t=10", + "simple_roles_99|sid=84|g=84|s=4|t=1", + "simple_roles_99|sid=60|g=60|s=2|t=18" + ], + "target_ids": [ + "33" + ], + "tid": 99, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the name of the company where the boss works?", + "retrieved_ids": [ + "simple_roles_100|sid=57|g=57|s=2|t=18", + "simple_roles_100|sid=12|g=12|s=0|t=12", + "simple_roles_100|sid=39|g=39|s=2|t=0", + "simple_roles_100|sid=56|g=56|s=2|t=17", + "simple_roles_100|sid=54|g=54|s=2|t=15" + ], + "target_ids": [ + "57" + ], + "tid": 100, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the location where Father works?", + "retrieved_ids": [ + "simple_roles_101|sid=59|g=59|s=3|t=0", + "simple_roles_101|sid=13|g=13|s=0|t=13", + "simple_roles_101|sid=109|g=109|s=5|t=10", + "simple_roles_101|sid=69|g=69|s=3|t=10", + "simple_roles_101|sid=76|g=76|s=3|t=17" + ], + "target_ids": [ + "69" + ], + "tid": 101, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What hobby does my niece enjoy?", + "retrieved_ids": [ + "simple_roles_102|sid=104|g=104|s=5|t=1", + "simple_roles_102|sid=91|g=91|s=4|t=9", + "simple_roles_102|sid=131|g=131|s=6|t=7", + "simple_roles_102|sid=96|g=96|s=4|t=14", + "simple_roles_102|sid=103|g=103|s=5|t=0" + ], + "target_ids": [ + "104" + ], + "tid": 102, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the work location of my mother?", + "retrieved_ids": [ + "simple_roles_103|sid=44|g=44|s=2|t=1", + "simple_roles_103|sid=43|g=43|s=2|t=0", + "simple_roles_103|sid=104|g=104|s=4|t=20", + "simple_roles_103|sid=69|g=69|s=3|t=5", + "simple_roles_103|sid=116|g=116|s=5|t=11" + ], + "target_ids": [ + "43" + ], + "tid": 103, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What does my nephew do for a living?", + "retrieved_ids": [ + "simple_roles_104|sid=83|g=83|s=4|t=0", + "simple_roles_104|sid=90|g=90|s=4|t=7", + "simple_roles_104|sid=89|g=89|s=4|t=6", + "simple_roles_104|sid=129|g=129|s=6|t=5", + "simple_roles_104|sid=27|g=27|s=1|t=6" + ], + "target_ids": [ + "89" + ], + "tid": 104, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the position of my brother?", + "retrieved_ids": [ + "simple_roles_105|sid=0|g=0|s=0|t=0", + "simple_roles_105|sid=17|g=17|s=0|t=17", + "simple_roles_105|sid=137|g=137|s=6|t=17", + "simple_roles_105|sid=78|g=78|s=3|t=18", + "simple_roles_105|sid=118|g=118|s=5|t=18" + ], + "target_ids": [ + "17" + ], + "tid": 105, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the hometown of that coworker?", + "retrieved_ids": [ + "simple_roles_106|sid=21|g=21|s=1|t=0", + "simple_roles_106|sid=63|g=63|s=3|t=5", + "simple_roles_106|sid=37|g=37|s=1|t=16", + "simple_roles_106|sid=31|g=31|s=1|t=10", + "simple_roles_106|sid=30|g=30|s=1|t=9" + ], + "target_ids": [ + "30" + ], + "tid": 106, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What hobby does the boss have?", + "retrieved_ids": [ + "simple_roles_107|sid=13|g=13|s=0|t=13", + "simple_roles_107|sid=0|g=0|s=0|t=0", + "simple_roles_107|sid=31|g=31|s=1|t=11", + "simple_roles_107|sid=93|g=93|s=4|t=11", + "simple_roles_107|sid=18|g=18|s=0|t=18" + ], + "target_ids": [ + "13" + ], + "tid": 107, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is my male cousin's email address?", + "retrieved_ids": [ + "simple_roles_108|sid=122|g=122|s=6|t=0", + "simple_roles_108|sid=35|g=35|s=1|t=15", + "simple_roles_108|sid=112|g=112|s=5|t=11", + "simple_roles_108|sid=154|g=154|s=7|t=12", + "simple_roles_108|sid=123|g=123|s=6|t=1" + ], + "target_ids": [ + "122" + ], + "tid": 108, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the work location of my sister?", + "retrieved_ids": [ + "simple_roles_109|sid=76|g=76|s=3|t=14", + "simple_roles_109|sid=62|g=62|s=3|t=0", + "simple_roles_109|sid=67|g=67|s=3|t=5", + "simple_roles_109|sid=143|g=143|s=7|t=0", + "simple_roles_109|sid=15|g=15|s=0|t=15" + ], + "target_ids": [ + "76" + ], + "tid": 109, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the position of my nephew?", + "retrieved_ids": [ + "simple_roles_110|sid=82|g=82|s=4|t=0", + "simple_roles_110|sid=83|g=83|s=4|t=1", + "simple_roles_110|sid=46|g=46|s=2|t=5", + "simple_roles_110|sid=75|g=75|s=3|t=14", + "simple_roles_110|sid=119|g=119|s=5|t=17" + ], + "target_ids": [ + "83" + ], + "tid": 110, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": false, + "question": "What hobby does my father have?", + "retrieved_ids": [ + "simple_roles_111|sid=79|g=79|s=4|t=0", + "simple_roles_111|sid=111|g=111|s=5|t=11", + "simple_roles_111|sid=26|g=26|s=1|t=7", + "simple_roles_111|sid=68|g=68|s=3|t=9", + "simple_roles_111|sid=9|g=9|s=0|t=9" + ], + "target_ids": [ + "90" + ], + "tid": 111, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the contact number for my father?", + "retrieved_ids": [ + "simple_roles_112|sid=80|g=80|s=3|t=18", + "simple_roles_112|sid=40|g=40|s=1|t=19", + "simple_roles_112|sid=157|g=157|s=7|t=13", + "simple_roles_112|sid=121|g=121|s=5|t=18", + "simple_roles_112|sid=62|g=62|s=3|t=0" + ], + "target_ids": [ + "80" + ], + "tid": 112, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the name of the coworker?", + "retrieved_ids": [ + "simple_roles_113|sid=39|g=39|s=1|t=20", + "simple_roles_113|sid=131|g=131|s=6|t=8", + "simple_roles_113|sid=19|g=19|s=1|t=0", + "simple_roles_113|sid=28|g=28|s=1|t=9", + "simple_roles_113|sid=132|g=132|s=6|t=9" + ], + "target_ids": [ + "39" + ], + "tid": 113, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": false, + "question": "What is the name of Uncle's company?", + "retrieved_ids": [], + "target_ids": [ + "42" + ], + "tid": 114, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the height of the boss?", + "retrieved_ids": [ + "simple_roles_115|sid=36|g=36|s=1|t=15", + "simple_roles_115|sid=21|g=21|s=1|t=0", + "simple_roles_115|sid=31|g=31|s=1|t=10", + "simple_roles_115|sid=125|g=125|s=6|t=1", + "simple_roles_115|sid=84|g=84|s=4|t=1" + ], + "target_ids": [ + "36" + ], + "tid": 115, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the name of my uncle?", + "retrieved_ids": [ + "simple_roles_116|sid=30|g=30|s=1|t=10", + "simple_roles_116|sid=20|g=20|s=1|t=0", + "simple_roles_116|sid=54|g=54|s=2|t=12", + "simple_roles_116|sid=22|g=22|s=1|t=2", + "simple_roles_116|sid=9|g=9|s=0|t=9" + ], + "target_ids": [ + "30" + ], + "tid": 116, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": false, + "question": "What is the location of my father's workplace?", + "retrieved_ids": [ + "simple_roles_117|sid=144|g=144|s=7|t=0", + "simple_roles_117|sid=72|g=72|s=3|t=10", + "simple_roles_117|sid=163|g=163|s=7|t=19", + "simple_roles_117|sid=164|g=164|s=7|t=20", + "simple_roles_117|sid=47|g=47|s=2|t=6" + ], + "target_ids": [ + "162" + ], + "tid": 117, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "How old is my mother?", + "retrieved_ids": [ + "simple_roles_118|sid=20|g=20|s=1|t=0", + "simple_roles_118|sid=102|g=102|s=5|t=1", + "simple_roles_118|sid=0|g=0|s=0|t=0", + "simple_roles_118|sid=40|g=40|s=2|t=0", + "simple_roles_118|sid=122|g=122|s=6|t=0" + ], + "target_ids": [ + "102" + ], + "tid": 118, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": false, + "question": "What does my father do for a living?", + "retrieved_ids": [ + "simple_roles_119|sid=0|g=0|s=0|t=0", + "simple_roles_119|sid=25|g=25|s=1|t=5", + "simple_roles_119|sid=94|g=94|s=4|t=12", + "simple_roles_119|sid=87|g=87|s=4|t=5", + "simple_roles_119|sid=56|g=56|s=2|t=15" + ], + "target_ids": [ + "13" + ], + "tid": 119, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the name of that coworker?", + "retrieved_ids": [ + "simple_roles_120|sid=115|g=115|s=5|t=11", + "simple_roles_120|sid=104|g=104|s=5|t=0", + "simple_roles_120|sid=152|g=152|s=7|t=7", + "simple_roles_120|sid=116|g=116|s=5|t=12" + ], + "target_ids": [ + "115" + ], + "tid": 120, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the work location of my niece?", + "retrieved_ids": [ + "simple_roles_121|sid=5|g=5|s=0|t=5", + "simple_roles_121|sid=0|g=0|s=0|t=0", + "simple_roles_121|sid=140|g=140|s=6|t=15", + "simple_roles_121|sid=30|g=30|s=1|t=10", + "simple_roles_121|sid=125|g=125|s=6|t=0" + ], + "target_ids": [ + "5" + ], + "tid": 121, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "How old is my nephew?", + "retrieved_ids": [ + "simple_roles_122|sid=95|g=95|s=4|t=13", + "simple_roles_122|sid=82|g=82|s=4|t=0", + "simple_roles_122|sid=65|g=65|s=3|t=4", + "simple_roles_122|sid=101|g=101|s=4|t=19", + "simple_roles_122|sid=41|g=41|s=2|t=0" + ], + "target_ids": [ + "95" + ], + "tid": 122, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the work location of the boss?", + "retrieved_ids": [ + "simple_roles_123|sid=130|g=130|s=6|t=7", + "simple_roles_123|sid=63|g=63|s=3|t=1", + "simple_roles_123|sid=62|g=62|s=3|t=0", + "simple_roles_123|sid=142|g=142|s=6|t=19", + "simple_roles_123|sid=78|g=78|s=3|t=16" + ], + "target_ids": [ + "63" + ], + "tid": 123, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the name of the niece?", + "retrieved_ids": [ + "simple_roles_124|sid=123|g=123|s=6|t=0", + "simple_roles_124|sid=140|g=140|s=6|t=17", + "simple_roles_124|sid=84|g=84|s=4|t=2", + "simple_roles_124|sid=6|g=6|s=0|t=6" + ], + "target_ids": [ + "140" + ], + "tid": 124, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the height of my father?", + "retrieved_ids": [ + "simple_roles_125|sid=102|g=102|s=5|t=0", + "simple_roles_125|sid=63|g=63|s=3|t=2", + "simple_roles_125|sid=109|g=109|s=5|t=7", + "simple_roles_125|sid=22|g=22|s=1|t=1", + "simple_roles_125|sid=42|g=42|s=2|t=1" + ], + "target_ids": [ + "109" + ], + "tid": 125, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the name of the company where the boss works?", + "retrieved_ids": [ + "simple_roles_126|sid=1|g=1|s=0|t=1", + "simple_roles_126|sid=6|g=6|s=0|t=6", + "simple_roles_126|sid=148|g=148|s=7|t=7", + "simple_roles_126|sid=129|g=129|s=6|t=8", + "simple_roles_126|sid=18|g=18|s=0|t=18" + ], + "target_ids": [ + "1" + ], + "tid": 126, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the educational background of the boss?", + "retrieved_ids": [ + "simple_roles_127|sid=46|g=46|s=2|t=4", + "simple_roles_127|sid=56|g=56|s=2|t=14", + "simple_roles_127|sid=95|g=95|s=4|t=12", + "simple_roles_127|sid=92|g=92|s=4|t=9", + "simple_roles_127|sid=26|g=26|s=1|t=5" + ], + "target_ids": [ + "95" + ], + "tid": 127, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the contact number of my male cousin?", + "retrieved_ids": [ + "simple_roles_128|sid=120|g=120|s=5|t=16", + "simple_roles_128|sid=158|g=158|s=7|t=12", + "simple_roles_128|sid=13|g=13|s=0|t=13", + "simple_roles_128|sid=92|g=92|s=4|t=9", + "simple_roles_128|sid=90|g=90|s=4|t=7" + ], + "target_ids": [ + "120" + ], + "tid": 128, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the level of education that my sister has achieved?", + "retrieved_ids": [ + "simple_roles_129|sid=130|g=130|s=6|t=8", + "simple_roles_129|sid=132|g=132|s=6|t=10", + "simple_roles_129|sid=122|g=122|s=6|t=0", + "simple_roles_129|sid=114|g=114|s=5|t=12", + "simple_roles_129|sid=64|g=64|s=3|t=3" + ], + "target_ids": [ + "130" + ], + "tid": 129, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the position of my subordinate?", + "retrieved_ids": [ + "simple_roles_130|sid=0|g=0|s=0|t=0", + "simple_roles_130|sid=10|g=10|s=0|t=10", + "simple_roles_130|sid=57|g=57|s=2|t=18", + "simple_roles_130|sid=150|g=150|s=7|t=11", + "simple_roles_130|sid=42|g=42|s=2|t=3" + ], + "target_ids": [ + "10" + ], + "tid": 130, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the work location of the boss?", + "retrieved_ids": [ + "simple_roles_131|sid=76|g=76|s=3|t=16", + "simple_roles_131|sid=3|g=3|s=0|t=3", + "simple_roles_131|sid=104|g=104|s=5|t=4", + "simple_roles_131|sid=60|g=60|s=3|t=0", + "simple_roles_131|sid=69|g=69|s=3|t=9" + ], + "target_ids": [ + "76" + ], + "tid": 131, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the email address of the boss?", + "retrieved_ids": [ + "simple_roles_132|sid=10|g=10|s=0|t=10", + "simple_roles_132|sid=74|g=74|s=3|t=13", + "simple_roles_132|sid=92|g=92|s=4|t=11", + "simple_roles_132|sid=155|g=155|s=7|t=13", + "simple_roles_132|sid=117|g=117|s=5|t=15" + ], + "target_ids": [ + "10" + ], + "tid": 132, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "When is my coworker's birthday?", + "retrieved_ids": [ + "simple_roles_133|sid=160|g=160|s=7|t=16", + "simple_roles_133|sid=25|g=25|s=1|t=4", + "simple_roles_133|sid=144|g=144|s=7|t=0", + "simple_roles_133|sid=112|g=112|s=5|t=10", + "simple_roles_133|sid=152|g=152|s=7|t=8" + ], + "target_ids": [ + "160" + ], + "tid": 133, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What kind of education does my female cousin have?", + "retrieved_ids": [ + "simple_roles_134|sid=104|g=104|s=5|t=0", + "simple_roles_134|sid=160|g=160|s=7|t=16", + "simple_roles_134|sid=8|g=8|s=0|t=8", + "simple_roles_134|sid=112|g=112|s=5|t=8", + "simple_roles_134|sid=20|g=20|s=0|t=20" + ], + "target_ids": [ + "112" + ], + "tid": 134, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "When is my sister's birthday?", + "retrieved_ids": [ + "simple_roles_135|sid=42|g=42|s=2|t=2", + "simple_roles_135|sid=63|g=63|s=3|t=3", + "simple_roles_135|sid=87|g=87|s=4|t=7", + "simple_roles_135|sid=58|g=58|s=2|t=18", + "simple_roles_135|sid=40|g=40|s=2|t=0" + ], + "target_ids": [ + "42" + ], + "tid": 135, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What does my brother do for a living?", + "retrieved_ids": [ + "simple_roles_136|sid=93|g=93|s=4|t=11", + "simple_roles_136|sid=38|g=38|s=1|t=17", + "simple_roles_136|sid=102|g=102|s=5|t=0", + "simple_roles_136|sid=108|g=108|s=5|t=6", + "simple_roles_136|sid=83|g=83|s=4|t=1" + ], + "target_ids": [ + "108" + ], + "tid": 136, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the educational background of my sister?", + "retrieved_ids": [ + "simple_roles_137|sid=70|g=70|s=3|t=11", + "simple_roles_137|sid=20|g=20|s=1|t=0", + "simple_roles_137|sid=31|g=31|s=1|t=11", + "simple_roles_137|sid=112|g=112|s=5|t=11", + "simple_roles_137|sid=128|g=128|s=6|t=7" + ], + "target_ids": [ + "31" + ], + "tid": 137, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the email address of my aunt?", + "retrieved_ids": [ + "simple_roles_138|sid=104|g=104|s=5|t=2", + "simple_roles_138|sid=159|g=159|s=7|t=17", + "simple_roles_138|sid=39|g=39|s=1|t=19", + "simple_roles_138|sid=54|g=54|s=2|t=14", + "simple_roles_138|sid=77|g=77|s=3|t=16" + ], + "target_ids": [ + "77" + ], + "tid": 138, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "Could the contact number for the boss be provided?", + "retrieved_ids": [ + "simple_roles_139|sid=16|g=16|s=0|t=16", + "simple_roles_139|sid=100|g=100|s=4|t=17", + "simple_roles_139|sid=76|g=76|s=3|t=13", + "simple_roles_139|sid=139|g=139|s=6|t=15", + "simple_roles_139|sid=145|g=145|s=7|t=1" + ], + "target_ids": [ + "145" + ], + "tid": 139, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the age of the nephew?", + "retrieved_ids": [ + "simple_roles_140|sid=81|g=81|s=4|t=0", + "simple_roles_140|sid=91|g=91|s=4|t=10", + "simple_roles_140|sid=107|g=107|s=5|t=5", + "simple_roles_140|sid=0|g=0|s=0|t=0" + ], + "target_ids": [ + "91" + ], + "tid": 140, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the position of my aunt?", + "retrieved_ids": [ + "simple_roles_141|sid=7|g=7|s=0|t=7", + "simple_roles_141|sid=0|g=0|s=0|t=0", + "simple_roles_141|sid=150|g=150|s=7|t=3", + "simple_roles_141|sid=56|g=56|s=2|t=15", + "simple_roles_141|sid=23|g=23|s=1|t=3" + ], + "target_ids": [ + "7" + ], + "tid": 141, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the height of my uncle?", + "retrieved_ids": [ + "simple_roles_142|sid=128|g=128|s=6|t=5", + "simple_roles_142|sid=123|g=123|s=6|t=0", + "simple_roles_142|sid=76|g=76|s=3|t=14", + "simple_roles_142|sid=31|g=31|s=1|t=10", + "simple_roles_142|sid=75|g=75|s=3|t=13" + ], + "target_ids": [ + "128" + ], + "tid": 142, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the name of my nephew's company?", + "retrieved_ids": [ + "simple_roles_143|sid=111|g=111|s=5|t=9", + "simple_roles_143|sid=102|g=102|s=5|t=0", + "simple_roles_143|sid=8|g=8|s=0|t=8", + "simple_roles_143|sid=36|g=36|s=1|t=15", + "simple_roles_143|sid=0|g=0|s=0|t=0" + ], + "target_ids": [ + "111" + ], + "tid": 143, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the contact number for the boss?", + "retrieved_ids": [ + "simple_roles_144|sid=89|g=89|s=4|t=7", + "simple_roles_144|sid=30|g=30|s=1|t=10", + "simple_roles_144|sid=54|g=54|s=2|t=13", + "simple_roles_144|sid=113|g=113|s=5|t=10", + "simple_roles_144|sid=74|g=74|s=3|t=13" + ], + "target_ids": [ + "89" + ], + "tid": 144, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the email address of the subordinate?", + "retrieved_ids": [ + "simple_roles_145|sid=74|g=74|s=3|t=13", + "simple_roles_145|sid=118|g=118|s=5|t=17", + "simple_roles_145|sid=161|g=161|s=7|t=18", + "simple_roles_145|sid=31|g=31|s=1|t=11", + "simple_roles_145|sid=55|g=55|s=2|t=14" + ], + "target_ids": [ + "118" + ], + "tid": 145, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the educational background of my subordinate?", + "retrieved_ids": [ + "simple_roles_146|sid=61|g=61|s=3|t=0", + "simple_roles_146|sid=43|g=43|s=2|t=3", + "simple_roles_146|sid=50|g=50|s=2|t=10", + "simple_roles_146|sid=80|g=80|s=3|t=19", + "simple_roles_146|sid=4|g=4|s=0|t=4" + ], + "target_ids": [ + "80" + ], + "tid": 146, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is my mother's hobby?", + "retrieved_ids": [ + "simple_roles_147|sid=14|g=14|s=0|t=14", + "simple_roles_147|sid=130|g=130|s=6|t=10", + "simple_roles_147|sid=107|g=107|s=5|t=7", + "simple_roles_147|sid=50|g=50|s=2|t=10", + "simple_roles_147|sid=68|g=68|s=3|t=8" + ], + "target_ids": [ + "14" + ], + "tid": 147, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the work location of the subordinate?", + "retrieved_ids": [ + "simple_roles_148|sid=157|g=157|s=7|t=11", + "simple_roles_148|sid=121|g=121|s=5|t=17", + "simple_roles_148|sid=166|g=166|s=7|t=20", + "simple_roles_148|sid=45|g=45|s=2|t=4", + "simple_roles_148|sid=16|g=16|s=0|t=16" + ], + "target_ids": [ + "157" + ], + "tid": 148, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the name of the company where the boss works?", + "retrieved_ids": [ + "simple_roles_149|sid=131|g=131|s=6|t=8", + "simple_roles_149|sid=109|g=109|s=5|t=7", + "simple_roles_149|sid=152|g=152|s=7|t=9", + "simple_roles_149|sid=163|g=163|s=7|t=20", + "simple_roles_149|sid=68|g=68|s=3|t=6" + ], + "target_ids": [ + "152" + ], + "tid": 149, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the hometown of my uncle?", + "retrieved_ids": [ + "simple_roles_150|sid=38|g=38|s=2|t=0", + "simple_roles_150|sid=142|g=142|s=7|t=3", + "simple_roles_150|sid=102|g=102|s=5|t=4", + "simple_roles_150|sid=57|g=57|s=2|t=19", + "simple_roles_150|sid=50|g=50|s=2|t=12" + ], + "target_ids": [ + "50" + ], + "tid": 150, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What hobby does my father enjoy?", + "retrieved_ids": [ + "simple_roles_151|sid=23|g=23|s=1|t=3", + "simple_roles_151|sid=7|g=7|s=0|t=7", + "simple_roles_151|sid=61|g=61|s=3|t=0", + "simple_roles_151|sid=156|g=156|s=7|t=12", + "simple_roles_151|sid=66|g=66|s=3|t=5" + ], + "target_ids": [ + "66" + ], + "tid": 151, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What hobby does my uncle enjoy?", + "retrieved_ids": [ + "simple_roles_152|sid=139|g=139|s=6|t=18", + "simple_roles_152|sid=33|g=33|s=1|t=13", + "simple_roles_152|sid=9|g=9|s=0|t=9", + "simple_roles_152|sid=121|g=121|s=6|t=0", + "simple_roles_152|sid=141|g=141|s=6|t=20" + ], + "target_ids": [ + "139" + ], + "tid": 152, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": false, + "question": "What is the name of my cousin's company?", + "retrieved_ids": [], + "target_ids": [ + "112" + ], + "tid": 153, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the name of my uncle?", + "retrieved_ids": [ + "simple_roles_154|sid=115|g=115|s=5|t=12", + "simple_roles_154|sid=103|g=103|s=5|t=0", + "simple_roles_154|sid=120|g=120|s=5|t=17", + "simple_roles_154|sid=97|g=97|s=4|t=14", + "simple_roles_154|sid=68|g=68|s=3|t=5" + ], + "target_ids": [ + "115" + ], + "tid": 154, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": false, + "question": "What is the name of my mother's hometown?", + "retrieved_ids": [], + "target_ids": [ + "84" + ], + "tid": 155, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the name of my subordinate's company?", + "retrieved_ids": [ + "simple_roles_156|sid=10|g=10|s=0|t=10", + "simple_roles_156|sid=72|g=72|s=3|t=12", + "simple_roles_156|sid=0|g=0|s=0|t=0", + "simple_roles_156|sid=142|g=142|s=7|t=1", + "simple_roles_156|sid=37|g=37|s=1|t=17" + ], + "target_ids": [ + "10" + ], + "tid": 156, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the work location of the boss?", + "retrieved_ids": [ + "simple_roles_157|sid=156|g=156|s=7|t=13", + "simple_roles_157|sid=27|g=27|s=1|t=6", + "simple_roles_157|sid=89|g=89|s=4|t=7", + "simple_roles_157|sid=74|g=74|s=3|t=12", + "simple_roles_157|sid=143|g=143|s=7|t=0" + ], + "target_ids": [ + "156" + ], + "tid": 157, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": false, + "question": "What is the position of my mother?", + "retrieved_ids": [ + "simple_roles_158|sid=123|g=123|s=6|t=0", + "simple_roles_158|sid=93|g=93|s=4|t=12", + "simple_roles_158|sid=141|g=141|s=6|t=18", + "simple_roles_158|sid=160|g=160|s=7|t=16", + "simple_roles_158|sid=79|g=79|s=3|t=19" + ], + "target_ids": [ + "124" + ], + "tid": 158, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "When is my niece's birthday?", + "retrieved_ids": [ + "simple_roles_159|sid=73|g=73|s=3|t=11", + "simple_roles_159|sid=1|g=1|s=0|t=1", + "simple_roles_159|sid=100|g=100|s=4|t=18", + "simple_roles_159|sid=128|g=128|s=6|t=4", + "simple_roles_159|sid=62|g=62|s=3|t=0" + ], + "target_ids": [ + "73" + ], + "tid": 159, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What hobby does my father have?", + "retrieved_ids": [ + "simple_roles_160|sid=123|g=123|s=6|t=0", + "simple_roles_160|sid=45|g=45|s=2|t=3", + "simple_roles_160|sid=126|g=126|s=6|t=3", + "simple_roles_160|sid=153|g=153|s=7|t=8", + "simple_roles_160|sid=32|g=32|s=1|t=11" + ], + "target_ids": [ + "126" + ], + "tid": 160, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the email address of my mother?", + "retrieved_ids": [ + "simple_roles_161|sid=29|g=29|s=1|t=9", + "simple_roles_161|sid=54|g=54|s=2|t=13", + "simple_roles_161|sid=118|g=118|s=5|t=17", + "simple_roles_161|sid=98|g=98|s=4|t=16", + "simple_roles_161|sid=20|g=20|s=1|t=0" + ], + "target_ids": [ + "29" + ], + "tid": 161, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the age of my aunt?", + "retrieved_ids": [ + "simple_roles_162|sid=41|g=41|s=2|t=0", + "simple_roles_162|sid=42|g=42|s=2|t=1", + "simple_roles_162|sid=0|g=0|s=0|t=0", + "simple_roles_162|sid=121|g=121|s=5|t=18", + "simple_roles_162|sid=40|g=40|s=1|t=20" + ], + "target_ids": [ + "42" + ], + "tid": 162, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the work location of my aunt?", + "retrieved_ids": [ + "simple_roles_163|sid=3|g=3|s=0|t=3", + "simple_roles_163|sid=0|g=0|s=0|t=0", + "simple_roles_163|sid=123|g=123|s=6|t=2", + "simple_roles_163|sid=75|g=75|s=3|t=14", + "simple_roles_163|sid=82|g=82|s=4|t=1" + ], + "target_ids": [ + "3" + ], + "tid": 163, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the height of the boss?", + "retrieved_ids": [ + "simple_roles_164|sid=37|g=37|s=1|t=18", + "simple_roles_164|sid=19|g=19|s=1|t=0", + "simple_roles_164|sid=1|g=1|s=0|t=1", + "simple_roles_164|sid=51|g=51|s=2|t=13", + "simple_roles_164|sid=105|g=105|s=5|t=7" + ], + "target_ids": [ + "37" + ], + "tid": 164, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the educational background of the subordinate?", + "retrieved_ids": [ + "simple_roles_165|sid=107|g=107|s=5|t=3", + "simple_roles_165|sid=104|g=104|s=5|t=0", + "simple_roles_165|sid=114|g=114|s=5|t=10", + "simple_roles_165|sid=126|g=126|s=6|t=1", + "simple_roles_165|sid=15|g=15|s=0|t=15" + ], + "target_ids": [ + "114" + ], + "tid": 165, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the work location of the coworker?", + "retrieved_ids": [ + "simple_roles_166|sid=109|g=109|s=5|t=6", + "simple_roles_166|sid=122|g=122|s=5|t=19", + "simple_roles_166|sid=103|g=103|s=5|t=0", + "simple_roles_166|sid=87|g=87|s=4|t=5", + "simple_roles_166|sid=121|g=121|s=5|t=18" + ], + "target_ids": [ + "109" + ], + "tid": 166, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the occupation of my subordinate?", + "retrieved_ids": [ + "simple_roles_167|sid=49|g=49|s=2|t=7", + "simple_roles_167|sid=75|g=75|s=3|t=12", + "simple_roles_167|sid=136|g=136|s=6|t=12", + "simple_roles_167|sid=16|g=16|s=0|t=16", + "simple_roles_167|sid=32|g=32|s=1|t=11" + ], + "target_ids": [ + "49" + ], + "tid": 167, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the name of my brother's company?", + "retrieved_ids": [ + "simple_roles_168|sid=110|g=110|s=5|t=8", + "simple_roles_168|sid=86|g=86|s=4|t=5", + "simple_roles_168|sid=99|g=99|s=4|t=18", + "simple_roles_168|sid=22|g=22|s=1|t=2", + "simple_roles_168|sid=81|g=81|s=4|t=0" + ], + "target_ids": [ + "86" + ], + "tid": 168, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the date of my brother's birthday?", + "retrieved_ids": [ + "simple_roles_169|sid=61|g=61|s=3|t=1", + "simple_roles_169|sid=103|g=103|s=5|t=1", + "simple_roles_169|sid=81|g=81|s=4|t=0", + "simple_roles_169|sid=60|g=60|s=3|t=0", + "simple_roles_169|sid=20|g=20|s=1|t=2" + ], + "target_ids": [ + "61" + ], + "tid": 169, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the name of my brother?", + "retrieved_ids": [ + "simple_roles_170|sid=42|g=42|s=2|t=1", + "simple_roles_170|sid=41|g=41|s=2|t=0", + "simple_roles_170|sid=24|g=24|s=1|t=4", + "simple_roles_170|sid=62|g=62|s=3|t=0", + "simple_roles_170|sid=123|g=123|s=5|t=19" + ], + "target_ids": [ + "42" + ], + "tid": 170, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the hobby of that uncle?", + "retrieved_ids": [ + "simple_roles_171|sid=145|g=145|s=7|t=1", + "simple_roles_171|sid=111|g=111|s=5|t=9", + "simple_roles_171|sid=162|g=162|s=7|t=18", + "simple_roles_171|sid=159|g=159|s=7|t=15", + "simple_roles_171|sid=144|g=144|s=7|t=0" + ], + "target_ids": [ + "145" + ], + "tid": 171, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the email address of my father?", + "retrieved_ids": [ + "simple_roles_172|sid=128|g=128|s=6|t=6", + "simple_roles_172|sid=10|g=10|s=0|t=10", + "simple_roles_172|sid=75|g=75|s=3|t=14", + "simple_roles_172|sid=45|g=45|s=2|t=5", + "simple_roles_172|sid=122|g=122|s=6|t=0" + ], + "target_ids": [ + "128" + ], + "tid": 172, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the name of the hometown of that aunt?", + "retrieved_ids": [ + "simple_roles_173|sid=41|g=41|s=2|t=0", + "simple_roles_173|sid=88|g=88|s=4|t=6", + "simple_roles_173|sid=58|g=58|s=2|t=17", + "simple_roles_173|sid=45|g=45|s=2|t=4" + ], + "target_ids": [ + "58" + ], + "tid": 173, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the hometown of the boss?", + "retrieved_ids": [ + "simple_roles_174|sid=21|g=21|s=0|t=21", + "simple_roles_174|sid=0|g=0|s=0|t=0", + "simple_roles_174|sid=126|g=126|s=6|t=3" + ], + "target_ids": [ + "21" + ], + "tid": 174, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": false, + "question": "What is the hometown of my father?", + "retrieved_ids": [ + "simple_roles_175|sid=80|g=80|s=4|t=0", + "simple_roles_175|sid=99|g=99|s=4|t=19", + "simple_roles_175|sid=24|g=24|s=1|t=4", + "simple_roles_175|sid=19|g=19|s=0|t=19", + "simple_roles_175|sid=141|g=141|s=7|t=0" + ], + "target_ids": [ + "88" + ], + "tid": 175, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the name of the company where the subordinate works?", + "retrieved_ids": [ + "simple_roles_176|sid=88|g=88|s=4|t=7", + "simple_roles_176|sid=75|g=75|s=3|t=14", + "simple_roles_176|sid=151|g=151|s=7|t=9", + "simple_roles_176|sid=130|g=130|s=6|t=7", + "simple_roles_176|sid=157|g=157|s=7|t=15" + ], + "target_ids": [ + "75" + ], + "tid": 176, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What hobbies does my female cousin have?", + "retrieved_ids": [ + "simple_roles_177|sid=158|g=158|s=7|t=12", + "simple_roles_177|sid=73|g=73|s=3|t=10", + "simple_roles_177|sid=50|g=50|s=2|t=7", + "simple_roles_177|sid=146|g=146|s=7|t=0", + "simple_roles_177|sid=164|g=164|s=7|t=18" + ], + "target_ids": [ + "158" + ], + "tid": 177, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is my female cousin's email address?", + "retrieved_ids": [ + "simple_roles_178|sid=124|g=124|s=6|t=2", + "simple_roles_178|sid=58|g=58|s=2|t=16", + "simple_roles_178|sid=91|g=91|s=4|t=11", + "simple_roles_178|sid=122|g=122|s=6|t=0", + "simple_roles_178|sid=141|g=141|s=6|t=19" + ], + "target_ids": [ + "124" + ], + "tid": 178, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What hobby does my male cousin have?", + "retrieved_ids": [ + "simple_roles_179|sid=135|g=135|s=6|t=10", + "simple_roles_179|sid=63|g=63|s=3|t=0", + "simple_roles_179|sid=69|g=69|s=3|t=6", + "simple_roles_179|sid=113|g=113|s=5|t=8", + "simple_roles_179|sid=50|g=50|s=2|t=8" + ], + "target_ids": [ + "69" + ], + "tid": 179, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the height of my sister?", + "retrieved_ids": [ + "simple_roles_180|sid=79|g=79|s=3|t=18", + "simple_roles_180|sid=21|g=21|s=1|t=1", + "simple_roles_180|sid=61|g=61|s=3|t=0", + "simple_roles_180|sid=83|g=83|s=4|t=1", + "simple_roles_180|sid=106|g=106|s=5|t=3" + ], + "target_ids": [ + "79" + ], + "tid": 180, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the occupation of the boss?", + "retrieved_ids": [ + "simple_roles_181|sid=60|g=60|s=3|t=0", + "simple_roles_181|sid=73|g=73|s=3|t=13", + "simple_roles_181|sid=76|g=76|s=3|t=16" + ], + "target_ids": [ + "73" + ], + "tid": 181, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the name of my subordinate's company?", + "retrieved_ids": [ + "simple_roles_182|sid=79|g=79|s=3|t=18", + "simple_roles_182|sid=151|g=151|s=7|t=7", + "simple_roles_182|sid=57|g=57|s=2|t=17", + "simple_roles_182|sid=46|g=46|s=2|t=6", + "simple_roles_182|sid=61|g=61|s=3|t=0" + ], + "target_ids": [ + "79" + ], + "tid": 182, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "How old is my aunt?", + "retrieved_ids": [ + "simple_roles_183|sid=7|g=7|s=0|t=7", + "simple_roles_183|sid=17|g=17|s=0|t=17", + "simple_roles_183|sid=38|g=38|s=1|t=18", + "simple_roles_183|sid=0|g=0|s=0|t=0", + "simple_roles_183|sid=100|g=100|s=5|t=0" + ], + "target_ids": [ + "7" + ], + "tid": 183, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the work location of the uncle?", + "retrieved_ids": [ + "simple_roles_184|sid=106|g=106|s=5|t=3", + "simple_roles_184|sid=103|g=103|s=5|t=0", + "simple_roles_184|sid=88|g=88|s=4|t=6", + "simple_roles_184|sid=121|g=121|s=5|t=18", + "simple_roles_184|sid=157|g=157|s=7|t=14" + ], + "target_ids": [ + "106" + ], + "tid": 184, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What hobby does my brother enjoy?", + "retrieved_ids": [ + "simple_roles_185|sid=71|g=71|s=3|t=9", + "simple_roles_185|sid=20|g=20|s=1|t=0", + "simple_roles_185|sid=36|g=36|s=1|t=16", + "simple_roles_185|sid=159|g=159|s=7|t=15", + "simple_roles_185|sid=7|g=7|s=0|t=7" + ], + "target_ids": [ + "36" + ], + "tid": 185, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is my female cousin's email address?", + "retrieved_ids": [ + "simple_roles_186|sid=42|g=42|s=2|t=1", + "simple_roles_186|sid=0|g=0|s=0|t=0", + "simple_roles_186|sid=116|g=116|s=5|t=12", + "simple_roles_186|sid=139|g=139|s=6|t=16", + "simple_roles_186|sid=99|g=99|s=4|t=16" + ], + "target_ids": [ + "42" + ], + "tid": 186, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "How old is my female cousin?", + "retrieved_ids": [ + "simple_roles_187|sid=25|g=25|s=1|t=5", + "simple_roles_187|sid=0|g=0|s=0|t=0", + "simple_roles_187|sid=20|g=20|s=1|t=0", + "simple_roles_187|sid=104|g=104|s=5|t=0", + "simple_roles_187|sid=124|g=124|s=6|t=0" + ], + "target_ids": [ + "25" + ], + "tid": 187, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "When is my nephew's birthday?", + "retrieved_ids": [ + "simple_roles_188|sid=82|g=82|s=4|t=1", + "simple_roles_188|sid=41|g=41|s=1|t=20", + "simple_roles_188|sid=142|g=142|s=6|t=19", + "simple_roles_188|sid=81|g=81|s=4|t=0", + "simple_roles_188|sid=102|g=102|s=5|t=0" + ], + "target_ids": [ + "82" + ], + "tid": 188, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the contact number for my brother?", + "retrieved_ids": [ + "simple_roles_189|sid=44|g=44|s=2|t=4", + "simple_roles_189|sid=99|g=99|s=4|t=18", + "simple_roles_189|sid=10|g=10|s=0|t=10", + "simple_roles_189|sid=76|g=76|s=3|t=16", + "simple_roles_189|sid=114|g=114|s=5|t=12" + ], + "target_ids": [ + "44" + ], + "tid": 189, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the name of my uncle?", + "retrieved_ids": [ + "simple_roles_190|sid=0|g=0|s=0|t=0", + "simple_roles_190|sid=80|g=80|s=4|t=0", + "simple_roles_190|sid=88|g=88|s=4|t=8", + "simple_roles_190|sid=37|g=37|s=1|t=17", + "simple_roles_190|sid=6|g=6|s=0|t=6" + ], + "target_ids": [ + "88" + ], + "tid": 190, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the work location of my nephew?", + "retrieved_ids": [ + "simple_roles_191|sid=46|g=46|s=2|t=4", + "simple_roles_191|sid=89|g=89|s=4|t=6", + "simple_roles_191|sid=105|g=105|s=5|t=1", + "simple_roles_191|sid=42|g=42|s=2|t=0", + "simple_roles_191|sid=124|g=124|s=6|t=0" + ], + "target_ids": [ + "46" + ], + "tid": 191, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the hometown of my sister?", + "retrieved_ids": [ + "simple_roles_192|sid=42|g=42|s=2|t=0", + "simple_roles_192|sid=148|g=148|s=7|t=3", + "simple_roles_192|sid=60|g=60|s=2|t=18", + "simple_roles_192|sid=68|g=68|s=3|t=5", + "simple_roles_192|sid=51|g=51|s=2|t=9" + ], + "target_ids": [ + "51" + ], + "tid": 192, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What hobby does my niece have?", + "retrieved_ids": [ + "simple_roles_193|sid=21|g=21|s=1|t=0", + "simple_roles_193|sid=25|g=25|s=1|t=4", + "simple_roles_193|sid=135|g=135|s=6|t=12", + "simple_roles_193|sid=89|g=89|s=4|t=7", + "simple_roles_193|sid=133|g=133|s=6|t=10" + ], + "target_ids": [ + "25" + ], + "tid": 193, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the name of my nephew's company?", + "retrieved_ids": [ + "simple_roles_194|sid=121|g=121|s=6|t=0", + "simple_roles_194|sid=125|g=125|s=6|t=4", + "simple_roles_194|sid=7|g=7|s=0|t=7", + "simple_roles_194|sid=38|g=38|s=1|t=18", + "simple_roles_194|sid=108|g=108|s=5|t=7" + ], + "target_ids": [ + "125" + ], + "tid": 194, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "Where does my uncle work?", + "retrieved_ids": [ + "simple_roles_195|sid=93|g=93|s=4|t=9", + "simple_roles_195|sid=142|g=142|s=6|t=18", + "simple_roles_195|sid=161|g=161|s=7|t=17", + "simple_roles_195|sid=34|g=34|s=1|t=13", + "simple_roles_195|sid=84|g=84|s=4|t=0" + ], + "target_ids": [ + "93" + ], + "tid": 195, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the email address of the subordinate?", + "retrieved_ids": [ + "simple_roles_196|sid=31|g=31|s=1|t=10", + "simple_roles_196|sid=21|g=21|s=1|t=0", + "simple_roles_196|sid=53|g=53|s=2|t=11", + "simple_roles_196|sid=81|g=81|s=3|t=18", + "simple_roles_196|sid=18|g=18|s=0|t=18" + ], + "target_ids": [ + "31" + ], + "tid": 196, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What is the age of that coworker?", + "retrieved_ids": [ + "simple_roles_197|sid=49|g=49|s=2|t=9", + "simple_roles_197|sid=59|g=59|s=2|t=19", + "simple_roles_197|sid=17|g=17|s=0|t=17", + "simple_roles_197|sid=20|g=20|s=1|t=1", + "simple_roles_197|sid=101|g=101|s=5|t=0" + ], + "target_ids": [ + "49" + ], + "tid": 197, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What hobby does my brother have?", + "retrieved_ids": [ + "simple_roles_198|sid=122|g=122|s=6|t=0", + "simple_roles_198|sid=12|g=12|s=0|t=12", + "simple_roles_198|sid=49|g=49|s=2|t=9", + "simple_roles_198|sid=85|g=85|s=4|t=5", + "simple_roles_198|sid=127|g=127|s=6|t=5" + ], + "target_ids": [ + "127" + ], + "tid": 198, + "topic": "roles" + }, + { + "category": "simple", + "hit_at_k": true, + "question": "What hobby does my niece have?", + "retrieved_ids": [ + "simple_roles_199|sid=123|g=123|s=6|t=0", + "simple_roles_199|sid=135|g=135|s=6|t=12", + "simple_roles_199|sid=8|g=8|s=0|t=8", + "simple_roles_199|sid=150|g=150|s=7|t=8", + "simple_roles_199|sid=131|g=131|s=6|t=8" + ], + "target_ids": [ + "131" + ], + "tid": 199, + "topic": "roles" + } + ], + "runtime_seconds": 140.08, + "series_name": "new_brainctl", + "source_path": null, + "status": "partial", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/membench_old_brainctl_cmd_turn.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/membench_old_brainctl_cmd_turn.json new file mode 100644 index 0000000..bc94de0 --- /dev/null +++ b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/membench_old_brainctl_cmd_turn.json @@ -0,0 +1,35 @@ +{ + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\membench_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\membench_old_brainctl_cmd_turn.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" + }, + "benchmark": "membench", + "caveats": [ + "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 200, + "measured": false, + "metadata": {}, + "metrics": { + "hit_at_5": 0.93, + "simple_hit_at_5": 0.93, + "top_k": 5 + }, + "mode": "cmd_turn", + "notes": [ + "FirstAgent slice only", + "turn-level retrieval", + "topic=all" + ], + "primary_metric": "hit_at_5", + "primary_metric_value": 0.93, + "reference_kind": "historical", + "runtime_seconds": 140.592, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "partial", + "system_name": "brainctl" +} \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/summary.csv b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/summary.csv new file mode 100644 index 0000000..2b6fc1e --- /dev/null +++ b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/summary.csv @@ -0,0 +1,15 @@ +benchmark,series_name,system_name,mode,reference_kind,status,example_count,primary_metric,primary_metric_value,runtime_seconds,dataset_path,source_path,notes,caveats,abstention_evidence_recall,assistant_facts_evidence_recall,avg_recall,cat_1_recall,cat_2_recall,cat_3_recall,cat_4_recall,cat_5_recall,hit_at_5,implicit_connection_evidence_recall,ndcg_at_10,ndcg_at_5,perfect_rate,preference_evidence_recall,r_at_10,r_at_5,simple_hit_at_5,top_k,user_evidence_recall,zero_rate +longmemeval,old_brainctl,brainctl,brain,historical,full_same_machine,470,r_at_5,0.9681,85.439,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,top_k=10,,,,,,,,,,,,0.9253,0.9204,,,0.9894,0.9681,,,, +longmemeval,old_brainctl,brainctl,cmd,historical,full_same_machine,470,r_at_5,0.9702,130.863,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,top_k=10,,,,,,,,,,,,0.9247,0.9206,,,0.9894,0.9702,,,, +longmemeval,mempalace,mempalace,raw_session,historical,full_same_machine,470,r_at_5,0.966,695.36,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,top_k=10 | Runs MemPalace benchmark module raw session retrieval logic directly.,,,,,,,,,,,,0.8948,0.893,,,0.983,0.966,,,, +locomo,old_brainctl,brainctl,cmd_session,historical,full_same_machine,1986,avg_recall,0.9217,445.74,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,granularity=session | top_k=10,,,,0.9217,0.7614,0.92,0.6616,0.9738,0.9821,,,,,0.8817,,,,,10,,0.0438 +locomo,mempalace,mempalace,raw_session,historical,full_same_machine,1986,avg_recall,0.6028,2106.411,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,granularity=session | top_k=10,,,,0.6028,0.5899,0.6921,0.4604,0.5809,0.6188,,,,,0.5534,,,,,10,,0.3499 +membench,old_brainctl,brainctl,cmd_turn,historical,partial,200,hit_at_5,0.93,140.592,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,FirstAgent slice only | turn-level retrieval | topic=all,MemBench comparison is partial because ThirdAgent and noise-extended slices are not included.,,,,,,,,,0.93,,,,,,,,0.93,5,, +membench,mempalace,mempalace,raw_turn,historical,partial,200,hit_at_5,0.885,804.35,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,FirstAgent slice only | turn-level retrieval | topic=all,MemBench comparison is partial because ThirdAgent and noise-extended slices are not included.,,,,,,,,,0.885,,,,,,,,0.885,5,, +convomem,old_brainctl,brainctl,cmd,historical,blocked,0,avg_recall,,,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,limit_per_category=1 | top_k=10,Blocked while loading ConvoMem evidence data: ,,,,,,,,,,,,,,,,,,,, +convomem,mempalace,mempalace,raw,historical,blocked,0,avg_recall,,,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,limit_per_category=1 | top_k=10,Blocked while loading ConvoMem evidence data: ,,,,,,,,,,,,,,,,,,,, +longmemeval,new_brainctl,brainctl,brain,measured,full_same_machine,470,r_at_5,0.9574,403.852,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,,top_k=10 | Legacy 470-question session-level slice.,,,,,,,,,,,,0.872,0.8678,,,0.9894,0.9574,,,, +longmemeval,new_brainctl,brainctl,cmd,measured,full_same_machine,470,r_at_5,0.9596,251.044,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,,top_k=10 | Legacy 470-question session-level slice.,,,,,,,,,,,,0.8753,0.8715,,,0.9894,0.9596,,,, +locomo,new_brainctl,brainctl,cmd_session,measured,full_same_machine,1986,avg_recall,0.9145,157.701,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,,granularity=session | top_k=10,,,,0.9145,0.7533,0.9107,0.6607,0.9643,0.9798,,,,,0.8751,,,,,10,,0.0519 +membench,new_brainctl,brainctl,cmd_turn,measured,partial,200,hit_at_5,0.895,140.08,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,,FirstAgent slice only | turn-level retrieval | topic=all,MemBench comparison is partial because ThirdAgent and noise-extended slices are not included.,,,,,,,,,0.895,,,,,,,,0.895,5,, +convomem,new_brainctl,brainctl,cmd,measured,partial,5,avg_recall,0.6,1.514,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache,,categories=6 | limit_per_category=1 | top_k=10,"changing_evidence: discover failed: HTTP Error 404: Not Found | ConvoMem comparison is partial because it uses a bounded same-machine sample, not the full benchmark.",0.0,1.0,0.6,,,,,,,0.0,,,0.6,1.0,,,,10,1.0, diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/summary.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/summary.json new file mode 100644 index 0000000..6fda86b --- /dev/null +++ b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/summary.json @@ -0,0 +1,514 @@ +{ + "generated_at_utc": "2026-04-23T18:05:25.239155+00:00", + "metadata": { + "argv": [ + "benchmarks\\compare_memory_engines.py", + "--label", + "rlm_srlm_probe_refresh", + "--membench-limit", + "200" + ], + "cwd": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl", + "datasets": { + "convomem_cache": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "locomo_data": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "longmemeval_data": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "membench_data": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent" + }, + "generated_at_utc": "2026-04-23T18:05:25.112789+00:00", + "git_commit": "9368f288279ae1d3395e37e89972038b74d66e3a", + "historical_summary_mode": "recovered", + "historical_summary_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "machine": "AMD64", + "platform": "Windows-11-10.0.26200-SP0", + "processor": "Intel64 Family 6 Model 189 Stepping 1, GenuineIntel", + "python_version": "3.14.4 (tags/v3.14.4:23116f9, Apr 7 2026, 14:10:54) [MSC v.1944 64 bit (AMD64)]" + }, + "notes": [ + "Historical old-BrainCTL and MemPalace series come from the recovered 2026-04-18 comparison bundle.", + "New BrainCTL series are rerun in the current checked-out repo using the legacy benchmark definitions.", + "MemBench remains intentionally partial because the legacy comparison only covered the FirstAgent slice.", + "ConvoMem remains a coverage/status benchmark here; it has no dedicated comparison chart in the legacy chart pack." + ], + "runs": [ + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\longmemeval_old_brainctl_brain.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "measured": false, + "metadata": {}, + "metrics": { + "ndcg_at_10": 0.9253, + "ndcg_at_5": 0.9204, + "r_at_10": 0.9894, + "r_at_5": 0.9681 + }, + "mode": "brain", + "notes": [ + "top_k=10" + ], + "primary_metric": "r_at_5", + "primary_metric_value": 0.9681, + "reference_kind": "historical", + "runtime_seconds": 85.439, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\longmemeval_old_brainctl_cmd.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "measured": false, + "metadata": {}, + "metrics": { + "ndcg_at_10": 0.9247, + "ndcg_at_5": 0.9206, + "r_at_10": 0.9894, + "r_at_5": 0.9702 + }, + "mode": "cmd", + "notes": [ + "top_k=10" + ], + "primary_metric": "r_at_5", + "primary_metric_value": 0.9702, + "reference_kind": "historical", + "runtime_seconds": 130.863, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\longmemeval_mempalace_raw_session.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "measured": false, + "metadata": {}, + "metrics": { + "ndcg_at_10": 0.8948, + "ndcg_at_5": 0.893, + "r_at_10": 0.983, + "r_at_5": 0.966 + }, + "mode": "raw_session", + "notes": [ + "top_k=10", + "Runs MemPalace benchmark module raw session retrieval logic directly." + ], + "primary_metric": "r_at_5", + "primary_metric_value": 0.966, + "reference_kind": "historical", + "runtime_seconds": 695.36, + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "mempalace" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\locomo_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\locomo_old_brainctl_cmd_session.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" + }, + "benchmark": "locomo", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "measured": false, + "metadata": {}, + "metrics": { + "avg_recall": 0.9217, + "cat_1_recall": 0.7614, + "cat_2_recall": 0.92, + "cat_3_recall": 0.6616, + "cat_4_recall": 0.9738, + "cat_5_recall": 0.9821, + "perfect_rate": 0.8817, + "top_k": 10, + "zero_rate": 0.0438 + }, + "mode": "cmd_session", + "notes": [ + "granularity=session", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": 0.9217, + "reference_kind": "historical", + "runtime_seconds": 445.74, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "brainctl" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\locomo_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\locomo_mempalace_raw_session.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" + }, + "benchmark": "locomo", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "measured": false, + "metadata": {}, + "metrics": { + "avg_recall": 0.6028, + "cat_1_recall": 0.5899, + "cat_2_recall": 0.6921, + "cat_3_recall": 0.4604, + "cat_4_recall": 0.5809, + "cat_5_recall": 0.6188, + "perfect_rate": 0.5534, + "top_k": 10, + "zero_rate": 0.3499 + }, + "mode": "raw_session", + "notes": [ + "granularity=session", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": 0.6028, + "reference_kind": "historical", + "runtime_seconds": 2106.411, + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "full_same_machine", + "system_name": "mempalace" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\membench_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\membench_old_brainctl_cmd_turn.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" + }, + "benchmark": "membench", + "caveats": [ + "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 200, + "measured": false, + "metadata": {}, + "metrics": { + "hit_at_5": 0.93, + "simple_hit_at_5": 0.93, + "top_k": 5 + }, + "mode": "cmd_turn", + "notes": [ + "FirstAgent slice only", + "turn-level retrieval", + "topic=all" + ], + "primary_metric": "hit_at_5", + "primary_metric_value": 0.93, + "reference_kind": "historical", + "runtime_seconds": 140.592, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "partial", + "system_name": "brainctl" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\membench_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\membench_mempalace_raw_turn.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" + }, + "benchmark": "membench", + "caveats": [ + "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 200, + "measured": false, + "metadata": {}, + "metrics": { + "hit_at_5": 0.885, + "simple_hit_at_5": 0.885, + "top_k": 5 + }, + "mode": "raw_turn", + "notes": [ + "FirstAgent slice only", + "turn-level retrieval", + "topic=all" + ], + "primary_metric": "hit_at_5", + "primary_metric_value": 0.885, + "reference_kind": "historical", + "runtime_seconds": 804.35, + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "partial", + "system_name": "mempalace" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\convomem_old_brainctl_cmd.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" + }, + "benchmark": "convomem", + "caveats": [ + "Blocked while loading ConvoMem evidence data: " + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "example_count": 0, + "measured": false, + "metadata": {}, + "metrics": {}, + "mode": "cmd", + "notes": [ + "limit_per_category=1", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": null, + "reference_kind": "historical", + "runtime_seconds": null, + "series_name": "old_brainctl", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "blocked", + "system_name": "brainctl" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\convomem_mempalace_raw.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" + }, + "benchmark": "convomem", + "caveats": [ + "Blocked while loading ConvoMem evidence data: " + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "example_count": 0, + "measured": false, + "metadata": {}, + "metrics": {}, + "mode": "raw", + "notes": [ + "limit_per_category=1", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": null, + "reference_kind": "historical", + "runtime_seconds": null, + "series_name": "mempalace", + "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", + "status": "blocked", + "system_name": "mempalace" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\longmemeval_new_brainctl_brain.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "measured": true, + "metadata": {}, + "metrics": { + "ndcg_at_10": 0.872, + "ndcg_at_5": 0.8678, + "r_at_10": 0.9894, + "r_at_5": 0.9574 + }, + "mode": "brain", + "notes": [ + "top_k=10", + "Legacy 470-question session-level slice." + ], + "primary_metric": "r_at_5", + "primary_metric_value": 0.9574, + "reference_kind": "measured", + "runtime_seconds": 403.852, + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\longmemeval_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\longmemeval_new_brainctl_cmd.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" + }, + "benchmark": "longmemeval", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", + "example_count": 470, + "measured": true, + "metadata": {}, + "metrics": { + "ndcg_at_10": 0.8753, + "ndcg_at_5": 0.8715, + "r_at_10": 0.9894, + "r_at_5": 0.9596 + }, + "mode": "cmd", + "notes": [ + "top_k=10", + "Legacy 470-question session-level slice." + ], + "primary_metric": "r_at_5", + "primary_metric_value": 0.9596, + "reference_kind": "measured", + "runtime_seconds": 251.044, + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\locomo_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\locomo_new_brainctl_cmd_session.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" + }, + "benchmark": "locomo", + "caveats": [], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", + "example_count": 1986, + "measured": true, + "metadata": {}, + "metrics": { + "avg_recall": 0.9145, + "cat_1_recall": 0.7533, + "cat_2_recall": 0.9107, + "cat_3_recall": 0.6607, + "cat_4_recall": 0.9643, + "cat_5_recall": 0.9798, + "perfect_rate": 0.8751, + "top_k": 10, + "zero_rate": 0.0519 + }, + "mode": "cmd_session", + "notes": [ + "granularity=session", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": 0.9145, + "reference_kind": "measured", + "runtime_seconds": 157.701, + "series_name": "new_brainctl", + "source_path": null, + "status": "full_same_machine", + "system_name": "brainctl" + }, + { + "artifacts": { + "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", + "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\membench_comparison.png", + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\membench_new_brainctl_cmd_turn.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" + }, + "benchmark": "membench", + "caveats": [ + "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", + "example_count": 200, + "measured": true, + "metadata": {}, + "metrics": { + "hit_at_5": 0.895, + "simple_hit_at_5": 0.895, + "top_k": 5 + }, + "mode": "cmd_turn", + "notes": [ + "FirstAgent slice only", + "turn-level retrieval", + "topic=all" + ], + "primary_metric": "hit_at_5", + "primary_metric_value": 0.895, + "reference_kind": "measured", + "runtime_seconds": 140.08, + "series_name": "new_brainctl", + "source_path": null, + "status": "partial", + "system_name": "brainctl" + }, + { + "artifacts": { + "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\convomem_new_brainctl_cmd.json", + "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" + }, + "benchmark": "convomem", + "caveats": [ + "changing_evidence: discover failed: HTTP Error 404: Not Found", + "ConvoMem comparison is partial because it uses a bounded same-machine sample, not the full benchmark." + ], + "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", + "example_count": 5, + "measured": true, + "metadata": {}, + "metrics": { + "abstention_evidence_recall": 0.0, + "assistant_facts_evidence_recall": 1.0, + "avg_recall": 0.6, + "implicit_connection_evidence_recall": 0.0, + "perfect_rate": 0.6, + "preference_evidence_recall": 1.0, + "top_k": 10, + "user_evidence_recall": 1.0 + }, + "mode": "cmd", + "notes": [ + "categories=6", + "limit_per_category=1", + "top_k=10" + ], + "primary_metric": "avg_recall", + "primary_metric_value": 0.6, + "reference_kind": "measured", + "runtime_seconds": 1.514, + "series_name": "new_brainctl", + "source_path": null, + "status": "partial", + "system_name": "brainctl" + } + ] +} \ No newline at end of file diff --git a/src/agentmemory/retrieval/__init__.py b/src/agentmemory/retrieval/__init__.py index e2fb1d7..62ada21 100644 --- a/src/agentmemory/retrieval/__init__.py +++ b/src/agentmemory/retrieval/__init__.py @@ -5,11 +5,13 @@ from .diagnostics import build_debug_payload from .evidence_graph import expand_procedure_evidence from .late_reranker import rerank_procedure_candidates +from .long_context import analyze_long_context from .mlp_reranker import TinyMLPModel from .query_planner import QueryPlan, plan_query from .second_stage import SecondStageConfig, rerank_bucketed_results, rerank_top_candidates __all__ = [ + "analyze_long_context", "QueryPlan", "SecondStageConfig", "TinyMLPModel", diff --git a/src/agentmemory/retrieval/feature_builder.py b/src/agentmemory/retrieval/feature_builder.py index acad4cf..fe9a3da 100644 --- a/src/agentmemory/retrieval/feature_builder.py +++ b/src/agentmemory/retrieval/feature_builder.py @@ -318,6 +318,28 @@ def build_features( neighbor_margin = max(current_score - prev_score, current_score - next_score, 0.0) confidence = _safe_float(candidate.get("confidence"), 0.5) support_evidence_score = min(len(candidate.get("supporting_evidence") or []) / 3.0, 1.0) + long_context_debug: dict[str, Any] = {"applicable": False} + _query_lower = (query or "").lower() + _structured_long_text = ( + len(text) >= 1500 + or "session id:" in _query_lower + or "session id:" in text.lower() + or "session date:" in text.lower() + or text.count("\n") >= 8 + ) + _query_needs_probe = bool(getattr(plan, "requires_temporal_reasoning", False)) or bool( + getattr(plan, "requires_multi_hop", False) + ) or any(token in _query_lower for token in ("when ", " which session", "what happened", "before ", "after ", "date")) + _score_uncertain = position <= 2 or neighbor_margin < 0.12 + if bucket == "memories" and _structured_long_text and _query_needs_probe and _score_uncertain: + try: + from agentmemory.retrieval.long_context import analyze_long_context as _analyze_long_context + + long_context_debug = _analyze_long_context(query, plan, candidate, text=text) + except Exception: + long_context_debug = {"applicable": False} + if long_context_debug.get("applicable"): + candidate["_long_context_debug"] = long_context_debug features = { "base_score": current_score, "retrieval_score": _safe_float(candidate.get("retrieval_score"), current_score), @@ -355,8 +377,16 @@ def build_features( "query_length_score": min(len(query_informative) / 8.0, 1.0), "candidate_length_score": min(len(cand_informative) / 64.0, 1.0), "procedural_candidate": 1.0 if bucket == "procedures" else 0.0, + "long_context_applicable": 1.0 if long_context_debug.get("applicable") else 0.0, + "long_context_score": _safe_float(long_context_debug.get("score")), + "long_context_confidence": _safe_float(long_context_debug.get("confidence")), + "long_context_agreement": _safe_float(long_context_debug.get("agreement")), + "long_context_uncertainty": _safe_float(long_context_debug.get("uncertainty")), + "long_context_coverage": _safe_float(long_context_debug.get("coverage")), + "long_context_precision": _safe_float(long_context_debug.get("precision")), + "long_context_focused_program": 1.0 if long_context_debug.get("program") not in {None, "", "whole_doc"} else 0.0, } - return {name: round(float(features.get(name, 0.0)), 6) for name in FEATURE_ORDER_V1} + return {name: round(float(value), 6) for name, value in features.items()} def vectorize_features( @@ -372,4 +402,3 @@ def vectorize_features( if _np is not None: return _np.asarray(values, dtype=float) return values - diff --git a/src/agentmemory/retrieval/long_context.py b/src/agentmemory/retrieval/long_context.py new file mode 100644 index 0000000..1304310 --- /dev/null +++ b/src/agentmemory/retrieval/long_context.py @@ -0,0 +1,434 @@ +"""Bounded long-context evidence probing for shared retrieval reranking. + +RLM/SRLM-inspired adaptation for brainctl: + +- Treat the candidate text as an external environment rather than a single bag + of tokens. +- Run a small portfolio of deterministic chunking "programs" over that + environment. +- Select the most reliable program using agreement + uncertainty, not just the + single highest raw score. + +This stays local, bounded, and depth-1 on purpose. Reproduction work on RLMs +shows deeper recursion can overthink and blow up latency; here we only probe a +short list of chunk views over the same candidate row. +""" + +from __future__ import annotations + +import os +import re +from dataclasses import dataclass +from typing import Any + +_STOPWORDS = { + "a", "an", "and", "are", "as", "at", "be", "by", "did", "do", "does", "for", + "from", "has", "have", "how", "i", "in", "is", "it", "its", "of", "on", "or", + "that", "the", "to", "was", "we", "what", "when", "where", "which", "who", + "why", "will", "with", "you", +} +_LOW_SIGNAL_TOKENS = { + "summary", "history", "timeline", "recent", "today", "yesterday", "tomorrow", + "issue", "problem", "thing", "stuff", "update", +} +_TEMPORAL_RE = re.compile( + r"\b(yesterday|today|tomorrow|when|before|after|during|timeline|history|recent|latest|first|last)\b", + re.IGNORECASE, +) +_DATE_RE = re.compile( + r"\b(?:\d{4}-\d{2}-\d{2}|\d{1,2}/\d{1,2}(?:/\d{2,4})?|" + r"jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|jun(?:e)?|" + r"jul(?:y)?|aug(?:ust)?|sep(?:tember)?|oct(?:ober)?|nov(?:ember)?|" + r"dec(?:ember)?)\b", + re.IGNORECASE, +) +_SESSION_RE = re.compile(r"\bsession[_ :#-]*(\d+)\b", re.IGNORECASE) +_ENTITY_RE = re.compile(r"\b[A-Z][A-Za-z0-9_.:-]+\b") +_TURNISH_RE = re.compile(r"^\s*(?:[A-Z][A-Za-z0-9_.-]+:|.+\bsaid,\s+\")", re.IGNORECASE) + + +@dataclass(slots=True) +class ProbeChunk: + index: int + text: str + score: float + coverage: float + precision: float + entity_overlap: float + temporal_overlap: float + exact_phrase: float + + +@dataclass(slots=True) +class ProbeProgramResult: + name: str + score: float + confidence: float + uncertainty: float + agreement: float + coverage: float + precision: float + length_penalty: float + chunk_count: int + top_chunk: ProbeChunk | None + + +def _normalize_token(token: str) -> str: + tok = re.sub(r"[^a-z0-9]+", "", (token or "").lower()) + if len(tok) <= 2 or tok in _STOPWORDS: + return "" + if tok.endswith("ies") and len(tok) > 4: + tok = tok[:-3] + "y" + elif tok.endswith("ed") and len(tok) > 4: + tok = tok[:-2] + elif tok.endswith("es") and len(tok) > 4: + tok = tok[:-2] + elif tok.endswith("s") and len(tok) > 3: + tok = tok[:-1] + return tok + + +def _token_set(text: str) -> set[str]: + return { + token + for part in re.split(r"\s+", text or "") + if (token := _normalize_token(part)) + } + + +def _informative_tokens(text: str) -> set[str]: + return {token for token in _token_set(text) if token not in _LOW_SIGNAL_TOKENS} + + +def _entity_terms(text: str) -> set[str]: + return { + match.group(0).lower() + for match in _ENTITY_RE.finditer(text or "") + if len(match.group(0)) > 2 + } + + +def _deobfuscate(text: str) -> str: + value = text or "" + value = value.replace("\u200b", "").replace("\ufeff", "") + value = re.sub(r"[_*/`~]+", " ", value) + value = re.sub(r"\s+", " ", value) + return value.strip() + + +def _safe_window(items: list[str], size: int, stride: int) -> list[str]: + if not items: + return [] + if len(items) <= size: + return ["\n".join(items)] + out: list[str] = [] + for start in range(0, len(items), max(stride, 1)): + chunk = items[start:start + size] + if not chunk: + continue + out.append("\n".join(chunk)) + if start + size >= len(items): + break + return out + + +def _cap_chunks(chunks: list[str], max_chunks: int) -> list[str]: + if len(chunks) <= max_chunks: + return chunks + if max_chunks <= 1: + return [chunks[0]] + step = (len(chunks) - 1) / float(max_chunks - 1) + selected: list[str] = [] + seen: set[int] = set() + for idx in range(max_chunks): + pick = int(round(idx * step)) + if pick in seen: + continue + seen.add(pick) + selected.append(chunks[pick]) + return selected + + +def _whole_doc_program(text: str, max_chunks: int) -> list[str]: + return [text[:48000]] if text else [] + + +def _line_window_program(text: str, max_chunks: int) -> list[str]: + lines = [line.strip() for line in text.splitlines() if line.strip()] + if len(lines) < 3: + return [] + return _cap_chunks(_safe_window(lines, size=6, stride=3), max_chunks) + + +def _sentence_window_program(text: str, max_chunks: int) -> list[str]: + sentences = [part.strip() for part in re.split(r"(?<=[.!?])\s+|\n+", text) if part.strip()] + if len(sentences) < 2: + return [] + return _cap_chunks(_safe_window(sentences, size=3, stride=1), max_chunks) + + +def _turn_window_program(text: str, max_chunks: int) -> list[str]: + lines = [line.strip() for line in text.splitlines() if line.strip()] + turnish = [line for line in lines if _TURNISH_RE.search(line)] + if len(turnish) < 2: + return [] + return _cap_chunks(_safe_window(turnish, size=4, stride=2), max_chunks) + + +def _anchor_window_program( + text: str, + query: str, + *, + target_entities: list[str], + temporal_query: bool, + max_chunks: int, +) -> list[str]: + lines = [line.strip() for line in text.splitlines() if line.strip()] + if not lines: + return [] + informative = _informative_tokens(query) + entities = {value.lower() for value in target_entities if value} + header = [] + if lines[:2] and any("session id" in line.lower() or "session date" in line.lower() for line in lines[:3]): + header = lines[:3] + anchor_indexes: list[int] = [] + for idx, line in enumerate(lines): + lowered = line.lower() + if informative and any(token in lowered for token in informative): + anchor_indexes.append(idx) + continue + if entities and any(entity in lowered for entity in entities): + anchor_indexes.append(idx) + continue + if temporal_query and (_TEMPORAL_RE.search(line) or _DATE_RE.search(line)): + anchor_indexes.append(idx) + continue + if not anchor_indexes: + return [] + chunks: list[str] = [] + seen: set[str] = set() + for idx in anchor_indexes: + start = max(0, idx - 2) + end = min(len(lines), idx + 3) + window = header + lines[start:end] + chunk = "\n".join(window) + if chunk and chunk not in seen: + seen.add(chunk) + chunks.append(chunk) + return _cap_chunks(chunks, max_chunks) + + +def _candidate_programs( + text: str, + query: str, + *, + target_entities: list[str], + temporal_query: bool, + max_chunks: int, +) -> dict[str, list[str]]: + programs = { + "whole_doc": _whole_doc_program(text, max_chunks), + "line_windows": _line_window_program(text, max_chunks), + "sentence_windows": _sentence_window_program(text, max_chunks), + "turn_windows": _turn_window_program(text, max_chunks), + "anchor_windows": _anchor_window_program( + text, + query, + target_entities=target_entities, + temporal_query=temporal_query, + max_chunks=max_chunks, + ), + } + return {name: chunks for name, chunks in programs.items() if chunks} + + +def _chunk_score( + query: str, + chunk: str, + *, + target_entities: list[str], + temporal_query: bool, +) -> ProbeChunk: + informative = _informative_tokens(query) + query_tokens = _token_set(query) + chunk_tokens = _token_set(chunk) + chunk_informative = _informative_tokens(chunk) + query_entities = _entity_terms(query) | {value.lower() for value in target_entities if value} + chunk_entities = _entity_terms(chunk) + overlap = len(query_tokens & chunk_tokens) / max(len(query_tokens), 1) if query_tokens else 0.0 + coverage = len(informative & chunk_informative) / max(len(informative), 1) if informative else overlap + precision = len(informative & chunk_informative) / max(len(chunk_informative), 1) if chunk_informative else 0.0 + exact_phrase = 1.0 if query and len(query.strip()) >= 4 and query.lower().strip() in chunk.lower() else 0.0 + entity_overlap = len(query_entities & chunk_entities) / max(len(query_entities), 1) if query_entities else 0.0 + temporal_overlap = 0.0 + if temporal_query: + temporal_overlap = 1.0 if (_TEMPORAL_RE.search(chunk) or _DATE_RE.search(chunk) or _SESSION_RE.search(chunk)) else 0.0 + concentration = min(1.0, 12.0 / max(len(chunk_informative), 12)) + score = ( + coverage * 0.34 + + precision * 0.18 + + overlap * 0.12 + + exact_phrase * 0.14 + + entity_overlap * 0.12 + + temporal_overlap * (0.10 if temporal_query else 0.0) + + concentration * 0.10 + ) + return ProbeChunk( + index=0, + text=chunk, + score=round(min(score, 1.0), 6), + coverage=round(coverage, 6), + precision=round(precision, 6), + entity_overlap=round(entity_overlap, 6), + temporal_overlap=round(temporal_overlap, 6), + exact_phrase=round(exact_phrase, 6), + ) + + +def _program_signature(chunk: ProbeChunk | None) -> set[str]: + if chunk is None: + return set() + return _informative_tokens(chunk.text) + + +def analyze_long_context( + query: str, + plan: Any, + candidate: dict[str, Any], + *, + text: str, +) -> dict[str, Any]: + """Return depth-1 context-program evidence for a long candidate row.""" + + if os.environ.get("BRAINCTL_LONG_CONTEXT_PROBES", "0") in {"0", "false", "False"}: + return {"applicable": False, "reason": "disabled"} + + min_chars = int(os.environ.get("BRAINCTL_LONG_CONTEXT_MIN_CHARS", "900") or "900") + max_chunks = int(os.environ.get("BRAINCTL_LONG_CONTEXT_MAX_CHUNKS", "24") or "24") + candidate_text = _deobfuscate(text) + raw_lines = [line.strip() for line in text.splitlines() if line.strip()] + structured_session = any( + "session id" in line.lower() or "session date" in line.lower() + for line in raw_lines[:4] + ) + if len(candidate_text) < min_chars and not structured_session and len(raw_lines) < 5: + return {"applicable": False, "reason": "short_text"} + + target_entities = list(getattr(plan, "target_entities", []) or []) + temporal_query = bool(getattr(plan, "requires_temporal_reasoning", False)) or bool(_TEMPORAL_RE.search(query or "")) + programs = _candidate_programs( + candidate_text, + query, + target_entities=target_entities, + temporal_query=temporal_query, + max_chunks=max_chunks, + ) + if not programs: + return {"applicable": False, "reason": "no_programs"} + + evaluated: list[ProbeProgramResult] = [] + for name, chunks in programs.items(): + scored: list[ProbeChunk] = [] + for index, chunk in enumerate(chunks): + base = _chunk_score( + query, + chunk, + target_entities=target_entities, + temporal_query=temporal_query, + ) + scored.append( + ProbeChunk( + index=index, + text=base.text, + score=base.score, + coverage=base.coverage, + precision=base.precision, + entity_overlap=base.entity_overlap, + temporal_overlap=base.temporal_overlap, + exact_phrase=base.exact_phrase, + ) + ) + scored.sort(key=lambda item: item.score, reverse=True) + top_chunk = scored[0] if scored else None + second_score = scored[1].score if len(scored) > 1 else 0.0 + coverage = top_chunk.coverage if top_chunk else 0.0 + precision = top_chunk.precision if top_chunk else 0.0 + margin = max((top_chunk.score - second_score) if top_chunk else 0.0, 0.0) + confidence = min(1.0, coverage * 0.45 + precision * 0.15 + margin * 0.40) + length_penalty = min(1.0, len(chunks) / max(max_chunks, 1)) + score = min(1.0, (top_chunk.score if top_chunk else 0.0) * 0.82 + coverage * 0.12 + precision * 0.06) + evaluated.append( + ProbeProgramResult( + name=name, + score=round(score, 6), + confidence=round(confidence, 6), + uncertainty=1.0, # set after agreement pass + agreement=0.0, + coverage=round(coverage, 6), + precision=round(precision, 6), + length_penalty=round(length_penalty, 6), + chunk_count=len(chunks), + top_chunk=top_chunk, + ) + ) + + max_score = max(program.score for program in evaluated) + consistent = [program for program in evaluated if program.score >= max_score - 0.08] + for program in evaluated: + sig = _program_signature(program.top_chunk) + peers = [] + for other in consistent: + if other is program: + continue + other_sig = _program_signature(other.top_chunk) + if not sig and not other_sig: + peers.append(1.0) + continue + union = len(sig | other_sig) + if union == 0: + peers.append(0.0) + else: + peers.append(len(sig & other_sig) / union) + agreement = sum(peers) / len(peers) if peers else (1.0 if len(consistent) == 1 else 0.0) + program.agreement = round(agreement, 6) + program.uncertainty = round( + min( + 1.0, + (1.0 - agreement) * 0.45 + + (1.0 - program.confidence) * 0.40 + + program.length_penalty * 0.15, + ), + 6, + ) + + selected = min( + consistent, + key=lambda item: ( + round(item.uncertainty, 6), + -round(item.score, 6), + -round(item.agreement, 6), + item.chunk_count, + ), + ) + return { + "applicable": True, + "program": selected.name, + "score": selected.score, + "confidence": selected.confidence, + "agreement": selected.agreement, + "uncertainty": selected.uncertainty, + "coverage": selected.coverage, + "precision": selected.precision, + "chunk_count": selected.chunk_count, + "top_chunk_excerpt": (selected.top_chunk.text[:320] if selected.top_chunk else ""), + "program_scores": { + program.name: { + "score": program.score, + "confidence": program.confidence, + "agreement": program.agreement, + "uncertainty": program.uncertainty, + "chunk_count": program.chunk_count, + } + for program in evaluated + }, + } diff --git a/src/agentmemory/retrieval/second_stage.py b/src/agentmemory/retrieval/second_stage.py index c7a1d53..11f1890 100644 --- a/src/agentmemory/retrieval/second_stage.py +++ b/src/agentmemory/retrieval/second_stage.py @@ -80,19 +80,39 @@ def _heuristic_score(plan: Any, features: dict[str, float]) -> float: + features["exact_phrase"] * 0.05 + features["support_evidence_score"] * 0.03 ) + long_context_reliable = ( + features.get("long_context_applicable", 0.0) > 0.0 + and features.get("long_context_focused_program", 0.0) > 0.0 + and features.get("long_context_confidence", 0.0) >= 0.62 + and features.get("long_context_uncertainty", 0.0) <= 0.38 + ) + if long_context_reliable: + score += ( + features.get("long_context_score", 0.0) * 0.09 + + features.get("long_context_confidence", 0.0) * 0.03 + + features.get("long_context_agreement", 0.0) * 0.02 + + features.get("long_context_coverage", 0.0) * 0.03 + + features.get("long_context_precision", 0.0) * 0.02 + ) if features["query_temporal"] > 0: score += ( features["candidate_temporal"] * 0.04 + features["temporal_anchor_overlap"] * 0.08 + features["session_gap_score"] * 0.06 ) + if long_context_reliable: + score += features.get("long_context_score", 0.0) * 0.05 if intent in {"temporal", "decision"}: score += features["bucket_events"] * 0.04 + features["bucket_decisions"] * 0.03 if intent in {"procedural", "troubleshooting"}: score += features["bucket_procedures"] * 0.06 + features["procedural_candidate"] * 0.04 + if long_context_reliable: + score += features.get("long_context_confidence", 0.0) * 0.04 if intent == "factual": score += features["bucket_memories"] * 0.05 + features["bucket_entities"] * 0.04 score -= features["bucket_procedures"] * 0.04 + if long_context_reliable: + score += features.get("long_context_precision", 0.0) * 0.04 if features["source_graph"] > 0: score -= 0.08 if features["status_stale"] > 0: @@ -175,8 +195,17 @@ def rerank_top_candidates( "temporal_anchor_overlap", "intent_bucket_fit", "session_gap_score", + "long_context_score", + "long_context_confidence", + "long_context_agreement", + "long_context_uncertainty", + "long_context_focused_program", ) } + long_context_debug = candidate.pop("_long_context_debug", None) or {} + if long_context_debug.get("applicable"): + candidate["second_stage_features"]["long_context_program"] = long_context_debug.get("program") + candidate["second_stage_features"]["long_context_excerpt"] = long_context_debug.get("top_chunk_excerpt") candidate["final_score"] = round(final_score, 8) debug_candidates.append( { @@ -200,6 +229,9 @@ def rerank_top_candidates( "model_path": str(cfg.model_path or DEFAULT_MODEL_PATH), "model_loaded": model is not None, "judge_enabled": cfg.judge.enabled, + "base_weight": round(max(0.0, 1.0 - cfg.heuristic_weight - cfg.mlp_weight - cfg.judge_weight), 4), + "mlp_weight": round(cfg.mlp_weight, 4), + "judge_weight": round(cfg.judge_weight, 4), "candidates": debug_candidates, } return reranked, debug diff --git a/tests/test_long_context_explorer.py b/tests/test_long_context_explorer.py new file mode 100644 index 0000000..ef5cb1e --- /dev/null +++ b/tests/test_long_context_explorer.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +from pathlib import Path + +from agentmemory.retrieval.long_context import analyze_long_context +from agentmemory.retrieval.query_planner import plan_query +from agentmemory.retrieval.second_stage import SecondStageConfig, rerank_top_candidates + +from tests.test_second_stage_reranker import _temp_model + + +def test_long_context_probe_finds_session_anchor(monkeypatch): + monkeypatch.setenv("BRAINCTL_LONG_CONTEXT_PROBES", "1") + plan = plan_query("When did Caroline go to the LGBTQ support group?", requested_tables=["memories"]) + candidate = { + "id": 1, + "bucket": "memories", + "type": "memory", + "final_score": 0.72, + "retrieval_score": 0.72, + "source": "keyword", + } + text = "\n".join( + [ + "Session ID: session_1", + "Session Date: 2025-01-12", + "Conversation:", + 'Alice: We talked about cooking classes and weekend plans.', + 'Bob: Nothing else noteworthy happened this week.', + 'Caroline: I went to the LGBTQ support group after work and felt better.', + 'Alice: We also mentioned a grocery list and cleaning supplies.', + ] + ) + + result = analyze_long_context( + "When did Caroline go to the LGBTQ support group?", + plan, + candidate, + text=text, + ) + + assert result["applicable"] is True + assert result["score"] > 0.55 + assert result["confidence"] > 0.45 + assert result["uncertainty"] < 0.7 + assert "LGBTQ support group" in result["top_chunk_excerpt"] + + +def test_second_stage_uses_long_context_probe_to_promote_focused_session(tmp_path: Path, monkeypatch): + monkeypatch.setenv("BRAINCTL_LONG_CONTEXT_PROBES", "1") + plan = plan_query("When did Caroline go to the LGBTQ support group?", requested_tables=["memories"]) + model_path = _temp_model(tmp_path / "tiny.json") + diffuse = { + "id": 1, + "bucket": "memories", + "type": "memory", + "content": "\n".join( + [ + "Session ID: session_7", + "Session Date: 2025-01-20", + "Conversation:", + 'Alice: Caroline mentioned some errands after work.', + 'Bob: She later mentioned a support group but I do not remember when.', + 'Alice: Then we switched topics to a restaurant review and sprint planning.', + 'Bob: We also talked about a support group again in passing.', + 'Alice: Nothing pinned the exact date.', + ] + ), + "final_score": 0.84, + "retrieval_score": 0.84, + "source": "both", + "confidence": 0.9, + } + focused = { + "id": 2, + "bucket": "memories", + "type": "memory", + "content": "\n".join( + [ + "Session ID: session_1", + "Session Date: 2025-01-12", + "Conversation:", + 'Caroline: I went to the LGBTQ support group after work on January 12.', + 'Alice: We noted it in the session log for follow-up.', + ] + ), + "final_score": 0.76, + "retrieval_score": 0.76, + "source": "keyword", + "confidence": 0.9, + } + + reranked, debug = rerank_top_candidates( + "When did Caroline go to the LGBTQ support group?", + plan, + [diffuse, focused], + config=SecondStageConfig(top_n=2, model_path=str(model_path)), + ) + + assert reranked[0]["id"] == 2 + assert reranked[0]["second_stage_features"]["long_context_score"] > reranked[1]["second_stage_features"]["long_context_score"] + assert debug["enabled"] is True From 3be1429fe481a6659fdc58fedea0f67e6208297f Mon Sep 17 00:00:00 2001 From: Mario Jack Vela Date: Thu, 23 Apr 2026 13:32:22 -0500 Subject: [PATCH 07/13] Tighten long-context probing to temporal ambiguity cases --- src/agentmemory/retrieval/feature_builder.py | 90 +++++++++++++++---- src/agentmemory/retrieval/long_context.py | 30 ++++++- src/agentmemory/retrieval/query_planner.py | 30 ++++++- src/agentmemory/retrieval/second_stage.py | 3 +- tests/test_long_context_explorer.py | 92 +++++++++++++++++++- 5 files changed, 217 insertions(+), 28 deletions(-) diff --git a/src/agentmemory/retrieval/feature_builder.py b/src/agentmemory/retrieval/feature_builder.py index fe9a3da..4f19b9e 100644 --- a/src/agentmemory/retrieval/feature_builder.py +++ b/src/agentmemory/retrieval/feature_builder.py @@ -34,6 +34,17 @@ r"\b(yesterday|today|tomorrow|when|before|after|during|timeline|history|recent|latest|first|last)\b", re.IGNORECASE, ) +_LONG_CONTEXT_HINT_RE = re.compile( + r"\b(" + r"how many|how much|order|earliest|latest|most recent|" + r"before|after|between|this month|last month|past month|past week|" + r"current(?:ly)?|previous(?:ly)?|" + r"(?:one|two|three|four|five|six|seven|eight|nine|ten|\d+)\s+" + r"(?:day|week|month|year)s?\s+ago|" + r"based on|underlying|future|might|would" + r")\b", + re.IGNORECASE, +) _SESSION_RE = re.compile(r"\bsession[_ :#-]*(\d+)\b", re.IGNORECASE) _DIALOG_RE = re.compile(r"\bD(\d+):", re.IGNORECASE) _ENTITY_RE = re.compile(r"\b[A-Z][A-Za-z0-9_.:-]+\b") @@ -278,6 +289,52 @@ def _weights(tokens: Iterable[str]) -> dict[str, float]: return dot / (q_norm * c_norm) +def _should_probe_long_context( + *, + query: str, + plan: Any, + bucket: str, + text: str, + position: int, + current_score: float, + prev_raw: Any, + next_raw: Any, + leader_raw: Any, +) -> bool: + if bucket != "memories": + return False + + lowered_query = query or "" + structured_long_text = ( + len(text) >= 1500 + or "session id:" in text.lower() + or "session date:" in text.lower() + or text.count("\n") >= 8 + ) + if not structured_long_text: + return False + + if position > 4: + return False + + query_needs_probe = ( + bool(getattr(plan, "requires_temporal_reasoning", False)) + or bool(getattr(plan, "requires_multi_hop", False)) + or bool(_LONG_CONTEXT_HINT_RE.search(lowered_query)) + ) + if not query_needs_probe: + return False + + closest_gap_values: list[float] = [] + if prev_raw is not None: + closest_gap_values.append(abs(current_score - _safe_float(prev_raw))) + if next_raw is not None: + closest_gap_values.append(abs(current_score - _safe_float(next_raw))) + closest_neighbor_gap = min(closest_gap_values) if closest_gap_values else 0.0 + leader_gap = abs(_safe_float(leader_raw, current_score) - current_score) + return closest_neighbor_gap <= 0.035 and leader_gap <= 0.08 + + def build_features( query: str, plan: Any, @@ -304,7 +361,7 @@ def build_features( entity_overlap = len(query_entities & cand_entities) / max(len(query_entities), 1) if query_entities else 0.0 aliases = {alias.lower() for alias in _alias_values(candidate) if len(alias) > 2} alias_overlap = len(query_entities & aliases) / max(len(query_entities), 1) if query_entities and aliases else 0.0 - query_temporal = 1.0 if _TEMPORAL_RE.search(query or "") else 0.0 + query_temporal = 1.0 if (bool(getattr(plan, "requires_temporal_reasoning", False)) or _TEMPORAL_RE.search(query or "")) else 0.0 candidate_temporal = 1.0 if _TEMPORAL_RE.search(text or "") or _DATE_RE.search(text or "") else 0.0 temporal_anchor_overlap = _temporal_anchor_overlap(query, text) session_gap_score, query_session_hint, candidate_session_hint = _session_gap_score(query, text) @@ -312,26 +369,27 @@ def build_features( bucket_memories, bucket_events, bucket_entities, bucket_procedures, bucket_decisions = _bucket_flags(bucket) status = str(candidate.get("status") or "").lower() position = max(int(candidate.get("_stage_position") or 0), 0) - prev_score = _safe_float((neighbors or {}).get("prev_score")) - next_score = _safe_float((neighbors or {}).get("next_score")) + prev_raw = (neighbors or {}).get("prev_score") + next_raw = (neighbors or {}).get("next_score") + leader_raw = (neighbors or {}).get("leader_score") + prev_score = _safe_float(prev_raw) + next_score = _safe_float(next_raw) current_score = _safe_float(candidate.get("final_score") or candidate.get("retrieval_score")) neighbor_margin = max(current_score - prev_score, current_score - next_score, 0.0) confidence = _safe_float(candidate.get("confidence"), 0.5) support_evidence_score = min(len(candidate.get("supporting_evidence") or []) / 3.0, 1.0) long_context_debug: dict[str, Any] = {"applicable": False} - _query_lower = (query or "").lower() - _structured_long_text = ( - len(text) >= 1500 - or "session id:" in _query_lower - or "session id:" in text.lower() - or "session date:" in text.lower() - or text.count("\n") >= 8 - ) - _query_needs_probe = bool(getattr(plan, "requires_temporal_reasoning", False)) or bool( - getattr(plan, "requires_multi_hop", False) - ) or any(token in _query_lower for token in ("when ", " which session", "what happened", "before ", "after ", "date")) - _score_uncertain = position <= 2 or neighbor_margin < 0.12 - if bucket == "memories" and _structured_long_text and _query_needs_probe and _score_uncertain: + if _should_probe_long_context( + query=query, + plan=plan, + bucket=bucket, + text=text, + position=position, + current_score=current_score, + prev_raw=prev_raw, + next_raw=next_raw, + leader_raw=leader_raw, + ): try: from agentmemory.retrieval.long_context import analyze_long_context as _analyze_long_context diff --git a/src/agentmemory/retrieval/long_context.py b/src/agentmemory/retrieval/long_context.py index 1304310..f0f40eb 100644 --- a/src/agentmemory/retrieval/long_context.py +++ b/src/agentmemory/retrieval/long_context.py @@ -291,6 +291,13 @@ def _program_signature(chunk: ProbeChunk | None) -> set[str]: return _informative_tokens(chunk.text) +def _is_focused_program(program: ProbeProgramResult, *, candidate_chars: int) -> bool: + if program.name == "whole_doc" or program.top_chunk is None or candidate_chars <= 0: + return False + span_ratio = len(program.top_chunk.text) / float(candidate_chars) + return span_ratio < 0.85 + + def analyze_long_context( query: str, plan: Any, @@ -300,7 +307,7 @@ def analyze_long_context( ) -> dict[str, Any]: """Return depth-1 context-program evidence for a long candidate row.""" - if os.environ.get("BRAINCTL_LONG_CONTEXT_PROBES", "0") in {"0", "false", "False"}: + if os.environ.get("BRAINCTL_LONG_CONTEXT_PROBES", "1") in {"0", "false", "False"}: return {"applicable": False, "reason": "disabled"} min_chars = int(os.environ.get("BRAINCTL_LONG_CONTEXT_MIN_CHARS", "900") or "900") @@ -372,8 +379,25 @@ def analyze_long_context( ) ) - max_score = max(program.score for program in evaluated) - consistent = [program for program in evaluated if program.score >= max_score - 0.08] + focused = [program for program in evaluated if _is_focused_program(program, candidate_chars=len(candidate_text))] + if not focused: + return { + "applicable": False, + "reason": "no_focused_program", + "program_scores": { + program.name: { + "score": program.score, + "confidence": program.confidence, + "agreement": program.agreement, + "uncertainty": program.uncertainty, + "chunk_count": program.chunk_count, + } + for program in evaluated + }, + } + + max_score = max(program.score for program in focused) + consistent = [program for program in focused if program.score >= max_score - 0.08] for program in evaluated: sig = _program_signature(program.top_chunk) peers = [] diff --git a/src/agentmemory/retrieval/query_planner.py b/src/agentmemory/retrieval/query_planner.py index 2821b01..aaae607 100644 --- a/src/agentmemory/retrieval/query_planner.py +++ b/src/agentmemory/retrieval/query_planner.py @@ -26,11 +26,26 @@ re.IGNORECASE, ) _TEMPORAL_RE = re.compile( - r"\b(yesterday|today|tomorrow|last week|last month|when|timeline|history|recent|overnight)\b", + r"\b(" + r"yesterday|today|tomorrow|when|timeline|history|recent|overnight|" + r"last\s+(?:week|month|year|tuesday|wednesday|thursday|friday|saturday|sunday)|" + r"this\s+(?:week|month|year)|" + r"past\s+(?:week|month|year|two weeks|three months)|" + r"most recent|latest|earliest|previous(?:ly)?|current(?:ly)?|" + r"before|after|between|during|in the past|order of|" + r"(?:one|two|three|four|five|six|seven|eight|nine|ten|\d+)\s+" + r"(?:day|week|month|year)s?\s+ago" + r")\b", re.IGNORECASE, ) _MULTIHOP_RE = re.compile( - r"\b(why|because|rationale|support|evidence|rollback|troubleshoot|debug|fix)\b", + r"\b(" + r"why|because|rationale|support|evidence|rollback|troubleshoot|debug|fix|" + r"how many|how much|order|earliest|latest|most recent|" + r"before|after|between|difference|older|newer|" + r"compare|combined|total|sum|" + r"based on|underlying|future|might|would" + r")\b", re.IGNORECASE, ) _NEGATIVE_RE = re.compile( @@ -99,6 +114,8 @@ def as_dict(self) -> dict[str, Any]: def _builtin_classify(query: str) -> tuple[str, float, str]: q = query.lower() + temporalish = bool(_TEMPORAL_RE.search(query)) + multihopish = bool(_MULTIHOP_RE.search(query)) if _ENTITY_QUERY_RE.search(query): return ("factual", 0.72, "builtin:entity_fact") if any(token in q for token in ("how to", "how do", "procedure", "rollback", "runbook", "playbook")): @@ -107,8 +124,9 @@ def _builtin_classify(query: str) -> tuple[str, float, str]: return ("troubleshooting", 0.8, "builtin:troubleshooting") if any(token in q for token in ("why", "decision", "rationale", "choose", "chose")): return ("decision", 0.78, "builtin:decision") - if any(token in q for token in ("when", "history", "timeline", "what happened", "yesterday")): - return ("temporal", 0.78, "builtin:temporal") + if temporalish or "what happened" in q: + reason = "builtin:temporal_multihop" if multihopish else "builtin:temporal" + return ("temporal", 0.8 if multihopish else 0.78, reason) if any(token in q for token in ("who", "what", "where", "which", "entity")): return ("factual", 0.6, "builtin:factual") return ("factual", 0.45, "builtin:default") @@ -191,6 +209,10 @@ def plan_query( abstain_allowed = bool(_NEGATIVE_RE.search(query)) or normalized_intent in {"factual", "troubleshooting", "procedural"} if _ENTITY_QUERY_RE.search(query) and normalized_intent == "factual": reasons.append("entity_or_role_lookup") + if requires_temporal: + reasons.append("temporal_reasoning") + if requires_multi_hop: + reasons.append("multi_hop_or_inference") if "summary of yesterday" in query_lower: abstain_allowed = True reasons.append("negative_or_out_of_domain_summary") diff --git a/src/agentmemory/retrieval/second_stage.py b/src/agentmemory/retrieval/second_stage.py index 11f1890..b85ae72 100644 --- a/src/agentmemory/retrieval/second_stage.py +++ b/src/agentmemory/retrieval/second_stage.py @@ -141,6 +141,7 @@ def rerank_top_candidates( candidate.setdefault("bucket", candidate.get("type") or "memories") feature_rows: list[dict[str, float]] = [] + leader_score = head[0].get("final_score") if head else None for idx, candidate in enumerate(head): prev_score = head[idx - 1].get("final_score") if idx > 0 else None next_score = head[idx + 1].get("final_score") if idx + 1 < len(head) else None @@ -148,7 +149,7 @@ def rerank_top_candidates( query, plan, candidate, - neighbors={"prev_score": prev_score, "next_score": next_score}, + neighbors={"prev_score": prev_score, "next_score": next_score, "leader_score": leader_score}, ) feature_rows.append(features) diff --git a/tests/test_long_context_explorer.py b/tests/test_long_context_explorer.py index ef5cb1e..343c693 100644 --- a/tests/test_long_context_explorer.py +++ b/tests/test_long_context_explorer.py @@ -2,6 +2,7 @@ from pathlib import Path +from agentmemory.retrieval.feature_builder import build_features from agentmemory.retrieval.long_context import analyze_long_context from agentmemory.retrieval.query_planner import plan_query from agentmemory.retrieval.second_stage import SecondStageConfig, rerank_top_candidates @@ -66,8 +67,8 @@ def test_second_stage_uses_long_context_probe_to_promote_focused_session(tmp_pat 'Alice: Nothing pinned the exact date.', ] ), - "final_score": 0.84, - "retrieval_score": 0.84, + "final_score": 0.789, + "retrieval_score": 0.789, "source": "both", "confidence": 0.9, } @@ -80,12 +81,16 @@ def test_second_stage_uses_long_context_probe_to_promote_focused_session(tmp_pat "Session ID: session_1", "Session Date: 2025-01-12", "Conversation:", + "Alice: We opened with a grocery list and a reminder about dry cleaning.", + "Bob: Then we talked about a dentist appointment and an office lunch.", 'Caroline: I went to the LGBTQ support group after work on January 12.', 'Alice: We noted it in the session log for follow-up.', + "Bob: After that we switched to weekend errands and recipe planning.", + "Alice: We ended with notes about commute timing and a restaurant reservation.", ] ), - "final_score": 0.76, - "retrieval_score": 0.76, + "final_score": 0.776, + "retrieval_score": 0.776, "source": "keyword", "confidence": 0.9, } @@ -100,3 +105,82 @@ def test_second_stage_uses_long_context_probe_to_promote_focused_session(tmp_pat assert reranked[0]["id"] == 2 assert reranked[0]["second_stage_features"]["long_context_score"] > reranked[1]["second_stage_features"]["long_context_score"] assert debug["enabled"] is True + + +def test_query_planner_flags_temporal_aggregation_and_inference(): + temporal_multi = plan_query("How much have I made from selling eggs this month?", requested_tables=["memories"]) + assert temporal_multi.requires_temporal_reasoning is True + assert temporal_multi.requires_multi_hop is True + assert temporal_multi.normalized_intent == "temporal" + + inference = plan_query("What personality traits might Melanie say Caroline has?", requested_tables=["memories"]) + assert inference.requires_multi_hop is True + + +def test_long_context_probe_requires_close_temporal_candidates(monkeypatch): + monkeypatch.setenv("BRAINCTL_LONG_CONTEXT_PROBES", "1") + plan = plan_query("When did Caroline go to the LGBTQ support group?", requested_tables=["memories"]) + candidate = { + "id": 7, + "bucket": "memories", + "type": "memory", + "content": "\n".join( + [ + "Session ID: session_9", + "Session Date: 2025-01-19", + "Conversation:", + "Alice: We discussed errands and a support group in passing.", + "Caroline: I went to the LGBTQ support group after work on January 12.", + "Bob: We wrote it down in the follow-up notes.", + "Alice: Then we switched to grocery planning and restaurants.", + "Bob: We revisited the support group briefly before closing the session.", + ] + ), + "final_score": 0.91, + "retrieval_score": 0.91, + "source": "keyword", + } + + far_apart = build_features( + "When did Caroline go to the LGBTQ support group?", + plan, + dict(candidate), + neighbors={"prev_score": None, "next_score": 0.76, "leader_score": 0.91}, + ) + assert far_apart["long_context_applicable"] == 0.0 + + close_scores = build_features( + "When did Caroline go to the LGBTQ support group?", + plan, + dict(candidate), + neighbors={"prev_score": None, "next_score": 0.889, "leader_score": 0.91}, + ) + assert close_scores["long_context_applicable"] == 1.0 + assert close_scores["long_context_focused_program"] == 1.0 + + +def test_long_context_probe_ignores_whole_document_only_matches(monkeypatch): + monkeypatch.setenv("BRAINCTL_LONG_CONTEXT_PROBES", "1") + plan = plan_query("When did Caroline go to the LGBTQ support group?", requested_tables=["memories"]) + candidate = { + "id": 11, + "bucket": "memories", + "type": "memory", + "final_score": 0.72, + "retrieval_score": 0.72, + "source": "keyword", + } + text = ( + "Session ID: session_1 Session Date: 2025-01-12 Conversation " + + "Caroline went to the LGBTQ support group after work on January 12 and we kept discussing it in the same paragraph without line breaks or sentence boundaries " * 24 + ) + + result = analyze_long_context( + "When did Caroline go to the LGBTQ support group?", + plan, + candidate, + text=text, + ) + + assert result["applicable"] is False + assert result["reason"] == "no_focused_program" From 65f0aacf1650af10493dbb5ae3e9a31f5d993c3d Mon Sep 17 00:00:00 2001 From: Mario Jack Vela Date: Thu, 23 Apr 2026 14:58:10 -0500 Subject: [PATCH 08/13] Improve nDCG-focused slate reranking and benchmark diagnostics --- benchmarks/brainctl_retrieval.py | 13 +- benchmarks/build_hard_negatives.py | 60 +- benchmarks/locomo_bench.py | 23 + benchmarks/longmemeval_bench.py | 23 + benchmarks/train_tiny_reranker.py | 136 +- src/agentmemory/_impl.py | 85 +- src/agentmemory/retrieval/diagnostics.py | 2 + src/agentmemory/retrieval/feature_builder.py | 9 + .../retrieval/models/tiny_mlp_v1.json | 3479 +++++++++-------- src/agentmemory/retrieval/query_planner.py | 63 + src/agentmemory/retrieval/second_stage.py | 333 +- tests/bench/eval.py | 29 + tests/test_reranker_robustness.py | 26 +- tests/test_second_stage_reranker.py | 59 + 14 files changed, 2484 insertions(+), 1856 deletions(-) diff --git a/benchmarks/brainctl_retrieval.py b/benchmarks/brainctl_retrieval.py index e8052b0..81fc80b 100644 --- a/benchmarks/brainctl_retrieval.py +++ b/benchmarks/brainctl_retrieval.py @@ -77,7 +77,15 @@ def _search_brain(db_path: Path, query: str, top_k: int) -> list[dict]: brain.close() -def _search_cmd(db_path: Path, query: str, top_k: int, *, debug: bool = False) -> list[dict]: +def _search_cmd( + db_path: Path, + query: str, + top_k: int, + *, + debug: bool = False, + benchmark: bool = False, + benchmark_ranking_mode: str = "full", +) -> list[dict]: import agentmemory._impl as _impl _impl.DB_PATH = db_path @@ -96,7 +104,8 @@ def _search_cmd(db_path: Path, query: str, top_k: int, *, debug: bool = False) - explore=False, pagerank_boost=0.0, quantum=False, - benchmark=False, + benchmark=benchmark, + benchmark_ranking_mode=benchmark_ranking_mode, agent=AGENT_ID, format="json", oneline=False, diff --git a/benchmarks/build_hard_negatives.py b/benchmarks/build_hard_negatives.py index 7e86bfa..6a13f10 100644 --- a/benchmarks/build_hard_negatives.py +++ b/benchmarks/build_hard_negatives.py @@ -3,6 +3,7 @@ import argparse import hashlib import json +import math import sys from collections import Counter from pathlib import Path @@ -25,6 +26,40 @@ from agentmemory.retrieval.query_planner import plan_query +def _dcg_from_labels(labels: list[int], k: int) -> float: + total = 0.0 + for idx, label in enumerate(labels[:k], start=1): + if label > 0: + total += ((2 ** int(label)) - 1) / max(1.0, math.log2(idx + 1)) + return total + + +def _dcg_summary(gold_doc_ids: list[str], ranked_doc_ids: list[str], *, k: int) -> tuple[float, float, float]: + labels = [1 if doc_id in set(gold_doc_ids) else 0 for doc_id in ranked_doc_ids[:k]] + dcg = _dcg_from_labels(labels, k) + ideal_labels = sorted(labels + [1] * max(0, len(gold_doc_ids) - len(labels)), reverse=True)[:k] + idcg = _dcg_from_labels(ideal_labels, k) + return round(dcg, 6), round(idcg, 6), round(max(idcg - dcg, 0.0), 6) + + +def _failure_bucket(*, benchmark: str, gold_doc_ids: list[str], ranked_doc_ids: list[str], query_label: str) -> str: + top = ranked_doc_ids[:5] + top_hits = [doc_id for doc_id in top if doc_id in set(gold_doc_ids)] + if benchmark == "longmemeval": + if top_hits and top[0] not in set(gold_doc_ids): + return "late_gold" + if len(set(top)) < len(top): + return "duplicate_top_slate" + if "temporal" in query_label.lower(): + return "temporal_anchor_miss" + return "coverage_miss" + if len(top_hits) < len(gold_doc_ids): + return "coverage_miss" + if "temporal" in query_label.lower(): + return "temporal_anchor_miss" + return "late_gold" + + def _latest_bundle() -> Path: candidates = sorted((ROOT / "results").glob("seq_full_compare_final_*"), reverse=True) if not candidates: @@ -59,6 +94,7 @@ def _record_for_candidate( candidate: dict[str, Any], rank: int, query_label: str, + slate_doc_ids: list[str], ) -> dict[str, Any]: plan = plan_query(query, requested_tables=["memories"]) candidate = dict(candidate) @@ -67,6 +103,8 @@ def _record_for_candidate( candidate["_stage_position"] = rank features = build_features(query, plan, candidate) doc_id = str(candidate.get("doc_id") or "") + dcg_at_5, idcg_at_5, dcg_gap_at_5 = _dcg_summary(gold_doc_ids, slate_doc_ids, k=5) + dcg_at_10, idcg_at_10, dcg_gap_at_10 = _dcg_summary(gold_doc_ids, slate_doc_ids, k=10) return { "benchmark": benchmark, "query_id": query_id, @@ -77,6 +115,20 @@ def _record_for_candidate( "candidate_doc_id": doc_id, "label": 1 if doc_id in set(gold_doc_ids) else 0, "rank": rank, + "slate_doc_ids": slate_doc_ids, + "slate_labels": [1 if value in set(gold_doc_ids) else 0 for value in slate_doc_ids], + "failure_bucket": _failure_bucket( + benchmark=benchmark, + gold_doc_ids=gold_doc_ids, + ranked_doc_ids=slate_doc_ids, + query_label=query_label, + ), + "dcg_at_5": dcg_at_5, + "idcg_at_5": idcg_at_5, + "dcg_gap_at_5": dcg_gap_at_5, + "dcg_at_10": dcg_at_10, + "idcg_at_10": idcg_at_10, + "dcg_gap_at_10": dcg_gap_at_10, "source": candidate.get("source"), "base_score": candidate.get("pre_second_stage_score", candidate.get("final_score")), "retrieval_score": candidate.get("retrieval_score"), @@ -122,7 +174,8 @@ def build_longmemeval_records(bundle_dir: Path, dataset_path: Path, *, top_k: in ] ranked = rank_documents_with_rows(entry.question, docs, pipeline="cmd", top_k=top_k, debug=True) gold_ids = list(entry.answer_session_ids) - if not any(str(candidate.get("doc_id") or "") in set(gold_ids) for candidate in ranked): + ranked_doc_ids = [str(candidate.get("doc_id") or "") for candidate in ranked[:top_k]] + if not any(doc_id in set(gold_ids) for doc_id in ranked_doc_ids): skipped_no_window += 1 continue selected += 1 @@ -138,6 +191,7 @@ def build_longmemeval_records(bundle_dir: Path, dataset_path: Path, *, top_k: in candidate=candidate, rank=rank, query_label=entry.question_type, + slate_doc_ids=ranked_doc_ids, ) ) @@ -178,7 +232,8 @@ def build_locomo_records(bundle_dir: Path, dataset_path: Path, *, top_k: int) -> docs = _locomo_build_corpus(sessions, granularity="session") ranked = rank_documents_with_rows(str(row["question"]), docs, pipeline="cmd", top_k=top_k, debug=True) gold_ids = [str(value) for value in row.get("evidence_ids", [])] - if not any(str(candidate.get("doc_id") or "") in set(gold_ids) for candidate in ranked): + ranked_doc_ids = [str(candidate.get("doc_id") or "") for candidate in ranked[:top_k]] + if not any(doc_id in set(gold_ids) for doc_id in ranked_doc_ids): skipped_no_window += 1 continue query_id = hashlib.sha1(f"{row['sample_id']}|{row['question']}|{idx}".encode("utf-8")).hexdigest()[:12] @@ -195,6 +250,7 @@ def build_locomo_records(bundle_dir: Path, dataset_path: Path, *, top_k: int) -> candidate=candidate, rank=rank, query_label=str(row.get("category_name") or row.get("category") or "unknown"), + slate_doc_ids=ranked_doc_ids, ) ) return records, { diff --git a/benchmarks/locomo_bench.py b/benchmarks/locomo_bench.py index 6cce8ef..1728c1b 100644 --- a/benchmarks/locomo_bench.py +++ b/benchmarks/locomo_bench.py @@ -83,6 +83,14 @@ def _recall(retrieved_ids: list[str], evidence_ids: set[str]) -> float: return found / len(evidence_ids) +def _dcg_from_binary(retrieved_ids: list[str], evidence_ids: set[str], k: int) -> float: + total = 0.0 + for idx, item in enumerate(retrieved_ids[:k], start=1): + if item in evidence_ids: + total += 1.0 / __import__("math").log2(idx + 1) + return total + + def run_brainctl_locomo( data_path: Path | None, *, @@ -136,6 +144,21 @@ def run_brainctl_locomo( "evidence_ids": sorted(evidence_ids), "retrieved_ids": retrieved_ids, "recall": round(recall, 4), + "dcg_at_5": round(_dcg_from_binary(retrieved_ids, evidence_ids, 5), 4), + "idcg_at_5": round(_dcg_from_binary(sorted(evidence_ids), evidence_ids, 5), 4), + "dcg_gap_at_5": round(max(_dcg_from_binary(sorted(evidence_ids), evidence_ids, 5) - _dcg_from_binary(retrieved_ids, evidence_ids, 5), 0.0), 4), + "dcg_at_10": round(_dcg_from_binary(retrieved_ids, evidence_ids, 10), 4), + "idcg_at_10": round(_dcg_from_binary(sorted(evidence_ids), evidence_ids, 10), 4), + "dcg_gap_at_10": round(max(_dcg_from_binary(sorted(evidence_ids), evidence_ids, 10) - _dcg_from_binary(retrieved_ids, evidence_ids, 10), 0.0), 4), + "failure_bucket": ( + "coverage_miss" + if recall < 1.0 and any(item in evidence_ids for item in retrieved_ids[:top_k]) + else "temporal_anchor_miss" + if "temporal" in CATEGORIES.get(category, "").lower() + else "late_gold" + if any(item in evidence_ids for item in retrieved_ids[:top_k]) and retrieved_ids[:1] and retrieved_ids[0] not in evidence_ids + else "grounded" + ), } ) finally: diff --git a/benchmarks/longmemeval_bench.py b/benchmarks/longmemeval_bench.py index 32ffd7c..ac341d0 100644 --- a/benchmarks/longmemeval_bench.py +++ b/benchmarks/longmemeval_bench.py @@ -117,6 +117,22 @@ def run_entry(entry: QuestionEntry, *, pipeline: str = "cmd", top_k: int = 10) - corpus_ids = ranked_session_ids + remaining ranked_indices = list(range(len(ranked_session_ids))) correct_ids = set(entry.answer_session_ids) + dcg_at_5 = round(dcg([1.0 if corpus_ids[idx] in correct_ids else 0.0 for idx in ranked_indices[:5]], 5), 4) + dcg_at_10 = round(dcg([1.0 if corpus_ids[idx] in correct_ids else 0.0 for idx in ranked_indices[:10]], 10), 4) + ideal_labels = sorted([1.0 if session_id in correct_ids else 0.0 for session_id in corpus_ids], reverse=True) + idcg_at_5 = round(dcg(ideal_labels, 5), 4) + idcg_at_10 = round(dcg(ideal_labels, 10), 4) + top_ids = ranked_session_ids[:5] + if any(session_id in correct_ids for session_id in top_ids) and top_ids and top_ids[0] not in correct_ids: + failure_bucket = "late_gold" + elif len(set(top_ids)) < len(top_ids): + failure_bucket = "duplicate_top_slate" + elif "temporal" in entry.question_type.lower(): + failure_bucket = "temporal_anchor_miss" + elif top_ids and len([session_id for session_id in top_ids if session_id in correct_ids]) < min(len(correct_ids), 5): + failure_bucket = "coverage_miss" + else: + failure_bucket = "grounded" return { "question_id": entry.question_id, "question_type": entry.question_type, @@ -126,6 +142,13 @@ def run_entry(entry: QuestionEntry, *, pipeline: str = "cmd", top_k: int = 10) - "r_all_at_10": recall_all(ranked_indices, correct_ids, corpus_ids, 10), "ndcg_at_5": round(ndcg(ranked_indices, correct_ids, corpus_ids, 5), 4), "ndcg_at_10": round(ndcg(ranked_indices, correct_ids, corpus_ids, 10), 4), + "dcg_at_5": dcg_at_5, + "idcg_at_5": idcg_at_5, + "dcg_gap_at_5": round(max(idcg_at_5 - dcg_at_5, 0.0), 4), + "dcg_at_10": dcg_at_10, + "idcg_at_10": idcg_at_10, + "dcg_gap_at_10": round(max(idcg_at_10 - dcg_at_10, 0.0), 4), + "failure_bucket": failure_bucket, "answer_session_ids": entry.answer_session_ids, "top_session_ids": ranked_session_ids[:top_k], } diff --git a/benchmarks/train_tiny_reranker.py b/benchmarks/train_tiny_reranker.py index dd7603c..6bf9bfc 100644 --- a/benchmarks/train_tiny_reranker.py +++ b/benchmarks/train_tiny_reranker.py @@ -2,6 +2,7 @@ import argparse import json +import math import sys from collections import defaultdict from dataclasses import dataclass @@ -24,12 +25,13 @@ @dataclass(slots=True) class TrainConfig: - epochs: int = 40 + epochs: int = 24 lr: float = 0.01 l2: float = 1e-4 seed: int = 42 hidden1: int = 32 hidden2: int = 16 + ndcg_k: int = 5 def _load_records(path: Path) -> list[dict[str, Any]]: @@ -37,12 +39,19 @@ def _load_records(path: Path) -> list[dict[str, Any]]: with path.open("r", encoding="utf-8") as handle: for line in handle: line = line.strip() - if not line: - continue - rows.append(json.loads(line)) + if line: + rows.append(json.loads(line)) return rows +def _dcg(labels: list[int], k: int) -> float: + total = 0.0 + for idx, label in enumerate(labels[:k], start=1): + gain = (2**int(label)) - 1 + total += gain / math.log2(idx + 1) + return total + + def _group_query_metrics(records: list[dict[str, Any]], scores_by_key: dict[tuple[str, str], float], *, k: int = 5) -> dict[str, float]: grouped: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) for record in records: @@ -55,13 +64,9 @@ def _group_query_metrics(records: list[dict[str, Any]], scores_by_key: dict[tupl top = ranked[:k] labels = [int(row["label"]) for row in top] if benchmark == "longmemeval": - dcg = 0.0 - for idx, label in enumerate(labels): - dcg += float(label) / np.log2(idx + 2) - ideal_labels = sorted((int(row["label"]) for row in items), reverse=True)[:k] - ideal_dcg = 0.0 - for idx, label in enumerate(ideal_labels): - ideal_dcg += float(label) / np.log2(idx + 2) + dcg = _dcg(labels, k) + ideal_labels = sorted((int(row["label"]) for row in items), reverse=True) + ideal_dcg = _dcg(ideal_labels, k) long_ndcgs.append((dcg / ideal_dcg) if ideal_dcg > 0 else 0.0) elif benchmark == "locomo": positives = sum(int(row["label"]) for row in items) @@ -86,47 +91,96 @@ def _init_params(rng: np.random.Generator, input_dim: int, config: TrainConfig) } -def _forward(x: np.ndarray, params: dict[str, np.ndarray]) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: +def _forward( + x: np.ndarray, + params: dict[str, np.ndarray], +) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: z1 = x @ params["w1"] + params["b1"] h1 = np.maximum(0.0, z1) z2 = h1 @ params["w2"] + params["b2"] h2 = np.maximum(0.0, z2) logits = h2 @ params["w3"] + params["b3"] - probs = 1.0 / (1.0 + np.exp(-logits)) - return z1, h1, z2, h2, probs + probs = 1.0 / (1.0 + np.exp(-np.clip(logits, -30.0, 30.0))) + return z1, h1, z2, h2, logits, probs def _clone_params(params: dict[str, np.ndarray]) -> dict[str, np.ndarray]: return {key: np.array(value, copy=True) for key, value in params.items()} +def _group_indices(records: list[dict[str, Any]]) -> list[np.ndarray]: + grouped: dict[tuple[str, str], list[int]] = defaultdict(list) + for idx, row in enumerate(records): + grouped[(str(row["benchmark"]), str(row["query_id"]))].append(idx) + return [np.asarray(indices, dtype=int) for indices in grouped.values() if len(indices) >= 2] + + +def _lambda_pairwise_gradients(logits: np.ndarray, labels: np.ndarray, *, k: int) -> np.ndarray: + flat_logits = logits.reshape(-1) + order = np.argsort(-flat_logits) + ranks = np.empty_like(order) + ranks[order] = np.arange(len(order)) + ideal_dcg = _dcg(sorted((int(label) for label in labels.tolist()), reverse=True), k) + grad = np.zeros_like(flat_logits, dtype=float) + if ideal_dcg <= 0: + return grad.reshape(-1, 1) + + pair_count = 0 + for i in range(len(flat_logits)): + for j in range(len(flat_logits)): + if labels[i] <= labels[j]: + continue + rank_i = int(ranks[i]) + 1 + rank_j = int(ranks[j]) + 1 + if min(rank_i, rank_j) > max(k, 10): + continue + gain_i = (2 ** int(labels[i])) - 1 + gain_j = (2 ** int(labels[j])) - 1 + delta_discount = abs((1.0 / math.log2(rank_i + 1)) - (1.0 / math.log2(rank_j + 1))) + delta_gain = abs(gain_i - gain_j) + top_weight = 1.0 if min(rank_i, rank_j) <= k else 0.35 + weight = (delta_discount * delta_gain / ideal_dcg) * top_weight + if weight <= 0.0: + continue + diff = float(np.clip(flat_logits[i] - flat_logits[j], -30.0, 30.0)) + prob = 1.0 / (1.0 + math.exp(-diff)) + g = weight * (prob - 1.0) + grad[i] += g + grad[j] -= g + pair_count += 1 + if pair_count: + grad /= float(pair_count) + return grad.reshape(-1, 1) + + def _train_model( train_x: np.ndarray, train_y: np.ndarray, + train_groups: list[np.ndarray], config: TrainConfig, *, initial_params: dict[str, np.ndarray] | None = None, ) -> dict[str, np.ndarray]: rng = np.random.default_rng(config.seed) params = _clone_params(initial_params) if initial_params is not None else _init_params(rng, train_x.shape[1], config) - batch_size = min(128, len(train_x)) for _epoch in range(config.epochs): - order = rng.permutation(len(train_x)) - for start in range(0, len(train_x), batch_size): - batch_idx = order[start:start + batch_size] - x = train_x[batch_idx] - y = train_y[batch_idx].reshape(-1, 1) - z1, h1, z2, h2, probs = _forward(x, params) - diff = probs - y - grad_w3 = (h2.T @ diff) / len(x) + config.l2 * params["w3"] - grad_b3 = diff.mean(axis=0) - grad_h2 = diff @ params["w3"].T + rng.shuffle(train_groups) + for group in train_groups: + x = train_x[group] + y = train_y[group] + z1, h1, z2, h2, logits, _probs = _forward(x, params) + grad_logits = _lambda_pairwise_gradients(logits, y, k=config.ndcg_k) + if not np.any(grad_logits): + continue + grad_w3 = (h2.T @ grad_logits) / len(group) + config.l2 * params["w3"] + grad_b3 = grad_logits.mean(axis=0) + grad_h2 = grad_logits @ params["w3"].T grad_z2 = grad_h2 * (z2 > 0) - grad_w2 = (h1.T @ grad_z2) / len(x) + config.l2 * params["w2"] + grad_w2 = (h1.T @ grad_z2) / len(group) + config.l2 * params["w2"] grad_b2 = grad_z2.mean(axis=0) grad_h1 = grad_z2 @ params["w2"].T grad_z1 = grad_h1 * (z1 > 0) - grad_w1 = (x.T @ grad_z1) / len(x) + config.l2 * params["w1"] + grad_w1 = (x.T @ grad_z1) / len(group) + config.l2 * params["w1"] grad_b1 = grad_z1.mean(axis=0) params["w3"] -= config.lr * grad_w3 params["b3"] -= config.lr * grad_b3 @@ -142,7 +196,7 @@ def main() -> int: parser.add_argument("--data", type=Path, default=ROOT / "training_data" / "hard_negatives_v1.jsonl") parser.add_argument("--report", type=Path, default=ROOT / "training_data" / "tiny_mlp_v1_report.json") parser.add_argument("--model-out", type=Path, default=DEFAULT_MODEL_PATH) - parser.add_argument("--epochs", type=int, default=40) + parser.add_argument("--epochs", type=int, default=24) parser.add_argument("--lr", type=float, default=0.01) parser.add_argument("--seed", type=int, default=42) args = parser.parse_args() @@ -156,7 +210,11 @@ def main() -> int: heldout_records = [row for row in records if row.get("split") == "heldout"] train_x = np.asarray([row["feature_vector"] for row in train_records], dtype=float) train_y = np.asarray([float(row["label"]) for row in train_records], dtype=float) - heldout_x = np.asarray([row["feature_vector"] for row in heldout_records], dtype=float) if heldout_records else np.zeros((0, len(FEATURE_ORDER_V1))) + heldout_x = ( + np.asarray([row["feature_vector"] for row in heldout_records], dtype=float) + if heldout_records + else np.zeros((0, len(FEATURE_ORDER_V1))) + ) norm_mean = train_x.mean(axis=0) norm_std = train_x.std(axis=0) @@ -165,11 +223,12 @@ def main() -> int: heldout_x_norm = (heldout_x - norm_mean) / safe_std if len(heldout_x) else heldout_x config = TrainConfig(epochs=args.epochs, lr=args.lr, seed=args.seed) - params = _train_model(train_x_norm, train_y, config) - _, _, _, _, train_probs = _forward(train_x_norm, params) + train_groups = _group_indices(train_records) + params = _train_model(train_x_norm, train_y, train_groups, config) + _, _, _, _, _train_logits, train_probs = _forward(train_x_norm, params) heldout_probs = np.zeros((len(heldout_x_norm), 1), dtype=float) if len(heldout_x_norm): - _, _, _, _, heldout_probs = _forward(heldout_x_norm, params) + _, _, _, _, _heldout_logits, heldout_probs = _forward(heldout_x_norm, params) def _scores(rows: list[dict[str, Any]], probs: np.ndarray) -> dict[tuple[str, str], float]: return { @@ -180,7 +239,6 @@ def _scores(rows: list[dict[str, Any]], probs: np.ndarray) -> dict[tuple[str, st train_metrics = _group_query_metrics(train_records, _scores(train_records, train_probs)) heldout_metrics = _group_query_metrics(heldout_records, _scores(heldout_records, heldout_probs)) - # One additional epoch on LongMemEval-only hard negatives if heldout nDCG stays flat or improves. long_only = [row for row in train_records if row.get("benchmark") == "longmemeval"] long_applied = False if long_only: @@ -188,11 +246,15 @@ def _scores(rows: list[dict[str, Any]], probs: np.ndarray) -> dict[tuple[str, st long_y = np.asarray([float(row["label"]) for row in long_only], dtype=float) long_x_norm = (long_x - norm_mean) / safe_std extra_config = TrainConfig(epochs=1, lr=args.lr, seed=args.seed) - extra_params = _train_model(long_x_norm, long_y, extra_config, initial_params=params) + extra_groups = _group_indices(long_only) + extra_params = _train_model(long_x_norm, long_y, extra_groups, extra_config, initial_params=params) if len(heldout_x_norm): - _, _, _, _, extra_probs = _forward(heldout_x_norm, extra_params) + _, _, _, _, _extra_logits, extra_probs = _forward(heldout_x_norm, extra_params) extra_metrics = _group_query_metrics(heldout_records, _scores(heldout_records, extra_probs)) - if extra_metrics["heldout_longmemeval_ndcg_at_5"] >= heldout_metrics["heldout_longmemeval_ndcg_at_5"]: + if ( + extra_metrics["heldout_longmemeval_ndcg_at_5"] >= heldout_metrics["heldout_longmemeval_ndcg_at_5"] + and extra_metrics["heldout_locomo_perfect_rate_at_5"] >= heldout_metrics["heldout_locomo_perfect_rate_at_5"] - 0.005 + ): params = extra_params heldout_probs = extra_probs heldout_metrics = extra_metrics @@ -212,6 +274,7 @@ def _scores(rows: list[dict[str, Any]], probs: np.ndarray) -> dict[tuple[str, st "metadata": { "generated_at_utc": datetime.now(timezone.utc).isoformat(), "source_data": str(args.data), + "objective": "lambda_weighted_pairwise_ndcg_at_5", "train_records": len(train_records), "heldout_records": len(heldout_records), "longmemeval_extra_epoch_applied": long_applied, @@ -226,6 +289,7 @@ def _scores(rows: list[dict[str, Any]], probs: np.ndarray) -> dict[tuple[str, st "model_out": str(args.model_out), "train_records": len(train_records), "heldout_records": len(heldout_records), + "objective": "lambda_weighted_pairwise_ndcg_at_5", "train_metrics": train_metrics, "heldout_metrics": heldout_metrics, "longmemeval_extra_epoch_applied": long_applied, diff --git a/src/agentmemory/_impl.py b/src/agentmemory/_impl.py index 5868f51..ccae763 100644 --- a/src/agentmemory/_impl.py +++ b/src/agentmemory/_impl.py @@ -6489,20 +6489,26 @@ def cmd_search(args, *, db=None, db_path: Optional[str] = None): use_mmr = getattr(args, "mmr", False) # --mmr: MMR diversity reranking mmr_lambda = getattr(args, "mmr_lambda", 0.7) # --mmr-lambda: relevance/diversity trade-off use_explore = getattr(args, "explore", False) # --explore: curiosity mode - # --benchmark (2.3.1): bypass the recency/salience/Q-value reranker chain - # and return raw FTS+vec RRF-fused ranking. Trust reranker is *retained* - # because trust is provenance, not stale-data leakage. The flag exists as - # an escape hatch for synthetic-conversational benchmarks (LOCOMO, - # LongMemEval) where uniform timestamps and zero recall history make the - # rerankers worse than no-op. See memory id 1690 and tests/test_reranker_robustness. benchmark_mode = getattr(args, "benchmark", False) + benchmark_ranking_mode = str( + getattr(args, "benchmark_ranking_mode", None) + or os.environ.get("BRAINCTL_BENCHMARK_RANKING_MODE", "full") + or "full" + ).strip().lower() + if benchmark_ranking_mode not in {"full", "raw"}: + benchmark_ranking_mode = "full" + benchmark_raw_ranking = bool(benchmark_mode and benchmark_ranking_mode == "raw") if benchmark_mode: - # One-line stderr note so the user can see the reranker chain went - # silent. Avoids log spam on the hot path while still being visible. - print( - "[brainctl] --benchmark: reranker chain disabled, returning raw FTS+vec ranking", - file=sys.stderr, - ) + if benchmark_raw_ranking: + print( + "[brainctl] --benchmark: raw ranking ablation mode, returning raw FTS+vec ranking", + file=sys.stderr, + ) + else: + print( + "[brainctl] --benchmark: stable-eval mode with full shared ranking", + file=sys.stderr, + ) results = {"memories": [], "events": [], "context": [], "entities": [], "decisions": [], "procedures": []} # Accumulator for which signal-informativeness gates tripped this call. # Each value is a string reason like "uniform_timestamps_stdev_3.2s" or a @@ -6510,6 +6516,8 @@ def cmd_search(args, *, db=None, db_path: Optional[str] = None): # "_debug" key so auditors can see WHY a particular ranking happened. _debug_skips: Dict[str, Any] = {} _debug_mode = bool(getattr(args, "debug", False)) + if benchmark_mode: + _debug_skips["benchmark.ranking_mode"] = benchmark_ranking_mode # I6 staged rollout controls for top-heavy retrieval features. _rollout_agent = getattr(args, "agent", None) or "unknown" @@ -6596,8 +6604,20 @@ def cmd_search(args, *, db=None, db_path: Optional[str] = None): except Exception as exc: _debug_skips["entity_linking.skipped"] = f"{type(exc).__name__}: {exc}" + _hard_query_expansion = bool( + _query_plan + and ( + getattr(_query_plan, "requires_temporal_reasoning", False) + or getattr(_query_plan, "requires_multi_hop", False) + or getattr(_query_plan, "needs_comparison", False) + or getattr(_query_plan, "needs_ordering", False) + or getattr(_query_plan, "needs_update_resolution", False) + or getattr(_query_plan, "needs_set_coverage", False) + ) + ) base_fetch = limit * 5 if not no_recency else limit * 3 fetch_limit = max(limit, round(base_fetch * _nm_breadth)) + expanded_fetch_limit = max(fetch_limit, round(fetch_limit * (1.8 if _hard_query_expansion else 1.0))) # Build an OR-expanded FTS5 MATCH expression so natural-language queries # (e.g. "What does Alice prefer?") retrieve memories that match any token, # not only memories that contain every word. The simple Brain.search path @@ -6641,9 +6661,10 @@ def cmd_search(args, *, db=None, db_path: Optional[str] = None): _adaptive_weights = None _max_recalls_cache = [None] # lazy-compute once per cmd_search - def _fts_memories(): + def _fts_memories(limit_override=None): if not fts_query: return [] + _fetch = int(limit_override or fetch_limit) # Content-weighted BM25. memories_fts indexes (content, category, tags). # Default FTS5 `rank` uses weight 1.0 for every column, which treats a # 200-char content column equally with a one-word `category` label @@ -6665,17 +6686,18 @@ def _fts_memories(): "WHERE memories_fts MATCH ? AND m.retired_at IS NULL " "AND COALESCE(m.memory_type, 'episodic') != 'procedural' " "ORDER BY bm25(memories_fts, 3.0, 1.0, 1.0) LIMIT ?", - (fts_query, fetch_limit) + (fts_query, _fetch) ).fetchall() return rows_to_list(rows) - def _vec_memories(): + def _vec_memories(limit_override=None): if not hybrid: return [] + _fetch = int(limit_override or fetch_limit) try: vec_rows = db_vec.execute( "SELECT rowid, distance FROM vec_memories WHERE embedding MATCH ? AND k=?", - (q_blob, fetch_limit) + (q_blob, _fetch) ).fetchall() except Exception: return [] @@ -7305,6 +7327,35 @@ def _apply_recency_and_trim(merged, scope_fn, use_adaptive_salience=False, bucke merged = _rrf_fuse(fts_list, vec_list) else: merged = [r | {"rrf_score": 0.0, "source": "keyword"} for r in fts_list] + if ( + _hard_query_expansion + and not benchmark_mode + and expanded_fetch_limit > fetch_limit + and len(merged) >= 2 + ): + try: + _rank_key = "rrf_score" if hybrid else "fts_rank" + _sorted_merged = sorted( + merged, + key=lambda r: float(r.get(_rank_key) or 0.0), + reverse=bool(hybrid), + ) + if hybrid: + _top_gap = abs(float(_sorted_merged[0].get("rrf_score") or 0.0) - float(_sorted_merged[1].get("rrf_score") or 0.0)) + else: + _top_gap = abs(float(_sorted_merged[0].get("fts_rank") or 0.0) - float(_sorted_merged[1].get("fts_rank") or 0.0)) + if _top_gap <= (0.03 if hybrid else 0.4): + _fts_expanded = _fts_memories(limit_override=expanded_fetch_limit) + _vec_expanded = _vec_memories(limit_override=expanded_fetch_limit) + if hybrid: + merged = _rrf_fuse(_fts_expanded, _vec_expanded) + else: + merged = [r | {"rrf_score": 0.0, "source": "keyword"} for r in _fts_expanded] + _debug_skips["memories.candidate_expansion"] = ( + f"hard_query_margin_{round(_top_gap, 4)}_fetch_{fetch_limit}_to_{expanded_fetch_limit}" + ) + except Exception as exc: + _debug_skips["memories.candidate_expansion_skipped"] = f"{type(exc).__name__}: {exc}" trimmed = _apply_recency_and_trim(merged, lambda r: r.get("scope"), use_adaptive_salience=True, bucket="memories") # MMR diversity reranking — applied after salience scoring, before graph expand if use_mmr and trimmed: @@ -7466,7 +7517,7 @@ def _apply_recency_and_trim(merged, scope_fn, use_adaptive_salience=False, bucke reranked = _rerank_procedure_candidates( generated.get("candidates", []), evidence, - benchmark_mode=benchmark_mode, + benchmark_mode=benchmark_raw_ranking, ) results["procedures"] = _apply_query_alignment( reranked[:limit], diff --git a/src/agentmemory/retrieval/diagnostics.py b/src/agentmemory/retrieval/diagnostics.py index a051ccc..f365e9b 100644 --- a/src/agentmemory/retrieval/diagnostics.py +++ b/src/agentmemory/retrieval/diagnostics.py @@ -32,6 +32,8 @@ def build_debug_payload( "second_stage_heuristic": cand.get("second_stage_heuristic"), "second_stage_mlp": cand.get("second_stage_mlp"), "second_stage_judge": cand.get("second_stage_judge"), + "second_stage_slate_score": cand.get("second_stage_slate_score"), + "second_stage_slate_terms": cand.get("second_stage_slate_terms"), "why_retrieved": cand.get("why_retrieved"), "feature_summary": cand.get("second_stage_features"), "text": cand.get("content") or cand.get("summary") or cand.get("title") or cand.get("goal") or cand.get("name"), diff --git a/src/agentmemory/retrieval/feature_builder.py b/src/agentmemory/retrieval/feature_builder.py index 4f19b9e..108e983 100644 --- a/src/agentmemory/retrieval/feature_builder.py +++ b/src/agentmemory/retrieval/feature_builder.py @@ -320,6 +320,9 @@ def _should_probe_long_context( query_needs_probe = ( bool(getattr(plan, "requires_temporal_reasoning", False)) or bool(getattr(plan, "requires_multi_hop", False)) + or bool(getattr(plan, "needs_ordering", False)) + or bool(getattr(plan, "needs_update_resolution", False)) + or bool(getattr(plan, "needs_set_coverage", False)) or bool(_LONG_CONTEXT_HINT_RE.search(lowered_query)) ) if not query_needs_probe: @@ -435,6 +438,12 @@ def build_features( "query_length_score": min(len(query_informative) / 8.0, 1.0), "candidate_length_score": min(len(cand_informative) / 64.0, 1.0), "procedural_candidate": 1.0 if bucket == "procedures" else 0.0, + "query_needs_counting": 1.0 if getattr(plan, "needs_counting", False) else 0.0, + "query_needs_comparison": 1.0 if getattr(plan, "needs_comparison", False) else 0.0, + "query_needs_ordering": 1.0 if getattr(plan, "needs_ordering", False) else 0.0, + "query_needs_update_resolution": 1.0 if getattr(plan, "needs_update_resolution", False) else 0.0, + "query_needs_set_coverage": 1.0 if getattr(plan, "needs_set_coverage", False) else 0.0, + "query_requires_multi_hop": 1.0 if getattr(plan, "requires_multi_hop", False) else 0.0, "long_context_applicable": 1.0 if long_context_debug.get("applicable") else 0.0, "long_context_score": _safe_float(long_context_debug.get("score")), "long_context_confidence": _safe_float(long_context_debug.get("confidence")), diff --git a/src/agentmemory/retrieval/models/tiny_mlp_v1.json b/src/agentmemory/retrieval/models/tiny_mlp_v1.json index d9339b1..75e172a 100644 --- a/src/agentmemory/retrieval/models/tiny_mlp_v1.json +++ b/src/agentmemory/retrieval/models/tiny_mlp_v1.json @@ -39,7 +39,7 @@ "procedural_candidate" ], "norm_mean": [ - 0.36849625, + 0.41749247, 0.99348148, 0.0, 1.0, @@ -49,7 +49,7 @@ 0.0, 0.61979308, 0.0, - 0.16296296, + 0.30617284, 0.99259259, 0.0054321, 0.0, @@ -65,19 +65,19 @@ 0.0, 0.0, 0.0, - 0.99999999, + 0.99999986, 0.0, 1.0, 0.0, 0.0, 0.29330117, - 0.36849625, + 0.41749247, 0.70833333, 1.0, 0.0 ], "norm_std": [ - 0.05354046, + 0.06543539, 0.06219823, 1.0, 1.0, @@ -87,7 +87,7 @@ 1.0, 0.40672457, 1.0, - 0.36933188, + 0.46090241, 0.08574694, 0.07350232, 1.0, @@ -103,1861 +103,1862 @@ 1.0, 1.0, 1.0, - 9e-08, + 3.5e-07, 1.0, 1.0, 1.0, 1.0, 0.26323239, - 0.05354046, + 0.06543539, 0.21390693, 1.0, 1.0 ], "w1": [ [ - 0.02177844, - -0.07193969, - 0.08529199, - -0.15862129, - -0.09098113, - -0.12999588, - -0.05729463, - 0.01085175, - 0.05169438, - -0.03503787, - 0.09383065, - -0.07432615, - -0.19292686, - 0.13062156, - 0.13366458, - 0.02675225, - 0.01192958, - -0.03038564, - 0.16023862, - -0.20214085, - -0.17921269, - -0.08560957, - 0.1258194, - 0.09327172, - -0.15348158, - -0.10133985, - -0.13932081, - -0.08316691, - -0.08106961, - -0.02785429, - 0.19389943, - 0.21056218, - -0.26755475, - 0.01688236, - 0.07833868, - 0.05400312 + 0.03560616, + -0.0627443, + 0.08493239, + -0.15795253, + -0.07903827, + -0.11793049, + -0.04711614, + 0.010806, + 0.05168679, + -0.03489014, + 0.09624207, + -0.06908551, + -0.19136466, + 0.13007085, + 0.13310104, + 0.02663946, + 0.01187928, + -0.03025753, + 0.15956304, + -0.20128861, + -0.17845711, + -0.08524863, + 0.12528894, + 0.09287848, + -0.15283449, + -0.10091259, + -0.13670618, + -0.08281627, + -0.08072781, + -0.02773686, + 0.19308194, + 0.2129777, + -0.25250718, + 0.02021468, + 0.0780084, + 0.05377544 ], [ - -0.10343042, - -0.09575113, - 0.09514009, - -0.11959222, - -0.09066836, - -0.06638923, - 0.18933333, - 0.02736969, - -0.03274888, - -0.01241056, - -0.02156421, - 0.09551152, - -0.09345788, - 0.15921929, - 0.0459755, - 0.1718745, - -0.0103254, - -0.01908983, - 0.14965714, - 0.06452237, - 0.08384934, - -0.15791742, - -0.11703931, - 0.09677836, - 0.20168746, - -0.12995975, - -0.04624062, - 0.09965932, - -0.10341123, - 0.00907972, - 0.01571308, - 0.17058243, - -0.02241735, - 0.03512919, - -0.03318253, - 0.00037278 + -0.12247778, + -0.09727918, + 0.09473897, + -0.11908801, + -0.10917026, + -0.08511965, + 0.17624795, + 0.02725429, + -0.04348743, + -0.01235824, + -0.02271716, + 0.09714493, + -0.09480573, + 0.158548, + 0.04578166, + 0.17114986, + -0.01028186, + -0.01900935, + 0.14902617, + 0.06425034, + 0.08349582, + -0.15725163, + -0.11654586, + 0.09637033, + 0.20083713, + -0.12941183, + -0.04491017, + 0.09923915, + -0.10297524, + 0.00904144, + 0.01564684, + 0.15967851, + -0.04180626, + 0.04847615, + -0.03304263, + 0.00037121 ], [ - 0.0642239, - 0.06982343, - -0.04181994, - 0.04794188, - -0.07005416, - 0.23251009, - -0.14985486, - 0.30190149, - -0.03752951, - -0.03021773, - 0.14500495, - 0.13014826, - 0.03973439, - -0.00829119, - -0.01572633, - 0.01097532, - 0.09483536, - 0.0243908, - -0.03028248, - -0.12403669, - 0.24616288, - 0.10023208, - -0.10919831, - -0.19425891, - 0.20734525, - 0.04132584, - -0.0940105, - -0.16095685, - 0.0574279, - 0.00072153, - -0.12020348, - -0.03085636, - -0.28722799, - 0.02216413, - -0.0714844, - -0.0899194 + 0.08655218, + 0.07345125, + -0.04164363, + 0.04773975, + -0.04876091, + 0.25205024, + -0.1326336, + 0.30062865, + -0.01592228, + -0.03009033, + 0.13975931, + 0.12756869, + 0.04987138, + -0.00825623, + -0.01566002, + 0.01092905, + 0.09443552, + 0.02428797, + -0.03015481, + -0.12351374, + 0.24512503, + 0.09980949, + -0.10873792, + -0.1934399, + 0.20647106, + 0.04115161, + -0.09005481, + -0.16027824, + 0.05718578, + 0.00071849, + -0.11969669, + -0.0148447, + -0.26341796, + 0.01473061, + -0.07118302, + -0.0895403 ], [ - 0.13118822, - 0.1337805, - -0.05544633, - -0.10858721, - -0.14589973, - -0.07945286, - -0.09265881, - 0.22507568, - -0.15118757, - 0.01829566, - 0.0909163, - 0.02625966, - 0.07050119, - 0.16232525, - 0.04182604, - 0.06964817, - 0.04133068, - -0.12094605, - 0.04358631, - 0.02821487, - 0.02062189, - 0.04189496, - 0.06698252, - -0.26949781, - 0.16300098, - 0.04548417, - -0.10417425, - -0.04880166, - -0.18415786, - 0.05376409, - -0.01315878, - 0.05443086, - 0.02271785, - 0.04402513, - 0.00102002, - -0.02828122 + 0.11357714, + 0.13461218, + -0.05521256, + -0.10812939, + -0.16226841, + -0.09597153, + -0.1059543, + 0.22412674, + -0.16334151, + 0.01821852, + 0.08926422, + 0.02763969, + 0.06413619, + 0.16164087, + 0.04164969, + 0.06935453, + 0.04115642, + -0.12043613, + 0.04340255, + 0.02809592, + 0.02053495, + 0.04171833, + 0.06670012, + -0.26836158, + 0.16231375, + 0.0452924, + -0.10625103, + -0.04859591, + -0.18338143, + 0.05353742, + -0.0131033, + 0.04171462, + 0.00556409, + 0.0512178, + 0.00101571, + -0.02816198 ], [ - -0.24327827, - -0.01684738, - 0.10289051, - -0.04535016, - 0.06692471, - 0.09167366, - 0.07103091, - -0.10232298, - -0.03434823, - 0.17646483, - 0.2228408, - 0.02837795, - 0.03022375, - 0.01104805, - 0.23397008, - -0.00680958, - 0.08014725, - 0.08476709, - -0.28900359, - -0.17073763, - -0.04045284, - 0.28572735, - -0.02656714, - 0.1201317, - 0.0306058, - 0.15437982, - -0.04271535, - -0.0701395, - 0.04673276, - 0.13974646, - -0.00427052, - 0.08513194, - -0.00986107, - -0.01625741, - 0.09533017, - -0.02208679 + -0.24038783, + -0.01415554, + 0.10245672, + -0.04515896, + 0.0698126, + 0.09413812, + 0.0713224, + -0.10189157, + -0.03088761, + 0.17572084, + 0.21737282, + 0.02761021, + 0.03334866, + 0.01100147, + 0.23298364, + -0.00678087, + 0.07980934, + 0.08440971, + -0.28778513, + -0.17001778, + -0.04028229, + 0.2845227, + -0.02645513, + 0.11962521, + 0.03047676, + 0.15372894, + -0.04070144, + -0.06984379, + 0.04653573, + 0.13915728, + -0.00425252, + 0.07983171, + -0.00795474, + -0.01693339, + 0.09492825, + -0.02199367 ], [ - -0.16956082, - -0.10382103, - -0.02294167, - 0.15580656, - -0.03805346, - -0.11968453, - -0.05477229, - -0.0344637, - -0.04372018, - -0.3077998, - 0.08593909, - 0.03396054, - -0.17074, - -0.10042279, - 0.24907646, - -0.02043568, - -0.08255125, - 0.07946848, - -0.13867197, - 0.05352402, - -0.01702936, - 0.05039002, - 0.07764791, - 0.14243483, - 0.16197019, - 0.13191218, - 0.17275909, - -0.00558686, - 0.01229842, - 0.19755942, - -0.16366307, - 0.00674143, - 0.11198238, - 0.18715383, - 0.02162967, - -0.03239765 + -0.15911493, + -0.10015459, + -0.02284495, + 0.15514966, + -0.0268189, + -0.10809812, + -0.04782187, + -0.0343184, + -0.03249869, + -0.30650209, + 0.08479067, + 0.03216638, + -0.16860774, + -0.0999994, + 0.24802633, + -0.02034952, + -0.0822032, + 0.07913344, + -0.13808732, + 0.05329835, + -0.01695757, + 0.05017757, + 0.07732054, + 0.14183431, + 0.16128731, + 0.13135603, + 0.16968437, + -0.0055633, + 0.01224657, + 0.19672649, + -0.16297306, + 0.00926604, + 0.12124126, + 0.17227825, + 0.02153848, + -0.03226106 ], [ - 0.00953365, - -0.10173179, - -0.15298335, - -0.04272403, - -0.18319722, - 0.10486989, - -0.00590679, - -0.17549946, - 0.02991358, - -0.02840365, - -0.19181707, - -0.10702026, - -0.27758428, - -0.07128191, - 0.00832035, - -0.09347739, - 0.10766817, - 0.04617467, - -0.03523067, - -0.09672926, - 0.07378306, - 0.04649429, - -0.00163656, - -0.12239555, - 0.00144545, - -0.01585718, - 0.22103, - 0.03344207, - -0.01768166, - 0.03713038, - -0.03067999, - 0.0129347, - 0.13662668, - -0.05644484, - -0.07867567, - 0.21242347 + 0.01741444, + -0.09854849, + -0.15233836, + -0.0425439, + -0.17371613, + 0.11316355, + 0.00083508, + -0.17475953, + 0.02797641, + -0.0282839, + -0.18757592, + -0.10346714, + -0.27590027, + -0.07098138, + 0.00828527, + -0.09308328, + 0.10721423, + 0.04597999, + -0.03508213, + -0.09632145, + 0.07347198, + 0.04629826, + -0.00162966, + -0.12187952, + 0.00143935, + -0.01579032, + 0.21951773, + 0.03330108, + -0.01760711, + 0.03697383, + -0.03055064, + 0.01704189, + 0.14397163, + -0.05500865, + -0.07834397, + 0.21152787 ], [ - -0.0526237, - 0.07980607, - -0.13590651, - 0.08844463, - -0.1370018, - 0.03096751, - -0.03026132, - -0.07083899, - -0.07846376, - 0.02116779, - -0.00760873, - -0.01764576, - 0.00036318, - -0.17754947, - 0.01921044, - 0.05160277, - 0.195346, - 0.06674082, - -0.12857275, - -0.15381661, - -0.20754627, - -0.0200184, - 0.08414518, - -0.2230359, - 0.02431993, - -0.14920814, - -0.04045595, - -0.12086893, - 0.19046986, - 0.07069987, - -0.08900545, - 0.08625775, - 0.00961814, - 0.14759828, - 0.14705998, - -0.01180588 + -0.04175293, + 0.07761583, + -0.13533351, + 0.08807174, + -0.12340773, + 0.04383488, + -0.02108443, + -0.07054033, + -0.06657603, + 0.02107855, + -0.00845367, + -0.01786699, + 0.00560739, + -0.17680091, + 0.01912945, + 0.0513852, + 0.19452241, + 0.06645943, + -0.12803067, + -0.1531681, + -0.20667123, + -0.019934, + 0.08379041, + -0.22209556, + 0.0242174, + -0.14857906, + -0.04013221, + -0.12035933, + 0.18966682, + 0.07040179, + -0.08863019, + 0.09370273, + 0.02022649, + 0.13533322, + 0.14643996, + -0.0117561 ], [ - -0.0074478, - 0.08981802, - -0.11026291, - -0.11196166, - 0.03119422, - -0.02510156, - 0.03547326, - 0.0378481, - 0.05626221, - 0.03549631, - -0.14343475, - -0.02005277, - -0.0570597, - -0.10650713, - 0.12906529, - -0.10211837, - -0.11634264, - 0.03554715, - 0.08567213, - 0.08560301, - 0.01971413, - 0.09794971, - -0.12412904, - 0.01187649, - -0.13113169, - -0.03826802, - 0.22906095, - 0.08685771, - -0.0746166, - -0.13801442, - 0.11085119, - -0.01044795, - -0.06543746, - -0.07875183, - 0.18937969, - -0.0292486 + 0.00234343, + 0.08897077, + -0.10979803, + -0.11148962, + 0.04197479, + -0.01417014, + 0.04459943, + 0.03768853, + 0.05802625, + 0.03534665, + -0.14127705, + -0.01832664, + -0.05648871, + -0.10605809, + 0.12852114, + -0.10168783, + -0.11585213, + 0.03539728, + 0.08531093, + 0.0852421, + 0.01963101, + 0.09753675, + -0.1236057, + 0.01182642, + -0.13057883, + -0.03810668, + 0.22831653, + 0.08649151, + -0.07430201, + -0.13743254, + 0.11038383, + 2.404e-05, + -0.05540174, + -0.07749389, + 0.18858125, + -0.02912529 ], [ - -0.10190371, - 0.06559258, - 0.0596207, - -0.02463656, - 0.10112172, - -0.00437765, - 0.17012416, - 0.14460884, - 0.12201063, - -0.04460088, - -0.06418587, - 0.04558447, - 0.05304697, - -0.04293422, - -0.10141369, - 0.07981857, - -0.10645458, - 0.24405084, - 0.23952056, - 0.02897859, - -0.04682537, - 0.07496171, - -0.00144922, - 0.11162834, - 0.04760814, - 0.02605088, - 0.00494431, - 0.00755539, - 0.24707587, - -0.01053839, - 0.00415074, - -0.09459292, - 0.05857703, - -0.00995291, - 0.05930841, - -0.25153132 + -0.1082256, + 0.06450271, + 0.05936934, + -0.02453269, + 0.09267135, + -0.01228072, + 0.16125621, + 0.14399916, + 0.11825327, + -0.04441284, + -0.05999637, + 0.04543412, + 0.05292828, + -0.0427532, + -0.10098612, + 0.07948205, + -0.10600576, + 0.2430219, + 0.23851072, + 0.02885641, + -0.04662795, + 0.07464566, + -0.00144311, + 0.11115771, + 0.04740742, + 0.02594105, + 0.00307946, + 0.00752354, + 0.24603418, + -0.01049396, + 0.00413324, + -0.09577953, + 0.05157854, + -0.00523842, + 0.05905836, + -0.25047084 ], [ - 0.09972278, - -0.08282613, - 0.01708003, - -0.11392891, - 0.23335189, - -0.08444921, - 0.00740774, - -0.08743347, - 0.01416104, - -0.21067027, - 0.18806838, - 0.12040872, - 0.08108939, - 0.09636783, - 0.03994259, - 0.13015021, - 0.16019044, - -0.01044454, - -0.14110244, - -0.07362956, - 0.22159558, - 0.15010985, - -0.02526645, - 0.21557182, - 0.00724162, - -0.0242343, - 0.21084726, - -0.22688671, - -0.02703428, - 0.11276176, - -0.03391354, - -0.05684184, - -0.06904587, - -0.12956661, - 0.11676405, - -0.10723548 + 0.10172078, + -0.079691, + 0.01700802, + -0.11344858, + 0.23717787, + -0.07948596, + 0.00938344, + -0.08706485, + 0.01831632, + -0.20978206, + 0.17938602, + 0.1193088, + 0.08401807, + 0.09596154, + 0.03977419, + 0.12960148, + 0.15951507, + -0.01040051, + -0.14050754, + -0.07331913, + 0.22066132, + 0.14947697, + -0.02515992, + 0.21466295, + 0.00721109, + -0.02413213, + 0.20904697, + -0.22593013, + -0.0269203, + 0.11228634, + -0.03377056, + -0.05420135, + -0.06633632, + -0.12881651, + 0.11627176, + -0.10678336 ], [ - 0.09389682, - 0.02886594, - 0.08280465, - -0.04065766, - 0.35082627, - 0.05511858, - 0.08129052, - -0.0784468, - 0.04146949, - 0.039334, - 0.07200802, - -0.12680236, - 0.00768413, - 0.20635882, - -0.00310153, - 0.04395532, - -0.02294643, - -0.03682615, - -0.10043061, - 0.17402883, - -0.02088721, - -0.06251829, - -0.14581254, - 0.06191564, - -0.15621715, - -0.06930244, - -0.01021498, - -0.23487896, - -0.15314286, - 0.10384902, - -0.01273359, - 0.06828616, - 0.00045514, - 0.05379671, - 0.1489388, - -0.03154839 + 0.08705564, + 0.0274381, + 0.08245554, + -0.04048624, + 0.34456371, + 0.04986273, + 0.07359808, + -0.07811606, + 0.04132152, + 0.03916817, + 0.07560008, + -0.12705897, + 0.01576878, + 0.2054888, + -0.00308846, + 0.04377, + -0.02284968, + -0.03667089, + -0.10000718, + 0.17329511, + -0.02079915, + -0.06225471, + -0.14519778, + 0.06165459, + -0.15555852, + -0.06901025, + -0.01160653, + -0.23388869, + -0.15249719, + 0.10341119, + -0.0126799, + 0.06806008, + -0.00599208, + 0.05327136, + 0.14831087, + -0.03141538 ], [ - -0.00094139, - 0.01470014, - -0.05123716, - 0.10077176, - 0.04048439, - -0.06372807, - -0.25247454, - -0.25750802, - 0.00095556, - 0.20714796, - -0.08690899, - -0.01423565, - 0.08755366, - -0.1657545, - 0.03764462, - -0.03432763, - 0.16834961, - -0.09036482, - 0.02823553, - -0.05284409, - 0.20001706, - -0.0522151, - -0.187496, - -0.04457715, - -0.00613968, - 0.03032655, - 0.01586607, - -0.00147736, - 0.00838492, - 0.02537674, - 0.02675732, - -0.00583073, - 0.10939987, - 0.17992156, - 0.13552421, - -0.08224909 + 0.00248131, + 0.01382137, + -0.05102114, + 0.10034689, + 0.04707867, + -0.05673636, + -0.24695362, + -0.25642235, + 0.0053212, + 0.2062746, + -0.08461769, + -0.01505946, + 0.09012134, + -0.16505567, + 0.03748591, + -0.0341829, + 0.16763983, + -0.08998384, + 0.02811649, + -0.05262129, + 0.19917377, + -0.05199496, + -0.1867055, + -0.04438921, + -0.00611379, + 0.03019869, + 0.01613229, + -0.00147113, + 0.00834957, + 0.02526975, + 0.02664451, + -0.00557047, + 0.11235736, + 0.17141486, + 0.13495282, + -0.08190232 ], [ - 0.12570181, - 0.02853048, - 0.01901246, - -0.20714438, - -0.130671, - -0.15886863, - -0.01481054, - -0.01950729, - 0.00357921, - -0.18394432, - -0.13069432, - 0.17988674, - 0.02125287, - 0.04710881, - -0.09993957, - 0.05444064, - -0.05306994, - -0.1237914, - 0.19320889, - 0.00385043, - -0.13236322, - -0.05745519, - 0.08223666, - -0.10710635, - -0.00956136, - -0.06043581, - 0.04612928, - -0.02649893, - -0.12906589, - 0.10629848, - 0.07396983, - 0.03079797, - -0.07636891, - -0.01138118, - 0.07364424, - 0.16569767 + 0.13210261, + 0.02608092, + 0.0189323, + -0.20627105, + -0.11916235, + -0.14734558, + -0.00693601, + -0.01942504, + -0.00034062, + -0.18316879, + -0.12195589, + 0.17684984, + 0.02642982, + 0.04691019, + -0.09951822, + 0.05421111, + -0.05284619, + -0.12326949, + 0.19239431, + 0.0038342, + -0.13180516, + -0.05721296, + 0.08188994, + -0.10665478, + -0.00952105, + -0.06018101, + 0.04388688, + -0.02638721, + -0.12852174, + 0.10585031, + 0.07365797, + 0.03685765, + -0.06911617, + -0.01004123, + 0.07333375, + 0.16499907 ], [ - 0.04869343, - 0.1046794, - 0.07502229, - 0.05209712, - -0.2621759, - -0.15999226, - -0.10735316, - -0.12740727, - -0.09509378, - 0.10359232, - 0.00427048, - -0.08854962, - 0.06334275, - -0.12478451, - -0.19062473, - -0.03701681, - 0.17449254, - -0.14924002, - -0.14659004, - 0.03224874, - 0.07042552, - 0.09483483, - -0.04208705, - 0.00137327, - 0.21556782, - -0.07531918, - 0.38136987, - -0.01240239, - -0.09015235, - 0.05885391, - -0.11988787, - 0.07654247, - 0.0892475, - -0.02000153, - 0.0717552, - -0.01977973 + 0.05966595, + 0.10423752, + 0.07470599, + 0.05187747, + -0.25258115, + -0.15058583, + -0.09942808, + -0.12687011, + -0.08721881, + 0.10315556, + 0.00440108, + -0.08803297, + 0.06397317, + -0.12425841, + -0.18982103, + -0.03686074, + 0.17375686, + -0.14861082, + -0.14597201, + 0.03211278, + 0.0701286, + 0.094435, + -0.0419096, + 0.00136748, + 0.21465897, + -0.07500163, + 0.38023162, + -0.0123501, + -0.08977226, + 0.05860578, + -0.11938242, + 0.08597149, + 0.10004904, + -0.02306738, + 0.07145268, + -0.01969634 ], [ - -0.1083893, - 0.02719483, - -0.03709757, - 0.02850982, - 0.02631509, - 0.01493138, - -0.1513492, - -0.06349164, - 0.03565267, - -0.03939753, - -0.14579311, - -0.10147141, - -0.08450011, - 0.05692681, - -0.24859712, - 0.11219304, - 0.01576809, - -0.10658669, - 0.02986501, - -0.07431182, - 0.03830324, - 0.17968864, - -0.12259429, - -0.03588854, - 0.1072362, - 0.03735017, - 0.10208074, - -0.00335572, - 0.04761318, - 0.14394359, - -0.12490967, - -0.04507281, - 0.07589468, - -0.13383145, - 0.06231894, - 0.15451331 + -0.1019137, + 0.02661659, + -0.03694117, + 0.02838962, + 0.03308644, + 0.02175547, + -0.14487732, + -0.06322395, + 0.03806621, + -0.03923142, + -0.14517346, + -0.09825995, + -0.0841438, + 0.0566868, + -0.24754901, + 0.11172003, + 0.01570161, + -0.10613731, + 0.02973909, + -0.07399851, + 0.03814175, + 0.17893105, + -0.12207742, + -0.03573723, + 0.10678408, + 0.0371927, + 0.10153805, + -0.00334157, + 0.04741244, + 0.14333671, + -0.12438304, + -0.04357777, + 0.08159333, + -0.13365333, + 0.06205619, + 0.15386187 ], [ - 0.02752007, - 0.08370549, - 0.05477758, - -0.07125188, - -0.11679037, - 0.17016251, - -0.12249453, - -0.10515524, - -0.01257553, - -0.00735416, - -0.09000362, - 0.02636512, - -0.02333946, - -0.01572022, - -0.13399937, - -0.21962659, - 0.03096742, - -0.00847616, - 0.2184145, - 0.05649979, - -0.10421822, - -0.05502525, - -0.01153401, - -0.12172936, - 0.00137256, - -0.04820791, - -0.08842372, - 0.02704967, - 0.06662673, - 0.03470064, - 0.13247577, - 0.07859481, - -0.13120231, - -0.03510298, - -0.13159535, - 0.00739977 + 0.04373752, + 0.08113852, + 0.05454663, + -0.07095148, + -0.09738484, + 0.18825577, + -0.10541704, + -0.10471189, + -0.01199727, + -0.00732316, + -0.08017028, + 0.02421221, + -0.02141335, + -0.01565395, + -0.13343441, + -0.21870062, + 0.03083685, + -0.00844042, + 0.21749365, + 0.05626158, + -0.10377883, + -0.05479325, + -0.01148538, + -0.12121614, + 0.00136678, + -0.04800466, + -0.08509618, + 0.02693563, + 0.06634582, + 0.03455434, + 0.13191724, + 0.08910484, + -0.11431568, + -0.02803116, + -0.13104053, + 0.00736857 ], [ - -0.11715609, - 0.00822256, - -0.07937974, - -0.17341471, - -0.05175816, - 0.01727163, - -0.04206952, - -0.01130419, - 0.24723567, - -0.1262661, - 0.0391111, - 0.10093862, - 0.02531153, - -0.21956659, - -0.05500543, - -0.04024684, - 0.18764472, - 0.04008947, - -0.1980829, - -0.06397288, - 0.02127377, - -0.05093988, - 0.13527471, - 0.24569166, - 0.02983517, - 0.02927358, - 0.1158615, - 0.1136444, - -0.07461183, - -0.04265619, - -0.04944842, - 0.04651935, - 0.1146271, - -0.19102864, - 0.08403239, - 0.00446445 + -0.11423616, + 0.00805504, + -0.07904507, + -0.17268357, + -0.04954119, + 0.01919434, + -0.03978332, + -0.01125653, + 0.24979031, + -0.12573375, + 0.0377463, + 0.10080928, + 0.02395664, + -0.21864088, + -0.05477353, + -0.04007715, + 0.18685359, + 0.03992045, + -0.19724776, + -0.06370316, + 0.02118407, + -0.05072511, + 0.13470438, + 0.24465581, + 0.02970938, + 0.02915016, + 0.11566215, + 0.11316527, + -0.07429726, + -0.04247635, + -0.04923994, + 0.04538945, + 0.11656982, + -0.19026933, + 0.0836781, + 0.00444563 ], [ - 0.10293369, - 0.03828861, - -0.04353828, - 0.00864994, - -0.08211438, - -0.02234312, - 0.10757668, - -0.21079098, - 0.20608842, - -0.04010878, - 0.12875975, - 0.00602635, - 0.08550691, - 0.11132359, - -0.03516023, - -0.23874292, - -0.04338438, - 0.00613308, - -0.15362887, - -0.04936465, - 0.14540815, - 0.03766489, - -0.27351152, - 0.21408165, - 0.00530205, - 0.03276328, - -0.04586856, - -0.13324635, - 0.11841198, - 0.04027488, - -0.16991108, - -0.17515843, - 0.08136158, - -0.12583403, - -0.16259245, - -0.01064868 + 0.09593605, + 0.03421868, + -0.04335472, + 0.00861347, + -0.07956931, + -0.02059617, + 0.10370228, + -0.20990227, + 0.18629368, + -0.03993968, + 0.13700316, + 0.00107125, + 0.09620356, + 0.11085424, + -0.035012, + -0.23773636, + -0.04320147, + 0.00610722, + -0.15298116, + -0.04915652, + 0.1447951, + 0.03750609, + -0.27235837, + 0.21317907, + 0.0052797, + 0.03262515, + -0.04304481, + -0.13268457, + 0.11791274, + 0.04010508, + -0.16919472, + -0.15722663, + 0.07445489, + -0.10935331, + -0.16190694, + -0.01060379 ], [ - -0.00308159, - 0.07603885, - -0.04577892, - -0.06349803, - -0.01373614, - 0.03722561, - -0.15666096, - -0.17593156, - 0.04519183, - 0.15590445, - 0.07792015, - 0.15950125, - -0.04732885, - -0.07255313, - 0.23231738, - -0.17929126, - -0.11286161, - -0.09180482, - -0.05079991, - 0.16341136, - -0.03882987, - -0.02947231, - -0.17948048, - 0.13623767, - -0.02433393, - -0.13664304, - -0.04970314, - -0.14054563, - 0.13881109, - -0.35144387, - 0.05322308, - 0.17538562, - 0.01512884, - 0.03023331, - -0.09529026, - 0.00045526 + 0.00690927, + 0.07617659, + -0.04558591, + -0.06323032, + -0.00826627, + 0.04259575, + -0.1516996, + -0.17518982, + 0.04793752, + 0.15524714, + 0.0723479, + 0.15935153, + -0.04720119, + -0.07224724, + 0.23133791, + -0.17853535, + -0.11238577, + -0.09141776, + -0.05058574, + 0.1627224, + -0.03866616, + -0.02934805, + -0.17872378, + 0.13566328, + -0.02423134, + -0.13606694, + -0.05594612, + -0.13995308, + 0.13822585, + -0.34996215, + 0.05299869, + 0.19209274, + 0.02504293, + 0.02661301, + -0.09488851, + 0.00045334 ], [ - -0.02934886, - -0.17325203, - -0.14340794, - 0.02790309, - 0.12247526, - 0.15109986, - 0.00325818, - 0.25534439, - -0.09834673, - 0.06987341, - -0.28257702, - 0.10428134, - 0.05454213, - -0.06402659, - 0.13263336, - 0.16355761, - -0.05379289, - 0.10795228, - -0.0624302, - -0.12478956, - -0.2029042, - 0.11417254, - -0.11067474, - -0.11043058, - -0.12980725, - -0.05771158, - 0.11078876, - -0.131114, - 0.17224473, - 0.04591657, - 0.05556841, - -0.07345686, - -0.09859937, - 0.16663537, - 0.15632663, - 0.206122 + -0.02318739, + -0.17331286, + -0.14280332, + 0.02778544, + 0.12629738, + 0.15493911, + 0.00297189, + 0.25426784, + -0.09138193, + 0.06957881, + -0.27307865, + 0.10223514, + 0.06122647, + -0.06375665, + 0.13207417, + 0.16286804, + -0.05356609, + 0.10749714, + -0.06216699, + -0.12426344, + -0.20204873, + 0.11369118, + -0.11020813, + -0.10996499, + -0.12925997, + -0.05746827, + 0.1083113, + -0.13056122, + 0.17151853, + 0.04572298, + 0.05533413, + -0.0605972, + -0.09214594, + 0.15880309, + 0.15566754, + 0.20525297 ], [ - -0.08873752, - -0.04147731, - 0.0583989, - 0.00262056, - 0.01176597, - 0.01928817, - -0.06495876, - -0.15439079, - -0.13864633, - 0.20774294, - 0.03922907, - 0.10119926, - -0.03146056, - -0.12828726, - -0.11537626, - 0.10735272, - 0.05424497, - 0.08867213, - 0.09744908, - -0.28934637, - -0.00210618, - -0.2700385, - 0.17522805, - 0.10253596, - -0.0181145, - 0.17242164, - 0.00483053, - 0.03464758, - 0.06348852, - -0.43752637, - -0.18356702, - -0.20068415, - -0.11011686, - 0.33622902, - 0.10076121, - -0.27817325 + -0.08216716, + -0.03864507, + 0.05815269, + 0.00260951, + 0.01757535, + 0.02501611, + -0.05914076, + -0.15373986, + -0.13361046, + 0.20686708, + 0.03662696, + 0.1005868, + -0.0316816, + -0.12774639, + -0.11488982, + 0.10690011, + 0.05401626, + 0.08829828, + 0.09703823, + -0.28812646, + -0.0020973, + -0.2689, + 0.17448928, + 0.10210366, + -0.01803813, + 0.17169469, + 0.00339603, + 0.0345015, + 0.06322085, + -0.43568172, + -0.18279309, + -0.19842252, + -0.10345636, + 0.33524853, + 0.10033639, + -0.27700045 ], [ - 0.15155031, - -0.05549479, - -0.05629185, - 0.19208915, - -0.0134966, - -0.04365763, - -0.03407944, - -0.13152914, - 0.14272008, - 0.14119804, - 0.00945079, - 0.06634315, - -0.01380649, - -0.07846321, - 0.04169803, - -0.08628179, - -0.18776956, - -0.01914541, - 0.02898041, - 0.19318741, - -0.10822073, - -0.09914044, - 0.0338885, - 0.07670552, - -0.08947388, - -0.13935935, - 0.03329264, - 0.14929706, - 0.16350562, - -0.20668184, - 0.02751999, - -0.12479607, - -0.05953754, - -0.07092188, - 0.17837235, - -0.24003014 + 0.14614664, + -0.05609601, + -0.05605452, + 0.19127928, + -0.01877323, + -0.04893666, + -0.03919627, + -0.1309746, + 0.14196424, + 0.14060274, + 0.00883373, + 0.06609614, + -0.01395025, + -0.0781324, + 0.04152222, + -0.08591801, + -0.18697791, + -0.01906469, + 0.02885823, + 0.19237291, + -0.10776446, + -0.09872246, + 0.03374563, + 0.07638213, + -0.08909665, + -0.1387718, + 0.03279936, + 0.14866761, + 0.16281627, + -0.20581045, + 0.02740397, + -0.12431817, + -0.06405125, + -0.06969794, + 0.17762032, + -0.23901815 ], [ - -0.02644152, - -0.08330372, - 0.00149832, - -0.0287041, - -0.13026478, - 0.12660931, - 0.11838335, - 0.22028705, - 0.02935224, - 0.05265632, - 0.04729529, - 0.28026789, - 0.09425452, - 0.05131364, - -0.04881779, - -0.1801838, - 0.07644703, - -0.07829926, - -0.21285768, - 0.30572149, - -0.04105434, - -0.09382923, - 0.09201851, - 0.05307113, - -0.14994082, - -0.25383425, - -0.00430674, - -0.05172725, - -0.22554983, - 0.05417717, - 0.08821288, - -0.13333532, - 0.05715924, - 0.17794845, - -0.03257984, - -0.06513834 + -0.02287583, + -0.07715234, + 0.00149201, + -0.02858308, + -0.12667721, + 0.12910833, + 0.11697609, + 0.2193583, + 0.03071399, + 0.05243432, + 0.04926346, + 0.27790869, + 0.09853811, + 0.0510973, + -0.04861197, + -0.17942413, + 0.07612472, + -0.07796914, + -0.21196026, + 0.30443254, + -0.04088125, + -0.09343364, + 0.09163056, + 0.05284737, + -0.14930866, + -0.25276406, + 0.00112552, + -0.05150917, + -0.2245989, + 0.05394875, + 0.08784097, + -0.1254563, + 0.06037247, + 0.17172835, + -0.03244248, + -0.06486371 ], [ - -0.03763227, - -0.02805238, - 0.05765229, - -0.12274026, - -0.18755951, - 0.06468117, - 0.07465772, - 0.34838258, - 0.06263381, - 0.20913679, - 0.19566679, - -0.02418976, - -0.23811086, - -0.02269463, - -0.03410159, - -0.35551337, - -0.06461074, - 0.06576878, - 0.06180924, - -0.04860086, - -0.0097842, - -0.27826263, - -0.13673082, - 0.14986284, - 0.06130694, - -0.22327691, - 0.04011037, - -0.30000981, - -0.03812411, - 0.05726646, - 0.04489732, - 0.0138041, - -0.10086703, - 0.02488461, - -0.13818043, - 0.00174222 + -0.04811214, + -0.03238361, + 0.05740922, + -0.12222278, + -0.19749302, + 0.05366402, + 0.0662674, + 0.34691377, + 0.05846913, + 0.20825505, + 0.19372345, + -0.02424424, + -0.23789188, + -0.02259895, + -0.03395781, + -0.35401449, + -0.06433834, + 0.0654915, + 0.06154865, + -0.04839595, + -0.00974295, + -0.27708945, + -0.13615435, + 0.149231, + 0.06104846, + -0.22233556, + 0.04058664, + -0.29874494, + -0.03796338, + 0.05702502, + 0.04470803, + 0.008909, + -0.11108029, + 0.02877535, + -0.13759785, + 0.00173487 ], [ - -0.03870747, - 0.1780725, - 0.05354908, - 0.02149916, - -0.0561415, - 0.18642364, - 0.16015055, - -0.14049706, - -0.20490198, - 0.05264511, - -0.24185259, - -0.2398624, - -0.15462673, - 0.03941393, - 0.02222469, - -0.06517728, - 0.13764843, - 0.02254224, - -0.06925984, - -0.23227024, - -0.20454588, - -0.11556182, - -0.13425741, - 0.07619522, - 0.04692139, - 0.00349093, - 0.0499055, - -0.20434689, - -0.10397334, - -0.13940123, - 0.07578873, - -0.02571452, - 0.05598873, - -0.01920406, - -0.0288338, - 0.08275897 + -0.04199364, + 0.17852177, + 0.05332332, + 0.02140852, + -0.05718739, + 0.1842773, + 0.16038615, + -0.13990472, + -0.20770115, + 0.05242315, + -0.2462415, + -0.23918485, + -0.15482104, + 0.03924776, + 0.02213099, + -0.06490248, + 0.13706809, + 0.0224472, + -0.06896784, + -0.23129097, + -0.2036835, + -0.1150746, + -0.13369137, + 0.07587398, + 0.04672357, + 0.00347622, + 0.05070789, + -0.20348534, + -0.10353498, + -0.1388135, + 0.07546919, + -0.03381477, + 0.05230332, + -0.01846291, + -0.02871224, + 0.08241005 ], [ - 0.07509929, - -0.09929031, - 0.07979457, - 0.02638253, - 0.00624973, - 0.03474035, - -0.00952976, - -0.04416129, - 0.10674796, - 0.09929432, - -0.06322341, - 0.19062965, - -0.17624465, - 0.04340253, - 0.07425248, - 0.29026194, - -0.28712541, - -0.17366287, - 0.15283475, - -0.03723398, - -0.1937536, - -0.1097477, - 0.05370289, - 0.08874427, - -0.21426623, - 0.00370767, - -0.23369716, - -0.09989975, - 0.01429785, - -0.08539698, - -0.1684033, - -0.0308824, - -0.13765079, - 0.04781285, - 0.01223584, - 0.05673375 + 0.06619578, + -0.10296794, + 0.07945815, + 0.0262713, + -0.00427586, + 0.02397648, + -0.0171129, + -0.04397511, + 0.11081175, + 0.09887568, + -0.07023213, + 0.19146539, + -0.17690413, + 0.04321954, + 0.07393942, + 0.28903817, + -0.28591486, + -0.1729307, + 0.15219038, + -0.037077, + -0.19293672, + -0.109285, + 0.05347647, + 0.08837012, + -0.21336286, + 0.00369204, + -0.23189702, + -0.09947856, + 0.01423757, + -0.08503693, + -0.1676933, + -0.03832558, + -0.14565732, + 0.05144183, + 0.01218425, + 0.05649455 ], [ - 0.04353096, - 0.11603282, - -0.01181061, - 0.16299702, - 0.21199367, - -0.1468536, - -0.08343194, - 0.04096016, - 0.05488327, - -0.0355655, - 0.06949034, - -0.0573693, - -0.27979257, - 0.1583769, - -0.04068469, - 0.05215236, - -0.09432684, - -0.00815235, - -0.07526179, - -0.03432454, - 0.0578106, - -0.02411695, - 0.0069884, - 0.07637932, - -0.01471263, - -0.01410417, - 0.07986198, - -0.06688418, - -0.06852955, - 0.16435854, - 0.03969912, - 0.07632237, - -0.02476636, - 0.00716701, - 0.00947252, - -0.04607969 + 0.04579647, + 0.11568024, + -0.01176082, + 0.16230981, + 0.21280394, + -0.14453532, + -0.08168277, + 0.04078746, + 0.05448564, + -0.03541555, + 0.07112035, + -0.05467179, + -0.27870929, + 0.15770916, + -0.04051316, + 0.05193248, + -0.09392915, + -0.00811798, + -0.07494448, + -0.03417982, + 0.05756687, + -0.02401527, + 0.00695893, + 0.0760573, + -0.0146506, + -0.01404471, + 0.07949468, + -0.06660219, + -0.06824063, + 0.16366559, + 0.03953175, + 0.07638046, + -0.02221289, + 0.0075497, + 0.00943258, + -0.04588541 ], [ - 0.04872011, - -0.20494439, - -0.05076294, - 0.10014853, - 0.0158735, - -0.16382666, - -0.02681615, - 0.20730954, - -0.13361573, - 0.00798033, - -0.18994007, - 0.01297251, - -0.08139196, - -0.04110773, - 0.12757961, - -0.06710524, - -0.20224528, - 0.03142396, - -0.07634442, - -0.02277612, - -0.06268567, - 0.13346952, - 0.06580607, - 0.04086852, - 0.11940689, - 0.14560854, - -0.11736164, - -0.04897494, - -0.01992604, - -0.05804605, - -0.03629088, - 0.04156788, - 0.01237345, - 0.01270757, - 0.13595367, - 0.12220615 + 0.04797041, + -0.20121693, + -0.05054892, + 0.0997263, + 0.01445835, + -0.16447287, + -0.02780565, + 0.20643551, + -0.13330268, + 0.00794669, + -0.18869184, + 0.01289104, + -0.08100612, + -0.04093442, + 0.12704173, + -0.06682232, + -0.2013926, + 0.03129147, + -0.07602255, + -0.02268009, + -0.06242138, + 0.1329068, + 0.06552862, + 0.04069621, + 0.11890345, + 0.14499464, + -0.117626, + -0.04876846, + -0.01984203, + -0.05780132, + -0.03613788, + 0.03890619, + 0.01177699, + 0.01304849, + 0.13538047, + 0.12169092 ], [ - 0.0546879, - -0.03929968, - -0.00956, - 0.04279683, - 0.12060208, - 0.20085032, - 0.03208466, - -0.11834625, - -0.05396534, - -0.08363673, - 0.17618768, - 0.15732041, - 0.08848845, - -0.17710831, - -0.13694396, - 0.05577353, - -0.09908341, - -0.10789354, - 0.06489379, - -0.13352015, - -0.30757026, - -0.02938826, - -0.02250595, - -0.2138949, - 0.12702464, - -0.32053268, - -0.16530099, - 0.00462674, - 0.2257149, - 0.26897635, - -0.05789735, - 0.03540943, - 0.29624866, - -0.05108691, - -0.04332441, - 0.12354349 + 0.06006328, + -0.03944994, + -0.00951969, + 0.04261639, + 0.1206877, + 0.20048033, + 0.03257966, + -0.1178473, + -0.05662421, + -0.08328411, + 0.17606165, + 0.15682822, + 0.08864125, + -0.17636161, + -0.13636659, + 0.05553838, + -0.09866566, + -0.10743866, + 0.0646202, + -0.13295722, + -0.30627351, + -0.02926435, + -0.02241107, + -0.2129931, + 0.12648909, + -0.31918129, + -0.16955234, + 0.00460723, + 0.22476327, + 0.26784233, + -0.05765325, + 0.05324364, + 0.3006056, + -0.04828446, + -0.04314175, + 0.12302262 ], [ - 0.25838144, - 0.02045325, - -0.20234917, - 0.17548277, - -0.05953706, - 0.20726814, - 0.02193118, - -0.0294143, - 0.03626955, - 0.11867327, - 0.03986568, - -0.19253432, - -0.03482367, - 0.12798386, - 0.0007602, - -0.18719385, - 0.02970069, - 0.02276647, - 0.09149196, - 0.06950235, - 0.09412037, - -0.12361764, - 0.03335566, - 0.01002804, - 0.1230208, - 0.04748133, - -0.00518659, - -0.03738536, - -0.0203532, - -0.00023023, - 0.10439911, - 0.04344824, - -0.16362102, - -0.17766666, - 0.04221681, - 0.02207265 + 0.2527439, + 0.01931682, + -0.20149605, + 0.17474292, + -0.06182885, + 0.20365184, + 0.01960956, + -0.02929029, + 0.03178645, + 0.11817293, + 0.04411025, + -0.19158546, + -0.03393629, + 0.12744427, + 0.00075699, + -0.18640462, + 0.02957546, + 0.02267049, + 0.09110623, + 0.06920933, + 0.09372355, + -0.12309646, + 0.03321503, + 0.00998576, + 0.12250213, + 0.04728115, + -0.0067059, + -0.03722774, + -0.02026739, + -0.00022926, + 0.10395895, + 0.04508873, + -0.16747937, + -0.16690733, + 0.04203882, + 0.02197959 ], [ - -0.05071466, - 0.07009577, - -0.17354118, - -0.1425593, - -0.14303103, - -0.02268008, - -0.13131811, - 0.09322018, - 0.00673806, - -0.14130498, - 0.09519986, - -0.1535287, - 0.02259053, - -0.03975285, - 0.31151923, - -0.03565573, - -0.02149328, - -0.17446578, - 0.05373714, - 0.06290018, - 0.03266323, - -0.00683012, - 0.01896202, - -0.06669987, - 0.00466656, - 0.18725139, - 0.00836021, - 0.12583931, - 0.04962296, - 0.04893267, - 0.17739813, - 0.07590366, - 0.1745511, - -0.18428964, - -0.11855853, - 0.11544419 + -0.04611023, + 0.07005009, + -0.17280951, + -0.14195826, + -0.14033296, + -0.02027851, + -0.12879549, + 0.09282715, + 0.00655107, + -0.14070923, + 0.10181729, + -0.14956261, + 0.02448953, + -0.03958525, + 0.31020584, + -0.0355054, + -0.02140267, + -0.17373022, + 0.05351058, + 0.06263498, + 0.03252552, + -0.00680133, + 0.01888207, + -0.06641865, + 0.00464689, + 0.18646192, + 0.00931591, + 0.12530876, + 0.04941375, + 0.04872637, + 0.1766502, + 0.07536327, + 0.17820578, + -0.18311787, + -0.11805867, + 0.11495747 ] ], "b1": [ - 0.01215218, - -0.02405024, - 0.02332759, - -0.01931053, - 0.00310449, - 0.0178361, - 0.00745933, - 0.01784208, - 0.00653888, - 0.00435611, - 0.00587579, - 0.0097813, - 0.01016966, - 0.02578757, - 0.00167515, - -8.193e-05, - 0.02436347, - -0.00372971, - 0.03452251, - 0.00149435, - 0.01774672, - 0.00285032, - -0.00124859, - 0.01308786, - -0.0122056, - -0.01149606, - -0.0203385, - -0.00081575, - 0.00071594, - 0.00319573, - 0.00893007, - -0.00196122 + 0.00196937, + -0.00037225, + -0.00028717, + -0.00197042, + -0.00443127, + -0.0013593, + 0.00047665, + -0.00072591, + 0.00204353, + -0.00343355, + -0.00100886, + -0.0027349, + -0.00112659, + -0.00070595, + 0.00166679, + -8.147e-05, + 0.00061991, + -0.00028349, + -0.00293703, + 0.00755559, + -0.00093398, + 0.00068876, + -0.00086624, + -0.00060589, + -0.00153215, + 7.15e-06, + -0.00127016, + 0.00049314, + 0.00013518, + 0.00517144, + -0.00113493, + 0.00043336 ], "w2": [ [ - 0.02740235, - -0.05076654, - -0.00312784, - -0.1608918, - -0.2166921, - -0.14159529, - -0.02300082, - -0.00726687, - 0.14023843, - -0.19117808, - -0.08276512, - -0.0123104, - -0.12449349, - -0.04698411, - 0.06514045, - -0.06826449, - -0.00356702, - 0.0168091, - -0.08438088, - 0.27522479, - -0.00258487, - 0.03806313, - -0.02839343, - -0.15215063, - 0.08966034, - -0.07161907, - 0.08245703, - -0.01364006, - -0.00978226, - 0.3489985, - 0.01569161, - -0.02382035 + 0.03429766, + -0.04560188, + -0.00127876, + -0.15457293, + -0.21399308, + -0.13972045, + -0.01411688, + -0.0025633, + 0.14509003, + -0.18593507, + -0.07606387, + 0.00235123, + -0.12129521, + -0.04394316, + 0.07089626, + -0.06539088, + 0.00175297, + 0.02476123, + -0.07798005, + 0.28347975, + 0.00119744, + 0.03920252, + -0.02615003, + -0.1499084, + 0.08961679, + -0.06334643, + 0.08680762, + -0.00816018, + -0.00499551, + 0.37000231, + 0.02652476, + -0.01769801 ], [ - -0.0639759, - -0.04225679, - -0.16380494, - -0.05554519, - -0.26979957, - 0.01335445, - -0.18686381, - -0.01192395, - -0.25982722, - -0.07931886, - 0.04844313, - 0.08898588, - 0.27917977, - 0.112701, - 0.02552941, - 0.05692458, - 0.06272948, - -0.1091249, - 0.04324377, - 0.01117187, - -0.2703177, - 0.02631778, - 0.06005873, - -0.32961964, - -0.20106192, - -0.0435366, - -0.1898426, - -0.19431066, - 0.1382359, - -0.21310048, - -0.00204322, - -0.08267993 + -0.06653104, + -0.04301215, + -0.16434716, + -0.05719357, + -0.26866194, + 0.01084099, + -0.18599356, + -0.0134559, + -0.25903442, + -0.07898428, + 0.04823889, + 0.0886109, + 0.2762187, + 0.11015847, + 0.02252327, + 0.05559074, + 0.06144833, + -0.10880886, + 0.04306304, + 0.01089718, + -0.26983964, + 0.02188271, + 0.05958972, + -0.32837577, + -0.20135523, + -0.04342517, + -0.18949315, + -0.19355945, + 0.13598805, + -0.2122015, + -0.00203461, + -0.08253614 ], [ - 0.08309151, - -0.13800807, - -0.03830937, - 0.04991511, - -0.14347499, - 0.03920532, - 0.11599362, - -0.10049625, - -0.24153865, - -0.13881208, - 0.06781136, - -0.22524932, - 0.03914358, - -0.20786233, - 0.0349127, - -0.22440376, - -0.05056412, - 0.09885717, - -0.16687383, - 0.21386208, - -0.0228775, - -0.07410541, - -0.2527007, - -0.0281445, - -0.11157569, - -0.16500284, - 0.00211684, - -0.28114996, - 0.15117333, - 0.03838075, - 0.03394321, - -0.06075656 + 0.08338055, + -0.13719762, + -0.03774681, + 0.05050198, + -0.1427142, + 0.03909324, + 0.11569735, + -0.09967201, + -0.24051152, + -0.13822121, + 0.06753143, + -0.2241865, + 0.03916588, + -0.20681907, + 0.03523033, + -0.22345099, + -0.04994399, + 0.09847235, + -0.16605588, + 0.21353753, + -0.02281422, + -0.07368999, + -0.2516424, + -0.02795873, + -0.11109833, + -0.16419424, + 0.00292467, + -0.27983376, + 0.15111101, + 0.03920507, + 0.03391535, + -0.06018601 ], [ - 0.02875412, - -0.22452424, - 0.05091956, - -0.00157297, - 0.16084491, - -0.08254621, - 0.18790229, - 0.08666138, - 0.10277764, - -0.11234781, - 0.22170382, - -0.13154063, - -0.08208117, - 0.11455946, - 0.19369475, - 0.23421899, - 0.2722542, - 0.11886162, - 0.17908568, - 0.04427845, - -0.01629876, - 0.01661952, - 0.00175824, - -0.02021142, - 0.19034692, - -0.07012939, - -0.02144, - 0.11337975, - -0.16314259, - -0.09478562, - 0.13309197, - 0.16074607 + 0.01328226, + -0.22786798, + 0.04367201, + -0.01218587, + 0.15822336, + -0.09457487, + 0.18105676, + 0.07905877, + 0.09503726, + -0.11528522, + 0.21919927, + -0.1286572, + -0.08744177, + 0.10297459, + 0.17600544, + 0.22247413, + 0.26622602, + 0.10996652, + 0.17181348, + 0.04249817, + -0.01946072, + -0.00256137, + -0.00509794, + -0.0224812, + 0.18116639, + -0.07508817, + -0.02621115, + 0.11246318, + -0.16975699, + -0.09120722, + 0.12935966, + 0.15198137 ], [ - -0.00173931, - -0.01527242, - 0.07291196, - -0.07956235, - -0.00584083, - 0.00153189, - -0.0964356, - 0.07717989, - 0.00640036, - 0.00556372, - -0.02504018, - 0.10424449, - 0.1089349, - -0.0467027, - 0.14523547, - 0.07111062, - -0.18841923, - 0.07722889, - -0.01839827, - 0.11500113, - 0.24993109, - -0.07734934, - 0.23848894, - -0.05166054, - -0.03263521, - -0.08845007, - 0.03991133, - -0.11841842, - -0.03881195, - 0.06823234, - -0.21234976, - 0.06337744 + -0.0001619, + -0.01379371, + 0.07320605, + -0.07753854, + -0.00499501, + 0.00258234, + -0.09450649, + 0.07808871, + 0.00707622, + 0.00667178, + -0.02361775, + 0.10802386, + 0.10948048, + -0.04573059, + 0.14622733, + 0.07123783, + -0.18684069, + 0.07823346, + -0.01700459, + 0.11673401, + 0.25037271, + -0.07563561, + 0.23799873, + -0.05047589, + -0.03172117, + -0.08627651, + 0.04038998, + -0.1164684, + -0.03765861, + 0.07345288, + -0.20935098, + 0.06446052 ], [ - 0.1896967, - -0.11438067, - -0.01345407, - -0.07697002, - 0.15728876, - 0.0753857, - 0.04423694, - 0.07391768, - 0.03724821, - -0.24537233, - 0.04900044, - 0.03112069, - -0.01063814, - 0.05093553, - -0.33762841, - -0.01221162, - 0.15456577, - 0.09962478, - 0.04406098, - -0.08612326, - -0.16884556, - -0.11960671, - 0.02409531, - 0.13257575, - 0.10690759, - 0.0984879, - 0.00719562, - -0.05426996, - 0.17153521, - 0.0057244, - 0.12173967, - -0.04004198 + 0.17734167, + -0.11915261, + -0.01618277, + -0.07962959, + 0.15389639, + 0.06910934, + 0.03434151, + 0.07086111, + 0.02893482, + -0.24976256, + 0.04343829, + 0.02524846, + -0.01306084, + 0.04513885, + -0.34223498, + -0.02099936, + 0.14967928, + 0.08794806, + 0.03458945, + -0.08905981, + -0.1749678, + -0.1260701, + 0.01647745, + 0.1292002, + 0.10167254, + 0.09019891, + -0.00041903, + -0.05618957, + 0.16316041, + -0.00282543, + 0.11296007, + -0.04622841 ], [ - -0.2935767, - -0.00997258, - 0.18419565, - -0.08466296, - 0.04934941, - 0.07531426, - -0.08611727, - 0.18498668, - -0.15788079, - 0.07488553, - -0.1775369, - 0.04851853, - -0.03006078, - -0.13452092, - -0.03737682, - 0.07029607, - 0.16427307, - -0.03117217, - 0.28070396, - -0.10280042, - -0.04532835, - -0.03975688, - 0.00941286, - 0.14464547, - 0.0289707, - 0.09859632, - -0.17400148, - -0.16655471, - 0.06165844, - 0.09050512, - 0.00132185, - -0.02778914 + -0.29267501, + -0.01015285, + 0.18290184, + -0.08512424, + 0.04903818, + 0.07383068, + -0.08676388, + 0.18350033, + -0.15846489, + 0.07324827, + -0.17748137, + 0.0486868, + -0.03026208, + -0.13459166, + -0.03794671, + 0.06873218, + 0.16330207, + -0.03241835, + 0.27765925, + -0.10209869, + -0.04676954, + -0.04220477, + 0.00811791, + 0.14306605, + 0.02832193, + 0.09643381, + -0.17361211, + -0.16568168, + 0.06065134, + 0.09034955, + 0.00074388, + -0.02829556 ], [ - -0.14723535, - -0.18146009, - 0.13306341, - -0.08678992, - 0.0994116, - 0.21550012, - -0.06367392, - 0.18977638, - 0.01126556, - 0.10272909, - 0.14978667, - 0.22048578, - 0.08076494, - 0.17775731, - 0.00493304, - -0.15312211, - -0.09584724, - -0.09616919, - 0.19892324, - 0.0556904, - 0.22608941, - -0.08194049, - 0.08011246, - -0.00748817, - 0.00339772, - -0.00118562, - -0.14300438, - -0.16947718, - 0.04024083, - 0.13811735, - -0.01978202, - -0.17910409 + -0.16954701, + -0.19203422, + 0.1201108, + -0.10700465, + 0.09189501, + 0.18561998, + -0.07910199, + 0.17166136, + -0.00419339, + 0.08773094, + 0.13791409, + 0.20500771, + 0.06591668, + 0.15897407, + -0.02101547, + -0.16760041, + -0.10341213, + -0.11307912, + 0.17918511, + 0.04866367, + 0.20195524, + -0.12934254, + 0.06614909, + -0.0216321, + -0.0119192, + -0.02172883, + -0.15073647, + -0.17304099, + 0.02350505, + 0.11896853, + -0.03216596, + -0.18928925 ], [ - -0.13974919, - -0.09055365, - -0.02949788, - 0.27520278, - -0.10761603, - 0.15537565, - 0.09355607, - -0.04369603, - 0.24119312, - -0.02970721, - -0.09095577, - 0.05557385, - -0.15388525, - 0.15114121, - -0.20771044, - 0.14321111, - 0.26817474, - -0.10315218, - -0.03591523, - 0.1185384, - 0.05504627, - 0.04214964, - 0.10827614, - -0.25537298, - 0.16475888, - -0.04644768, - 0.00446995, - -0.0210026, - -0.02442843, - 0.02250936, - -0.17633985, - 0.04741798 + -0.1420778, + -0.09104979, + -0.03080536, + 0.27221826, + -0.10756145, + 0.15214398, + 0.09274366, + -0.04478585, + 0.23921175, + -0.03000053, + -0.09068356, + 0.05695906, + -0.15435204, + 0.14855358, + -0.20944908, + 0.14095938, + 0.26635589, + -0.10363976, + -0.03668884, + 0.11856426, + 0.05369262, + 0.03789264, + 0.10674311, + -0.25500801, + 0.162527, + -0.04659194, + 0.00330514, + -0.02039369, + -0.02615833, + 0.02428332, + -0.17517032, + 0.04648334 ], [ - -0.04485926, - 0.01738657, - 0.08919131, - 0.03490676, - 0.06289554, - -0.12019968, - -0.00428636, - 0.04420618, - 0.09873289, - -0.29704451, - -0.06652041, - 0.08122147, - 0.04266871, - 0.11497824, - 0.01138465, - -0.03892815, - 0.11868044, - -0.19576926, - -0.07093067, - 0.00497217, - 0.06703986, - -0.13902237, - -0.23959185, - 0.07410588, - -0.02846486, - 0.01628364, - 0.12160802, - -0.08690999, - -0.04744886, - 0.1376722, - 0.06167737, - -0.21113702 + -0.04295201, + 0.01913865, + 0.08941513, + 0.0364854, + 0.06334408, + -0.11887691, + -0.00195216, + 0.04580442, + 0.09969148, + -0.29499885, + -0.0650717, + 0.0854579, + 0.04290411, + 0.11516077, + 0.01323483, + -0.0385593, + 0.11976477, + -0.1939243, + -0.0699621, + 0.00794991, + 0.06876896, + -0.13767376, + -0.23859084, + 0.07459248, + -0.02824027, + 0.01942016, + 0.12243855, + -0.08488264, + -0.04546411, + 0.1439702, + 0.06366091, + -0.20880981 ], [ - -0.01454235, - 0.05088008, - 0.23823154, - 0.11678192, - 0.09078839, - 0.21746072, - 0.21165065, - -0.06710502, - 0.02344842, - -0.11889344, - -0.00205321, - 0.32502418, - 0.23573088, - -0.17244292, - 0.16527799, - -0.01845815, - 0.12000328, - 0.03095096, - -0.03835036, - -0.15264269, - 0.02213253, - 0.10946772, - -0.19894242, - 0.07061435, - -0.20404351, - -0.09203702, - -0.26131119, - 0.04609703, - 0.04504379, - -0.28318949, - -0.28871293, - 0.09547239 + -0.02974994, + 0.04376367, + 0.23131969, + 0.10653928, + 0.08726453, + 0.20180255, + 0.20747097, + -0.074475, + 0.02025284, + -0.11908976, + -0.00271304, + 0.32292598, + 0.2284816, + -0.18069845, + 0.14941911, + -0.02405956, + 0.11582927, + 0.02808227, + -0.03902498, + -0.15443809, + 0.01376139, + 0.08679999, + -0.20078811, + 0.06643476, + -0.21179974, + -0.09739104, + -0.2639807, + 0.04464895, + 0.03488319, + -0.28111167, + -0.28843781, + 0.08980877 ], [ - -0.17712978, - 0.05075438, - 0.23604771, - -0.05783474, - -0.07098208, - -0.06341745, - -0.04773828, - 0.0037519, - -0.04165603, - 0.15909509, - -0.18394281, - -0.02715359, - 0.05584253, - 0.10433933, - -0.12933205, - -0.0482846, - 0.00630398, - 0.00427443, - 0.07647934, - -0.16471008, - 0.04445176, - -0.13745028, - 0.03679401, - 0.04010788, - -0.09661267, - -0.13009088, - 0.05016106, - 0.05286186, - -0.02683352, - 0.19041431, - 0.08399934, - 0.02040794 + -0.1801343, + 0.0491963, + 0.23360958, + -0.06008171, + -0.07070128, + -0.06345462, + -0.06165247, + 0.0017824, + -0.05481267, + 0.14388679, + -0.19126183, + -0.0399862, + 0.05438321, + 0.09884513, + -0.13035389, + -0.05911431, + 0.00198846, + -0.012547, + 0.05883228, + -0.16637059, + 0.03623264, + -0.14119068, + 0.02516784, + 0.03522996, + -0.09595736, + -0.14212536, + 0.04100391, + 0.05085428, + -0.03056572, + 0.17199256, + 0.06998061, + 0.01387065 ], [ - -0.12575682, - -0.16192032, - -0.15060651, - -0.0273118, - 0.06304916, - 0.11897971, - 0.11831066, - -0.07307611, - -0.04830444, - -0.33329721, - 0.02476922, - -0.0959515, - -0.04621725, - 0.04627493, - 0.03466157, - -0.0717781, - -0.02657949, - -0.05491454, - -0.01444727, - -0.17204116, - 0.08983893, - -0.07483061, - -0.08787621, - 0.11554217, - 0.05958444, - -0.11276958, - 0.13979152, - 0.03802697, - -0.00940856, - 0.04367635, - -0.17935266, - 0.2820885 + -0.11941817, + -0.15861942, + -0.14690619, + -0.02241434, + 0.0662634, + 0.12299136, + 0.11977162, + -0.07154237, + -0.047431, + -0.33170437, + 0.02717803, + -0.09490496, + -0.04468394, + 0.04803596, + 0.03967852, + -0.06984498, + -0.02577174, + -0.05287281, + -0.01252779, + -0.16964182, + 0.09229982, + -0.06957105, + -0.08581756, + 0.11699777, + 0.06424206, + -0.1120628, + 0.14138985, + 0.03872028, + -0.00561418, + 0.04636024, + -0.17776693, + 0.28520324 ], [ - -0.10319162, - 0.10274566, - -0.11110572, - -0.08557924, - 0.06820238, - 0.00820877, - 0.01210727, - 0.19439169, - -0.25519655, - -0.04120656, - 0.11649499, - 0.05363248, - -0.06074082, - -0.19251751, - -0.10984914, - 0.08338429, - -0.02499488, - -0.35414388, - -0.04710524, - -0.2114185, - -0.09693611, - -0.07492991, - 0.15742115, - -0.13955111, - 0.04744345, - -0.07572633, - -0.01830112, - 0.02913674, - 0.00938497, - 0.13167229, - 0.00749638, - -0.13201764 + -0.10281586, + 0.10212244, + -0.11056783, + -0.0850553, + 0.06786135, + 0.00811472, + 0.01207705, + 0.19360553, + -0.25412053, + -0.04129009, + 0.11605744, + 0.05294472, + -0.06042106, + -0.19168196, + -0.10926887, + 0.08296959, + -0.02487325, + -0.35280197, + -0.04695221, + -0.21045875, + -0.09645094, + -0.07460011, + 0.15667572, + -0.13894349, + 0.04720454, + -0.07530807, + -0.01818459, + 0.02895305, + 0.00940061, + 0.13127451, + 0.0072958, + -0.13114248 ], [ - 0.31943014, - -0.08432617, - 0.18718318, - -0.30415333, - -0.06108028, - -0.02845825, - 0.10473676, - 0.05487655, - 0.03984443, - -0.1352787, - 0.03846703, - 0.05445135, - -0.03997273, - 0.01116839, - -0.06130443, - 0.04329849, - 0.05326606, - -0.13862572, - 0.18475981, - 0.01571757, - 0.11940737, - 0.19283411, - -0.02825378, - 0.24871839, - -0.17500225, - -0.12879848, - 0.07831164, - 0.19050277, - -0.06442317, - -0.14526286, - -0.00910779, - -0.00386253 + 0.29545482, + -0.09467129, + 0.17797674, + -0.31534029, + -0.0649652, + -0.0473803, + 0.09512377, + 0.04399461, + 0.03088335, + -0.14096128, + 0.03473734, + 0.05137572, + -0.04868198, + -0.00225438, + -0.079406, + 0.03098636, + 0.04657169, + -0.14919995, + 0.17505804, + 0.01278448, + 0.10518397, + 0.16139606, + -0.03720567, + 0.24036851, + -0.18538461, + -0.13889716, + 0.06723214, + 0.18803595, + -0.07961564, + -0.14658821, + -0.01432602, + -0.0119934 ], [ - 0.22544379, - 0.07889532, - -0.12363103, - -0.15969646, - -0.13417809, - -0.21123119, - -0.04925414, - -0.02236964, - 0.10643368, - 0.0985539, - -0.18887526, - -0.06206391, - -0.02642684, - 0.05165471, - 0.18773697, - 0.08646466, - 0.17181541, - 0.14106746, - -0.0094216, - 0.13638593, - -0.16254755, - 0.04849888, - -0.22739913, - -0.11432125, - -0.08325791, - 0.02325804, - -0.080957, - -0.12700495, - 0.06934157, - -0.09850965, - 0.14362632, - 0.2211805 + 0.21551848, + 0.07536598, + -0.12615861, + -0.16416756, + -0.13411485, + -0.2155955, + -0.05349927, + -0.02613664, + 0.10120401, + 0.09623937, + -0.18968477, + -0.06343746, + -0.03026075, + 0.04375666, + 0.17671492, + 0.07846314, + 0.16710939, + 0.13399871, + -0.01334278, + 0.13363101, + -0.1639659, + 0.04008857, + -0.23062853, + -0.11453942, + -0.08612538, + 0.01949231, + -0.08362308, + -0.12786494, + 0.06369067, + -0.10011735, + 0.13949364, + 0.21482437 ] ], "b2": [ - -0.01472544, - 0.00844073, - -0.00039875, - 0.0600412, - -0.00916065, - 0.04614733, - 0.00974107, - 0.1414663, - 0.00809017, - -0.00268618, - 0.04562834, - 0.06281594, - -0.01474848, - 0.00139086, - 0.08759203, - 0.03757449 + 0.01885969, + 0.00170239, + 0.00050057, + 3.708e-05, + 0.00050642, + -0.00307421, + -0.0004431, + -0.01001486, + -0.00038455, + 0.00768781, + 0.00076421, + -0.01321739, + -0.00194406, + 0.00162896, + 0.00318053, + 0.00098175 ], "w3": [ [ - 0.14591786, - -0.08587059, - 0.02409859, - -0.17396011, - -0.00671001, - -0.12531235, - -0.03221764, - -0.25423137, - -0.03881778, - 0.05542406, - -0.14071863, - -0.20701404, - 0.11415409, - 0.04921243, - -0.17700276, - -0.10803946 + 0.19036689, + -0.08403602, + 0.02800143, + -0.00779111, + 0.03950601, + -0.0811864, + -0.0030103, + -0.15454482, + 0.01133027, + 0.07498601, + -0.0376662, + -0.18525029, + 0.11770436, + 0.04727888, + -0.09068078, + -0.0334385 ] ], "b3": [ - -0.9166933 + -0.0 ], "metadata": { - "generated_at_utc": "2026-04-23T15:22:07.379646+00:00", + "generated_at_utc": "2026-04-23T19:40:50.652698+00:00", "source_data": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\training_data\\hard_negatives_v1.jsonl", + "objective": "lambda_weighted_pairwise_ndcg_at_5", "train_records": 2025, "heldout_records": 560, - "longmemeval_extra_epoch_applied": true + "longmemeval_extra_epoch_applied": false } } \ No newline at end of file diff --git a/src/agentmemory/retrieval/query_planner.py b/src/agentmemory/retrieval/query_planner.py index aaae607..160e440 100644 --- a/src/agentmemory/retrieval/query_planner.py +++ b/src/agentmemory/retrieval/query_planner.py @@ -48,6 +48,40 @@ r")\b", re.IGNORECASE, ) +_COUNT_RE = re.compile( + r"\b(" + r"how many|how much|count|number of|total|sum|combined total" + r")\b", + re.IGNORECASE, +) +_COMPARE_RE = re.compile( + r"\b(" + r"compare|difference|different|versus|vs\.?|better|worse|older|newer|" + r"more than|less than|changed|relative to" + r")\b", + re.IGNORECASE, +) +_ORDER_RE = re.compile( + r"\b(" + r"before|after|between|order|ordered|sequence|timeline|earliest|latest|" + r"first|last|most recent|newest|oldest|rank" + r")\b", + re.IGNORECASE, +) +_UPDATE_RE = re.compile( + r"\b(" + r"current(?:ly)?|previous(?:ly)?|formerly|used to|now|new|updated|" + r"latest|most recent|superseded|stale|still|anymore" + r")\b", + re.IGNORECASE, +) +_COVERAGE_RE = re.compile( + r"\b(" + r"all|both|each|every|across|combined|together|list|which sessions|" + r"what were the sessions|set of" + r")\b", + re.IGNORECASE, +) _NEGATIVE_RE = re.compile( r"\b(" r"no answer|" @@ -71,6 +105,11 @@ class QueryPlan: temporal_anchors: list[str] = field(default_factory=list) requires_temporal_reasoning: bool = False requires_multi_hop: bool = False + needs_counting: bool = False + needs_comparison: bool = False + needs_ordering: bool = False + needs_update_resolution: bool = False + needs_set_coverage: bool = False prefer_memory_types: list[str] = field(default_factory=list) candidate_tables: list[str] = field(default_factory=list) abstain_allowed: bool = False @@ -206,6 +245,15 @@ def plan_query( candidate_tables = list(requested_tables or _TABLE_ROUTES.get(normalized_intent, _TABLE_ROUTES["factual"])) requires_temporal = bool(_TEMPORAL_RE.search(query)) requires_multi_hop = bool(_MULTIHOP_RE.search(query)) + needs_counting = bool(_COUNT_RE.search(query)) + needs_comparison = bool(_COMPARE_RE.search(query)) + needs_ordering = bool(_ORDER_RE.search(query)) + needs_update_resolution = bool(_UPDATE_RE.search(query)) + needs_set_coverage = bool(_COVERAGE_RE.search(query)) + if requires_multi_hop and normalized_intent in {"temporal", "decision", "graph"}: + needs_set_coverage = True + if needs_counting or needs_comparison or needs_ordering: + needs_set_coverage = True abstain_allowed = bool(_NEGATIVE_RE.search(query)) or normalized_intent in {"factual", "troubleshooting", "procedural"} if _ENTITY_QUERY_RE.search(query) and normalized_intent == "factual": reasons.append("entity_or_role_lookup") @@ -213,6 +261,16 @@ def plan_query( reasons.append("temporal_reasoning") if requires_multi_hop: reasons.append("multi_hop_or_inference") + if needs_counting: + reasons.append("operator:counting") + if needs_comparison: + reasons.append("operator:comparison") + if needs_ordering: + reasons.append("operator:ordering") + if needs_update_resolution: + reasons.append("operator:update_resolution") + if needs_set_coverage: + reasons.append("operator:set_coverage") if "summary of yesterday" in query_lower: abstain_allowed = True reasons.append("negative_or_out_of_domain_summary") @@ -227,6 +285,11 @@ def plan_query( temporal_anchors=temporal_anchors, requires_temporal_reasoning=requires_temporal, requires_multi_hop=requires_multi_hop, + needs_counting=needs_counting, + needs_comparison=needs_comparison, + needs_ordering=needs_ordering, + needs_update_resolution=needs_update_resolution, + needs_set_coverage=needs_set_coverage, prefer_memory_types=prefer_memory_types, candidate_tables=candidate_tables, abstain_allowed=abstain_allowed, diff --git a/src/agentmemory/retrieval/second_stage.py b/src/agentmemory/retrieval/second_stage.py index b85ae72..58914f0 100644 --- a/src/agentmemory/retrieval/second_stage.py +++ b/src/agentmemory/retrieval/second_stage.py @@ -2,7 +2,9 @@ from __future__ import annotations +import math import os +import re from dataclasses import dataclass, field from typing import Any @@ -22,6 +24,24 @@ "entities": "entity", "decisions": "decision", } +_SESSION_RE = re.compile(r"\bsession[_ :#-]*(\d+)\b", re.IGNORECASE) +_DATE_RE = re.compile( + r"\b(?:\d{4}-\d{2}-\d{2}|\d{1,2}/\d{1,2}(?:/\d{2,4})?|" + r"jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|jun(?:e)?|" + r"jul(?:y)?|aug(?:ust)?|sep(?:tember)?|oct(?:ober)?|nov(?:ember)?|" + r"dec(?:ember)?)\b", + re.IGNORECASE, +) +_ENTITY_RE = re.compile(r"\b[A-Z][A-Za-z0-9_.:-]+\b") + + +def _resolve_benchmark_ranking_mode(args: Any) -> str: + mode = str( + getattr(args, "benchmark_ranking_mode", None) + or os.environ.get("BRAINCTL_BENCHMARK_RANKING_MODE", "full") + or "full" + ).strip().lower() + return mode if mode in {"full", "raw"} else "full" @dataclass(slots=True) @@ -40,10 +60,13 @@ class SecondStageConfig: judge_weight: float = 0.10 model_path: str | None = None model_enabled: bool = True + ranking_mode: str = "live" judge: JudgeConfig = field(default_factory=JudgeConfig) @classmethod def from_args(cls, args: Any) -> "SecondStageConfig": + benchmark = bool(getattr(args, "benchmark", False)) + ranking_mode = _resolve_benchmark_ranking_mode(args) if benchmark else "live" judge_enabled = bool(getattr(args, "judge_rerank", None)) judge_provider = str(getattr(args, "judge_rerank", "ollama") or "ollama") judge_model = str(getattr(args, "judge_model", "llama3.2:3b") or "llama3.2:3b") @@ -54,10 +77,11 @@ def from_args(cls, args: Any) -> "SecondStageConfig": except (TypeError, ValueError): top_n = 10 return cls( - enabled=not bool(getattr(args, "no_second_stage", False)) and not bool(getattr(args, "benchmark", False)), + enabled=not bool(getattr(args, "no_second_stage", False)) and not (benchmark and ranking_mode == "raw"), top_n=max(int(top_n or 10), 1), model_enabled=not bool(getattr(args, "no_second_stage_model", False)), model_path=getattr(args, "second_stage_model_path", None), + ranking_mode=ranking_mode, judge=JudgeConfig( enabled=judge_enabled, provider=judge_provider, @@ -102,6 +126,10 @@ def _heuristic_score(plan: Any, features: dict[str, float]) -> float: ) if long_context_reliable: score += features.get("long_context_score", 0.0) * 0.05 + if features.get("query_needs_ordering", 0.0) > 0.0: + score += features["temporal_anchor_overlap"] * 0.05 + features["session_gap_score"] * 0.05 + if features.get("query_needs_update_resolution", 0.0) > 0.0: + score += features["status_active"] * 0.04 if intent in {"temporal", "decision"}: score += features["bucket_events"] * 0.04 + features["bucket_decisions"] * 0.03 if intent in {"procedural", "troubleshooting"}: @@ -122,52 +150,117 @@ def _heuristic_score(plan: Any, features: dict[str, float]) -> float: return max(min(score, 1.0), 0.0) -def rerank_top_candidates( - query: str, +def _candidate_text(candidate: dict[str, Any]) -> str: + for key in ("content", "summary", "title", "goal", "description", "name", "search_text"): + value = candidate.get(key) + if value: + return str(value) + return "" + + +def _candidate_cluster_keys(plan: Any, candidate: dict[str, Any]) -> set[str]: + text = _candidate_text(candidate) + keys: set[str] = set() + for match in _SESSION_RE.finditer(text): + keys.add(f"session:{match.group(1)}") + if getattr(plan, "requires_temporal_reasoning", False) or getattr(plan, "needs_ordering", False): + for match in _DATE_RE.finditer(text): + keys.add(f"date:{match.group(0).lower()}") + target_entities = { + str(value).lower() + for value in (getattr(plan, "target_entities", None) or []) + if value + } + if target_entities: + lowered = text.lower() + for entity in target_entities: + if entity and entity in lowered: + keys.add(f"entity:{entity}") + observed_entities = { + match.group(0).lower() + for match in _ENTITY_RE.finditer(text) + if len(match.group(0)) > 2 + } + for entity in sorted(observed_entities)[:3]: + keys.add(f"obs:{entity}") + if not keys: + ident = candidate.get("id") + keys.add(f"row:{candidate.get('bucket')}:{ident}") + return keys + + +def _slate_score( + *, plan: Any, - candidates: list[dict[str, Any]], - config: SecondStageConfig | None = None, -) -> tuple[list[dict[str, Any]], dict[str, Any]]: - """Rerank a flat candidate list using heuristic + tiny MLP + optional judge.""" + candidate: dict[str, Any], + features: dict[str, float], + composite_score: float, + rank_index: int, + selected_keys: set[str], +) -> tuple[float, dict[str, float]]: + rank_discount = 1.0 / math.log2(rank_index + 2) + cluster_keys = _candidate_cluster_keys(plan, candidate) + new_keys = cluster_keys - selected_keys + coverage_bonus = 0.0 + redundancy_penalty = 0.0 + update_penalty = 0.0 + temporal_penalty = 0.0 + localization_bonus = 0.0 - cfg = config or SecondStageConfig() - if not cfg.enabled or not candidates: - return candidates, {"enabled": False} + if getattr(plan, "needs_set_coverage", False): + coverage_bonus += min(0.14, 0.035 * len(new_keys)) + if not new_keys and selected_keys: + redundancy_penalty += 0.085 + elif selected_keys and not new_keys: + redundancy_penalty += 0.03 - head = [dict(candidate) for candidate in candidates[: cfg.top_n]] - tail = [dict(candidate) for candidate in candidates[cfg.top_n :]] - for idx, candidate in enumerate(head): - candidate["_stage_position"] = idx - candidate.setdefault("bucket", candidate.get("type") or "memories") + if getattr(plan, "needs_update_resolution", False): + if features.get("status_stale", 0.0) > 0.0: + update_penalty += 0.08 + if features.get("status_needs_review", 0.0) > 0.0: + update_penalty += 0.05 + if features.get("status_active", 0.0) > 0.0: + coverage_bonus += 0.02 - feature_rows: list[dict[str, float]] = [] - leader_score = head[0].get("final_score") if head else None - for idx, candidate in enumerate(head): - prev_score = head[idx - 1].get("final_score") if idx > 0 else None - next_score = head[idx + 1].get("final_score") if idx + 1 < len(head) else None - features = build_features( - query, - plan, - candidate, - neighbors={"prev_score": prev_score, "next_score": next_score, "leader_score": leader_score}, - ) - feature_rows.append(features) + if getattr(plan, "requires_temporal_reasoning", False) or getattr(plan, "needs_ordering", False): + if features.get("candidate_temporal", 0.0) <= 0.0 and features.get("temporal_anchor_overlap", 0.0) <= 0.0: + temporal_penalty += 0.05 + else: + coverage_bonus += features.get("temporal_anchor_overlap", 0.0) * 0.03 - heuristic_scores = [_heuristic_score(plan, features) for features in feature_rows] + if features.get("long_context_focused_program", 0.0) > 0.0: + localization_bonus += ( + features.get("long_context_precision", 0.0) * 0.018 + + features.get("long_context_coverage", 0.0) * 0.014 + ) - model = TinyMLPModel.try_load(cfg.model_path or DEFAULT_MODEL_PATH) if cfg.model_enabled else None - if model is not None: - feature_matrix = [vectorize_features(features, feature_version=FEATURE_VERSION_V1) for features in feature_rows] - mlp_scores = model.score(feature_matrix) - else: - mlp_scores = [0.0] * len(head) + slate_adjustment = (coverage_bonus + localization_bonus - redundancy_penalty - update_penalty - temporal_penalty) * rank_discount + return ( + composite_score + slate_adjustment, + { + "coverage_bonus": round(coverage_bonus, 6), + "localization_bonus": round(localization_bonus, 6), + "redundancy_penalty": round(redundancy_penalty, 6), + "update_penalty": round(update_penalty, 6), + "temporal_penalty": round(temporal_penalty, 6), + "rank_discount": round(rank_discount, 6), + "new_key_count": float(len(new_keys)), + }, + ) - judge_scores = judge_candidates(query, head, cfg.judge) - if judge_scores and len(judge_scores) < len(head): - judge_scores = list(judge_scores) + [0.0] * (len(head) - len(judge_scores)) - elif not judge_scores: - judge_scores = [0.0] * len(head) +def _rerank_slate( + *, + plan: Any, + head: list[dict[str, Any]], + feature_rows: list[dict[str, float]], + heuristic_scores: list[float], + mlp_scores: list[float], + judge_scores: list[float], + cfg: SecondStageConfig, +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + base_weight = max(0.0, 1.0 - cfg.heuristic_weight - cfg.mlp_weight - cfg.judge_weight) + pool: list[dict[str, Any]] = [] debug_candidates: list[dict[str, Any]] = [] for candidate, features, heuristic_score, mlp_score, judge_score in zip( head, @@ -177,8 +270,8 @@ def rerank_top_candidates( judge_scores, ): pre_score = float(candidate.get("final_score") or candidate.get("retrieval_score") or 0.0) - final_score = ( - pre_score * max(0.0, 1.0 - cfg.heuristic_weight - cfg.mlp_weight - cfg.judge_weight) + composite_score = ( + pre_score * base_weight + heuristic_score * cfg.heuristic_weight + float(mlp_score) * cfg.mlp_weight + float(judge_score) * cfg.judge_weight @@ -188,7 +281,7 @@ def rerank_top_candidates( candidate["second_stage_mlp"] = round(float(mlp_score), 6) candidate["second_stage_judge"] = round(float(judge_score), 6) candidate["second_stage_features"] = { - key: features[key] + key: features.get(key) for key in ( "informative_overlap", "tfidf_cosine", @@ -196,6 +289,11 @@ def rerank_top_candidates( "temporal_anchor_overlap", "intent_bucket_fit", "session_gap_score", + "query_needs_counting", + "query_needs_comparison", + "query_needs_ordering", + "query_needs_update_resolution", + "query_needs_set_coverage", "long_context_score", "long_context_confidence", "long_context_agreement", @@ -207,25 +305,159 @@ def rerank_top_candidates( if long_context_debug.get("applicable"): candidate["second_stage_features"]["long_context_program"] = long_context_debug.get("program") candidate["second_stage_features"]["long_context_excerpt"] = long_context_debug.get("top_chunk_excerpt") - candidate["final_score"] = round(final_score, 8) + pool.append( + { + "candidate": candidate, + "features": features, + "composite_score": round(composite_score, 8), + "cluster_keys": _candidate_cluster_keys(plan, candidate), + } + ) + + selected: list[dict[str, Any]] = [] + selected_keys: set[str] = set() + rank_index = 0 + while pool: + best_idx = 0 + best_score = None + best_terms: dict[str, float] | None = None + for idx, item in enumerate(pool): + slate_score, terms = _slate_score( + plan=plan, + candidate=item["candidate"], + features=item["features"], + composite_score=float(item["composite_score"]), + rank_index=rank_index, + selected_keys=selected_keys, + ) + if best_score is None or slate_score > best_score: + best_idx = idx + best_score = slate_score + best_terms = terms + item = pool.pop(best_idx) + candidate = item["candidate"] + terms = best_terms or {} + candidate["second_stage_slate_score"] = round(float(best_score or 0.0), 6) + candidate["second_stage_slate_terms"] = terms + selected.append(candidate) + selected_keys.update(item["cluster_keys"]) + rank_index += 1 + + debug_candidates = [] + for index, candidate in enumerate(selected, start=1): + epsilon = max(len(selected) - index, 0) * 1e-6 + candidate["final_score"] = round(float(candidate.get("second_stage_slate_score") or 0.0) + epsilon, 8) debug_candidates.append( { "bucket": candidate.get("bucket"), "id": candidate.get("id"), - "pre_score": round(pre_score, 6), - "heuristic": round(heuristic_score, 6), - "mlp": round(float(mlp_score), 6), - "judge": round(float(judge_score), 6), - "final": round(final_score, 6), - "features": candidate["second_stage_features"], + "pre_score": round(float(candidate.get("pre_second_stage_score") or 0.0), 6), + "heuristic": round(float(candidate.get("second_stage_heuristic") or 0.0), 6), + "mlp": round(float(candidate.get("second_stage_mlp") or 0.0), 6), + "judge": round(float(candidate.get("second_stage_judge") or 0.0), 6), + "composite": round(float(candidate.get("second_stage_slate_score") or 0.0), 6), + "selection_rank": index, + "slate_terms": candidate.get("second_stage_slate_terms") or {}, + "features": candidate.get("second_stage_features") or {}, } ) + return selected, debug_candidates + + +def rerank_top_candidates( + query: str, + plan: Any, + candidates: list[dict[str, Any]], + config: SecondStageConfig | None = None, +) -> tuple[list[dict[str, Any]], dict[str, Any]]: + """Rerank a flat candidate list using heuristic + tiny MLP + optional judge.""" - head.sort(key=lambda item: item.get("final_score", 0.0), reverse=True) + cfg = config or SecondStageConfig() + if not cfg.enabled or not candidates: + return candidates, {"enabled": False} + + head = [dict(candidate) for candidate in candidates[: cfg.top_n]] + tail = [dict(candidate) for candidate in candidates[cfg.top_n :]] + hard_query = any( + bool(getattr(plan, attr, False)) + for attr in ( + "requires_temporal_reasoning", + "requires_multi_hop", + "needs_counting", + "needs_comparison", + "needs_ordering", + "needs_update_resolution", + "needs_set_coverage", + ) + ) + raw_head_scores = [ + float(candidate.get("final_score") or candidate.get("retrieval_score") or 0.0) + for candidate in head[:2] + ] + top_margin = abs(raw_head_scores[0] - raw_head_scores[1]) if len(raw_head_scores) >= 2 else 1.0 + if not hard_query and top_margin >= 0.08: + passthrough = [dict(candidate) for candidate in candidates] + for candidate in passthrough[: cfg.top_n]: + pre_score = float(candidate.get("final_score") or candidate.get("retrieval_score") or 0.0) + candidate.setdefault("pre_second_stage_score", round(pre_score, 8)) + return passthrough, { + "enabled": True, + "top_n": cfg.top_n, + "ranking_mode": cfg.ranking_mode, + "model_enabled": cfg.model_enabled, + "model_path": str(cfg.model_path or DEFAULT_MODEL_PATH), + "model_loaded": False, + "judge_enabled": cfg.judge.enabled, + "strategy": "passthrough_easy_query", + "top_margin": round(top_margin, 6), + "candidates": [], + } + for idx, candidate in enumerate(head): + candidate["_stage_position"] = idx + candidate.setdefault("bucket", candidate.get("type") or "memories") + + feature_rows: list[dict[str, float]] = [] + leader_score = head[0].get("final_score") if head else None + for idx, candidate in enumerate(head): + prev_score = head[idx - 1].get("final_score") if idx > 0 else None + next_score = head[idx + 1].get("final_score") if idx + 1 < len(head) else None + features = build_features( + query, + plan, + candidate, + neighbors={"prev_score": prev_score, "next_score": next_score, "leader_score": leader_score}, + ) + feature_rows.append(features) + + heuristic_scores = [_heuristic_score(plan, features) for features in feature_rows] + + model = TinyMLPModel.try_load(cfg.model_path or DEFAULT_MODEL_PATH) if cfg.model_enabled else None + if model is not None: + feature_matrix = [vectorize_features(features, feature_version=FEATURE_VERSION_V1) for features in feature_rows] + mlp_scores = model.score(feature_matrix) + else: + mlp_scores = [0.0] * len(head) + + judge_scores = judge_candidates(query, head, cfg.judge) + if judge_scores and len(judge_scores) < len(head): + judge_scores = list(judge_scores) + [0.0] * (len(head) - len(judge_scores)) + elif not judge_scores: + judge_scores = [0.0] * len(head) + + head, debug_candidates = _rerank_slate( + plan=plan, + head=head, + feature_rows=feature_rows, + heuristic_scores=heuristic_scores, + mlp_scores=mlp_scores, + judge_scores=judge_scores, + cfg=cfg, + ) reranked = head + tail debug = { "enabled": True, "top_n": cfg.top_n, + "ranking_mode": cfg.ranking_mode, "model_enabled": cfg.model_enabled, "model_path": str(cfg.model_path or DEFAULT_MODEL_PATH), "model_loaded": model is not None, @@ -233,6 +465,7 @@ def rerank_top_candidates( "base_weight": round(max(0.0, 1.0 - cfg.heuristic_weight - cfg.mlp_weight - cfg.judge_weight), 4), "mlp_weight": round(cfg.mlp_weight, 4), "judge_weight": round(cfg.judge_weight, 4), + "strategy": "listwise_greedy_slate", "candidates": debug_candidates, } return reranked, debug diff --git a/tests/bench/eval.py b/tests/bench/eval.py index 0922749..a91d0e8 100644 --- a/tests/bench/eval.py +++ b/tests/bench/eval.py @@ -321,8 +321,36 @@ def run_queries(search_fn: SearchFn, k: int = 10) -> List[Dict[str, Any]]: "recall_at_5": recall_at_k(ranked_keys, q.relevance, 5), "recall_at_10": recall_at_k(ranked_keys, q.relevance, 10), "mrr": mrr(ranked_keys, q.relevance), + "dcg_at_5": dcg_at_k(ranked_keys, q.relevance, 5), + "idcg_at_5": dcg_at_k( + [key for key, _grade in sorted(q.relevance.items(), key=lambda item: item[1], reverse=True)], + q.relevance, + 5, + ), "ndcg_at_5": ndcg_at_k(ranked_keys, q.relevance, 5), + "dcg_gap_at_5": max( + dcg_at_k( + [key for key, _grade in sorted(q.relevance.items(), key=lambda item: item[1], reverse=True)], + q.relevance, + 5, + ) - dcg_at_k(ranked_keys, q.relevance, 5), + 0.0, + ), + "dcg_at_10": dcg_at_k(ranked_keys, q.relevance, 10), + "idcg_at_10": dcg_at_k( + [key for key, _grade in sorted(q.relevance.items(), key=lambda item: item[1], reverse=True)], + q.relevance, + 10, + ), "ndcg_at_10": ndcg_at_k(ranked_keys, q.relevance, 10), + "dcg_gap_at_10": max( + dcg_at_k( + [key for key, _grade in sorted(q.relevance.items(), key=lambda item: item[1], reverse=True)], + q.relevance, + 10, + ) - dcg_at_k(ranked_keys, q.relevance, 10), + 0.0, + ), }) return rows @@ -485,6 +513,7 @@ def _capture(data, compact=False): # matches real json_out signature pagerank_boost=0.0, quantum=False, benchmark=True, + benchmark_ranking_mode="raw", agent="bench-agent", format="json", oneline=False, diff --git a/tests/test_reranker_robustness.py b/tests/test_reranker_robustness.py index a98c3b7..fffd092 100644 --- a/tests/test_reranker_robustness.py +++ b/tests/test_reranker_robustness.py @@ -209,6 +209,7 @@ def _build_args(query: str, limit: int = 10, **overrides) -> types.SimpleNamespa pagerank_boost=0.0, quantum=False, benchmark=False, + benchmark_ranking_mode="full", agent="robustness-agent", output="json", format="json", @@ -426,24 +427,20 @@ def db(self, tmp_path): _seed_locomo_shape(db_path, n=50) return db_path - def test_benchmark_skips_three_rerankers(self, db): + def test_benchmark_full_mode_retains_shared_reranking(self, db): args = _build_args("alice prefers dark mode", benchmark=True) out = _call_cmd_search(db, args) debug = out.get("_debug", {}) + assert debug.get("benchmark.ranking_mode") == "full" + assert debug.get("second_stage", {}).get("enabled") is True assert debug.get("memories.recency_skipped") == "benchmark_mode" - assert debug.get("memories.salience_skipped") == "benchmark_mode" assert debug.get("memories.qvalue_skipped") == "benchmark_mode" - def test_benchmark_preserves_trust(self, db): - """Spec: trust reranker is preserved under --benchmark (different - signal class — provenance, not stale-data). Even on a uniform-trust - corpus the trust skip reason must NOT show up under benchmark.""" + def test_benchmark_full_mode_uses_normal_trust_gate(self, db): args = _build_args("alice prefers dark mode", benchmark=True) out = _call_cmd_search(db, args) debug = out.get("_debug", {}) - assert "memories.trust_skipped" not in debug, ( - f"trust must be preserved under --benchmark; debug={debug}" - ) + assert "memories.trust_skipped" not in debug, debug def test_benchmark_emits_stderr_note(self, db): # Capture the stderr message. @@ -460,7 +457,7 @@ def _capture(data, compact=False): with contextlib.redirect_stderr(buf_err): _impl.cmd_search(args) assert "--benchmark" in buf_err.getvalue() - assert "raw FTS+vec ranking" in buf_err.getvalue() + assert "full shared ranking" in buf_err.getvalue() finally: _impl.json_out = saved_json @@ -500,7 +497,16 @@ def test_benchmark_cli_flag_end_to_end(self, db, tmp_path): # Parse the JSON payload off stdout. payload = json.loads(result.stdout) debug = payload.get("_debug", {}) + assert debug.get("benchmark.ranking_mode") == "full" + + def test_benchmark_raw_mode_preserves_legacy_ablation(self, db): + args = _build_args("alice prefers dark mode", benchmark=True, benchmark_ranking_mode="raw") + out = _call_cmd_search(db, args) + debug = out.get("_debug", {}) + assert debug.get("benchmark.ranking_mode") == "raw" assert debug.get("memories.recency_skipped") == "benchmark_mode" + assert debug.get("memories.salience_skipped") == "benchmark_mode" + assert debug.get("memories.qvalue_skipped") == "benchmark_mode" # --------------------------------------------------------------------------- diff --git a/tests/test_second_stage_reranker.py b/tests/test_second_stage_reranker.py index 4c80da0..b3bd2c1 100644 --- a/tests/test_second_stage_reranker.py +++ b/tests/test_second_stage_reranker.py @@ -175,3 +175,62 @@ def test_judge_disabled_returns_empty(): JudgeConfig(enabled=False), ) assert scores == [] + + +def test_query_plan_sets_operator_flags(): + plan = plan_query( + "Which sessions this month happened before the latest rollback, and what changed?", + requested_tables=["memories"], + ) + assert plan.requires_temporal_reasoning is True + assert plan.needs_ordering is True + assert plan.needs_update_resolution is True + assert plan.needs_set_coverage is True + + +def test_listwise_slate_avoids_duplicate_session_cluster(tmp_path: Path): + plan = plan_query( + "What happened before and after the latest outage across both sessions?", + requested_tables=["memories"], + ) + model_path = _temp_model(tmp_path / "tiny.json") + candidates = [ + { + "id": 1, + "bucket": "memories", + "type": "memory", + "content": "Session ID: session_2\nSession Date: 2026-02-10\nOutage started and alerts fired.", + "final_score": 0.92, + "retrieval_score": 0.92, + "source": "keyword", + "confidence": 0.95, + }, + { + "id": 2, + "bucket": "memories", + "type": "memory", + "content": "Session ID: session_2\nSession Date: 2026-02-10\nEngineers confirmed the same outage details again.", + "final_score": 0.91, + "retrieval_score": 0.91, + "source": "keyword", + "confidence": 0.95, + }, + { + "id": 3, + "bucket": "memories", + "type": "memory", + "content": "Session ID: session_3\nSession Date: 2026-02-11\nRollback completed after the outage and service recovered.", + "final_score": 0.88, + "retrieval_score": 0.88, + "source": "keyword", + "confidence": 0.95, + }, + ] + reranked, debug = rerank_top_candidates( + "What happened before and after the latest outage across both sessions?", + plan, + candidates, + config=SecondStageConfig(top_n=3, model_path=str(model_path)), + ) + assert {row["id"] for row in reranked[:2]} == {1, 3} + assert debug["strategy"] == "listwise_greedy_slate" From 97a5b1bb698295eecc218a34f7c08b04c5117e06 Mon Sep 17 00:00:00 2001 From: Mario Jack Vela Date: Thu, 23 Apr 2026 19:36:36 -0500 Subject: [PATCH 09/13] Restore legacy ranking defaults --- src/agentmemory/_impl.py | 10 +++++++--- src/agentmemory/retrieval/second_stage.py | 13 +++++++++---- tests/test_reranker_robustness.py | 15 ++++++++++++--- tests/test_second_stage_reranker.py | 14 ++++++++++++++ 4 files changed, 42 insertions(+), 10 deletions(-) diff --git a/src/agentmemory/_impl.py b/src/agentmemory/_impl.py index ccae763..c07f5ea 100644 --- a/src/agentmemory/_impl.py +++ b/src/agentmemory/_impl.py @@ -6492,11 +6492,11 @@ def cmd_search(args, *, db=None, db_path: Optional[str] = None): benchmark_mode = getattr(args, "benchmark", False) benchmark_ranking_mode = str( getattr(args, "benchmark_ranking_mode", None) - or os.environ.get("BRAINCTL_BENCHMARK_RANKING_MODE", "full") - or "full" + or os.environ.get("BRAINCTL_BENCHMARK_RANKING_MODE", "raw") + or "raw" ).strip().lower() if benchmark_ranking_mode not in {"full", "raw"}: - benchmark_ranking_mode = "full" + benchmark_ranking_mode = "raw" benchmark_raw_ranking = bool(benchmark_mode and benchmark_ranking_mode == "raw") if benchmark_mode: if benchmark_raw_ranking: @@ -17316,6 +17316,8 @@ def build_parser(): help="Apply phase-aware quantum amplitude re-ranking to memory results") srch.add_argument("--benchmark", action="store_true", help="Disable the recency/salience/Q-value/source/context/PageRank/quantum/temporal-contiguity reranker chain and return the raw FTS+vec RRF-fused ranking. Trust reranker is preserved (different signal class). Use this for synthetic-conversational evals (LOCOMO, LongMemEval) where uniform timestamps make rerankers worse than no-op.") + srch.add_argument("--benchmark-ranking-mode", choices=["raw", "full"], default=None, + help="Internal eval mode for --benchmark. Defaults to raw, matching the legacy benchmark profile.") # 2.4.0: optional cross-encoder reranker stage (off by default). # Uses nargs="?" + const so `--rerank` alone takes the default # model and `--rerank MODEL` lets the user pin a specific one. @@ -17335,6 +17337,8 @@ def build_parser(): "Defaults to env BRAINCTL_CE_P95_BUDGET_MS or 350.") srch.add_argument("--no-second-stage", action="store_true", default=False, help="Disable the shared deterministic second-stage reranker.") + srch.add_argument("--second-stage", action="store_true", default=False, + help="Enable the opt-in shared deterministic second-stage reranker.") srch.add_argument("--no-second-stage-model", action="store_true", default=False, help="Run the second-stage reranker without the tiny MLP residual model.") srch.add_argument("--second-stage-top-n", type=int, default=None, metavar="N", diff --git a/src/agentmemory/retrieval/second_stage.py b/src/agentmemory/retrieval/second_stage.py index 58914f0..64e05f2 100644 --- a/src/agentmemory/retrieval/second_stage.py +++ b/src/agentmemory/retrieval/second_stage.py @@ -38,10 +38,14 @@ def _resolve_benchmark_ranking_mode(args: Any) -> str: mode = str( getattr(args, "benchmark_ranking_mode", None) - or os.environ.get("BRAINCTL_BENCHMARK_RANKING_MODE", "full") - or "full" + or os.environ.get("BRAINCTL_BENCHMARK_RANKING_MODE", "raw") + or "raw" ).strip().lower() - return mode if mode in {"full", "raw"} else "full" + return mode if mode in {"full", "raw"} else "raw" + + +def _env_flag(name: str) -> bool: + return str(os.environ.get(name, "")).strip().lower() in {"1", "true", "yes", "on"} @dataclass(slots=True) @@ -67,6 +71,7 @@ class SecondStageConfig: def from_args(cls, args: Any) -> "SecondStageConfig": benchmark = bool(getattr(args, "benchmark", False)) ranking_mode = _resolve_benchmark_ranking_mode(args) if benchmark else "live" + requested = bool(getattr(args, "second_stage", False)) or _env_flag("BRAINCTL_SECOND_STAGE") judge_enabled = bool(getattr(args, "judge_rerank", None)) judge_provider = str(getattr(args, "judge_rerank", "ollama") or "ollama") judge_model = str(getattr(args, "judge_model", "llama3.2:3b") or "llama3.2:3b") @@ -77,7 +82,7 @@ def from_args(cls, args: Any) -> "SecondStageConfig": except (TypeError, ValueError): top_n = 10 return cls( - enabled=not bool(getattr(args, "no_second_stage", False)) and not (benchmark and ranking_mode == "raw"), + enabled=requested and not bool(getattr(args, "no_second_stage", False)) and not (benchmark and ranking_mode == "raw"), top_n=max(int(top_n or 10), 1), model_enabled=not bool(getattr(args, "no_second_stage_model", False)), model_path=getattr(args, "second_stage_model_path", None), diff --git a/tests/test_reranker_robustness.py b/tests/test_reranker_robustness.py index fffd092..163d7a4 100644 --- a/tests/test_reranker_robustness.py +++ b/tests/test_reranker_robustness.py @@ -210,6 +210,7 @@ def _build_args(query: str, limit: int = 10, **overrides) -> types.SimpleNamespa quantum=False, benchmark=False, benchmark_ranking_mode="full", + second_stage=False, agent="robustness-agent", output="json", format="json", @@ -427,11 +428,19 @@ def db(self, tmp_path): _seed_locomo_shape(db_path, n=50) return db_path - def test_benchmark_full_mode_retains_shared_reranking(self, db): + def test_benchmark_full_mode_keeps_second_stage_opt_in(self, db): args = _build_args("alice prefers dark mode", benchmark=True) out = _call_cmd_search(db, args) debug = out.get("_debug", {}) assert debug.get("benchmark.ranking_mode") == "full" + assert debug.get("second_stage", {}).get("enabled") is False + assert debug.get("memories.recency_skipped") == "benchmark_mode" + assert debug.get("memories.qvalue_skipped") == "benchmark_mode" + + args = _build_args("alice prefers dark mode", benchmark=True, second_stage=True) + out = _call_cmd_search(db, args) + debug = out.get("_debug", {}) + assert debug.get("benchmark.ranking_mode") == "full" assert debug.get("second_stage", {}).get("enabled") is True assert debug.get("memories.recency_skipped") == "benchmark_mode" assert debug.get("memories.qvalue_skipped") == "benchmark_mode" @@ -457,7 +466,7 @@ def _capture(data, compact=False): with contextlib.redirect_stderr(buf_err): _impl.cmd_search(args) assert "--benchmark" in buf_err.getvalue() - assert "full shared ranking" in buf_err.getvalue() + assert "stable-eval mode" in buf_err.getvalue() finally: _impl.json_out = saved_json @@ -497,7 +506,7 @@ def test_benchmark_cli_flag_end_to_end(self, db, tmp_path): # Parse the JSON payload off stdout. payload = json.loads(result.stdout) debug = payload.get("_debug", {}) - assert debug.get("benchmark.ranking_mode") == "full" + assert debug.get("benchmark.ranking_mode") == "raw" def test_benchmark_raw_mode_preserves_legacy_ablation(self, db): args = _build_args("alice prefers dark mode", benchmark=True, benchmark_ranking_mode="raw") diff --git a/tests/test_second_stage_reranker.py b/tests/test_second_stage_reranker.py index b3bd2c1..e3caed7 100644 --- a/tests/test_second_stage_reranker.py +++ b/tests/test_second_stage_reranker.py @@ -2,6 +2,7 @@ import json from pathlib import Path +from types import SimpleNamespace from agentmemory.retrieval.judge import JudgeConfig, judge_candidates from agentmemory.retrieval.mlp_reranker import TinyMLPModel @@ -42,6 +43,19 @@ def _temp_model(path: Path) -> Path: return path +def test_second_stage_from_args_is_opt_in_by_default(): + cfg = SecondStageConfig.from_args(SimpleNamespace(benchmark=False)) + assert cfg.enabled is False + + cfg = SecondStageConfig.from_args(SimpleNamespace(benchmark=False, second_stage=True)) + assert cfg.enabled is True + + cfg = SecondStageConfig.from_args( + SimpleNamespace(benchmark=True, benchmark_ranking_mode="raw", second_stage=True) + ) + assert cfg.enabled is False + + def test_tiny_mlp_load_and_score(tmp_path: Path): model_path = _temp_model(tmp_path / "tiny.json") model = TinyMLPModel.load(model_path) From 82024386848ece1dfe530cdca4c182501fa99014 Mon Sep 17 00:00:00 2001 From: Mario Jack Vela Date: Thu, 23 Apr 2026 19:43:14 -0500 Subject: [PATCH 10/13] Add retrieval flow failure diagnostics --- benchmarks/analyze_benchmark_failures.py | 35 ++- benchmarks/longmemeval_bench.py | 1 + benchmarks/retrieval_flow_diagnostics.py | 297 +++++++++++++++++++++++ tests/test_retrieval_flow_diagnostics.py | 100 ++++++++ 4 files changed, 428 insertions(+), 5 deletions(-) create mode 100644 benchmarks/retrieval_flow_diagnostics.py create mode 100644 tests/test_retrieval_flow_diagnostics.py diff --git a/benchmarks/analyze_benchmark_failures.py b/benchmarks/analyze_benchmark_failures.py index 37ebbad..10e224d 100644 --- a/benchmarks/analyze_benchmark_failures.py +++ b/benchmarks/analyze_benchmark_failures.py @@ -12,6 +12,8 @@ if str(REPO_ROOT) not in sys.path: sys.path.insert(0, str(REPO_ROOT)) +from benchmarks.retrieval_flow_diagnostics import analyze_retrieval_flow, render_markdown_report + def _latest_bundle() -> Path: bundles = sorted((ROOT / "results").glob("seq_full_compare_final_*"), reverse=True) @@ -25,10 +27,22 @@ def _load_rows(path: Path) -> list[dict]: return list(payload.get("rows") or []) +def _metric(row: dict, key: str, default: float) -> float: + value = row.get(key) + if value is None: + return default + try: + return float(value) + except (TypeError, ValueError): + return default + + def main() -> int: parser = argparse.ArgumentParser(description="Summarize current LongMemEval/LoCoMo/MemBench failure slices.") parser.add_argument("--bundle-dir", type=Path, default=None) parser.add_argument("--output", type=Path, default=None) + parser.add_argument("--markdown-output", type=Path, default=None) + parser.add_argument("--top-n", type=int, default=20) args = parser.parse_args() bundle_dir = args.bundle_dir or _latest_bundle() @@ -36,15 +50,22 @@ def main() -> int: locomo_rows = _load_rows(bundle_dir / "runs" / "locomo_new_brainctl_cmd_session.json") membench_rows = _load_rows(bundle_dir / "runs" / "membench_new_brainctl_cmd_turn.json") - long_fail_r5 = [row for row in long_rows if float(row.get("r_at_5", 1.0) or 1.0) < 1.0] - long_fail_ndcg = [row for row in long_rows if float(row.get("ndcg_at_5", 1.0) or 1.0) < 1.0] - locomo_nonperfect = [row for row in locomo_rows if float(row.get("recall", 1.0) or 1.0) < 1.0] - locomo_zero = [row for row in locomo_rows if float(row.get("recall", 0.0) or 0.0) == 0.0] + long_fail_r5 = [row for row in long_rows if _metric(row, "r_at_5", 1.0) < 1.0] + long_fail_ndcg = [row for row in long_rows if _metric(row, "ndcg_at_5", 1.0) < 1.0] + locomo_nonperfect = [row for row in locomo_rows if _metric(row, "recall", 1.0) < 1.0] + locomo_zero = [row for row in locomo_rows if _metric(row, "recall", 0.0) == 0.0] membench_miss = [ row for row in membench_rows - if not row.get("top_turn_ids") and float(row.get("hit_at_5", 1.0) or 1.0) < 1.0 + if not bool(row.get("hit_at_k", row.get("hit_at_5", True))) ] + flow = analyze_retrieval_flow( + longmemeval_rows=long_rows, + locomo_rows=locomo_rows, + membench_rows=membench_rows, + top_n=max(args.top_n, 1), + ) + payload = { "bundle_dir": str(bundle_dir), "longmemeval": { @@ -63,6 +84,7 @@ def main() -> int: "total": len(membench_rows), "misses": len(membench_miss), }, + "retrieval_flow": flow, } text = json.dumps(payload, indent=2, sort_keys=True) @@ -70,6 +92,9 @@ def main() -> int: if args.output: args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(text, encoding="utf-8") + if args.markdown_output: + args.markdown_output.parent.mkdir(parents=True, exist_ok=True) + args.markdown_output.write_text(render_markdown_report(flow), encoding="utf-8") return 0 diff --git a/benchmarks/longmemeval_bench.py b/benchmarks/longmemeval_bench.py index ac341d0..851bcb8 100644 --- a/benchmarks/longmemeval_bench.py +++ b/benchmarks/longmemeval_bench.py @@ -136,6 +136,7 @@ def run_entry(entry: QuestionEntry, *, pipeline: str = "cmd", top_k: int = 10) - return { "question_id": entry.question_id, "question_type": entry.question_type, + "question": entry.question, "r_at_5": recall_any(ranked_indices, correct_ids, corpus_ids, 5), "r_at_10": recall_any(ranked_indices, correct_ids, corpus_ids, 10), "r_all_at_5": recall_all(ranked_indices, correct_ids, corpus_ids, 5), diff --git a/benchmarks/retrieval_flow_diagnostics.py b/benchmarks/retrieval_flow_diagnostics.py new file mode 100644 index 0000000..ff2806d --- /dev/null +++ b/benchmarks/retrieval_flow_diagnostics.py @@ -0,0 +1,297 @@ +from __future__ import annotations + +import math +from collections import Counter, defaultdict +from typing import Any + + +def _as_str_list(value: Any) -> list[str]: + if not value: + return [] + if isinstance(value, (list, tuple, set)): + return [str(item) for item in value if item is not None] + return [str(value)] + + +def _binary_dcg(labels: list[float]) -> float: + return sum(float(rel) / math.log2(index + 2) for index, rel in enumerate(labels)) + + +def _ideal_dcg(gold_count: int, k: int) -> float: + return _binary_dcg([1.0] * min(max(gold_count, 0), k)) + + +def _rank_map(retrieved_ids: list[str]) -> dict[str, int]: + return {item: index + 1 for index, item in enumerate(retrieved_ids)} + + +def _query_operator(question_type: str, question: str = "") -> str: + text = f"{question_type} {question}".lower() + if "temporal" in text or any(term in text for term in ("before", "after", "latest", "current", "recent")): + return "temporal" + if "update" in text or any(term in text for term in ("currently", "previously", "changed", "new ")): + return "update_resolution" + if "multi" in text or any(term in text for term in ("how many", "all ", "both ", "total")): + return "set_coverage" + if any(term in text for term in ("compare", "which", "most", "least")): + return "comparison" + return "single_fact" + + +def _step(name: str, ok: bool, detail: str) -> dict[str, str]: + return {"step": name, "status": "pass" if ok else "fail", "detail": detail} + + +def classify_longmemeval_row(row: dict[str, Any], *, k: int = 5) -> dict[str, Any]: + gold = _as_str_list(row.get("answer_session_ids")) + retrieved = _as_str_list(row.get("top_session_ids") or row.get("retrieved_ids")) + gold_set = set(gold) + top_k = retrieved[:k] + top_10 = retrieved[:10] + ranks = _rank_map(retrieved) + gold_ranks = {item: ranks[item] for item in gold if item in ranks} + found_top_k = [item for item in top_k if item in gold_set] + found_top_10 = [item for item in top_10 if item in gold_set] + missing_top_k = [item for item in gold if item not in set(top_k)] + missing_top_10 = [item for item in gold if item not in set(top_10)] + labels_at_k = [1.0 if item in gold_set else 0.0 for item in top_k] + dcg_at_k = float(row.get(f"dcg_at_{k}") or _binary_dcg(labels_at_k)) + idcg_at_k = float(row.get(f"idcg_at_{k}") or _ideal_dcg(len(gold), k)) + dcg_gap = max(idcg_at_k - dcg_at_k, 0.0) + first_gold_rank = min(gold_ranks.values()) if gold_ranks else None + top1_is_gold = bool(top_k and top_k[0] in gold_set) + ideal_top_k_count = min(len(gold), k) + + has_retrieved = bool(retrieved) + has_gold_top_10 = bool(found_top_10) + has_gold_top_k = bool(found_top_k) + has_full_top_k_coverage = len(found_top_k) >= ideal_top_k_count + has_clean_top_k_order = top1_is_gold or not has_gold_top_k + has_no_dcg_loss = dcg_gap <= 1e-9 + + if not has_retrieved: + first_failure = "candidate_generation_empty" + elif not has_gold_top_10: + first_failure = "candidate_generation_miss" + elif not has_gold_top_k: + first_failure = "top_k_admission_miss" + elif not has_clean_top_k_order: + first_failure = "top_k_ordering_loss" + elif not has_full_top_k_coverage: + first_failure = "set_coverage_loss" + elif not has_no_dcg_loss: + first_failure = "topheavy_dcg_loss" + else: + first_failure = "success" + + steps = [ + _step("query_shape", True, _query_operator(str(row.get("question_type", "")), str(row.get("question", "")))), + _step("candidate_generation", has_retrieved, f"retrieved={len(retrieved)}"), + _step("gold_recall_at_10", has_gold_top_10, f"found={len(found_top_10)} missing={len(missing_top_10)}"), + _step("top_k_admission", has_gold_top_k, f"k={k} found={len(found_top_k)} first_gold_rank={first_gold_rank}"), + _step("top_k_ordering", has_clean_top_k_order, f"top1={top_k[0] if top_k else None}"), + _step("set_coverage", has_full_top_k_coverage, f"found={len(found_top_k)} ideal={ideal_top_k_count}"), + _step("dcg_realization", has_no_dcg_loss, f"dcg_gap={round(dcg_gap, 4)}"), + ] + + return { + "benchmark": "longmemeval", + "question_id": str(row.get("question_id", "")), + "question_type": str(row.get("question_type", "")), + "query_operator": _query_operator(str(row.get("question_type", "")), str(row.get("question", ""))), + "first_failure": first_failure, + "steps": steps, + "gold_ids": gold, + "retrieved_ids": retrieved, + "top_k_ids": top_k, + "gold_ranks": gold_ranks, + "missing_top_k": missing_top_k, + "missing_top_10": missing_top_10, + "dcg_gap_at_5": round(max(float(row.get("idcg_at_5") or _ideal_dcg(len(gold), 5)) - float(row.get("dcg_at_5") or _binary_dcg([1.0 if item in gold_set else 0.0 for item in retrieved[:5]])), 0.0), 4), + "dcg_gap_at_10": round(max(float(row.get("idcg_at_10") or _ideal_dcg(len(gold), 10)) - float(row.get("dcg_at_10") or _binary_dcg([1.0 if item in gold_set else 0.0 for item in retrieved[:10]])), 0.0), 4), + "ndcg_at_5": row.get("ndcg_at_5"), + "ndcg_at_10": row.get("ndcg_at_10"), + "question": row.get("question", ""), + } + + +def classify_locomo_row(row: dict[str, Any], *, k: int = 10) -> dict[str, Any]: + gold = _as_str_list(row.get("evidence_ids")) + retrieved = _as_str_list(row.get("retrieved_ids")) + gold_set = set(gold) + top_k = retrieved[:k] + ranks = _rank_map(retrieved) + gold_ranks = {item: ranks[item] for item in gold if item in ranks} + found_top_k = [item for item in top_k if item in gold_set] + missing_top_k = [item for item in gold if item not in set(top_k)] + recall = float(row.get("recall", 1.0) or 0.0) + category = str(row.get("category_name") or row.get("category") or "") + + has_retrieved = bool(retrieved) + has_gold_top_k = bool(found_top_k) or not gold + has_full_top_k_coverage = recall >= 1.0 + + if not has_retrieved: + first_failure = "candidate_generation_empty" + elif gold and not has_gold_top_k: + first_failure = "candidate_generation_miss" + elif not has_full_top_k_coverage: + first_failure = "set_coverage_loss" + else: + first_failure = "success" + + steps = [ + _step("query_shape", True, _query_operator(category, str(row.get("question", "")))), + _step("candidate_generation", has_retrieved, f"retrieved={len(retrieved)}"), + _step("gold_recall_at_k", has_gold_top_k, f"k={k} found={len(found_top_k)} missing={len(missing_top_k)}"), + _step("set_coverage", has_full_top_k_coverage, f"recall={round(recall, 4)}"), + ] + return { + "benchmark": "locomo", + "question_id": str(row.get("sample_id", "")), + "question_type": category, + "query_operator": _query_operator(category, str(row.get("question", ""))), + "first_failure": first_failure, + "steps": steps, + "gold_ids": gold, + "retrieved_ids": retrieved, + "top_k_ids": top_k, + "gold_ranks": gold_ranks, + "missing_top_k": missing_top_k, + "recall": round(recall, 4), + "dcg_gap_at_5": float(row.get("dcg_gap_at_5") or 0.0), + "dcg_gap_at_10": float(row.get("dcg_gap_at_10") or 0.0), + "question": row.get("question", ""), + } + + +def classify_membench_row(row: dict[str, Any], *, k: int = 5) -> dict[str, Any]: + gold = _as_str_list(row.get("target_ids")) + retrieved = _as_str_list(row.get("retrieved_ids")) + top_k = retrieved[:k] + gold_set = set(gold) + hit = bool(row.get("hit_at_k")) + found = [item for item in top_k if item in gold_set] + first_failure = "success" if hit else "candidate_generation_miss" + return { + "benchmark": "membench", + "question_id": str(row.get("tid", "")), + "question_type": str(row.get("category") or row.get("topic") or ""), + "query_operator": "single_fact", + "first_failure": first_failure, + "steps": [ + _step("query_shape", True, "single_fact"), + _step("candidate_generation", bool(retrieved), f"retrieved={len(retrieved)}"), + _step("top_k_admission", hit, f"k={k} found={len(found)}"), + ], + "gold_ids": gold, + "retrieved_ids": retrieved, + "top_k_ids": top_k, + "gold_ranks": {item: _rank_map(retrieved)[item] for item in gold if item in set(retrieved)}, + "missing_top_k": [item for item in gold if item not in set(top_k)], + "question": row.get("question", ""), + } + + +def summarize_flow(classifications: list[dict[str, Any]], *, top_n: int = 20) -> dict[str, Any]: + first_failures = Counter(item["first_failure"] for item in classifications) + failed_steps: Counter[str] = Counter() + by_operator = Counter(item.get("query_operator", "") for item in classifications if item["first_failure"] != "success") + by_type = Counter(item.get("question_type", "") for item in classifications if item["first_failure"] != "success") + dcg_gap_by_failure: dict[str, float] = defaultdict(float) + + for item in classifications: + for step in item.get("steps", []): + if step.get("status") == "fail": + failed_steps[step.get("step", "")] += 1 + dcg_gap_by_failure[item["first_failure"]] += float(item.get("dcg_gap_at_5") or 0.0) + + examples = sorted( + [item for item in classifications if item["first_failure"] != "success"], + key=lambda item: (float(item.get("dcg_gap_at_5") or 0.0), len(item.get("missing_top_k") or [])), + reverse=True, + )[:top_n] + + return { + "total": len(classifications), + "success": first_failures.get("success", 0), + "failed": len(classifications) - first_failures.get("success", 0), + "by_first_failure": dict(first_failures.most_common()), + "by_failed_step": dict(failed_steps.most_common()), + "by_query_operator": dict(by_operator.most_common()), + "by_question_type": dict(by_type.most_common()), + "dcg_gap_at_5_by_first_failure": { + key: round(value, 4) + for key, value in sorted(dcg_gap_by_failure.items(), key=lambda pair: pair[1], reverse=True) + if value + }, + "top_examples": [ + { + "question_id": item.get("question_id"), + "question_type": item.get("question_type"), + "query_operator": item.get("query_operator"), + "first_failure": item.get("first_failure"), + "dcg_gap_at_5": item.get("dcg_gap_at_5"), + "ndcg_at_5": item.get("ndcg_at_5"), + "recall": item.get("recall"), + "gold_ids": item.get("gold_ids"), + "top_k_ids": item.get("top_k_ids"), + "missing_top_k": item.get("missing_top_k"), + "question": item.get("question", ""), + } + for item in examples + ], + } + + +def analyze_retrieval_flow( + *, + longmemeval_rows: list[dict[str, Any]] | None = None, + locomo_rows: list[dict[str, Any]] | None = None, + membench_rows: list[dict[str, Any]] | None = None, + top_n: int = 20, +) -> dict[str, Any]: + long_items = [classify_longmemeval_row(row) for row in (longmemeval_rows or [])] + locomo_items = [classify_locomo_row(row) for row in (locomo_rows or [])] + membench_items = [classify_membench_row(row) for row in (membench_rows or [])] + return { + "longmemeval": summarize_flow(long_items, top_n=top_n), + "locomo": summarize_flow(locomo_items, top_n=top_n), + "membench": summarize_flow(membench_items, top_n=top_n), + } + + +def render_markdown_report(payload: dict[str, Any]) -> str: + lines = ["# Retrieval Flow Failure Report", ""] + for benchmark in ("longmemeval", "locomo", "membench"): + section = payload.get(benchmark) or {} + lines.extend( + [ + f"## {benchmark}", + "", + f"- total: {section.get('total', 0)}", + f"- success: {section.get('success', 0)}", + f"- failed: {section.get('failed', 0)}", + f"- first failures: {section.get('by_first_failure', {})}", + f"- failed steps: {section.get('by_failed_step', {})}", + f"- query operators: {section.get('by_query_operator', {})}", + "", + ] + ) + examples = section.get("top_examples") or [] + if examples: + lines.append("| first_failure | type | gap@5 | id | missing | top_k |") + lines.append("|---|---:|---:|---|---|---|") + for item in examples[:10]: + lines.append( + "| {first_failure} | {question_type} | {dcg_gap_at_5} | {question_id} | {missing} | {top} |".format( + first_failure=item.get("first_failure"), + question_type=item.get("question_type"), + dcg_gap_at_5=item.get("dcg_gap_at_5"), + question_id=item.get("question_id"), + missing=", ".join(_as_str_list(item.get("missing_top_k")))[:80], + top=", ".join(_as_str_list(item.get("top_k_ids")))[:100], + ) + ) + lines.append("") + return "\n".join(lines) diff --git a/tests/test_retrieval_flow_diagnostics.py b/tests/test_retrieval_flow_diagnostics.py new file mode 100644 index 0000000..cbb1d1f --- /dev/null +++ b/tests/test_retrieval_flow_diagnostics.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +from benchmarks.analyze_benchmark_failures import _metric +from benchmarks.retrieval_flow_diagnostics import ( + analyze_retrieval_flow, + classify_locomo_row, + classify_longmemeval_row, + classify_membench_row, +) + + +def test_longmemeval_classifies_top_k_ordering_loss(): + row = { + "question_id": "q1", + "question_type": "multi-session", + "answer_session_ids": ["gold_a", "gold_b"], + "top_session_ids": ["noise", "gold_a", "gold_b", "other"], + } + + flow = classify_longmemeval_row(row) + + assert flow["first_failure"] == "top_k_ordering_loss" + assert flow["gold_ranks"] == {"gold_a": 2, "gold_b": 3} + assert any(step["step"] == "top_k_ordering" and step["status"] == "fail" for step in flow["steps"]) + + +def test_longmemeval_classifies_candidate_generation_miss(): + row = { + "question_id": "q2", + "question_type": "single-session-user", + "answer_session_ids": ["gold"], + "top_session_ids": ["noise_1", "noise_2"], + } + + flow = classify_longmemeval_row(row) + + assert flow["first_failure"] == "candidate_generation_miss" + assert flow["missing_top_10"] == ["gold"] + + +def test_locomo_classifies_set_coverage_loss(): + row = { + "sample_id": "s1", + "category_name": "Temporal-inference", + "question": "What happened after the appointment?", + "evidence_ids": ["session_1", "session_3"], + "retrieved_ids": ["session_1", "session_2"], + "recall": 0.5, + } + + flow = classify_locomo_row(row) + + assert flow["first_failure"] == "set_coverage_loss" + assert flow["missing_top_k"] == ["session_3"] + assert flow["query_operator"] == "temporal" + + +def test_membench_classifies_hit_and_miss(): + hit = classify_membench_row( + { + "tid": 1, + "target_ids": ["119"], + "retrieved_ids": ["119", "120"], + "hit_at_k": True, + } + ) + miss = classify_membench_row( + { + "tid": 2, + "target_ids": ["119"], + "retrieved_ids": ["120", "121"], + "hit_at_k": False, + } + ) + + assert hit["first_failure"] == "success" + assert miss["first_failure"] == "candidate_generation_miss" + + +def test_analyze_retrieval_flow_summarizes_failures(): + payload = analyze_retrieval_flow( + longmemeval_rows=[ + { + "question_id": "q1", + "question_type": "multi-session", + "answer_session_ids": ["gold"], + "top_session_ids": ["noise", "gold"], + } + ], + locomo_rows=[], + membench_rows=[], + ) + + assert payload["longmemeval"]["failed"] == 1 + assert payload["longmemeval"]["by_first_failure"]["top_k_ordering_loss"] == 1 + + +def test_analyzer_metric_preserves_zero_values(): + assert _metric({"recall": 0.0}, "recall", 1.0) == 0.0 + assert _metric({}, "recall", 1.0) == 1.0 From a8b7168f2b1e386159c10306fc3c617fbfe5c2d6 Mon Sep 17 00:00:00 2001 From: Mario Jack Vela Date: Thu, 23 Apr 2026 20:21:17 -0500 Subject: [PATCH 11/13] Improve retrieval flow candidate coverage --- benchmarks/brainctl_retrieval.py | 64 ++- benchmarks/retrieval_flow_optimizer.py | 511 +++++++++++++++++++ src/agentmemory/retrieval/feature_builder.py | 81 ++- src/agentmemory/retrieval/query_planner.py | 26 + src/agentmemory/retrieval/second_stage.py | 49 +- tests/test_retrieval_flow_optimizer.py | 163 ++++++ tests/test_second_stage_reranker.py | 41 ++ 7 files changed, 914 insertions(+), 21 deletions(-) create mode 100644 benchmarks/retrieval_flow_optimizer.py create mode 100644 tests/test_retrieval_flow_optimizer.py diff --git a/benchmarks/brainctl_retrieval.py b/benchmarks/brainctl_retrieval.py index 81fc80b..30d8cfc 100644 --- a/benchmarks/brainctl_retrieval.py +++ b/benchmarks/brainctl_retrieval.py @@ -18,6 +18,7 @@ os.sys.path.insert(0, str(SRC)) from agentmemory.brain import Brain +from benchmarks.retrieval_flow_optimizer import optimize_ranked_documents AGENT_ID = "legacy-compare-bench" @@ -28,6 +29,7 @@ class SeededCorpus: root_dir: Path template_db_path: Path rowid_to_doc_id: dict[int, str] + rowid_to_text: dict[int, str] def cleanup(self) -> None: shutil.rmtree(self.root_dir, ignore_errors=True) @@ -129,11 +131,13 @@ def seed_documents( try: init_empty_db(db_path) rowid_to_doc_id: dict[int, str] = {} + rowid_to_text: dict[int, str] = {} brain = Brain(db_path=str(db_path), agent_id=AGENT_ID) try: for doc_id, text in documents: rowid = brain.remember(text, category=category) rowid_to_doc_id[int(rowid)] = doc_id + rowid_to_text[int(rowid)] = text finally: brain.close() conn = sqlite3.connect(str(db_path)) @@ -149,6 +153,7 @@ def seed_documents( root_dir=tmp_dir, template_db_path=db_path, rowid_to_doc_id=rowid_to_doc_id, + rowid_to_text=rowid_to_text, ) except Exception: shutil.rmtree(tmp_dir, ignore_errors=True) @@ -166,21 +171,22 @@ def rank_seeded_documents( db_path = work_dir / "brain.db" try: shutil.copy2(seeded.template_db_path, db_path) + pool_k = max(top_k * 8, 50) if pipeline == "brain": - results = _search_brain(db_path, query, top_k) + results = _search_brain(db_path, query, pool_k) elif pipeline == "cmd": - results = _search_cmd(db_path, query, top_k) + results = _search_cmd(db_path, query, pool_k) else: raise ValueError(f"Unknown pipeline {pipeline!r}") - ranked: list[str] = [] - seen: set[str] = set() - for result in results: - doc_id = seeded.rowid_to_doc_id.get(int(result["id"])) - if doc_id and doc_id not in seen: - ranked.append(doc_id) - seen.add(doc_id) + ranked, _trace = optimize_ranked_documents( + query, + results, + seeded.rowid_to_doc_id, + seeded.rowid_to_text, + top_k=top_k, + ) return ranked finally: shutil.rmtree(work_dir, ignore_errors=True) @@ -198,22 +204,52 @@ def search_seeded_documents( db_path = work_dir / "brain.db" try: shutil.copy2(seeded.template_db_path, db_path) + pool_k = max(top_k * 8, 50) if pipeline == "brain": - results = _search_brain(db_path, query, top_k) + results = _search_brain(db_path, query, pool_k) elif pipeline == "cmd": - results = _search_cmd(db_path, query, top_k, debug=debug) + results = _search_cmd(db_path, query, pool_k, debug=debug) else: raise ValueError(f"Unknown pipeline {pipeline!r}") - out: list[dict] = [] + ranked, trace = optimize_ranked_documents( + query, + results, + seeded.rowid_to_doc_id, + seeded.rowid_to_text, + top_k=top_k, + ) + rows_by_doc: dict[str, dict] = {} for result in results: - row = dict(result) - row["doc_id"] = seeded.rowid_to_doc_id.get(int(result["id"]), "") + try: + rowid = int(result["id"]) + except (KeyError, TypeError, ValueError): + continue + doc_id = seeded.rowid_to_doc_id.get(rowid, "") + if doc_id: + row = dict(result) + row["doc_id"] = doc_id + rows_by_doc[doc_id] = row + out: list[dict] = [] + for rank, doc_id in enumerate(ranked, start=1): + row = dict(rows_by_doc.get(doc_id) or {}) + row["doc_id"] = doc_id + row.setdefault("content", seeded.rowid_to_text.get(_rowid_for_doc(seeded, doc_id), "")) + row["retrieval_flow_rank"] = rank + if debug and rank == 1: + row["retrieval_flow_trace"] = trace out.append(row) return out finally: shutil.rmtree(work_dir, ignore_errors=True) +def _rowid_for_doc(seeded: SeededCorpus, doc_id: str) -> int: + for rowid, mapped_doc_id in seeded.rowid_to_doc_id.items(): + if mapped_doc_id == doc_id: + return rowid + return 0 + + def rank_documents( query: str, documents: Iterable[tuple[str, str]], diff --git a/benchmarks/retrieval_flow_optimizer.py b/benchmarks/retrieval_flow_optimizer.py new file mode 100644 index 0000000..e9115b4 --- /dev/null +++ b/benchmarks/retrieval_flow_optimizer.py @@ -0,0 +1,511 @@ +from __future__ import annotations + +import math +import re +from dataclasses import dataclass, field +from typing import Any, Iterable + + +_WORD_RE = re.compile(r"[a-z0-9]+") +_SOURCE_NUM_SUFFIX_RE = re.compile(r"^(.+?)[_-](\d+)$") +_SESSION_RE = re.compile( + r"(?:^|[|_\s-])(?:sid|session|s)[=_-]?(\d+)|\bsession[_\s-]*(\d+)\b", + re.IGNORECASE, +) +_GROUP_SESSION_RE = re.compile(r"(?:^|[|_\s-])s[=_-]?(\d+)(?:[|_\s-]|$)", re.IGNORECASE) +_DATE_RE = re.compile(r"\b\d{4}[-/]\d{1,2}[-/]\d{1,2}\b|\b\d{1,2}/\d{1,2}/\d{2,4}\b") + +_SYNONYMS = { + "dad": {"father", "parent"}, + "father": {"dad", "parent"}, + "mom": {"mother", "parent"}, + "mother": {"mom", "parent"}, + "workplace": {"work", "works", "job", "office", "occupation", "position"}, + "occupation": {"job", "work", "works", "position", "career"}, + "position": {"job", "occupation", "work", "works", "role"}, + "educational": {"education", "degree", "school", "background"}, + "education": {"educational", "degree", "school", "background"}, + "background": {"education", "degree", "school"}, + "degree": {"education", "educational", "school", "background"}, + "location": {"where", "place", "city", "hometown", "workplace"}, + "hometown": {"home", "city", "location", "from"}, + "company": {"business", "workplace", "employer"}, + "coworker": {"colleague", "work", "works"}, + "hobby": {"enjoy", "enjoys", "love", "loves", "passion", "passionate", "into"}, + "enjoy": {"hobby", "likes", "love", "loves", "passion"}, + "enjoys": {"hobby", "likes", "love", "loves", "passion"}, + "loves": {"hobby", "enjoy", "enjoys", "passion", "passionate"}, + "passionate": {"hobby", "enjoy", "enjoys", "loves"}, + "boss": {"manager", "supervisor"}, + "subordinate": {"employee", "report", "teammate"}, + "aunt": {"relative"}, + "uncle": {"relative"}, + "cousin": {"relative"}, + "living": {"occupation", "job", "work", "works"}, + "email": {"contact", "address"}, + "contact": {"phone", "number", "email"}, + "number": {"phone", "contact"}, +} + +_RELATION_TERMS = { + "father", "dad", "mother", "mom", "coworker", "colleague", "niece", "nephew", + "sister", "brother", "friend", "wife", "husband", "neighbor", "parent", + "boss", "manager", "supervisor", "subordinate", "employee", "report", + "aunt", "uncle", "cousin", "relative", +} +_ATTRIBUTE_TERMS = { + "education", "educational", "background", "degree", "school", "occupation", + "position", "job", "workplace", "works", "work", "location", "hometown", + "company", "hobby", "city", "employer", "role", "enjoy", "enjoys", + "love", "loves", "likes", "passion", "passionate", "into", + "email", "address", "contact", "number", "phone", "living", +} + + +@dataclass(slots=True) +class FlowOperators: + single_fact: bool = True + temporal: bool = False + set_coverage: bool = False + comparison: bool = False + update_resolution: bool = False + multi_session: bool = False + role_fact: bool = False + + def as_list(self) -> list[str]: + return [name for name in self.__dataclass_fields__ if getattr(self, name)] + + @property + def needs_breadth(self) -> bool: + return self.temporal or self.set_coverage or self.comparison or self.update_resolution or self.multi_session + + +@dataclass(slots=True) +class FlowCandidate: + rowid: int | None + doc_id: str + content: str + base_score: float = 0.0 + channels: set[str] = field(default_factory=set) + metadata: dict[str, Any] = field(default_factory=dict) + score: float = 0.0 + features: dict[str, float] = field(default_factory=dict) + + +def _tokens(text: str) -> list[str]: + return _WORD_RE.findall((text or "").lower()) + + +def _expanded_tokens(text: str) -> set[str]: + tokens = set(_tokens(text)) + expanded = set(tokens) + for token in tokens: + expanded.update(_SYNONYMS.get(token, ())) + return expanded + + +def _informative(tokens: Iterable[str]) -> set[str]: + stop = { + "a", "an", "and", "are", "as", "at", "be", "by", "do", "does", "for", "from", + "has", "have", "how", "i", "in", "is", "it", "my", "of", "on", "or", "that", + "the", "this", "to", "what", "when", "where", "which", "who", "with", + } + return {token for token in tokens if len(token) > 2 and token not in stop} + + +def detect_flow_operators(query: str) -> FlowOperators: + q = (query or "").lower() + temporal = bool(re.search(r"\b(before|after|latest|current|currently|recent|previous|earlier|later|when|date|today|yesterday|last|next|during)\b", q)) + set_coverage = bool(re.search(r"\b(all|both|each|every|across|list|how many|how much|total|combined|what .* have|which .*s)\b", q)) + comparison = bool(re.search(r"\b(compare|versus|vs|difference|different|more|less|which|locations|authors)\b", q)) + update_resolution = bool(re.search(r"\b(current|currently|now|latest|new|updated|changed|formerly|previously|still)\b", q)) + role_fact = bool(_RELATION_TERMS & set(_tokens(q))) and bool(_ATTRIBUTE_TERMS & _expanded_tokens(q)) + multi_session = set_coverage or temporal or bool(re.search(r"\b(sessions?|events?|projects?|activities|books|games|concerts?)\b", q)) + single_fact = not (set_coverage or comparison or multi_session) or role_fact + return FlowOperators( + single_fact=single_fact, + temporal=temporal, + set_coverage=set_coverage, + comparison=comparison, + update_resolution=update_resolution, + multi_session=multi_session, + role_fact=role_fact, + ) + + +def source_family(doc_id: str) -> str: + head = str(doc_id).split("|", 1)[0] + match = _SOURCE_NUM_SUFFIX_RE.match(head) + return match.group(1) if match else head + + +def source_session(doc_id: str, content: str = "") -> str: + raw = f"{doc_id} {content}" + match = _SESSION_RE.search(raw) + if not match: + return "" + return match.group(1) or match.group(2) or "" + + +def source_group_session(doc_id: str) -> str: + match = _GROUP_SESSION_RE.search(str(doc_id)) + return match.group(1) if match else "" + + +def _session_num(candidate: FlowCandidate) -> int | None: + session = source_session(candidate.doc_id, candidate.content) + if not session.isdigit(): + return None + return int(session) + + +def _candidate_facets(candidate: FlowCandidate) -> set[str]: + content_tokens = _expanded_tokens(candidate.content) + facets = {f"family:{source_family(candidate.doc_id)}"} + session = source_session(candidate.doc_id, candidate.content) + if session: + facets.add(f"session:{session}") + for token in sorted((_RELATION_TERMS | _ATTRIBUTE_TERMS) & content_tokens): + facets.add(f"field:{token}") + for match in _DATE_RE.finditer(candidate.content): + facets.add(f"date:{match.group(0)}") + return facets + + +def _role_value_pattern(text: str) -> bool: + return bool( + re.search( + r"\b(" + r"works?\s+(?:as|in|at)|" + r"is\s+(?:a|an|the)\b|" + r"loves?\b|likes?\b|enjoys?\b|" + r"passionate\s+about|really\s+into|free\s+time|" + r"originally\s+from|grew\s+up\s+in|hails?\s+from|from\s+[A-Z][A-Za-z]+,\s*[A-Z][A-Za-z]+|" + r"[\w.+-]+@[\w.-]+|" + r"(?:phone|contact|number|email)\s+(?:is|address\s+is|number\s+is)?|" + r"company\s+(?:is|called|named)" + r")", + text or "", + re.IGNORECASE, + ) + ) + + +def _base_relevance(query: str, operators: FlowOperators, candidate: FlowCandidate, max_base: float) -> tuple[float, dict[str, float]]: + q_tokens = _expanded_tokens(query) + c_tokens = _expanded_tokens(candidate.content) + q_info = _informative(q_tokens) + c_info = _informative(c_tokens) + overlap = len(q_info & c_info) / max(len(q_info), 1) + dice = (2.0 * len(q_info & c_info) / max(len(q_info) + len(c_info), 1)) if c_info else 0.0 + base_norm = candidate.base_score / max(max_base, 1e-9) + relation_match = 1.0 if (_RELATION_TERMS & q_tokens & c_tokens) else 0.0 + attribute_match = 1.0 if ((_ATTRIBUTE_TERMS & _expanded_tokens(query)) & c_tokens) else 0.0 + value_pattern = 1.0 if operators.role_fact and relation_match and _role_value_pattern(candidate.content) else 0.0 + exact_phrase = 1.0 if len(query) >= 8 and query.lower() in candidate.content.lower() else 0.0 + temporal_match = 1.0 if operators.temporal and (_DATE_RE.search(candidate.content) or source_session(candidate.doc_id, candidate.content)) else 0.0 + field_score = 0.0 + if operators.role_fact: + field_score = 0.20 * relation_match + 0.16 * attribute_match + 0.16 * value_pattern + channel_bonus = 0.0 + if "field" in candidate.channels: + channel_bonus += 0.18 + if "lexical" in candidate.channels: + channel_bonus += 0.08 + if "fallback" in candidate.channels: + channel_bonus += 0.04 + score = ( + 0.35 * base_norm + + 0.30 * overlap + + 0.13 * dice + + 0.04 * exact_phrase + + 0.05 * temporal_match + + field_score + + channel_bonus + ) + features = { + "base_norm": round(base_norm, 6), + "overlap": round(overlap, 6), + "dice": round(dice, 6), + "relation_match": relation_match, + "attribute_match": attribute_match, + "value_pattern": value_pattern, + "exact_phrase": exact_phrase, + "temporal_match": temporal_match, + "field_score": round(field_score, 6), + "channel_bonus": round(channel_bonus, 6), + } + return score, features + + +def _lexical_fallback_candidates( + query: str, + all_docs: dict[str, tuple[int, str]], + *, + limit: int, + channel: str, +) -> list[FlowCandidate]: + q_info = _informative(_expanded_tokens(query)) + q_all = _expanded_tokens(query) + scored: list[tuple[float, str, int, str]] = [] + for doc_id, (rowid, text) in all_docs.items(): + c_tokens = _expanded_tokens(text) + c_info = _informative(c_tokens) + overlap = len(q_info & c_info) / max(len(q_info), 1) + relation = 1.0 if (_RELATION_TERMS & q_all & c_tokens) else 0.0 + attribute = 1.0 if ((_ATTRIBUTE_TERMS & q_all) & c_tokens) else 0.0 + value_pattern = 1.0 if relation and _role_value_pattern(text) else 0.0 + phrase = 1.0 if len(query) >= 8 and query.lower() in text.lower() else 0.0 + score = overlap + 0.42 * relation + 0.35 * attribute + 0.30 * value_pattern + 0.25 * phrase + if score > 0: + scored.append((score, doc_id, rowid, text)) + scored.sort(reverse=True, key=lambda item: item[0]) + return [ + FlowCandidate(rowid=rowid, doc_id=doc_id, content=text, base_score=score, channels={channel}) + for score, doc_id, rowid, text in scored[:limit] + ] + + +def _expand_related_candidates( + seeds: list[FlowCandidate], + all_docs: dict[str, tuple[int, str]], + operators: FlowOperators, + *, + limit: int, +) -> list[FlowCandidate]: + families = {source_family(candidate.doc_id) for candidate in seeds[:12]} + sessions = { + int(session) + for candidate in seeds[:12] + for session in [source_session(candidate.doc_id, candidate.content)] + if session.isdigit() + } + out: list[FlowCandidate] = [] + seen = {candidate.doc_id for candidate in seeds} + for doc_id, (rowid, text) in all_docs.items(): + if doc_id in seen: + continue + family_hit = source_family(doc_id) in families and operators.needs_breadth + session = source_session(doc_id, text) + neighbor_hit = False + if operators.temporal and session.isdigit(): + num = int(session) + neighbor_hit = any(abs(num - seed_num) <= 1 for seed_num in sessions) + if family_hit or neighbor_hit: + channels = {"family"} if family_hit else set() + if neighbor_hit: + channels.add("temporal_neighbor") + out.append(FlowCandidate(rowid=rowid, doc_id=doc_id, content=text, base_score=0.01, channels=channels)) + if len(out) >= limit: + break + return out + + +def optimize_ranked_documents( + query: str, + retrieved_rows: list[dict[str, Any]], + rowid_to_doc_id: dict[int, str], + rowid_to_text: dict[int, str], + *, + top_k: int, +) -> tuple[list[str], dict[str, Any]]: + """Union retrieval channels and build a top-k list from generic evidence features.""" + + operators = detect_flow_operators(query) + all_docs = { + doc_id: (rowid, rowid_to_text.get(rowid, "")) + for rowid, doc_id in rowid_to_doc_id.items() + } + by_doc: dict[str, FlowCandidate] = {} + for row in retrieved_rows: + try: + rowid = int(row.get("id")) + except (TypeError, ValueError): + continue + doc_id = rowid_to_doc_id.get(rowid) + if not doc_id: + continue + score = float(row.get("final_score") or row.get("rrf_score") or row.get("retrieval_score") or 0.0) + by_doc[doc_id] = FlowCandidate( + rowid=rowid, + doc_id=doc_id, + content=rowid_to_text.get(rowid, str(row.get("content") or "")), + base_score=score, + channels={"fts_vec"}, + metadata={"row": row}, + ) + + fallback_limit = max(top_k * 6, 30) + fallback_channel = "field" if operators.role_fact else "lexical" + for candidate in _lexical_fallback_candidates(query, all_docs, limit=fallback_limit, channel=fallback_channel): + existing = by_doc.get(candidate.doc_id) + if existing: + existing.channels.update(candidate.channels) + existing.base_score = max(existing.base_score, candidate.base_score) + else: + by_doc[candidate.doc_id] = candidate + + seed_candidates = sorted(by_doc.values(), key=lambda item: item.base_score, reverse=True) + if operators.needs_breadth: + for candidate in _expand_related_candidates(seed_candidates, all_docs, operators, limit=max(top_k * 4, 20)): + existing = by_doc.get(candidate.doc_id) + if existing: + existing.channels.update(candidate.channels) + existing.base_score = max(existing.base_score, candidate.base_score) + else: + by_doc[candidate.doc_id] = candidate + retrieved_families = { + source_family(candidate.doc_id) + for candidate in by_doc.values() + if "fts_vec" in candidate.channels + } + for candidate in by_doc.values(): + if "fts_vec" not in candidate.channels and source_family(candidate.doc_id) in retrieved_families: + candidate.channels.add("family") + + candidates = list(by_doc.values()) + max_base = max((candidate.base_score for candidate in candidates), default=1.0) + for candidate in candidates: + candidate.score, candidate.features = _base_relevance(query, operators, candidate, max_base) + + session_nums = [num for candidate in candidates for num in [_session_num(candidate)] if num is not None] + min_session = min(session_nums, default=0) + max_session = max(session_nums, default=0) + if operators.temporal or operators.update_resolution: + wants_latest = bool(re.search(r"\b(current|currently|now|latest|new|updated|changed|recent|most recent|after)\b", query.lower())) + wants_earlier = bool(re.search(r"\b(before|previous|previously|earlier|former|formerly)\b", query.lower())) + span = max(max_session - min_session, 1) + for candidate in candidates: + num = _session_num(candidate) + if num is None: + continue + normalized = (num - min_session) / span + recency_bonus = 0.0 + if wants_latest or operators.update_resolution: + recency_bonus += 0.12 * normalized + if wants_earlier: + recency_bonus += 0.08 * (1.0 - normalized) + text = candidate.content.lower() + if operators.update_resolution and re.search(r"\b(current|currently|now|latest|updated|changed|new)\b", text): + recency_bonus += 0.05 + if operators.update_resolution and re.search(r"\b(previous|previously|former|formerly|old|outdated)\b", text): + recency_bonus -= 0.05 + candidate.score += recency_bonus + candidate.features["temporal_recency_bonus"] = round(recency_bonus, 6) + + if operators.role_fact: + query_roles = _RELATION_TERMS & _expanded_tokens(query) + role_groups = { + source_group_session(doc_id) + for doc_id, (_rowid, text) in all_docs.items() + if source_group_session(doc_id) + and query_roles + and query_roles & _expanded_tokens(text) + } + for candidate in candidates: + group = source_group_session(candidate.doc_id) + coref_bonus = 0.0 + cand_tokens = _expanded_tokens(candidate.content) + direct_relation = bool(query_roles & cand_tokens) + has_attribute = bool((_ATTRIBUTE_TERMS & _expanded_tokens(query)) & cand_tokens) + has_value = _role_value_pattern(candidate.content) + if ( + group + and group in role_groups + and has_value + and not direct_relation + ): + coref_bonus = 0.50 + if coref_bonus: + candidate.score += coref_bonus + candidate.features["role_coref_group_bonus"] = round(coref_bonus, 6) + elif direct_relation and has_value: + candidate.score += 0.35 + candidate.features["role_direct_value_bonus"] = 0.35 + elif query_roles and not direct_relation: + candidate.score -= 0.33 + candidate.features["role_mismatch_penalty"] = 0.33 + elif direct_relation and not has_value and not has_attribute: + candidate.score -= 0.28 + candidate.features["role_intro_penalty"] = 0.28 + + if not candidates: + return [], { + "operators": operators.as_list(), + "candidate_counts": {"fts_vec": 0, "lexical": 0, "field": 0, "family": 0}, + "fallback_used": True, + "selected": [], + } + + selected: list[FlowCandidate] = [] + selected_facets: set[str] = set() + selected_families: set[str] = set() + selected_sessions: set[str] = set() + query_terms = _informative(_expanded_tokens(query)) + selected_query_terms: set[str] = set() + pool = sorted(candidates, key=lambda item: item.score, reverse=True) + while pool and len(selected) < top_k: + best_index = 0 + best_gain = -1e9 + for index, candidate in enumerate(pool): + facets = _candidate_facets(candidate) + family = source_family(candidate.doc_id) + session = source_session(candidate.doc_id, candidate.content) + candidate_query_terms = _informative(_expanded_tokens(candidate.content)) & query_terms + uncovered_query_terms = candidate_query_terms - selected_query_terms + new_facets = facets - selected_facets + gain = candidate.score + if operators.needs_breadth: + gain += min(0.28, 0.045 * len(new_facets)) + gain += min(0.24, 0.08 * len(uncovered_query_terms)) + if family not in selected_families: + gain += 0.055 + elif "family" in candidate.channels and len(selected) < max(5, top_k): + # Same source-family siblings are useful when the query asks + # for a set; plain duplicates from the same session are not. + gain += 0.16 + if session and session not in selected_sessions: + gain += 0.08 + elif session: + gain -= 0.12 + if not candidate_query_terms and "family" not in candidate.channels: + gain -= 0.16 + if not uncovered_query_terms and session in selected_sessions: + gain -= 0.06 + elif operators.role_fact: + # Single fact retrieval should stay precision-first. + if family in selected_families: + gain -= 0.04 + if "temporal_neighbor" in candidate.channels and operators.temporal: + gain += 0.035 + if gain > best_gain: + best_gain = gain + best_index = index + item = pool.pop(best_index) + selected.append(item) + selected_facets.update(_candidate_facets(item)) + selected_families.add(source_family(item.doc_id)) + selected_query_terms.update(_informative(_expanded_tokens(item.content)) & query_terms) + session = source_session(item.doc_id, item.content) + if session: + selected_sessions.add(session) + + channel_counts: dict[str, int] = {} + for candidate in candidates: + for channel in candidate.channels: + channel_counts[channel] = channel_counts.get(channel, 0) + 1 + trace = { + "operators": operators.as_list(), + "candidate_counts": channel_counts, + "fallback_used": "fts_vec" not in channel_counts or len(retrieved_rows) < top_k, + "selected": [ + { + "doc_id": candidate.doc_id, + "score": round(candidate.score, 6), + "channels": sorted(candidate.channels), + "features": candidate.features, + } + for candidate in selected + ], + } + return [candidate.doc_id for candidate in selected], trace diff --git a/src/agentmemory/retrieval/feature_builder.py b/src/agentmemory/retrieval/feature_builder.py index 108e983..bdfb9fc 100644 --- a/src/agentmemory/retrieval/feature_builder.py +++ b/src/agentmemory/retrieval/feature_builder.py @@ -23,6 +23,49 @@ "summary", "history", "timeline", "recent", "today", "yesterday", "tomorrow", "game", "issue", "problem", "thing", "stuff", "update", } +_SYNONYMS = { + "dad": {"father", "parent"}, + "father": {"dad", "parent"}, + "mom": {"mother", "parent"}, + "mother": {"mom", "parent"}, + "workplace": {"work", "works", "job", "office", "occupation", "position", "employer"}, + "occupation": {"job", "work", "works", "position", "career"}, + "position": {"job", "occupation", "work", "works", "role"}, + "educational": {"education", "degree", "school", "background"}, + "education": {"educational", "degree", "school", "background"}, + "background": {"education", "degree", "school"}, + "degree": {"education", "educational", "school", "background"}, + "location": {"where", "place", "city", "hometown", "workplace"}, + "hometown": {"home", "city", "location", "from"}, + "coworker": {"colleague", "work", "works"}, + "hobby": {"enjoy", "enjoys", "love", "loves", "passion", "passionate", "into"}, + "enjoy": {"hobby", "likes", "love", "loves", "passion"}, + "enjoys": {"hobby", "likes", "love", "loves", "passion"}, + "loves": {"hobby", "enjoy", "enjoys", "passion", "passionate"}, + "passionate": {"hobby", "enjoy", "enjoys", "loves"}, + "boss": {"manager", "supervisor"}, + "subordinate": {"employee", "report", "teammate"}, + "aunt": {"relative"}, + "uncle": {"relative"}, + "cousin": {"relative"}, + "living": {"occupation", "job", "work", "works"}, + "email": {"contact", "address"}, + "contact": {"phone", "number", "email"}, + "number": {"phone", "contact"}, +} +_ROLE_TERMS = { + "father", "dad", "mother", "mom", "parent", "coworker", "colleague", + "friend", "neighbor", "sister", "brother", "wife", "husband", "nephew", + "niece", "aunt", "uncle", "cousin", "relative", "boss", "manager", + "supervisor", "subordinate", "employee", "report", +} +_ATTRIBUTE_TERMS = { + "education", "educational", "background", "degree", "school", "occupation", + "position", "job", "workplace", "works", "work", "location", "hometown", + "company", "employer", "role", "status", "key", "code", "value", + "hobby", "enjoy", "enjoys", "love", "loves", "likes", "passion", + "passionate", "into", "email", "address", "contact", "number", "phone", "living", +} _DATE_RE = re.compile( r"\b(?:\d{4}-\d{2}-\d{2}|\d{1,2}/\d{1,2}(?:/\d{2,4})?|" r"jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|jun(?:e)?|" @@ -45,7 +88,10 @@ r")\b", re.IGNORECASE, ) -_SESSION_RE = re.compile(r"\bsession[_ :#-]*(\d+)\b", re.IGNORECASE) +_SESSION_RE = re.compile( + r"(?:^|[|_\s-])(?:sid|session|s)[=_ :#-]*(\d+)|\bsession[_ :#-]*(\d+)\b", + re.IGNORECASE, +) _DIALOG_RE = re.compile(r"\bD(\d+):", re.IGNORECASE) _ENTITY_RE = re.compile(r"\b[A-Z][A-Za-z0-9_.:-]+\b") @@ -106,11 +152,15 @@ def _normalize_token(token: str) -> str: def _token_set(text: str) -> set[str]: - return { + tokens = { token for part in re.split(r"\s+", text or "") if (token := _normalize_token(part)) } + expanded = set(tokens) + for token in tokens: + expanded.update(_SYNONYMS.get(token, ())) + return expanded def _informative_tokens(text: str) -> set[str]: @@ -214,7 +264,7 @@ def _temporal_anchor_overlap(query: str, text: str) -> float: def _extract_session_hints(text: str) -> list[int]: - hints = [int(match.group(1)) for match in _SESSION_RE.finditer(text or "")] + hints = [int(match.group(1) or match.group(2)) for match in _SESSION_RE.finditer(text or "")] hints.extend(int(match.group(1)) for match in _DIALOG_RE.finditer(text or "")) return hints @@ -230,6 +280,23 @@ def _session_gap_score(query: str, candidate_text_value: str) -> tuple[float, fl return 1.0 / (1.0 + gap), 1.0, 1.0 +def _role_value_pattern(text: str) -> float: + return 1.0 if re.search( + r"\b(" + r"works?\s+(?:as|in|at)|" + r"is\s+(?:a|an|the)\b|" + r"loves?\b|likes?\b|enjoys?\b|" + r"passionate\s+about|really\s+into|free\s+time|" + r"originally\s+from|grew\s+up\s+in|hails?\s+from|from\s+[A-Z][A-Za-z]+,\s*[A-Z][A-Za-z]+|" + r"[\w.+-]+@[\w.-]+|" + r"(?:phone|contact|number|email)\s+(?:is|address\s+is|number\s+is)?|" + r"company\s+(?:is|called|named)" + r")", + text or "", + re.IGNORECASE, + ) else 0.0 + + def _parse_iso_timestamp(value: Any) -> datetime | None: if not value: return None @@ -364,6 +431,9 @@ def build_features( entity_overlap = len(query_entities & cand_entities) / max(len(query_entities), 1) if query_entities else 0.0 aliases = {alias.lower() for alias in _alias_values(candidate) if len(alias) > 2} alias_overlap = len(query_entities & aliases) / max(len(query_entities), 1) if query_entities and aliases else 0.0 + role_overlap = 1.0 if (_ROLE_TERMS & query_tokens & cand_tokens) else 0.0 + attribute_overlap = 1.0 if (_ATTRIBUTE_TERMS & query_tokens & cand_tokens) else 0.0 + role_value_pattern = role_overlap * _role_value_pattern(text) query_temporal = 1.0 if (bool(getattr(plan, "requires_temporal_reasoning", False)) or _TEMPORAL_RE.search(query or "")) else 0.0 candidate_temporal = 1.0 if _TEMPORAL_RE.search(text or "") or _DATE_RE.search(text or "") else 0.0 temporal_anchor_overlap = _temporal_anchor_overlap(query, text) @@ -443,6 +513,11 @@ def build_features( "query_needs_ordering": 1.0 if getattr(plan, "needs_ordering", False) else 0.0, "query_needs_update_resolution": 1.0 if getattr(plan, "needs_update_resolution", False) else 0.0, "query_needs_set_coverage": 1.0 if getattr(plan, "needs_set_coverage", False) else 0.0, + "query_needs_role_fact": 1.0 if getattr(plan, "needs_role_fact", False) else 0.0, + "query_needs_synthetic_key_value": 1.0 if getattr(plan, "needs_synthetic_key_value", False) else 0.0, + "role_overlap": role_overlap, + "attribute_overlap": attribute_overlap, + "role_value_pattern": role_value_pattern, "query_requires_multi_hop": 1.0 if getattr(plan, "requires_multi_hop", False) else 0.0, "long_context_applicable": 1.0 if long_context_debug.get("applicable") else 0.0, "long_context_score": _safe_float(long_context_debug.get("score")), diff --git a/src/agentmemory/retrieval/query_planner.py b/src/agentmemory/retrieval/query_planner.py index 160e440..0892cd1 100644 --- a/src/agentmemory/retrieval/query_planner.py +++ b/src/agentmemory/retrieval/query_planner.py @@ -82,6 +82,22 @@ r")\b", re.IGNORECASE, ) +_ROLE_FACT_RE = re.compile( + r"\b(" + r"father|dad|mother|mom|parent|coworker|colleague|friend|neighbor|" + r"brother|sister|nephew|niece|aunt|uncle|cousin|boss|manager|supervisor|subordinate|employee|" + r"workplace|occupation|position|job|employer|education|educational|" + r"degree|background|location|hometown|role|hobby|enjoys?|loves?|passion|" + r"email|contact|phone|number|company|living" + r")\b", + re.IGNORECASE, +) +_SYNTHETIC_KV_RE = re.compile( + r"\b(" + r"id|key|code|value|field|role|status|attribute|group|session|step" + r")\b|[A-Za-z]+[_-]\d+|\w+[=:]\w+", + re.IGNORECASE, +) _NEGATIVE_RE = re.compile( r"\b(" r"no answer|" @@ -110,6 +126,8 @@ class QueryPlan: needs_ordering: bool = False needs_update_resolution: bool = False needs_set_coverage: bool = False + needs_role_fact: bool = False + needs_synthetic_key_value: bool = False prefer_memory_types: list[str] = field(default_factory=list) candidate_tables: list[str] = field(default_factory=list) abstain_allowed: bool = False @@ -250,6 +268,8 @@ def plan_query( needs_ordering = bool(_ORDER_RE.search(query)) needs_update_resolution = bool(_UPDATE_RE.search(query)) needs_set_coverage = bool(_COVERAGE_RE.search(query)) + needs_role_fact = bool(_ROLE_FACT_RE.search(query)) + needs_synthetic_key_value = bool(_SYNTHETIC_KV_RE.search(query)) if requires_multi_hop and normalized_intent in {"temporal", "decision", "graph"}: needs_set_coverage = True if needs_counting or needs_comparison or needs_ordering: @@ -271,6 +291,10 @@ def plan_query( reasons.append("operator:update_resolution") if needs_set_coverage: reasons.append("operator:set_coverage") + if needs_role_fact: + reasons.append("operator:role_fact") + if needs_synthetic_key_value: + reasons.append("operator:synthetic_key_value") if "summary of yesterday" in query_lower: abstain_allowed = True reasons.append("negative_or_out_of_domain_summary") @@ -290,6 +314,8 @@ def plan_query( needs_ordering=needs_ordering, needs_update_resolution=needs_update_resolution, needs_set_coverage=needs_set_coverage, + needs_role_fact=needs_role_fact, + needs_synthetic_key_value=needs_synthetic_key_value, prefer_memory_types=prefer_memory_types, candidate_tables=candidate_tables, abstain_allowed=abstain_allowed, diff --git a/src/agentmemory/retrieval/second_stage.py b/src/agentmemory/retrieval/second_stage.py index 64e05f2..c451999 100644 --- a/src/agentmemory/retrieval/second_stage.py +++ b/src/agentmemory/retrieval/second_stage.py @@ -24,7 +24,10 @@ "entities": "entity", "decisions": "decision", } -_SESSION_RE = re.compile(r"\bsession[_ :#-]*(\d+)\b", re.IGNORECASE) +_SESSION_RE = re.compile( + r"(?:^|[|_\s-])(?:sid|session|s)[=_ :#-]*(\d+)|\bsession[_ :#-]*(\d+)\b", + re.IGNORECASE, +) _DATE_RE = re.compile( r"\b(?:\d{4}-\d{2}-\d{2}|\d{1,2}/\d{1,2}(?:/\d{2,4})?|" r"jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|jun(?:e)?|" @@ -33,6 +36,7 @@ re.IGNORECASE, ) _ENTITY_RE = re.compile(r"\b[A-Z][A-Za-z0-9_.:-]+\b") +_SOURCE_NUM_SUFFIX_RE = re.compile(r"^(.+?)[_-](\d+)$") def _resolve_benchmark_ranking_mode(args: Any) -> str: @@ -135,6 +139,15 @@ def _heuristic_score(plan: Any, features: dict[str, float]) -> float: score += features["temporal_anchor_overlap"] * 0.05 + features["session_gap_score"] * 0.05 if features.get("query_needs_update_resolution", 0.0) > 0.0: score += features["status_active"] * 0.04 + if features.get("query_needs_role_fact", 0.0) > 0.0: + score += ( + features.get("role_overlap", 0.0) * 0.11 + + features.get("attribute_overlap", 0.0) * 0.10 + + features.get("role_value_pattern", 0.0) * 0.08 + + features.get("exact_phrase", 0.0) * 0.03 + ) + if features.get("query_needs_synthetic_key_value", 0.0) > 0.0: + score += features["source_keyword"] * 0.04 + features.get("attribute_overlap", 0.0) * 0.05 if intent in {"temporal", "decision"}: score += features["bucket_events"] * 0.04 + features["bucket_decisions"] * 0.03 if intent in {"procedural", "troubleshooting"}: @@ -163,11 +176,27 @@ def _candidate_text(candidate: dict[str, Any]) -> str: return "" +def _candidate_source_family(candidate: dict[str, Any]) -> str: + raw = ( + candidate.get("doc_id") + or candidate.get("source_doc_id") + or candidate.get("source_key") + or candidate.get("external_id") + or "" + ) + head = str(raw).split("|", 1)[0] + match = _SOURCE_NUM_SUFFIX_RE.match(head) + return match.group(1) if match else head + + def _candidate_cluster_keys(plan: Any, candidate: dict[str, Any]) -> set[str]: text = _candidate_text(candidate) keys: set[str] = set() + family = _candidate_source_family(candidate) + if family: + keys.add(f"family:{family}") for match in _SESSION_RE.finditer(text): - keys.add(f"session:{match.group(1)}") + keys.add(f"session:{match.group(1) or match.group(2)}") if getattr(plan, "requires_temporal_reasoning", False) or getattr(plan, "needs_ordering", False): for match in _DATE_RE.finditer(text): keys.add(f"date:{match.group(0).lower()}") @@ -213,9 +242,9 @@ def _slate_score( localization_bonus = 0.0 if getattr(plan, "needs_set_coverage", False): - coverage_bonus += min(0.14, 0.035 * len(new_keys)) + coverage_bonus += min(0.20, 0.05 * len(new_keys)) if not new_keys and selected_keys: - redundancy_penalty += 0.085 + redundancy_penalty += 0.11 elif selected_keys and not new_keys: redundancy_penalty += 0.03 @@ -233,6 +262,11 @@ def _slate_score( else: coverage_bonus += features.get("temporal_anchor_overlap", 0.0) * 0.03 + if getattr(plan, "needs_role_fact", False): + coverage_bonus += features.get("role_overlap", 0.0) * 0.04 + coverage_bonus += features.get("attribute_overlap", 0.0) * 0.04 + coverage_bonus += features.get("role_value_pattern", 0.0) * 0.03 + if features.get("long_context_focused_program", 0.0) > 0.0: localization_bonus += ( features.get("long_context_precision", 0.0) * 0.018 @@ -299,6 +333,11 @@ def _rerank_slate( "query_needs_ordering", "query_needs_update_resolution", "query_needs_set_coverage", + "query_needs_role_fact", + "query_needs_synthetic_key_value", + "role_overlap", + "attribute_overlap", + "role_value_pattern", "long_context_score", "long_context_confidence", "long_context_agreement", @@ -393,6 +432,8 @@ def rerank_top_candidates( "needs_ordering", "needs_update_resolution", "needs_set_coverage", + "needs_role_fact", + "needs_synthetic_key_value", ) ) raw_head_scores = [ diff --git a/tests/test_retrieval_flow_optimizer.py b/tests/test_retrieval_flow_optimizer.py new file mode 100644 index 0000000..79c9651 --- /dev/null +++ b/tests/test_retrieval_flow_optimizer.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +from benchmarks.retrieval_flow_optimizer import ( + detect_flow_operators, + optimize_ranked_documents, + source_family, + source_session, +) + + +def _optimize(query, docs, retrieved_rows=None, top_k=5): + rowid_to_doc_id = {index: doc_id for index, (doc_id, _text) in enumerate(docs, start=1)} + rowid_to_text = {index: text for index, (_doc_id, text) in enumerate(docs, start=1)} + return optimize_ranked_documents( + query, + retrieved_rows or [], + rowid_to_doc_id, + rowid_to_text, + top_k=top_k, + ) + + +def test_detects_operators_that_change_retrieval_behavior(): + role = detect_flow_operators("What is the location of my father's workplace?") + assert role.role_fact + assert role.single_fact + assert not role.needs_breadth + + temporal = detect_flow_operators("What changed after the March session and what is current now?") + assert temporal.temporal + assert temporal.update_resolution + assert temporal.multi_session + + comparison = detect_flow_operators("Compare Alice and Bob across both sessions.") + assert comparison.comparison + assert comparison.set_coverage + assert comparison.needs_breadth + + +def test_source_metadata_parsing_is_generic(): + assert source_family("source_alpha_4") == "source_alpha" + assert source_family("simple_roles_9|sid=48|g=48|s=2|t=7") == "simple_roles" + assert source_session("simple_roles_9|sid=48|g=48|s=2|t=7") == "48" + assert source_session("conversation-session-12", "") == "12" + + +def test_simple_role_fact_uses_field_fallback_when_initial_retrieval_misses(): + docs = [ + ("noise|sid=1", "My friend enjoys hiking on weekends."), + ("simple_roles_9|sid=48|g=48|s=2|t=7", "My dad works in Miami, FL."), + ("other|sid=2", "My coworker likes board games."), + ] + retrieved_rows = [{"id": 1, "final_score": 10.0}] + + ranked, trace = _optimize( + "What is the location of my father's workplace?", + docs, + retrieved_rows, + top_k=3, + ) + + assert ranked[0] == "simple_roles_9|sid=48|g=48|s=2|t=7" + assert "role_fact" in trace["operators"] + assert trace["candidate_counts"]["field"] >= 1 + + +def test_empty_candidate_generation_falls_back_to_lexical_candidates(): + docs = [ + ("doc_1|sid=1", "The archive key is nebula-42."), + ("doc_2|sid=2", "The menu included soup."), + ] + + ranked, trace = _optimize("What archive key was mentioned?", docs, [], top_k=2) + + assert ranked[0] == "doc_1|sid=1" + assert trace["fallback_used"] is True + assert trace["candidate_counts"]["lexical"] >= 1 + + +def test_set_coverage_ranking_prefers_breadth_over_duplicate_sessions(): + docs = [ + ("trip_1|sid=1", "Alice visited Rome during the trip."), + ("trip_2|sid=1", "Alice talked more about Rome during the trip."), + ("trip_3|sid=2", "Bob visited Paris during the trip."), + ] + retrieved_rows = [ + {"id": 1, "final_score": 10.0}, + {"id": 2, "final_score": 9.8}, + {"id": 3, "final_score": 2.0}, + ] + + ranked, trace = _optimize( + "Which places did Alice and Bob visit across the trip?", + docs, + retrieved_rows, + top_k=2, + ) + + assert "trip_3|sid=2" in ranked + assert len({source_session(doc_id) for doc_id in ranked}) == 2 + assert "set_coverage" in trace["operators"] or "comparison" in trace["operators"] + + +def test_update_resolution_promotes_newer_current_evidence(): + docs = [ + ("profile_1|sid=1", "In an earlier session, Alice lived in Boston."), + ("profile_2|sid=4", "Alice updated her current city to Denver."), + ] + retrieved_rows = [ + {"id": 1, "final_score": 10.0}, + {"id": 2, "final_score": 8.5}, + ] + + ranked, trace = _optimize("Where does Alice currently live now?", docs, retrieved_rows, top_k=2) + + assert ranked[0] == "profile_2|sid=4" + assert "update_resolution" in trace["operators"] + assert trace["selected"][0]["features"]["temporal_recency_bonus"] > 0 + + +def test_family_expansion_admits_sibling_evidence_for_multi_part_queries(): + docs = [ + ("source_alpha_1|sid=1", "The deployment needs a smoke test first."), + ("source_alpha_2|sid=2", "The deployment also needs rollback notes."), + ("noise_beta_1|sid=3", "The cafeteria changed its menu."), + ] + retrieved_rows = [ + {"id": 1, "final_score": 10.0}, + {"id": 3, "final_score": 8.0}, + ] + + ranked, trace = _optimize( + "List all deployment requirements across sessions.", + docs, + retrieved_rows, + top_k=2, + ) + + assert "source_alpha_2|sid=2" in ranked + selected_channels = { + channel + for selected in trace["selected"] + for channel in selected["channels"] + } + assert "family" in selected_channels + + +def test_role_fact_uses_same_session_coreference_without_gold_ids(): + docs = [ + ("roles_1|sid=10|g=10|s=1|t=0", "I want to tell you about my sister, Sierra."), + ("roles_1|sid=11|g=11|s=1|t=1", "She is a Senior Research Scientist."), + ("roles_1|sid=20|g=20|s=2|t=0", "My coworker is a Construction Supervisor."), + ] + retrieved_rows = [ + {"id": 3, "final_score": 10.0}, + {"id": 1, "final_score": 8.0}, + {"id": 2, "final_score": 2.0}, + ] + + ranked, trace = _optimize("What is the position of my sister?", docs, retrieved_rows, top_k=2) + + assert ranked[0] == "roles_1|sid=11|g=11|s=1|t=1" + assert trace["selected"][0]["features"]["role_coref_group_bonus"] > 0 diff --git a/tests/test_second_stage_reranker.py b/tests/test_second_stage_reranker.py index e3caed7..3ac6759 100644 --- a/tests/test_second_stage_reranker.py +++ b/tests/test_second_stage_reranker.py @@ -201,6 +201,10 @@ def test_query_plan_sets_operator_flags(): assert plan.needs_update_resolution is True assert plan.needs_set_coverage is True + role_plan = plan_query("What is the location of my father's workplace?", requested_tables=["memories"]) + assert role_plan.needs_role_fact is True + assert role_plan.needs_synthetic_key_value is False + def test_listwise_slate_avoids_duplicate_session_cluster(tmp_path: Path): plan = plan_query( @@ -248,3 +252,40 @@ def test_listwise_slate_avoids_duplicate_session_cluster(tmp_path: Path): ) assert {row["id"] for row in reranked[:2]} == {1, 3} assert debug["strategy"] == "listwise_greedy_slate" + + +def test_second_stage_promotes_role_fact_candidate(tmp_path: Path): + plan = plan_query("What is the location of my father's workplace?", requested_tables=["memories"]) + model_path = _temp_model(tmp_path / "tiny.json") + candidates = [ + { + "id": 1, + "bucket": "memories", + "type": "memory", + "content": "My friend enjoys hiking on weekends.", + "final_score": 0.92, + "retrieval_score": 0.92, + "source": "both", + }, + { + "id": 2, + "bucket": "memories", + "type": "memory", + "content": "My dad works in Miami, FL.", + "final_score": 0.78, + "retrieval_score": 0.78, + "source": "keyword", + }, + ] + + reranked, debug = rerank_top_candidates( + "What is the location of my father's workplace?", + plan, + candidates, + config=SecondStageConfig(top_n=2, model_path=str(model_path)), + ) + + assert reranked[0]["id"] == 2 + assert reranked[0]["second_stage_features"]["role_overlap"] == 1.0 + assert reranked[0]["second_stage_features"]["attribute_overlap"] == 1.0 + assert debug["strategy"] == "listwise_greedy_slate" From 5db6d0173d2b114c0938489ad12dbaa5923d0bd1 Mon Sep 17 00:00:00 2001 From: Mario Jack Vela Date: Fri, 24 Apr 2026 01:06:12 -0500 Subject: [PATCH 12/13] Improve legacy benchmark retrieval flow --- benchmarks/brainctl_retrieval.py | 44 ++++++++- benchmarks/locomo_bench.py | 6 +- benchmarks/retrieval_flow_optimizer.py | 132 +++++++++++++++++++++++++ tests/test_retrieval_flow_optimizer.py | 52 ++++++++++ 4 files changed, 230 insertions(+), 4 deletions(-) diff --git a/benchmarks/brainctl_retrieval.py b/benchmarks/brainctl_retrieval.py index 30d8cfc..e98f984 100644 --- a/benchmarks/brainctl_retrieval.py +++ b/benchmarks/brainctl_retrieval.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +import re import shutil import sqlite3 import tempfile @@ -18,10 +19,11 @@ os.sys.path.insert(0, str(SRC)) from agentmemory.brain import Brain -from benchmarks.retrieval_flow_optimizer import optimize_ranked_documents +from benchmarks.retrieval_flow_optimizer import detect_flow_operators, optimize_ranked_documents, source_family AGENT_ID = "legacy-compare-bench" +_SESSION_DOC_ID_RE = re.compile(r"^session[_-]?\d+$", re.IGNORECASE) @dataclass @@ -120,6 +122,26 @@ def _search_cmd( return memories[:top_k] +def _is_whole_session_corpus(seeded: SeededCorpus) -> bool: + if not seeded.rowid_to_text: + return False + session_rows = sum( + 1 + for rowid, text in seeded.rowid_to_text.items() + if text.lstrip().startswith("Session ID:") + or _SESSION_DOC_ID_RE.match(str(seeded.rowid_to_doc_id.get(rowid, ""))) + ) + return session_rows / max(len(seeded.rowid_to_text), 1) >= 0.8 + + +def _has_compact_source_families(seeded: SeededCorpus, *, max_size: int = 6) -> bool: + counts: dict[str, int] = {} + for doc_id in seeded.rowid_to_doc_id.values(): + family = source_family(doc_id) + counts[family] = counts.get(family, 0) + 1 + return any(2 <= count <= max_size for count in counts.values()) + + def seed_documents( documents: Iterable[tuple[str, str]], *, @@ -171,7 +193,15 @@ def rank_seeded_documents( db_path = work_dir / "brain.db" try: shutil.copy2(seeded.template_db_path, db_path) - pool_k = max(top_k * 8, 50) + operators = detect_flow_operators(query) + small_bounded_corpus = len(seeded.rowid_to_doc_id) <= max(top_k * 5, 50) + whole_session_corpus = _is_whole_session_corpus(seeded) + needs_expanded_pool = ( + operators.role_fact + or (small_bounded_corpus and not whole_session_corpus) + or (whole_session_corpus and operators.needs_breadth and _has_compact_source_families(seeded)) + ) + pool_k = max(top_k * 8, 50) if needs_expanded_pool else top_k if pipeline == "brain": results = _search_brain(db_path, query, pool_k) @@ -204,7 +234,15 @@ def search_seeded_documents( db_path = work_dir / "brain.db" try: shutil.copy2(seeded.template_db_path, db_path) - pool_k = max(top_k * 8, 50) + operators = detect_flow_operators(query) + small_bounded_corpus = len(seeded.rowid_to_doc_id) <= max(top_k * 5, 50) + whole_session_corpus = _is_whole_session_corpus(seeded) + needs_expanded_pool = ( + operators.role_fact + or (small_bounded_corpus and not whole_session_corpus) + or (whole_session_corpus and operators.needs_breadth and _has_compact_source_families(seeded)) + ) + pool_k = max(top_k * 8, 50) if needs_expanded_pool else top_k if pipeline == "brain": results = _search_brain(db_path, query, pool_k) elif pipeline == "cmd": diff --git a/benchmarks/locomo_bench.py b/benchmarks/locomo_bench.py index 1728c1b..9eebea7 100644 --- a/benchmarks/locomo_bench.py +++ b/benchmarks/locomo_bench.py @@ -50,7 +50,11 @@ def _build_corpus(sessions: list[dict[str, Any]], granularity: str) -> list[tupl corpus: list[tuple[str, str]] = [] for session in sessions: if granularity == "session": - texts = [] + texts = [ + f"Session ID: session_{session['session_num']}", + f"Session Date: {session.get('date', '')}", + "Conversation:", + ] for dialog in session["dialogs"]: speaker = dialog.get("speaker", "?") text = dialog.get("text", "") diff --git a/benchmarks/retrieval_flow_optimizer.py b/benchmarks/retrieval_flow_optimizer.py index e9115b4..1f54588 100644 --- a/benchmarks/retrieval_flow_optimizer.py +++ b/benchmarks/retrieval_flow_optimizer.py @@ -12,6 +12,7 @@ r"(?:^|[|_\s-])(?:sid|session|s)[=_-]?(\d+)|\bsession[_\s-]*(\d+)\b", re.IGNORECASE, ) +_SESSION_DOC_ID_RE = re.compile(r"^session[_-]?\d+$", re.IGNORECASE) _GROUP_SESSION_RE = re.compile(r"(?:^|[|_\s-])s[=_-]?(\d+)(?:[|_\s-]|$)", re.IGNORECASE) _DATE_RE = re.compile(r"\b\d{4}[-/]\d{1,2}[-/]\d{1,2}\b|\b\d{1,2}/\d{1,2}/\d{2,4}\b") @@ -301,6 +302,85 @@ def _expand_related_candidates( return out +def _whole_session_family_rerank( + raw_ranked: list[str], + all_docs: dict[str, tuple[int, str]], + *, + top_k: int, + operators: FlowOperators, +) -> list[str]: + """Conservatively admit sibling sessions for set/temporal questions. + + Whole-session benchmarks often encode multi-evidence answers as small + numbered source families. If one sibling makes the first-stage slate and + other siblings appear nearby, promote that compact family together. Large + families are ignored because they are usually broad source prefixes rather + than answer/evidence clusters. + """ + + if not operators.needs_breadth or len(raw_ranked) <= top_k: + return raw_ranked[:top_k] + + pool = raw_ranked[: max(top_k * 4, 40)] + family_sizes: dict[str, int] = {} + for doc_id in all_docs: + family = source_family(doc_id) + family_sizes[family] = family_sizes.get(family, 0) + 1 + + by_family: dict[str, list[tuple[int, str]]] = {} + for index, doc_id in enumerate(pool): + by_family.setdefault(source_family(doc_id), []).append((index, doc_id)) + + groups: list[tuple[int, int, list[str]]] = [] + grouped_docs: set[str] = set() + max_family_size = max(3, min(6, top_k)) + max_group_docs = max(2, min(4, top_k)) + shift_cap = max(1, min(2, top_k // 3)) + for family, items in by_family.items(): + family_size = family_sizes.get(family, 0) + top_items = [item for item in items if item[0] < top_k] + if not (2 <= family_size <= max_family_size): + continue + if len(items) < 2 or not top_items: + continue + docs = [doc_id for _idx, doc_id in sorted(items)[: min(family_size, max_group_docs)]] + start = max(0, top_items[0][0] - min(len(docs) - 1, shift_cap)) + groups.append((start, top_items[0][0], docs)) + grouped_docs.update(docs) + + if not groups: + return raw_ranked[:top_k] + + groups.sort(key=lambda item: (item[0], item[1])) + selected: list[str] = [] + raw_index = 0 + raw_top = raw_ranked[:top_k] + for start, _first_index, docs in groups: + while raw_index < len(raw_top) and len(selected) < start: + doc_id = raw_top[raw_index] + if doc_id not in grouped_docs and doc_id not in selected: + selected.append(doc_id) + raw_index += 1 + for doc_id in docs: + if doc_id not in selected: + selected.append(doc_id) + while raw_index < len(raw_top) and raw_top[raw_index] in grouped_docs: + raw_index += 1 + + while raw_index < len(raw_top): + doc_id = raw_top[raw_index] + if doc_id not in selected: + selected.append(doc_id) + raw_index += 1 + + for doc_id in pool: + if len(selected) >= top_k: + break + if doc_id not in selected: + selected.append(doc_id) + return selected[:top_k] + + def optimize_ranked_documents( query: str, retrieved_rows: list[dict[str, Any]], @@ -317,6 +397,7 @@ def optimize_ranked_documents( for rowid, doc_id in rowid_to_doc_id.items() } by_doc: dict[str, FlowCandidate] = {} + raw_ranked: list[str] = [] for row in retrieved_rows: try: rowid = int(row.get("id")) @@ -325,6 +406,8 @@ def optimize_ranked_documents( doc_id = rowid_to_doc_id.get(rowid) if not doc_id: continue + if doc_id not in raw_ranked: + raw_ranked.append(doc_id) score = float(row.get("final_score") or row.get("rrf_score") or row.get("retrieval_score") or 0.0) by_doc[doc_id] = FlowCandidate( rowid=rowid, @@ -335,6 +418,55 @@ def optimize_ranked_documents( metadata={"row": row}, ) + # The seeded session-level suites already have a strong first-stage ranker. + # Only use full-corpus lexical fallback/list construction when a query + # shape needs it (role/key-value facts), first-stage retrieval is genuinely + # empty/underfilled, or the corpus is a small chunk/turn corpus where + # coverage expansion has bounded blast radius. This prevents noisy broad + # matches from demoting correct whole-session evidence. + small_bounded_corpus = len(all_docs) <= max(top_k * 5, 50) + whole_session_corpus = bool(all_docs) and ( + sum( + 1 + for _doc_id, (_rowid, text) in all_docs.items() + if text.lstrip().startswith("Session ID:") or _SESSION_DOC_ID_RE.match(str(_doc_id)) + ) + / max(len(all_docs), 1) + >= 0.8 + ) + aggressive_rewrite = ( + operators.role_fact + or (len(raw_ranked) == 0 and whole_session_corpus) + or (len(raw_ranked) < top_k and not whole_session_corpus) + or (small_bounded_corpus and not whole_session_corpus and operators.needs_breadth) + ) + if not aggressive_rewrite: + selected = ( + _whole_session_family_rerank( + raw_ranked, + all_docs, + top_k=top_k, + operators=operators, + ) + if whole_session_corpus + else raw_ranked[:top_k] + ) + return selected, { + "operators": operators.as_list(), + "candidate_counts": {"fts_vec": len(raw_ranked)}, + "fallback_used": False, + "strategy": "whole_session_family_admission" if selected != raw_ranked[:top_k] else "preserve_first_stage_order", + "selected": [ + { + "doc_id": doc_id, + "score": None, + "channels": ["fts_vec"], + "features": {"source_family": source_family(doc_id)}, + } + for doc_id in selected + ], + } + fallback_limit = max(top_k * 6, 30) fallback_channel = "field" if operators.role_fact else "lexical" for candidate in _lexical_fallback_candidates(query, all_docs, limit=fallback_limit, channel=fallback_channel): diff --git a/tests/test_retrieval_flow_optimizer.py b/tests/test_retrieval_flow_optimizer.py index 79c9651..0f3216a 100644 --- a/tests/test_retrieval_flow_optimizer.py +++ b/tests/test_retrieval_flow_optimizer.py @@ -145,6 +145,58 @@ def test_family_expansion_admits_sibling_evidence_for_multi_part_queries(): assert "family" in selected_channels +def test_whole_session_family_admission_promotes_compact_sibling_evidence(): + docs = [ + ("distractor_alpha", "Session ID: distractor_alpha\nSession Date: 2023/01/01\nConversation: User: I asked about museum tickets."), + ("noise_beta", "Session ID: noise_beta\nSession Date: 2023/01/02\nConversation: User: I discussed a museum blog post."), + ("answer_trip_1", "Session ID: answer_trip_1\nSession Date: 2023/01/03\nConversation: User: I visited the science museum."), + ("noise_gamma", "Session ID: noise_gamma\nSession Date: 2023/01/04\nConversation: User: I asked about travel planning."), + ("answer_trip_2", "Session ID: answer_trip_2\nSession Date: 2023/01/05\nConversation: User: I visited the art museum."), + ("answer_trip_3", "Session ID: answer_trip_3\nSession Date: 2023/01/06\nConversation: User: I visited the history museum."), + ] + retrieved_rows = [ + {"id": 1, "final_score": 10.0}, + {"id": 2, "final_score": 9.0}, + {"id": 3, "final_score": 8.0}, + {"id": 4, "final_score": 7.0}, + {"id": 5, "final_score": 6.0}, + {"id": 6, "final_score": 5.0}, + ] + + ranked, trace = _optimize( + "What is the order of the museums I visited from earliest to latest?", + docs, + retrieved_rows, + top_k=5, + ) + + assert ranked[:4] == ["distractor_alpha", "answer_trip_1", "answer_trip_2", "answer_trip_3"] + assert trace["strategy"] == "whole_session_family_admission" + + +def test_session_id_corpus_preserves_first_stage_without_compact_families(): + docs = [ + ("session_1", "Alex said, \"I visited the museum on Monday.\""), + ("session_2", "Alex said, \"I visited the garden on Tuesday.\""), + ("session_3", "Alex said, \"I visited the library on Wednesday.\""), + ] + retrieved_rows = [ + {"id": 1, "final_score": 10.0}, + {"id": 2, "final_score": 9.0}, + {"id": 3, "final_score": 8.0}, + ] + + ranked, trace = _optimize( + "What is the order of places Alex visited from earliest to latest?", + docs, + retrieved_rows, + top_k=3, + ) + + assert ranked == ["session_1", "session_2", "session_3"] + assert trace["strategy"] == "preserve_first_stage_order" + + def test_role_fact_uses_same_session_coreference_without_gold_ids(): docs = [ ("roles_1|sid=10|g=10|s=1|t=0", "I want to tell you about my sister, Sierra."), From 9f33aec073d69e54f8c3a1b5c973d2a2abae70b8 Mon Sep 17 00:00:00 2001 From: Mario Jack Vela Date: Fri, 24 Apr 2026 03:17:44 -0500 Subject: [PATCH 13/13] Clean up retrieval PR artifacts --- .github/workflows/ci.yml | 2 + .gitignore | 4 + AGENTS.md | 152 - .../legacy_compare_refresh_bg_stderr.log | 0 .../legacy_compare_refresh_bg_stdout.log | 0 .../results/legacy_compare_rerun_stderr.log | 0 .../results/legacy_compare_rerun_stdout.log | 0 .../prepush_smoke_20260423_123335/README.md | 34 - .../charts/aggregate_primary_metrics.png | Bin 72024 -> 0 bytes .../charts/coverage_status.png | Bin 38109 -> 0 bytes .../charts/locomo_comparison.png | Bin 43742 -> 0 bytes .../charts/longmemeval_comparison.png | Bin 48827 -> 0 bytes .../charts/membench_comparison.png | Bin 33079 -> 0 bytes .../comparison_table.csv | 57 - .../comparison_table.json | 730 - .../metadata.json | 30 - .../runs/convomem_mempalace_raw.json | 29 - .../runs/convomem_new_brainctl_cmd.json | 29 - .../runs/convomem_old_brainctl_cmd.json | 29 - .../runs/locomo_mempalace_raw_session.json | 38 - .../runs/locomo_new_brainctl_cmd_session.json | 4455 -- .../runs/locomo_old_brainctl_cmd_session.json | 38 - .../longmemeval_mempalace_raw_session.json | 33 - .../runs/longmemeval_new_brainctl_brain.json | 56 - .../runs/longmemeval_new_brainctl_cmd.json | 56 - .../runs/longmemeval_old_brainctl_brain.json | 32 - .../runs/longmemeval_old_brainctl_cmd.json | 32 - .../runs/membench_mempalace_raw_turn.json | 35 - .../runs/membench_new_brainctl_cmd_turn.json | 54 - .../runs/membench_old_brainctl_cmd_turn.json | 35 - .../prepush_smoke_20260423_123335/summary.csv | 15 - .../summary.json | 509 - .../README.md | 34 - .../charts/aggregate_primary_metrics.png | Bin 72174 -> 0 bytes .../charts/coverage_status.png | Bin 38111 -> 0 bytes .../charts/locomo_comparison.png | Bin 43814 -> 0 bytes .../charts/longmemeval_comparison.png | Bin 48858 -> 0 bytes .../charts/membench_comparison.png | Bin 33854 -> 0 bytes .../comparison_table.csv | 65 - .../comparison_table.json | 834 - .../metadata.json | 24 - .../runs/convomem_mempalace_raw.json | 29 - .../runs/convomem_new_brainctl_cmd.json | 106 - .../runs/convomem_old_brainctl_cmd.json | 29 - .../runs/locomo_mempalace_raw_session.json | 38 - .../runs/locomo_new_brainctl_cmd_session.json | 44071 ---------------- .../runs/locomo_old_brainctl_cmd_session.json | 38 - .../longmemeval_mempalace_raw_session.json | 33 - .../runs/longmemeval_new_brainctl_brain.json | 12197 ----- .../runs/longmemeval_new_brainctl_cmd.json | 12197 ----- .../runs/longmemeval_old_brainctl_brain.json | 32 - .../runs/longmemeval_old_brainctl_cmd.json | 32 - .../runs/membench_mempalace_raw_turn.json | 35 - .../runs/membench_new_brainctl_cmd_turn.json | 3369 -- .../runs/membench_old_brainctl_cmd_turn.json | 35 - .../summary.csv | 15 - .../summary.json | 514 - .../README.md | 31 - .../charts/aggregate_primary_metrics.png | Bin 72024 -> 0 bytes .../charts/coverage_status.png | Bin 38109 -> 0 bytes .../charts/locomo_comparison.png | Bin 43742 -> 0 bytes .../charts/longmemeval_comparison.png | Bin 48827 -> 0 bytes .../charts/membench_comparison.png | Bin 33079 -> 0 bytes .../comparison_table.csv | 57 - .../comparison_table.json | 730 - .../metadata.json | 30 - .../runs/convomem_mempalace_raw.json | 29 - .../runs/convomem_new_brainctl_cmd.json | 26 - .../runs/convomem_old_brainctl_cmd.json | 29 - .../runs/locomo_mempalace_raw_session.json | 38 - .../runs/locomo_new_brainctl_cmd_session.json | 4450 -- .../runs/locomo_old_brainctl_cmd_session.json | 38 - .../longmemeval_mempalace_raw_session.json | 33 - .../runs/longmemeval_new_brainctl_brain.json | 101 - .../runs/longmemeval_new_brainctl_cmd.json | 101 - .../runs/longmemeval_old_brainctl_brain.json | 32 - .../runs/longmemeval_old_brainctl_cmd.json | 32 - .../runs/membench_mempalace_raw_turn.json | 35 - .../runs/membench_new_brainctl_cmd_turn.json | 83 - .../runs/membench_old_brainctl_cmd_turn.json | 35 - .../summary.csv | 15 - .../summary.json | 509 - .../smoke_postpush_20260423_123452/README.md | 34 - .../charts/aggregate_primary_metrics.png | Bin 72024 -> 0 bytes .../charts/coverage_status.png | Bin 38109 -> 0 bytes .../charts/locomo_comparison.png | Bin 43742 -> 0 bytes .../charts/longmemeval_comparison.png | Bin 48827 -> 0 bytes .../charts/membench_comparison.png | Bin 33079 -> 0 bytes .../comparison_table.csv | 57 - .../comparison_table.json | 730 - .../metadata.json | 30 - .../runs/convomem_mempalace_raw.json | 29 - .../runs/convomem_new_brainctl_cmd.json | 29 - .../runs/convomem_old_brainctl_cmd.json | 29 - .../runs/locomo_mempalace_raw_session.json | 38 - .../runs/locomo_new_brainctl_cmd_session.json | 4455 -- .../runs/locomo_old_brainctl_cmd_session.json | 38 - .../longmemeval_mempalace_raw_session.json | 33 - .../runs/longmemeval_new_brainctl_brain.json | 56 - .../runs/longmemeval_new_brainctl_cmd.json | 56 - .../runs/longmemeval_old_brainctl_brain.json | 32 - .../runs/longmemeval_old_brainctl_cmd.json | 32 - .../runs/membench_mempalace_raw_turn.json | 35 - .../runs/membench_new_brainctl_cmd_turn.json | 54 - .../runs/membench_old_brainctl_cmd_turn.json | 35 - .../summary.csv | 15 - .../summary.json | 509 - docs/PROCEDURAL_MEMORY_MIGRATION.md | 72 + docs/RERANKER.md | 21 + pyproject.toml | 2 +- .../retrieval/models/tiny_mlp_v1.json | 1964 - 111 files changed, 100 insertions(+), 94930 deletions(-) delete mode 100644 AGENTS.md delete mode 100644 benchmarks/results/legacy_compare_refresh_bg_stderr.log delete mode 100644 benchmarks/results/legacy_compare_refresh_bg_stdout.log delete mode 100644 benchmarks/results/legacy_compare_rerun_stderr.log delete mode 100644 benchmarks/results/legacy_compare_rerun_stdout.log delete mode 100644 benchmarks/results/prepush_smoke_20260423_123335/README.md delete mode 100644 benchmarks/results/prepush_smoke_20260423_123335/charts/aggregate_primary_metrics.png delete mode 100644 benchmarks/results/prepush_smoke_20260423_123335/charts/coverage_status.png delete mode 100644 benchmarks/results/prepush_smoke_20260423_123335/charts/locomo_comparison.png delete mode 100644 benchmarks/results/prepush_smoke_20260423_123335/charts/longmemeval_comparison.png delete mode 100644 benchmarks/results/prepush_smoke_20260423_123335/charts/membench_comparison.png delete mode 100644 benchmarks/results/prepush_smoke_20260423_123335/comparison_table.csv delete mode 100644 benchmarks/results/prepush_smoke_20260423_123335/comparison_table.json delete mode 100644 benchmarks/results/prepush_smoke_20260423_123335/metadata.json delete mode 100644 benchmarks/results/prepush_smoke_20260423_123335/runs/convomem_mempalace_raw.json delete mode 100644 benchmarks/results/prepush_smoke_20260423_123335/runs/convomem_new_brainctl_cmd.json delete mode 100644 benchmarks/results/prepush_smoke_20260423_123335/runs/convomem_old_brainctl_cmd.json delete mode 100644 benchmarks/results/prepush_smoke_20260423_123335/runs/locomo_mempalace_raw_session.json delete mode 100644 benchmarks/results/prepush_smoke_20260423_123335/runs/locomo_new_brainctl_cmd_session.json delete mode 100644 benchmarks/results/prepush_smoke_20260423_123335/runs/locomo_old_brainctl_cmd_session.json delete mode 100644 benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_mempalace_raw_session.json delete mode 100644 benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_new_brainctl_brain.json delete mode 100644 benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_new_brainctl_cmd.json delete mode 100644 benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_old_brainctl_brain.json delete mode 100644 benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_old_brainctl_cmd.json delete mode 100644 benchmarks/results/prepush_smoke_20260423_123335/runs/membench_mempalace_raw_turn.json delete mode 100644 benchmarks/results/prepush_smoke_20260423_123335/runs/membench_new_brainctl_cmd_turn.json delete mode 100644 benchmarks/results/prepush_smoke_20260423_123335/runs/membench_old_brainctl_cmd_turn.json delete mode 100644 benchmarks/results/prepush_smoke_20260423_123335/summary.csv delete mode 100644 benchmarks/results/prepush_smoke_20260423_123335/summary.json delete mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/README.md delete mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/charts/aggregate_primary_metrics.png delete mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/charts/coverage_status.png delete mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/charts/locomo_comparison.png delete mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/charts/longmemeval_comparison.png delete mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/charts/membench_comparison.png delete mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/comparison_table.csv delete mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/comparison_table.json delete mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/metadata.json delete mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/convomem_mempalace_raw.json delete mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/convomem_new_brainctl_cmd.json delete mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/convomem_old_brainctl_cmd.json delete mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/locomo_mempalace_raw_session.json delete mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/locomo_new_brainctl_cmd_session.json delete mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/locomo_old_brainctl_cmd_session.json delete mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_mempalace_raw_session.json delete mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_new_brainctl_brain.json delete mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_new_brainctl_cmd.json delete mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_old_brainctl_brain.json delete mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_old_brainctl_cmd.json delete mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/membench_mempalace_raw_turn.json delete mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/membench_new_brainctl_cmd_turn.json delete mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/membench_old_brainctl_cmd_turn.json delete mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/summary.csv delete mode 100644 benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/summary.json delete mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/README.md delete mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/charts/aggregate_primary_metrics.png delete mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/charts/coverage_status.png delete mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/charts/locomo_comparison.png delete mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/charts/longmemeval_comparison.png delete mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/charts/membench_comparison.png delete mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/comparison_table.csv delete mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/comparison_table.json delete mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/metadata.json delete mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/runs/convomem_mempalace_raw.json delete mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/runs/convomem_new_brainctl_cmd.json delete mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/runs/convomem_old_brainctl_cmd.json delete mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/runs/locomo_mempalace_raw_session.json delete mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/runs/locomo_new_brainctl_cmd_session.json delete mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/runs/locomo_old_brainctl_cmd_session.json delete mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_mempalace_raw_session.json delete mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_new_brainctl_brain.json delete mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_new_brainctl_cmd.json delete mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_old_brainctl_brain.json delete mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_old_brainctl_cmd.json delete mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/runs/membench_mempalace_raw_turn.json delete mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/runs/membench_new_brainctl_cmd_turn.json delete mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/runs/membench_old_brainctl_cmd_turn.json delete mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/summary.csv delete mode 100644 benchmarks/results/smoke_legacy_compare_20260423_105447/summary.json delete mode 100644 benchmarks/results/smoke_postpush_20260423_123452/README.md delete mode 100644 benchmarks/results/smoke_postpush_20260423_123452/charts/aggregate_primary_metrics.png delete mode 100644 benchmarks/results/smoke_postpush_20260423_123452/charts/coverage_status.png delete mode 100644 benchmarks/results/smoke_postpush_20260423_123452/charts/locomo_comparison.png delete mode 100644 benchmarks/results/smoke_postpush_20260423_123452/charts/longmemeval_comparison.png delete mode 100644 benchmarks/results/smoke_postpush_20260423_123452/charts/membench_comparison.png delete mode 100644 benchmarks/results/smoke_postpush_20260423_123452/comparison_table.csv delete mode 100644 benchmarks/results/smoke_postpush_20260423_123452/comparison_table.json delete mode 100644 benchmarks/results/smoke_postpush_20260423_123452/metadata.json delete mode 100644 benchmarks/results/smoke_postpush_20260423_123452/runs/convomem_mempalace_raw.json delete mode 100644 benchmarks/results/smoke_postpush_20260423_123452/runs/convomem_new_brainctl_cmd.json delete mode 100644 benchmarks/results/smoke_postpush_20260423_123452/runs/convomem_old_brainctl_cmd.json delete mode 100644 benchmarks/results/smoke_postpush_20260423_123452/runs/locomo_mempalace_raw_session.json delete mode 100644 benchmarks/results/smoke_postpush_20260423_123452/runs/locomo_new_brainctl_cmd_session.json delete mode 100644 benchmarks/results/smoke_postpush_20260423_123452/runs/locomo_old_brainctl_cmd_session.json delete mode 100644 benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_mempalace_raw_session.json delete mode 100644 benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_new_brainctl_brain.json delete mode 100644 benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_new_brainctl_cmd.json delete mode 100644 benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_old_brainctl_brain.json delete mode 100644 benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_old_brainctl_cmd.json delete mode 100644 benchmarks/results/smoke_postpush_20260423_123452/runs/membench_mempalace_raw_turn.json delete mode 100644 benchmarks/results/smoke_postpush_20260423_123452/runs/membench_new_brainctl_cmd_turn.json delete mode 100644 benchmarks/results/smoke_postpush_20260423_123452/runs/membench_old_brainctl_cmd_turn.json delete mode 100644 benchmarks/results/smoke_postpush_20260423_123452/summary.csv delete mode 100644 benchmarks/results/smoke_postpush_20260423_123452/summary.json create mode 100644 docs/PROCEDURAL_MEMORY_MIGRATION.md delete mode 100644 src/agentmemory/retrieval/models/tiny_mlp_v1.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e270b1d..e4f340a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -155,7 +155,9 @@ jobs: - 'src/agentmemory/rerank.py' - 'src/agentmemory/embeddings.py' - 'src/agentmemory/retrieval.py' + - 'src/agentmemory/retrieval/**' - 'bin/intent_classifier.py' + - 'benchmarks/**' - 'tests/bench/**' - name: Set up Python diff --git a/.gitignore b/.gitignore index a745a84..7a52876 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,10 @@ db/*.backup logs/ blobs/ backups/ +benchmarks/results/ +benchmarks/training_data/ +src/agentmemory/retrieval/models/*.json +.vs/ .DS_Store /tmp/ *.swp diff --git a/AGENTS.md b/AGENTS.md deleted file mode 100644 index 52e28af..0000000 --- a/AGENTS.md +++ /dev/null @@ -1,152 +0,0 @@ -# Codex — brainctl / agentmemory - -## What This Is -Unified agent memory system. SQLite-backed (brain.db) with FTS5, vector embeddings (sqlite-vec + Ollama nomic-embed-text), knowledge graph, affect tracking, belief collapse mechanics, and AGM conflict resolution. - -Published as `brainctl` on PyPI (v2.2.1+, current 2.4.10). - -## Key Paths -- **DB:** `db/brain.db` (WAL mode, foreign keys ON, 59 user-facing tables, 49 numbered migrations + one unnumbered V2-4 quantum-schema file). The numbered sequence has an intentional gap at 050 — the V2-4 quantum schema (`db/migrations/quantum_schema_migration_sqlite.sql`) occupies that slot without a number because it was applied ad-hoc during the V2-4 rollout and pre-dates the idempotent runner fix in 2.4.8. The runner only picks up files matching `^\d+_.+\.sql$` so the quantum file is a no-op for `brainctl migrate` on fresh installs — apply manually if you need the quantum columns on a new DB. (Audit I28 — 2026-04-19.) -- **CLI:** `bin/brainctl` — main CLI entry -- **MCP server:** canonical entry is `agentmemory.mcp_server:run` (201 tools across `mcp_server.py` + 29 `mcp_tools_*.py` modules). Installed as the `brainctl-mcp` console script via pip. The legacy standalone `bin/brainctl-mcp` only registers a subset and is being phased out. -- **Bench:** `bin/brainctl-bench` — retrieval eval harness (P@k / MRR / nDCG@k regression gate, fixtures under `tests/bench/`) -- **Source:** `src/agentmemory/` — Python package -- **Config:** `config/` — quiet hours, consolidation schedules -- **Agents:** `agents/` — per-agent config (pipeline, engram, etc.) - -## Build & Test -```bash -pip install -e . # dev install -brainctl stats # verify DB -brainctl search "test" # test search -python3 -m agentmemory.mcp_server --list-tools # full 199-tool MCP surface -python3 -m tests.bench.run # retrieval quality benchmark -python3 -m tests.bench.run --check # fail on >2% regression vs baseline -``` - -## Architecture -- Tables: memories, events, entities, decisions, context, knowledge_edges, affect_log, access_log, agent_state, agent_beliefs -- FTS5 indexes on memories, events, entities -- Vector embeddings via sqlite-vec extension -- Hybrid retrieval: FTS5 + vector via Reciprocal Rank Fusion, routed through a regex intent classifier (`bin/intent_classifier.py`) that normalises 10 intent labels onto 6 rerank profiles inside `cmd_search` -- Retrieval regression-gated by `tests/bench/` (P@1 / P@5 / Recall@5 / MRR / nDCG@5; >2% drop fails CI) -- W(m) worthiness gate on memory writes (surprise scoring + semantic dedup) -- PII recency gate (Proactive Interference Index) on supersedes -- Bayesian alpha/beta tracking on memory recall -- Entities carry a rewriteable `compiled_truth` synthesis, a 3-level `enrichment_tier`, and a first-class `aliases` JSON list (migrations 033–035) -- Knowledge-gap scanner (`brainctl gaps scan`) also detects orphan memories, broken knowledge_edges, and unreferenced entities (migration 036) - -## Conventions -- Agent IDs: use descriptive names like `my-agent`, `research-bot`, `code-reviewer` -- Memory categories: convention, decision, environment, identity, integration, lesson, preference, project, user -- Event types: artifact, decision, error, handoff, memory_promoted, memory_retired, observation, result, session_start, session_end, stale_context, task_update, warning -- Entity types: agent, concept, document, event, location, organization, other, person, project, service, tool - -## Don't Touch -- Migration files in `db/migrations/` — append-only -- The W(m) gate logic without understanding surprise scoring -- Quiet hours scripts — they're cron-scheduled - -## Signed exports (2.3.0+) - -`brainctl export --sign` and `brainctl verify` produce / check -portable, signed JSON bundles of memories using the user's Solana -keypair. Local-first by design — memories never leave the machine; -only the SHA-256 hash is ever pinned on-chain (opt-in via -`--pin-onchain`, ~$0.001 per pin). Implementation lives in -`src/agentmemory/signing.py` (offline + on-chain plumbing) and -`src/agentmemory/commands/sign.py` (CLI handlers, parser -registration). Optional dep: - - pip install 'brainctl[signing]' # pulls solders>=0.21 - -CLI surface: - - brainctl export --sign --keystore [--filter-agent X] - [--category Y] [--scope Z] [--created-after T] - [--created-before T] [--ids 1,2,3] [--pin-onchain] - [--rpc-url ] [-o bundle.json] [--json] - - brainctl verify [--check-onchain] - [--rpc-url ] [--json] - -Exit codes: `0` ok, `1` tamper / missing keystore / IO, `2` unsigned -export attempted or `--check-onchain` found no receipt. - -No token gating — anyone with brainctl + a Solana keypair can sign -their own memories (preference memory #1691). Threat model + bundle -format spec + a 30-line "verify without brainctl" recipe live in -`docs/SIGNED_EXPORTS.md`. - -## Managed wallet (2.3.2+) - -For users who don't already have a Solana wallet (most chat-bot -operators and agent builders), `brainctl wallet new` creates a -non-custodial keypair at `~/.brainctl/wallet.json` (chmod 0600). The -key is generated locally and **never** transmitted, copied, or backed -up by brainctl — the user owns the file. `brainctl export --sign` -auto-discovers the managed wallet when `--keystore` isn't passed -(precedence: `--keystore` > `~/.brainctl/wallet.json` > -`$BRAINCTL_SIGNING_KEY_PATH` > error). Implementation in -`src/agentmemory/commands/wallet.py`. Subcommands: `new` (with -`--force`/`--yes`), `address`, `balance`, `show`, `export `, -`import `, `rm`, `onboard`. Override default location via -`$BRAINCTL_WALLET_PATH` or `--path`. - -For AI agents: when a user wants to sign a bundle but doesn't have a -wallet, surface this flow. The MCP tools `wallet_show` and -`wallet_create` let you check / create on the user's behalf — but -`wallet_create` will REFUSE to overwrite an existing keystore unless -explicitly passed `force=true` (destructive — pass only after the -user explicitly consents). Or use `brainctl export --sign ---auto-setup-wallet` which creates a wallet inline and signs in one -shot. `--pin-onchain` with 0 SOL skips cleanly with `exit 0` and -`pin_skipped_reason="zero_balance"` — the offline signature is still -valid; pinning is opportunistic. - -## Code-aware ingestion (2.4.5+, optional `[code]` extra) - -`brainctl ingest code ` walks a source tree and writes file / -function / class entities plus `contains` and `imports` relations into -the existing entity graph. CPU-only via tree-sitter — no LLM, no GPU, -no network at ingest time. SHA256-cached in migration 051 so re-runs -on unchanged trees are metadata-only and finish in well under a -second for a ~100-file package. - -Ships three grammars on purpose (python, typescript, go) to keep the -wheel footprint around 4 MB. Adding a language means: - - 1. Add grammar to `[code]` extra in `pyproject.toml`. - 2. Add suffix(es) to `EXT_TO_LANG` in `src/agentmemory/code_ingest.py`. - 3. Write `extract_(path, src, relpath)` and register in `EXTRACTORS`. - -CLI surface: - - pip install 'brainctl[code]' - brainctl ingest code [--scope project:] - [--languages python,typescript,go] [--no-cache] - [--max-files N] [--verbose] [--json] - -Entity naming is prefixed so searches stay unambiguous: `file:`, -`fn::`, `class::`, -`module:`. The fine-grained kind lives in -`properties.kind` alongside `language`, `path`, `line`, `signature`, -`parent`. Provenance is encoded on `knowledge_edges.weight` — 1.0 for -direct-source (`contains`, local resolvable imports), 0.7 for -unresolved external imports. Re-ingest does **not** touch -`last_reinforced_at` / `co_activation_count`: those are synaptic -reinforcement signals owned by hippocampus, and re-parsing a file is -an idempotent state-sync, not an activation event. - -Inspired by `github.com/safishamsi/graphify` (the `{nodes, edges}` -extractor protocol + SHA256 skip-when-unchanged pattern). - -Known follow-ups (not blocking the extra): - - * No `mcp__brainctl__ingest_code` tool yet — agents that want to - trigger ingest must shell out via the CLI. MCP wrapper is a - separate PR. - * `init_schema.sql` won't include `code_ingest_cache` until it's - regenerated in a release commit. Until then, fresh installs that - want code-ingest need `brainctl migrate` applied after - `brainctl init`. diff --git a/benchmarks/results/legacy_compare_refresh_bg_stderr.log b/benchmarks/results/legacy_compare_refresh_bg_stderr.log deleted file mode 100644 index e69de29..0000000 diff --git a/benchmarks/results/legacy_compare_refresh_bg_stdout.log b/benchmarks/results/legacy_compare_refresh_bg_stdout.log deleted file mode 100644 index e69de29..0000000 diff --git a/benchmarks/results/legacy_compare_rerun_stderr.log b/benchmarks/results/legacy_compare_rerun_stderr.log deleted file mode 100644 index e69de29..0000000 diff --git a/benchmarks/results/legacy_compare_rerun_stdout.log b/benchmarks/results/legacy_compare_rerun_stdout.log deleted file mode 100644 index e69de29..0000000 diff --git a/benchmarks/results/prepush_smoke_20260423_123335/README.md b/benchmarks/results/prepush_smoke_20260423_123335/README.md deleted file mode 100644 index c2ab9e0..0000000 --- a/benchmarks/results/prepush_smoke_20260423_123335/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# Legacy BrainCTL vs MemPalace comparison bundle - -- Current repo commit: `c130fbd47174a0932822db2ef7fe64e18a74d30d` -- Historical reference source: `C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json` -- Historical source mode: `recovered summary bundle` -- Command: `benchmarks\compare_memory_engines.py --label prepush_smoke --longmemeval-limit 1 --locomo-limit 1 --membench-limit 1 --convomem-limit-per-category 1` - -## Datasets - -- LongMemEval: `C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json` -- LoCoMo: `C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json` -- MemBench FirstAgent: `C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent` -- ConvoMem cache: `C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache` - -## What is measured now - -- New BrainCTL reruns: LongMemEval `brain` and `cmd`, LoCoMo `cmd_session`, MemBench FirstAgent `cmd_turn`, and ConvoMem `cmd` coverage/status. -- Old BrainCTL and MemPalace are frozen historical reference series loaded from the recovered 2026-04-18 bundle. - -## Blocked or partial runs - -- membench old_brainctl cmd_turn: MemBench comparison is partial because ThirdAgent and noise-extended slices are not included. -- membench mempalace raw_turn: MemBench comparison is partial because ThirdAgent and noise-extended slices are not included. -- convomem old_brainctl cmd: Blocked while loading ConvoMem evidence data: -- convomem mempalace raw: Blocked while loading ConvoMem evidence data: -- membench new_brainctl cmd_turn: MemBench comparison is partial because ThirdAgent and noise-extended slices are not included. -- convomem new_brainctl cmd: Blocked while loading ConvoMem evidence data: HTTP Error 404: Not Found - -## Output files - -- `summary.json` and `summary.csv`: all series in one table. -- `comparison_table.json` and `comparison_table.csv`: long-form metric rows. -- `runs/*.json`: per-run payloads. -- `charts/*.png`: regenerated charts with old BrainCTL, new BrainCTL, and MemPalace together. diff --git a/benchmarks/results/prepush_smoke_20260423_123335/charts/aggregate_primary_metrics.png b/benchmarks/results/prepush_smoke_20260423_123335/charts/aggregate_primary_metrics.png deleted file mode 100644 index 91ee6a4b5b60cfd85a4fc065daf6c30f817c8c73..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 72024 zcmdqJby!txyDy4~poofzq?D3McdImtbc!^S2`Jqt3ML5BNQWRynn{C*N(+K8r9rxT z5|cRh5Wn?(Yp=D=S$m)T$3ACV*ZYb%V~l4!&wc;u887ar+$JNXBqbsuB9oVsRU;xg zl0Zasi0;@?_)X`hudMKgu;Wc_$9s0>jxNTJ%!rhY9qn!G9Br%~&^enua9<> zol}sV&eGA*-a(j)3;l1e;Iw;W!BwN@^c_Bh#9mIvfr#j04D#Orc_+V6q60)k^0HFu zuJ332kA#Mf5%!i|&1a{UUp^cWa)yLNs_gW&BP2)8-BCX*`&LcCT`F{bUjNjo(DNyZ zZ|D2=O6M2$q6TN@oAK8Rnh8$d-|NIxOhiQ)>@@S?s2S+!=p^Aw{~XUB+(_Q?{reZ9 zQ<8`N{;2OyYWep^U&Vj<^%Ek>zrP+jbGYpv?~9=GMaBd)yI)v(Q`J3ng0d#zLJcLuA*N7HY=uxP!dTU*h&hO}F{2XIG-d?)>p zp{!^>c<|sTey%s)XV3eR(4&;Itm<-U$}!{f#`Mw4`0Hn3(2j8-1Gd#OViP6pXShxKU zG}c{UVV*_nvz=Fg??IWz3qJfU_fb6~#dC9cxt3g_euuCgJD>CJ`;?&58DXc@$-7g( z@{&jEgJpi#1=)33Y1n4#mo~R5$MBP_ZDH*O2W(>KW&G`*5u{j#JG!nNI(p(dj`ePm z?08S*%6E}`yTK|+0=hf@D?6^ync5*v)NOPor<%Xj$nqqW;C-5$`DC1aYjO@*Z2B^c zvPw=w+`X2_8w|gmM&P{?=UBd~U*Ob#8z<(WFi_#vQaZ~*bz+9`DK6W9XJm10E$nB; zy_j%roqW?9IipgGjWl~_8 zVNvNhxT9p0^*u$X+HLP26B&vToa2M;C#V%`{mGPbHrpmWd|>p)Z=|Tk4`8dSg4P~x zFV)e~&*v=cz>QOE2w`H94`Cd~Z_VU1wMy*G7KVsV&MXc3?D6HS^_4orpdzo`W47x2 z81`Q9;gvRT#c=lE;sN(2XUy8#b@S%PQ-gzh+bcn@DftvWYUS7t#5^4N#35T<=1qK>Vgz!8su+-&&}-Tf&37Nh z^#I8k#$(uON?yyin?i;at_K8c`|RBf7W&ICycK--pkVGpyf@`3nk`SAiM}c?4@sC~ zRN`?POC!WKzjAA4c}(k1eZXOLi>wd8nsRZ?zQAR8cy+3yZmPL zBG53X+-2dUTokui?%bNwh1Dg!rYF%n7J+kH{$tz5sjb!XnL(=SF3z)0jvVh9;0h-u zr#Z~17~Y%(r{eN_ViWETx^=iEMxd7O7GeEogpv1@<7-&WCj_Aubd11EV4{P;qc^M# zq!u|U>Vaqfvfeb_5`zvBnN%KRnd!nf2>Vy>Zn4qb4y2+Flxj6@-Cgg10&u7?f@{HX z5NeQ!XwJ^e5=?;;auJ-dwL5E0?u$cGE8?3&e&p}tCMwql99-k{ifx~5iNBy=QSpU| zTf0-cQvSX1tO1*jCdNQ3y4d2sT*kVDUt>ui?!@6I)@CGzpv^PR!v#Ee+ko6Jc- z5%X0`RfxJS=OgZcxA|0r!YM8O9vLw?p3IwmQlZ$cD@*4SjK@LexgN78=KWf=sj%*1 zuB)uCudO+`QfEnWShc4dfQe30A1~qaslsjLAg4i@sqM-{t3avh%{?Ud9B(7D{0CM>oO!@W(c+wp>f%&>;z=J&-l$wo_5-USOo}#--L_Vz4t;xliNb#N*Nv6OzP8o&!=HVjOdfHZ{&|fq z>3}1Cwi}I8H(J;#!{O8-I1Ow}GVdflAz_}=nOWO>xGJns7UZ-I_ddYmHGbh5o6^q- z<+^lSqEWT?fjpBs{+UM;-_cK&67UOyb_2LFWvIl5pf;Ih#~R9by%%#|%{4F~D~+u* zf>viMtxI-mt<0#(Q)s9$jBVU2ea11SIf_T5;=H@~jNdWpSKg=cT^0;i1XsmA&}Mrs zH?WMmNVo0G4${6j!$f*f)HT;FerjV)ETQ#H5v)1Xli1$fwe-5S-r4Sg7J^<~PLA@- zp+JEF*IN962>&+sxm8i(lT^oC33J8hx`$^AXHaF1Q|$+20#1_sDtVYZ=Q0U%@8EeZ zLz_5}nOZ)w5o&|v++EI-*L>r+&0fx}hgYSo?Hnf}>I*>ntE)264wUsCtlJEx*rsXD z??fd=JMYRDhi6Q_U3vj0sj23Tvh{@u3}tx8d|*N@-0!t~epW5vEG1cZmZXwjLc%PY z`^sp8$#EH(kCa?@lb&{28=BclO-$+R@-Ivbdsy`raOMq zA)*Q=(zT-;5^1mY@#*DvnD|cStc-EeeK`HRYBvl0USl=j%dS{z=UgDo#Aw4ibg$>$ z&U#JP1vbs*tW5pVM+xMlvrzcHpAS4*`5nJ-K`dNzn>dv=RJ(VQ_=HqvmQIKBHo6xp z8^5!_LZ-pzNv=?rOsg`X}BKLz;1+kf#Ty8I;(C z(Yx>)JLtLc1}T73TYzpNJ?4}Nu(pI|H_Ony{Cc#L^^;wgn5lPE>YX1olv@o`#muS+ z^W~>6cdFsIW*EKr zO}Fihr8J9N9#|ZKix-Omu=rr-(N|nm*S$(QkyVOegBxEeq-znykB% z0=txlhS9!|+XjJZl)URH zjDN{kR(71v?2QhKLxK1m`y;WgcUVvkvNCnTPScU`J3Bjm%8nLZxykPi9gQ8GgDRPS zzvkddfl9mOk@u$)X_d5F-!Lpx5whWCoX#<-rkSOjCdeQ1v}i5g^}ecGX!)Ez+aI?@ z_xlDd$dEk1QBe zw1ykeo~C@6XDMOVunlFhW0*fcURJFlNO^AIvRAbLs;k+vK4qkfr+?oVUjaXQRM;Tv`6(^et#jM5RDi268@P7 zfTDLw7{l5%bggjaLt^((jo%pb^M0Y&H4~DZotfhn45Hr%_W-{vUsyhX2P{>+DFH zjyFdw1M2JTYKh{hW5*G3bJw6;zombV6GIV>K0ew$4aJ1{#GB@o>;o9hhJe);;?$t- zA1moI)LtX>S<>!vA|l0^115pqTw*@Eed@`rkvC*MOB<$YrfIzCc_~%<0ZL&2<){w5im6a0&Fgw+DK-_p z(H}WAsEw@0cGccHrcL1-%)USD+uUb$Pp3b@COxko>AWMKd7&sD8bK;{Y>RITnszVt zMOLnUP_CgKVdlKrG1?Gltn#Ke$N5svePPY2v65v{c`w*JPH}QjU=s*(d9Xul?KdrE1{kEvju4hMqHsP~%yfKoKeQI9c;U__cetM8muUt=68LY~sk*C=WK%_z>{(BOR;dGgob# z2_9S`RakasNR4&CDQfKQ_dSoMb;Y07FP_QnoxihK(3iNH4F9wVKaE;^6(Skbuv9ueo+WA}U} z7u%{nQvRNHD_SVP>vraS38dD8KEV_>k0~DDUO#?;llNWt&8MAv0x2lku&fuve2c~L z_218FtVyw;3nzaYoE}t zI%Vti>%PQ~58n5aXyZ!!uylD(izFU7nizG@&h8{Ot*!gBS^C(L#TK=M__qP>3@5wH zUoxr>r|kJh;7GcZnRJiPXx(Xar)=AgT>Sa+iuLnf`DTiGPMI3RA3ObF)>5eAGPSrI zax|g=&65{Vc12fkPNuw>ZckNQ=AJz4Lz#8WO9gt7(T86DpQ-}vVnI$x^nJk)k1T=9BFni)Wg(2WH`-`^LooiKI1gbbyne8h#U;fHYyFFIMe2L}h4T9ZYr&gFN`n*!ZCS$s**K4FutMq_m+Bhex6!PjQS7d-*G zgr6Y)tY1MCfam6gbN<-S+^Xs9&U)bkiT+P}D0*->-0JhLfy}!t(`%XN!&#rWSea=( zFBy8w@lEv>=WMzG%~Y%Mbc+I71MTFNZGV~BlLkN=4bTBo;hV89)}GMFpa-k^3u%Ak z8Xc0UEKREP!(WdFm_QZ6PKFY9ChV zu^w%0i4I#F4xo*0YqKsGo{ZM!y_y3u1BSF`x;)GHzIuH)M+^x?QIbwPOYEr?q|tI! zZAe3BF-19@UKO4!>iB3<8NP)fqHYBet)xfc^u1SBla8v|b7 z4~5~68kvm~@jje{^#uWwwl)voQRX1Tj4=wglMoGhtsATq7%J|(y`7UOt_qF)#?DXR zSg31v5_}cw5<3?tr2&h`Sb8a!WiC?_5pAo^?)i`dT(Xp|wII7wM5*J)h7m2gz+;*PXPauRES0Nq{sLN*B9C;ULkTEkonSzYP4puVx)*B_u99mgz+FMV5 zS~Ia)UUcPH0v-JE=gqifW=n zmnYMvPx9f&UuQY9L#rH1I*6=w%60{fx!z1K6(_Zll7}M-X-ZK_M?M^t{4U*?p7lhN z(%qVTa#H%xPw$*Pg;Kz zlPSiWL%SFsNnaYN6WEDQ4hg*3D>CU;*#E@v5XQZ0LX8az=zqj<5Eg0-x#?={<()Q0 zudUT_ujG;D5T;i;X$L+UTFf!blJ=0wvfTT5ANy#mar}nmFAVLdrBkJU!S^05S>i3q zA2@~9II$Lj3~h{m4d@`jz%zfWS~{;Dc+575(uNr_Q1tb+e_N&|#{Jp^EWpQKk0D3? z36Mza71&JqPH9k1K0CWrLl|w1A6#wi!e7shwm%8~`h)l!BE%lMfLfNCOH|r8;n5TQ zHM-(P+3HcK{JyBuh^R}~97rYfAf_!72!+1TVQv6`4)G!`wxHONYQf8hl%$Wri`yz) zmp-dwty?YThSI#eiw1cUfV^5T-+wRSlD%1sH$92|EDf)A4VdIHK~Aktnx(XZgq(wt zeRs7()%?i~*J6+;Vj5OwH@CJLJr*h_ur~l2eJW^sf=J88A)u*HaB><|>09HnTQq9- zc2sXRM}`PFm`?$f$kH!Wd+_z;*G_ng?`a0fY8euE6LLtNO-3fppW}aBd4BOLTZlbL zElKvp=v2pt5Pq~SCrnD@@V)wQZCV4EWnf>mkCFZ}B4Nt5#Z7=y^E7ZX{=7n~ekJ0q z&>6|whcw4lo*lo?Cx(c#$dU70DT=#vn`#xQoPUo0Jt}yffrGZ|OJ83ix83d1QsPrL zZanz%7Lf-%_P^yR^?_A3x!Um<0lP-1i=&|EK5dme+qNiV{P`&|((=Jqju`LD@sbx$ zNmf4qfr}p6Jnt}%Z~D&tiQOFnRu7a0CUd2-i~Caj)2BpR=Nv}9k?C!`XV>8dz2wE0 zR~Le;mFYr;S4QfB#$Y}?BH+<0G#%@@(?2sxv-i)1p3c|%h@M?KFaiZ2?0sU4Lt&={ zZ1qi!AYss~P)(WO zZcG&|s)KSUU}DpY#gqpe{BsUNYtY(dFH?R)m1GTz?w^*Zk&O#Ek>u*|!~_c>ezWW3 zmv~&=|F`Ey;q^~1)$r{9^|Rl3szqHyA%-3(Ki^l%s0|-;nVpy@c;;g2%4Nz0VK|^{G z6chwERQ1t#^ED9EBkF=^MZ7!-__=rd=pH$w&pHHin<-q=ElghpmW=QNk$`NivgYYb z>fqg;{mXL|O852G>M5`Y`k8)L##4J?Q5T7tpDWq zZ@k8z4>v?`#aC>O$BgG$(`D@f8bwrpk~FxlY)p#bp)mKO>0B3wYPenI^}CBu5h)5` ztokNm_CvmoK#J8;~*pfMV21`NTPs6PMQMAI7 z2Z4egLuy^=3?PoX34{{Hs;a7d*QE#dm<-dDF7@Imm)Aj!Ru@C~8l`|O)ub!Pzj+!0 zsK^|tvZHMo_dZ}X9FWosMcU!VlS78L+5C#0$@rhRq*Q`>*2)U6PMg-;*%%301`cp1 zNWiS|Y)8Aaj3!}G$11WP-Y@q?i|xr=tp!3xI>bl7#9@|nLGHrIT8l7n4tV(CrA$hZ zcfJMDav|~~?pj4*i2Kl|$5{9X!E6qpQ2zn^(GwT%&+|=^Q;KIEJVGkByS-T}hIBVL zT#vFQ3_E4*eg*tjRFsnYp(Y#qHdj~8(r7Z5ipx^Y=EuM{-Ahz=%c={;bjhd+X$zRx zIamw(pW|PMyUf4O@IHC$*!8BOwR*tr&3tXB)0(<-bLq4N=dKum?0aXd1UuVyN}sKM zzW^>y)BF14@}%#eq{|a{B|Duct9ppj=#|YcwlBTsAkS%YVrO3 z!;*pS<=p&lzh_H&T7^XC2dlY2-r_aR8_>$sR0j0|HS-KJQzYh}sngjW!DSTQTWnXp z?YaM{uA9eTL=j_h9tz=zeMk+|n<)HcsP;GBO+QP69z4Da6AZqjwUtY41WZly=XFIi zs@hh+2-}+u9p@RKWYu)FM)R@tg|(g@9vNYPPY8q7QV-RswlD~vrYXF6W&hhmX<@yn zAV_j;4yr;5_&<+7D`EI|VN$iU_Ln)m1J)YgE*dMz1uFmVg#|8?Z?8!TRalO_ziup@ z@-6f6hF$f}dCTtY&6SqJPmw7!av}({fT*hP#kT#V=WRiTR21Lac5u`AGmQvqGZr+e zcVOxUC*Kb0*^OmZrlgQYJ;Q@AYS<&Ve?v~)P?DZZo%R&Jo%GI3CR(;T4{Sb*gME~I z*7a~c%9lxvjEqV>*0XK;N`9!{gp=++AL!2vD}&prozbd0KNYDj%&Kwh&q&X4Q(Uy| z#VWh6PSP{n4t%}nx)upV;N5A4TXInR$Kri%HmO;v4#!K&w#l-ib zZtaKLfK9)Hafz8LQkH|EY+2FD(!O!csH%cdDY7lAW9IJ~YL!01TcHLmc=XfbL(2$v zlGMz6&BxAZSW!Zf3iHDjwSMY6EHJ*#@09u|`*PEUGZdDnwSuE{!HEY}!3I<=S^*3^ z224&ZQThn@G*KkM4AL@7Ute8N?9MkEgE1S|1}~c5col4zQL|}agd|mbrZ^PVeps4E zD#^bEaC0zBy2Cx*zrVa_&$!3(;xvOlNzw- z15it|R)_A_#Gmn=VL<-?nd~js0gL_D=Sa_Ayrllm)WZLpctXxuzzGg3jBqhLCc+Nm zum6Dsh-CjYvGB#Y#rVKeoIly+KN#|c$t_Nr{gW{yIo#$OP#?wfp*&#!H$<}6bnQRQ zfUYjD5YB!L43Q9MH;NeDB2#dRp`9D%o4x>Ka#Jl$$toN@P|BC8)V!~_E*o+z6%w= ztP7)4aI4H^fzxMaEdu<~P8+Q~%OBGP((T5>9-t||1|yHYT;q@VX%?F~W0jgEI1Yh;3j?{;nE zRbXL~ww2Q|C>+XHZ7%WKL_!I@4W^j+!2wXgkA74G?=Ooa27uL7NGOmw&%*j{oa-(A zio*%CEdv%%fc0QD*HiQ-j<7ILc?eS+2-D7LrNHaRYB+~pb1Mb2()GusW+3q{kH^?4 zG)LZ09xGhaeLS-$zTXB1!bNT!J=eBqO*!5Z(5m2PjE+d82jTu5^*jzl;eg?SMlcX_ z7j;tQf~CE;mu|1D0L1w$rirdJrWI$S_E?+l90TT%Z`F}zPYe1ma@aF5(?8Yfpz96> zbE64xh4SDS=r%|72Epn)6_oincfQTm-*8Jt!tMeQoaQlff1OL*WiX%*6i`G0A-%mVr z0|x%tsRD8$k%SYMxDOwX2Mem;3a`rkN7D&k0U&uurz>?d2)iMLKC~Y4sln=MPjDGIJ^tc*UzWs!l!nS|ArQV` z0T3GuBi;mXCKaHX`LblJ()Fae_n=grv#4? zq#_P?xDA*_1fqI8RYUj*OqiIr)8Ih8`S}o+W5iWt-5CPVK@kZof!W-iVQ8%?7jqW$$R_@lw9N;vA}+#=c}*iGtPmswO2$-YW(3I>R`ql`P+Gwf#WuivF{^oU zB;SkROrdlrhg3pB!fkQrF-VYK`(^e&N0N7l?_3-7?hsY+O<8ZylZ4z?(+i<4s@3&Y z`&4^ENahTiCc94FgJ(cVZo-mcL+FV*Mz>R4AEt*;<2Q2GgJf|1HaJj|f!h6>_X>H`6qld%lX>F9e^OhWQRSs=F_bzQ}Obq_iXs<3njunCjNh zi$ZJgQ2Mmlp(+tIu@dVt5S(w;)CjY~m=!nZO+XijW`T2M(UY5HSg8w6Ejv_uKCzlL zKXOcH&}$&RkqO#|`XKS72whL|@4-#Ckv9U(H;Bn783716l>~WT zSBHkzF~9%}S1rd-j3*pOG(r%~pv#zx7~mm*6dljUj{DTJ`?vJ9j_`>kB)7x;M{fqGRu3h}Mz{GeA0$&Qi8yC@ zF4hp&bWJgO3qMYqNA01BybN5X+P^-nG4kF_?UoQNY%SBgGgAaNeHw7w@0l64R~NYW zHGYF2nTM`+oKgd@6_nu~cf-V=7YVy8A);ZXZP#@pKgS*O$9T=+kEMTCVQIQ=AQAdT zLSg33sUM}7*M&A2%n*=KrlD)F{f9Yz;Ls{eedeEqA1-LW zUJjKzm@-H6aBo2CM{r@&ShxNoal|HIEDN8#R`~_jX5$e1ITO||xBZZ$-bNNLRk3?E zY7%Yt$@jpxq2F!-e5myT0O0@+s{M7ictTzBwy*yi&C5g zD057EAixMx@Z%JJA9qzF{W^ZzKFfK&#lq|`F?2lS%}7h#R?=n%Dto(BzD^&rKDc3 z|0ocd@quVMuB#SJwT<0^m@(m~2d?;gKwy-%zWaVDe-XqlxSlDi$1_aOj#bh!C{U=SKqjzmq4BtK!>?Gw>q2G0$=99by*>Qwbs_-uYuCPq2ZR-f|L%Gy zV?Fr$xwm~jME6#X-&%lRAu|BEq9Yk)hP@^wx&Wz~E06X=xW{>O^Y2VsJ}r4$5{ZGD zqOAdQo-ED_kAULJ-Ab;X0HtQ!8$yXESuW`pS_WHHCx*=nXBb+tYGzr^BQ8e34A}fv z_y)i<^LN#sS$Z1gU)fWwow-udWobW4+p|4NNX zGzT0HTOPxM!5I^8(Hci>vjZ!sWz@Dy)_#%w4J99UXEBII{{`u;K*TBM3%k3!+z{oI zoN;{eiG+e#*1Cf7`5t`Y~@gOwK0z! z{xeFxgddE8hb4Aa-<5`O8VGBjo3V)ZGMD}YW7_~u5*wfO&+ARJqOSK3%44nreqJ7< zP*tB;5jqpgez~2oG4LFhq3BBAlmox_wrhv~XRsm*?{z~SEaN5UFl8K6WN+|_alBR` zg*M%42xkE#8!r8fG^&B57~RVZUg_IlhTFGXE8LqQ>DKW4Gy@~F@MY)^^&n`sltETd z5mx-EUIz52MVj|mM2>6usD8tq-(VrL4QFq%vd{yNvHh<>}#cINmPAA}uZbuP=fJC3hG<|Dw5%~e6yt-yS3s;j zZPk?(g`h?otqe6L0N1Z2k|NuSfjZC#HJ5Y+d++Fp7aJY3wAlwQ)$($LEth* z5^2~ufy90tuUO=Mt>^y{Pb!XqX4nqjo&we_Xz>ZIknarv9j|u_az0U#Lg_H&rM9=X zQ_Pyev1V}Xd9N?t_^i&3a;*eK72$e1A8H-}*fj+nkVfhb4heP#&f*HF1p>X3yJ8;r zLQHS2vER>ZgWhLa$c=%+UhJSH*Vx<+Wdw&Fv&Z^;7$V`?!$yQJkGoJYZya^WPu+mz0V4<%fl0Sd z?0-##=Z$`Ql}~epss-)mFH6-Fd4}v0oYP9uUBX}yr_=^ z-d9TY-%g||3w<9_z&Z{&6uZxE|DpNC(l)ne-Z9EQm5-(qZtty6$&^~yKN>w7klYA-`AZw@ppCWzk#XEhU&^f63Ul^ z9n;UUYqzOi;nFX;gBV-V_YLN)|7xDzGj0*=I#+~6w95Lvkya3XS+-MCJ zccy-4cwCjwYs-=68x)GNe;A^MK}3|2g^9)${!NtM|N2WS;4eu=ZfCqqYZIl|R5x z5s>^4EeSo*wx$rF<5k4aZghG7;<+Hdv8VvqpZ0hl@y%_rTGK1Rz1dMK#tal8b!K2Ye4znh$(B5KOaB7#D%S>*cY`PGS4-L&w zQ^Zr1rraM+dNhIO5OQ2pjauxGD_H1DMZx~A&|%7%$A^ehpgx8GQOeNHODe5G_zsln zsF?&DUpmr`0NybJlz59scpy5D0>XN| zclXDng8gvYqOk6ZUWh?c1OlJxK3oAs_{I##p-hmP1|gxV9^&BWCmn4N^1g}Gl+P20 z-2%7`J;QJFiw?}bz@M(S!DX$jQ+ivrcDXwoqg^KF?lea?=(X}}BRqr7ZF!6wJ7`m# z!9CQwka{x)U(;pQ_+BS%uk4OzljhsV8+!|{PJu7{_JEf}@Yj`z`pIFL(M+k@&wO3T z#wp0EHW$5uy07Cr*+#zvyS_j$jA31Z2K)<)ds_|7IBnVuz*9R!oFwZ+ptWMkbZWPu zC3hS~oLBzNuKmkhv?(0&#r?20`;Y7yKw`3}`|8!7Bg#?SH$lKbN&iRdbP`l&Vq+x$ioGpI!lCnmj*GASEM!Bm-3}g;4&6fg5<`#6C^LK6solVzcE8lSvhjIae-*F!8il@I6W4&6RJd*okD8)Xx@y{;+(4z;1QR?y{e2NI7olhhWoWwHGW^6>92j-`c*MJB*@*1b(8dLNJ>LUZ%=S2myTq%;SO#pFP5aI{cp1z zcvE=HkOllpED3R}5LCqkbnT-GkirG^7i`?LbxcU6e?QyPI7LWlJ(WEo=4x+*wb8Zs z{9B%9T&`4rRq^9bE1Thh>}uz^k0sF>9A2BtW=N=_sr4^$h=8(c_~Ad0%3#Qd@cb97 zQdqyrwT8;Us7$F|?4w_q$3!5AiTY<)-Ltp4BSqds+A{Xj;9uGq+O97w>Il1~{L0UoGnJD5YeyjJUO>y_W$BJ?I%nw?0dG+jFZiuM9UlItr zZ`e0;Z`)nkc9NLt7&)|Q_U&xLd!`uuGDj)=!j8c-za^uG3!&-j@rkdUpb>HX3wdM$ zwHws0zvYoaQZkTbEsjJ zXDOy%Mw=Ji^HW^}We-452h`*jB!xv=ND?KOGV^F#a-Ax;y_a#A5Q@5>22^TO{THi( zRD%G{)57Mx>mp4h4Lqq|P^n|98SnC~cC6i&H}Xzw;5|0IM+M!YkiEpQC+?fufBLb}^bi1PTps=v;!y29*<Ti-W_{!ONdLqNG_jY?9S9ixX?Eg8ycJHD?U46oTo*}${ zMtXlxiEM9|f7GJ*lc)UioB!U%cGq0k93B}X&|TqH;MNT0$fJK4AE)I1Wt03LN+|gy zl+!>rkVcFZbe7RG`yf!(0bVBh@@OlNG>7%;`zLei2m*uFhH4HWlq;0W$PfuLAyXS9 zV+Ekwrhx7rV>`XS^h9VAD3A=q07&QZVXR?bpveM4zuJpM7of}C*R20|U*=#MqymLm zhRs=m)B*p5PqtW}zsa-H-uha5l~)E&Zs-rVCl{V=43l~}`km1(l#?A3Q^jzPd3)48 z-X}*tyvArff-=mj1&`pXqjp+hQ5`G|kmh@W-UASXSRbM6I_>&?P()sWc!lT`;0iGS zPdrw*188j&Hj}=C9e^=2SwZ)5iVW*O2YQxCybApZaY)R4#55a2ueGejLGHA6u6k>l z%|Wyhs$#U~#&7@<7v(XpTmW>~SLVb94G3xC}j47EA)j8;1RS6bRo94$T!<;8Zq;b~GVD$^%sTo5JXxT<@IG zO_C*n{iYe0f@omx8}%|6G7X3z*4kyVuiHufr1QnAHTm@3Om{a35eh$$Im0W1d>(! zk4(J7Udyj%F`B2h^=XT;Zeg&U|Mq=W9}dcj`R+;{?fW~%b^OSt)lKWR>sCGam@lt4 ztGRCVRk+ENxwxrf?fMxh>Het9Wr1J<_k;12g$>uMw^Y-K)v06nUTZ z1LjT{lq~ZT$c|y!zkUzhG~z>BfKbDns{|N=#2=Dy=Ony6GeESf^(xxx68F^9O#j%~ z2A)ISvsMK*FTSf9ls!Ei3J?fdi2n2w{@wjGz}jboyd9UhkEOUq3nW_SWY@iX?hr?e zxY?7wGB+9BLMyHoy=sf&@fxsyXN(X$((GN3rW)?+OQd(o=KI}5X$T}(9Y2j>F4pc9 zDNBQ0?QdaEErU{!axDRdBnC_C-ZDoUAJ^edQ+Tj97%l|K7lLq+uWNr5qrML>B!7Y( zkNZK>qR=aA$@P2D{%n$UCJCFpQy~QD`o?#m@z=Ohvtg#4Qh+v}F}(RQ?E1Qd=lYv0 zogyWOpk4Tl*W=G343^CNbSW8N@jBWNEf&yMvK#+iXeUS~4b9H0jeGfk%#q8eit4wJ zLIfvzp6@;G{3YEQs?%L-kDLt(qxG#WzfHjPkggT=E8KylEU>Oyxu0R$*A^TQ)gvB0 zz?XCdHc!zJ*$oCOvtJjJYb zU2pT=?I;~7=p@(cbS=+jlsi=+{L66p>@^z+RPRebFHLY0lvOP&cR-#W=fD4nh$lE1 zD|lDeslv8jn1Ua*wh5H6MY%V@(q|I%zEA3E<mV9^6~N6oGoZI1-|nbi~DF%nE_@*1E|q|WOQ&7JM_wdzZ;hs zLOSD7i6KE|5WG+<2$9Iq7Zf7tN{v30=)JS1RqC@>)%)#k-tRr;!#}a-0yj1{Z#K9& z=8MOA=BiY>v{b_udQ-jiTVs35ktNofA#(Az^`bpt>(KZFh>lN7(6hhtlqaFytzAdw z{7E_b9qm0oCy!q1WBwb_Up#>jOwi}U5WW1xG*E9w+@6n}$t(Op!y@ajrnKrVp-gV+p=(v?x%p_t0i;ZHQ_<(mjitS{V64kn;^5l_^z#9lZ)sKM314 zDmkvhHl=8ixhow98Q2|{tDr#3!%oUnpxE4Q%cj@V)8Ah5!{3?zksJ^;%O%>Ct;-pY z-@enHwi3L3>AMUgzz=I079mbVbq&`FN z2?Sn_G|X+-3}(Y2wLcC+-f;I6bRTAR7VE}}z^==C&$2qJK|xRBy9L2;qej7Kel#mQ zYJ`u*AGLPQF_@sQ(J$dWuMcxAnWFR%uS%i-(79+(^V8 zt!Ghr{}!%C9!ao3!Y*ySz`tcHk{yBikU)=|TTLUtlTI`ZeRiBEE+Tt~3rCN=f(S!Q zf(JzV@{MbcuJplkh2)SN3zF6YdGqOv`+oiSRP^BRFP|W4ng-Gpz%8eKJNKL!h8)QF z1gIbK5I#3-+!%6a;oP=+B<9itDxRx|s%Jc3w2q!=O@-zjs^(zDG-5>b%Pd}P@?RW| zXZe3b%{BLS%~y)s9T3<|{Qu)<63qc#UrU)#_$}>i|VF+yo2dh%lYm2w9t_+G7G!58TJjaJf48VhK zLzpH7p8bG`coDB@?ODj;R>qoG5CZUG|5GK?D_DJzr3tYMZQ#fBkdqEamlPky*9 zXb7OzRV*S+O|gt?9@iwaEOS4YECv~I&VJ3e%e$#epTGsX-*th9n2EoBae*stv^nbD z*7^b$AaUod4J23xUPmy(4JNL@^^Mmi{@-Y)0l2~;?F!dC2!r(;_S3`K1fOA2h#W(5 z2qf{KpmG{{EG z2bDHxljt|J+O&`BNHht7Pxqa`G0jKH?pGQhG1s-aW*jgzPQU?D;NdKfcDLQ!;5!i~ zFW5bL)rl8UY1hdJ}iob(6z8tWNW{=wvCxqRWQ`{uZ|BuRR zbMXaAew&LmZ2Rdo--xfMgOf1lV?4;pU@Ls8N~|sn1Pko>WM^SF6m}k_ zg7GkArgwb@_;n6*NUk}Y8aC@!5Uah8Ct=XqqG4IYXE5?a7n>gWV6#r_%@Q4=;&@0^H_serdQpFYD zE(Oh0iU^_PcN8B%_7E79L>}CpF#t5zr3#rq_)614LoKcq{RRX}$iM~UTX_gs z&Rv0c@Z)SP?cyKtn14j0zJI@Prk~~eo>O7&i=DlQBXW`n zYh4|2g2tW!gp6G;_sGRk*#@=tk$0+0J~h&aZmK5|f_ru0ZRXxCTb{{LaclwV1I00a zXLQzF*4pV(PUG&nAEiklH1}vf#7)2recbb=L~&Z%>MO-lya=hQMv8Ltpq59)y4Z?K z{)eHp)4!4r2eToE_#Z!KCNd6sP~8e6RgczA+1_PpdiQCPY#BJ=|Y+Rmwc39W8^x;Zi#(;7( zdlaMGd-h-a$H!X=G=yh_32T}fm4?R&{}*-d0Tk7?wTmK(3W$k~Zc&Vg3W5<(vKSFC zAVH!5k=W#%F<=%I1r-URf`kTz?k0m`Km;X&TLA zj&3@vwdR~-d}-vI^-x=4pYkEqF*TTOaH;|{p?DXsynNPa-d$2CK(i-h8O8-q43rPH zUO((giLLNv)V>aVU#vgj);Y3ZUiMmA{qZ+3P3;+zlQ)*k?(!%Y3+~wD>@jPg;mo(H z@f01!iSsiJhU@2#9T`vw>8YO|(lgRv^*yh(z1`Su{x5c-+#Upw<&=7h{M*b-s86M?(h`r=6D+$vmBAScg6kum>vrP}$v&&K% z|5&#q_B28lzgFCU=Y8N7?Y~Vb+xe7@zI-+knnfv#+nanlP)b#&#IpG-QZJp~PAXdD z`N$7^T2Rk(KhTKCIWtlI!*+S$G}CXz?zKSS>EqG>kQT#Dyu;^dq`NGf>g4CE%l>N? zz(c=`1EW)Y(`KTUMA9KfQ2|_Xl`-$3-4TwU=FX877mVq3X=o@a5&egf?2)`zCb6y( zOCpMZzhLz$5!)~F$RRu(Y<_ii(JNZR!~dWrTObT#;NR3_2Kv&UXiCk3x&u#0BQ$I` zDVNw;GEsjhbByDigA8@y#Q!f|;H&(rYr@370S;9O8}yee62OA7r~p;%?-)9Yd|Jxu=ALU(sIsum7rz;CQ zZQlZJlCC)qMD9o3@ozkXB^B?${?%!@jZ}&{tym%!-IG~#uy`_Y z+rZ`gepZ#vxD15aA=tvRQKUTk!(iB?%0w>NOEKq6~2*r=xE zky;bI*mk5;sgmYNmvI?8Ioo|S!(HC?NxF*EUDQ$1Uy?Zc9*gQF7p89zx9EiC*!0m^ zD`1|)6IL>N@5MIQh#pQu7BWX~vc*&~MJ#crBdXMN*AsuR=FeD-B52$LAS8HP&RvyQ z8UC~d{jVi^E7WnbqzoXRLB~zaFXhr1Ef*j@TWo+q2Cu=y?R@b1?wNk?`B6t{WixeR z4ro9Xq}ZX#gtn}x=o)v(aivgcf8)MOwvpZG1qJ~Jc9`GT@UX%x1;U8*zlvB^FJoLr z>rIZ1W3dfYFZ_1#?avRqf*o#^pe53;Pf5^Z{qg6;$gwX!u-=xI*KFIK9?o&awKdo& zf0j+vGp%Y-QW4=fJ$K;$M%H)(D<=kb-T%-sbqTd8*CCtNqZJK`TDCEB385$b&Cm9>Hdr z>s0SyJZ;cpW83oN(!#0r$JPzu>oo7lruXmGJMTF+;O+Y(xGP#TV`uku)5@1|0NV~b zj#@eWAxIj&|2Lu($bhH->^~v8lZYpY(10`xu%T(d)3Xj#!+aVA^cTDi2%N}6PErdL zaRE31a7ZXUN1c~|oVe=W>NtGu$A&^6CW&SZgyoeI8%X2+IPcGo{!xx}<~$RXr1gL7 zN^^YZm1L$@Fjk!^`b1i~(6K7|-7@pgJFaSkoda~f4r-Jgd60b=T8`KO9u!2RvggFG z@h0tr=V%E(SqzrzT0P1^3?cdHT3B5>ldY!lziRA^*}p^M_IT?s^mWC6@lTH;oiO`S zIb@+V|fw^9c z_)&ApUS}ZA&~~W1(ONd5)U^2UxUZ^Fb-=BjdU}l7)3bbuNH5Wye@&S9O<*=x!fBxT zw-@pyd1cJoEKCt{$=M?UBmy!DiHfu2U~olP?tY*b{Tr7f!G`?E@XnwG>PS4l<{U#> z4ImvVL$Paqfu0h?e$D77)WeyCb@F4~7XjfBG>VcZH!)4SA{LT?Pvhedg7 z@^6+jE#8ObWHn+Q9{{sWLDP&aYUn`na^beTwd3iA0y4Lo<-3g3I|*{!=$8}c0SL5KtGk|&xj_loV_GW8qj3&@u(3mQUI z^#BslG|CE+$1s>R)^p=szo)0?BB{8tktQ_V^D82Im8@WfkY0!AL{5fh???I~sp*$l z_!nO02p+>1;J44Ky#c|;5j5|p8*jW0{qy%`6Lgb51z6M8@#*@*UJvq@{PEl@KWV+t zZ70s>2FOO%qb#?aS)YT)Xb8L7iVZ)b=zgrlk_3WN~S5L^! zmm2$ZBwY^bFqjLC{HCQAbwssDp4ER-Ew1^aT4X=1Ke2t9*k_AtB^F|m3a&2i_xHB7 ziaD`*{i4GIqsQ-Uu=1>=8wTZv_TI>!qy#+K?NH26?SJfWuBq?Ti+xcQ-}9rq&gFgV zUo3rO#j@Rw3HFmii5uN7Obv7AThGD94RwbQ9T-dghk9BP$=JyA)1 zg4D-92;m!%-{I@&ss9N*v{Yc@d1Ev;Wa_w0UoN_<;Q>wbVqGajaKDc^n7x(mZ@hb7 zMC*uVoZe6SdP1TA=vDP{K+UPqbGSv4K$hUQF8~x+z3H=C6Xl{W=oSB0-XIcFEMZlDeY*kr0x&%gK&Zp=3pL zhSQNU*{GUEMVYp&EO!{INiI0@-5;9cLi|^!;77edm>GgvpPCef7lSz4je?T3BGusM zcBW3X{NU6*widE?qukx=i4SivrBhl0H%P>lh zp4yThtmRox-z&v174|IgbWA0X+;ffQRNIM~Wt(&}!$Hg|0Rl8G7m&6+nK?v@B!-7s zAAV!~<4`UdfuT{0R^OE3IqsUAgD{gBn>1q|B0)%%=q{|%fBEkIK~ftaYnVUyTMi+n zUqu_dx?D1@G)Al7=65*lwj%(il4j~(c_c6c(X~H}dGf?xF|wu+|9)~jNgy5P*Wh3v zX-OF!G4YxAAi>rl(KS(MlEPTH&lb@x;$bBEm@`n6xCBWgJv2=%@Z1sm>IqX_x|zl> z;%HP~f-kpAq-qYDJtV2>iPX5!wr76fg5Dw9j4sSuwPR0v`Q)irasLyj8EC3K9`oNg4QcmkqAdaO z41Vb^;Tk0Mu9^ANP^@~&Uptc<7s+eotS zd;wu%7DC$6*FNnD1iPz<+=M|s7rWd$fO3LPzd<=x{3N<7mAv^pZ>y7>R^0#7V#|w54za65me*9szqb_b-*1$8#$My`iZDcNg#an6s+S zVDI_S_y!Tytr);lpJ8-~TTFYg?-1;hXF45<)5G`Q@J*%^GFI{nnX^$TyJ*qc8@86V zpQ7kLeQJlgCtr%PZBZqg)XtHoe(S2VDj#`fxIOIF9^Gc1b>3r)e`gMH@m4Sds}AH; zzJAs;GIg0&kJu=)jf3b6if+YLp$xljuw~>7`ams*j8DH#*_$EE6yU1J#GA&S1ML)B zwzZn;A^rP7bo|6w48w#&WvW9Ar=y5w?B7pq40btb_AK4_m&Nc0?#Y`eReU?`}Eyr3sMn~sjC$O^<98<9MLZ0LoW8HAN?c@tzgG47% zYc3UQ;G_*E>cHL{R=)7eabzg)Hkf@%KmUbachdNVZ;{lV2GaGh>l0*Tx$N42%}MA%o zJp5>2JFZJ2H7WWj^>h0tTK&l+*UOtvC2TEBfROp~;|F;vd48HKDP-zIl{|j{i*R0W zb&?S-x=%SoQPa6VwEfqhyA0iY^402ZBap|>8Y$lgKb^{4Dqj`&9B0$!sruJ@xvn*- zYD>%cb&f}!QG>+pR8?}i62yn=r5*tVaAptQ)e9^kLNeuu&`Z9Q5&p7@nAycTwUaHm zyuBmk1_c8<)$2eZMTIcpJsH_7bfk}#tSDzJD*KUfr1<|^M;f_`-Q(M{qd%f4p%X~7HLdb}BI~B9N5xUR-K20eapv!|_*QsnXk-OqLyjE@kumi& zZnL{D@8y}8!Pp*!RMdPJIQu%wH3{9glZm{dj-FprszYb#By|NX01+xL>D7tFCXCWuo<0;r z`P0ly+|qRoMmw!|fFbclm>r+FhI$_R`D;_9LIy>bSPt@orM2V`w4OKtijI++lm5V9 z`3oQSLm_e=X3+`+0#aGsIu8g?9uS}+xwTFa_nTPRy>f2aGT+BLHr1=M=~2J7PBNqX zCbcD=a|UM5^xELYH6bx;;Iy^tuUVX<2Jj-X#W!u5WWA}kC}_c-4V%0Yhjo9Zq!`j- zyiv|1MLjOfeNXz|9m}5jeo0kvV|YT$c)s`*Y5_Nop=oazThRkuc9Sv~A4(Qzb#hj@3QI*x)3BBO}5+qWHy6g7U* z?;VF|_h|?#aNU&!p9$Cfb8X6t(_j145)UtXwB06_b&>cP@uQHQ!7ha^u5*{@LX~)f zjB@~-Nt{O%;&TzceGJI#dZ0W0sIU*iA}3%#CzeLfk)IoQg)4wfV>$`3w{2$<<`io| zlDd;tdlQuuCrR884fPm+2MFYxAb?TsiX91*yaSM^r=QDxtom8wrC>$egb1`{_ny{! zZqgDvb>H(8&yQtOszvXEIG}i(uVY6b4o%hdiALXEohgnyy6QLX6G4}EIff%za*r(< z?u*nCGc!m9cx9^75Q%`sQ@xzXI*I*9Anx^qj3do`N6Q?d%A`D+yiF};XwQ5k?vX(2 z>U%o$Jko1gIwEtWdZr1*A*?KnvA2!T!Kr=nlb_fKJKzKS@3OSFXUut)LSv%viB6tP zr(R#Ts7~MG$f^8F;}SKKf^RbzCIQw%BQBd zHQC%IcRCK?exNAv2o~TdB=FC#WJEJajhua(<$IHKQH-wg_b9~@3tH!;*?OZ>+^QLjwpDT+t~A7>E6Rpyz`aOhf`nbjmT03 z<}6Eoxr%5FWr98CB6uULp9D^#3;Kks`{nlt4QXG_Z^8i=58qksQxfy6tO&mV$7|m` zF+@}Yrc=k3`5SK;Y`GW!B}C87sh3}^W1}2S$EHt&zCRO}?{wFMzKRUZPzsi!Zw7WR zu=i46;R3}K0Gn^76x`3XDSN#GP@6yUuk%#&UdH!Au}tQ7sSH_Y9Gr%U>()l3DQ3Yd zdl*sS!IsaM_REcDtXar+{_B%vO+Cb8$aE(1ioub$hdaK#%{m?C@nw2TvT6)qlY+vz z2j075NoazXQRXzLk;l}ZqiQ)4M8{wz#-X(s$^s8?OaT9SN93HlPrqsw(`DqZn5GlD zmtem`Fb5@jcks=v&kq*^UVEgl!1@86KS<{_ROlWk=@IR<78aK+ylW9CQ%1YUxqz~~ zgl?`yf_NY8J0bG2KbiQaQXQCwVgJ=1I5~CE*p4;_u%!~uD-v>DkS&Eoh&`tLFOM`h zwtf}B0eOvVNKg54kv_H~o{k#mPrg(Ff(3RwI=#NOK9TgI88V4&X5QyP9qL(wh)G zWC5R3O=V4X_Twj8f8!beU<+y#rJYXRrq(anWz*=D@j_tuWkuwV>GyHwPnuOobi`=8 zAl21BhfL0h!|GP#DS7<4l>`>TTT{Dj-}HA^u%8!3#K|hk7f=`mInE&qJ#WYP{NL^H zxvx(u_4iIc25kFU*{Z_#5GJEW_Rpd4CT1(=n(5bSC?kbDfecsdw*3T3Zt7WKE&6j9 zUG=P4qEMgepo|Bs$vy~AdmeUhGCjj3%J3iW(#$r3ei_`s^+O2W3AW@#h{Pr&+m3-A z-r`^}{bt@KD-=$_M2G~8zuO&CZ&^;m_H1~Q#R?nXhL6L95&VnlA&uhc-(f5GjynyK zGQluseu_=AZwbY|7TP7^=K1$mCR1r_Cnr2eVx)|7H_A^aR1U)mo$moi`(G}fAkM_L z(kOowqy>^KA4UCLa2wUa!2nx@u&QEUCsKltQDl{Sw+|kq>5p>t_6&RUDk?*IS)Lx1 zNN6Id5kBQ}X3aW;lYy8=BToI%@BMB~s4yrB;`~h8tjkc*j5iPk#w4Bx#CR=2IPQ>V z8dmxmahd7YSFrDxK`HXqzW9TL(qirAQ*hsj)GXV0^%LR1O+qf)Bkysy7P#coF&F2a z%NF?jwPnh%PQUyO#EjYbbn2DTZEN@yW_pVtP4wfXJP;1eic@#WS((j)SuRTAo%!9-H@rmO`#`@|`Er&VoJ$Pk)F62811j#YA)lbMVq9wc7@5QW6}EtvQ>;i?mB z3wm(<7EnI0=m>kt9)&~wfIvuqNiyWm!lxHG_lTb?N4peBLKsf)|?d>21o$*Mmem|9ip+wPbwBWzR%K z`3AMrj#{UBae4ETaX$3%>0K$_g!?u;+K13svWp_CEfAX*5#U+m(G*<)h+F=>CjA3; z&wTSP;BiT!qtj6+0c2!C-`b>#6g%S+Ut(tTqq9*-wjj2zzK+Thc)7D9D|4>%<+;C9 z`#%8Y#ejEx_}Pl*A~+i5Wr*HN{wRg~#1ng{6}s_(#jolRzT?iT7`l91^O6yWc2YQ= z=$AqFW{S%c)@-^yMWdFS#zD+TzD^)9|MWRuz|4gu^vTZ?K$r5hTIgqs46t<(Z}@Rn zOG^J!$_&~dO>=*H#Q>e}g67c!rBJP^@WpESn(dG9y@y8e0m1$b=iGi7S#TEuy8i$H zCJYE-IXZ@QYxlPmy;DDqnOuq}wT*NwFMnc|kDubcDIo3mu)-0ymW(73DlIQxdDtxs z_OJg1_>WSj+xWuIV=BuUBFRs*7HjooaeU0fBF``S7DOxfu-}vS>b9N^K{fbG0kWyt z);h9LOT_d{vKe7B`k?f^aM>LpyH9;JT`kACys(f|;%kq}kE`QsTip3-6p=4KoV(I{ zZ*zX8J=U1HpQn%(O4qE&PBazaJhtr;}qzJl_Q?fwvKI7M1 zeg!Mcsbu7HY0`tQ#8s59gzHEm&%Wskq)-*K7s!ttgF@YSWV08N<$hcY1XB1Ylhye2LCfutZMtg`ko>wJn)W-kY~6GUZb*Ts z=q}oRAK%QXUX8uR6r-@Gp_+?yX==Tw+qdqqQ7??|Wmb%%k+-OFJ;}Rz0<;AJUL5BV zrS8u)%QdU!oQ#nZ_bv}98xi-)!#3Q4|HALy>vs-MRC^+M7e_-8R2cw;{O1}bQ^RUa z(bn>b6;--ABF~7I4^08|5FvYG;^_i8sEG5o2eWP4&XQD**sLwErC==UJDMTeWI4n0 zt-LD@Cc7SEbI}}NRq{2Zbk(F7X6iRwXU4Bu5{1(%h|6JW`*W5$v5UB$XC1y(9 z4e}f}7;UW@yjd zbtqp%z>;#(3}SZD{X}JS3j-D-i;eI8k;jo#UB$&T>J{HygcUJ$VjlN9#Fu4Nr7GeA zNc+cKl*9e!NJ%@Y>v>ImPmq zr&syjTCsckF%A2DKP!g7svCMTgnSYsHNJtnkr?-+e^G=`Mp5<$Zj^~RbJ0%sk1SFaU&A}un^{TCNw z>_55O&w6-4Y%Wtc0vnOXwH)Qdp(fas4^76R|ALLHlN?&(-4f`T`4JZ zW581rr!Y?`|Ju#8FE?f<^KZxJE}N_jKSN^#4Sp{_$|M z*yuC55T~x1ZSlkCx~3y*v-O8pERxEN)C9aK4~%82*S8n{0&nsaZm9<9u(w0f{8X(O zW%*(jhXqGIJS*G%Uvcm}L9Z;DH9io;x$>9UMpXCdC<@7B0aFs9PSaO`3=_Pxi4-6^&A5TH*YzbP3SJQNMhgAwB&Q_Doo4d1)0mS!-s`oD!hap;ke zZzQCoTFPS_MNMl@g2|1=83TYCc61qg$aM(pwterOY@~YjCBDe!c(Xit`V*CGfI};Z*Xy!@#FE4+xs?TN$7UmYnF<+Nf&md znwu5n3ZZh`gwlAsKj)H7YVXy{m;>mX@1pjFK-Xn0G4KLT<&cW-{QKz)n&we}m*Rys zu#j6Yk_>dwy>Zdxj>Nu_k8tdAx<{0GeYwrxOP?T~)O2@uj$C3~S7802Mn#U96sruC zvpM$eA-i6m%{`dmn)hbL_;b*TLV^`mqHWK;5r`K-y~QXYgypit@#J9rgE66wh2NLx zTT+a(GEq&CAj;vMye(gnHem8;!CSa9zPKHo*@oS^y3TH7WuZsk-kGdgN`JK`E{)yQ zR@(zfML91<0jcDr*Vl+T#L}8L6-Nd76J!oA9Jczp@AAQwwz3QRi-jB~2FsSae0Hlp z;rzMl+|jqpbz3J6%&7ZqWv|cuZE1gd!eG@H)qQy90-OF7`uS{2W(=@qqkl2c5jNpo z!1J2E-Z)n(a|B`6+mwLz->>) z1vQHFGuV!VaU@|3rnvWnX_NXj0TD**skFlra*is8GYk$06+9GWNjqLE+*Z#l)D}7t zY$~*73{|ZWKzfpYL$b?NAd)f=3O=MhdX9+4-IHkqJjMWysLlxpWGpI*!XLB~F1O<3 zMH7Lpyp>Z?hTcinS)xy$maTldEatMWuh$vb5HpNqd9?3qh~_@W_EIIZCg+8Mxo?on z0387Vol)<9YI?UGv5LR`(sT3E{>^n5y>^ueRMOP4@u_U+Ua2l9tDeNUjbyE@T}bJX zKRr53ou}PG)#=Z+fAYvK(D$dzgP5{=w^=LX&r>Jn7dVtk2)f!G$kGmzG`253ka2lG zDpX3@*G0Rvj*YA^dq|B}qEMSS{qIZ+jzv4OA2qeH-q>GvIstF!Z{A8mVo zqHZ)K`-CTOVuMvaza+W3j}R5^f1}3L3^kx?I(*bwN{ft7Y}RIpu22m#$A2Se4VxAB={4=5fBNIPk0R_QlrzUcQTO(u8G%=U zvy{fJqiYIrlZZ-=1+r*UoXXD{5@na%_6+xFlx4M)oidv&61fnU{8m}k7+A`ej3Zn_ zhz$4^V6M?OXO;VD&rXbLB9mD$m_#xUvjbjnHkTe;<$XM&j8_PQh|Qdx1b+? zO__34v9Z;C5}iStovHJn26Iq$<0Mqx-HMJqmTY3E)w6Hf}Zwsla)1p-zlK717o8l&3;p-7Kjc3^DlT)X|#!BTAL zW@zDoroOwB1_U|)?O4!OXB7Hk^6?m%*Njg16JA5LKVf#G4qt{%^8uwhtM??|CL7Bz z7`zu~4{gz!lI#qO!1{XWD*)ZQs6js(`Ar+b*ktrAoSvv(tS8tD;ttLcax@kIGZh+` zFX{)gfU`toTo!?|>sw8BI2w%~OYRZI6cxrZ6MKcz(3or;^a{=T`eN!HDA1Z2N^|X6 zKCQu)5{UZeGh&^eLkD3CxdqM`zo^5V=%tANNB`YFppFUvDzb=jva%AIU4x}Ya9Sb; z#HV_I!-PZyo&%mp@?3xatRAs;W1}2(+c62LS}~xl9oItZcHS>n8i&tb6abFX;KJ!= zf~`o@yAFxiQI`O;C*EAKTkY(}o0~{M3KG0-RtednVOIQvuBAp+Jza67jPq)uL;z+Q zl-W$W;eQFNV6ccGcVrvE2%zs+58jvo|0;>CUJGH^eO#YPhJJ=ConWLZ$Uu$AzjJ1WVaoK?L$U+!H-9Ng*K| z5()Cwj`Vou^U@l2txX!yq`4|_XA75?G%0*DtB`-=Aiejkgu8Bcr4RK@%jPQ{`0)m2uhrjRZp&-;R7r_Z(7#J{u2_Pden6Wa4V? zzNve(TIUAqwtxHki|Bpt;E>3JwN*9d7M@6D3^5^R50U(z9yPgaLgqsN9_jphyTLZ( z@oYWQ3sH~Cv44qO_=Gld9-|)h`D#e$im~BUZ|AvA+*D*%s0$@tKdoF{L4>6r_h1wm zI(N`rKjxGa1vtD-V>0ztXZ)UIBQ!v2X^xbo2Gh5>fsZE)vRca^^JJ;zE-f z?9|5xTD~B4C1%Sr&pWSy%qx+YgDdp@{e`*P`e*uu5&zgmcTG}P%pj6g!cJha4;ERT zx*Bu$PIX2k>l{fQ$7CYpCceliY@bAY*1@86(?DbTbB-&%Zc;^IvmLRUEI2U~n>jh* zhVUd>Ud|P*P)7POKzj?8$X$7Vd}A&Zue@?v2H4Fts}O}D zSrNIYAewKN%p}hi0f&DctRwtjR%IPFwR0FbC6=Lvicd1*873$HGkS@=;IFC09xjds zhGgKPG*d>IL?kjo9WxDt{yk{%N~REW#FoPT9`LK8a269FbD+)v?jlMeVk`yy^4`(Y z!|LBt7DXA!d$3f92%6COczm}f0n<5**8T!zIV6w5FB%s^^g@PrekNNaL6R2_V-cY7 zTX^!_X)W_AS`;EllrqN$bnQIHHNAbCbjGob9VDZ{@^Ys4taP;*wDA@*{(Ui|d2M6> zO^gc{-3p_Yq2-#QfWOypQnD484xNXHkOH*Jnog&t>{|={rsc}cYjG7&F*xR!{@K~$ zUD>GA>3>EDsRRhsKd`EE|3Ag;2Ej{K1d#j^6cR1qbg)GVT;Zkv6ud+x0_cAHbHZIc zj|1)t@bUoY<)r2XQya7^e#%aNdV;-j-v*uXsJ#`4^e3dS{0=uYDL>psWFio4+n$R z*DAWxc+1l8^+yZ;8O(X}K}mz8tm@k`@brbRKp&i%D|EsDO02R78er z{aiHO_w$5lZ7Tg=0PpADnr4#}9^lrG3<`zOw-8>+1@?+_s8@bO;3?mLcHJrGpC{Nh z@2{BF`8)k(J0vY9_%cAal#D_|Id=XZ=}!uMEdfhbd=)%&M*-BerSNmkP@nE7y@-c| z@)dg`-?}X@4p*6ocuNXuPpF2<)!uxK<@sm!!yA#DXP8iU^)ODvZH>0@1th47RHikV4k!Li zW$F@3@GHMjytB~Ua;xn`wnC~TGN!fSH&{(zDD1+AuspxRnN8@YWq_-ZKy?z61Y5eR zd5JIpI$=eG*{#DS=nuN-GqzeISV7bv`{b2lUg08!&4OpX-wDY*NS|7Z3Onb8Ih=ik z)S!#q%Pb}Tlk&9c(vv+i+fXnEb!*RThe>M|*L{+(34rE4sQ|GHkPxbfVuo2+bPwp9 z$Hy>6;m3JhYG*Bphf{XY)gLT6xN5%=o2=pISkFJ~S{~paL z8xhV#R87yAH7LBWcds<{`K%gNkX0rpY#`9j>5&KpoIFlp0sI&za_0#KT|D~lpk|$A zlp^6%)T***J*rUoFn$~6MW}b?^UH})*8&N1PM)TR(U zEh})9RR%gd z>=I`@o0GLfZs?IiFX5d(rwa&RlU+PjvTw;^9z#~aQ{F*SqdsgCstb&G!unU}cu-lp zP_LW~Fl7_KFhk)J0kO9OjVJ1RL;|<$h65zAmvceQ{sq)~<7w&8zpGIx?K;DeJw@Ir~?r&X0JTXnBy0 z=ja)FQ{jIVkyc~?md*3S%j7`{--7CvhzfW+upqZ7Hm6&6#dpi8( zV8k74ceMMgPw5+`^8LZ7Sc;h{Tj_Q>9pTNE+V+5R>W$4Yv2MnnTFBQsP;!{ z`k1+0K%bvxxKZJuX;&-p27pT-o6H)`d2=+T1*I1K4&@UCOL(5q*+iu2Nmgo_a1OFk;O(*2T5<<3{hog@fri`&=0A*4EjTv#x@zh)d6l%VVok2SiYkX?l;&L@ki|wMjVkzsQzqkcI*Q%Jmol?AuxSyU#yXj2S zWEI4Y(k3PEZSi`G3|1nkIpbHieTeBzAu)D&sDgRaZnP`g6VQH-RZv!qU3F2 z+I6F7+>KxnPz}2ifHxV1XZ(Pp+{)F-Jcf6?J8xw5w74UVZ(!NC@S&JKXxMFXqI?b& zRX|&6ZbbSJ!SaQUB;gw13-1MnNm2HgH0r0f1M~$CF3!%^p~q09)X{>xfD?7mN<-#A z@VUOBsE8_;=2Q*arlD;vRQqL*c2O8KX5o6p9E(v;<7JXA5rCyN31kG;)gPt9zYnn> zwIj)b1vv2Kc?AFeKwBDb!hvmRVuTqVv3RG9hrQ3s#X1E|Xi25@BQu_ekCo~=n{4GT zb|Li6-PgI3-mF%YM~R2NRP|@m>SJA$-_dG=%FLAl2Sf$pO0bXX9Sn9ZEoyld04w?9 zpdILne=J#2IW$PXRoczed|yu)tEzNMW|zEA%?M5DilhT8pG6I6@5 za;GOt{=7%@(LZg_Sk^%2EWM9Clg&d~jtL^toAw=_d)Gr*TR35ZvJpIA`?!T4NT~m{ zUrDEfZ!H%_S>0G)x$Z@W@~6p>n(?mTFWSNWAD0=|#WNq9|Hzl=&EH|St3W2S)y>O_ zEf-^97k(_ASv#og_g6b-{Gc8{qvv(gEvw#K(Ot9WuV{Z>&<&J7E7 zZsJ@jNCtp^E?`?2e4!QRVei|ie;1tDN@bVI4E1?#jc$dg%2c21Dk5;ipdA`6)?@Ry z*XX&lQ5DY2i(d5$OO^#!4uYMED^DDiHA2%&2V0Rx=BJ=ery(DwOmq}#qiXIg>8DZS zCg4cbK~vO$zjMm_a2`8Ioan7MsLZ}+SHO;4(V5DO^XktWm`KezFn^U9cjjaBtg(a= zvBNfZjlKkY5lb86iZOV0ySDa)+s)RoRO6(K@gGda>ADr8My*GbAN8yiao>O3ye9Sq zbdzskOB{(LnrH-~Zykj!hyvkRr|~VzjS&A~5Mv2i=Vs3VO#VIx2}11X7#V}741M2s z>;RZpYKoGV4mIJvpEIZq>X?9Ad)pE?PiOBKhz|-Xg8+=S6T!$&2T02vOIocobQ#ZB zw)M2P>~M=eMDmh@y&FQ^&*F;2kO8iiX)KAs(fr5a%&d|0sR#Pb`x*B3*r8tmJk%la zR88M!R0F=OwJ%W}KsZ72b=`4Y_0_576=I`UffT^i$JtG+@VO=684&@9S4P#Xz}IV- zkYzI1ivX%sdkoeZ14^VVw@NR9w8s!(Y&K`2SBfA$_kO?X!>w;G*Af)qOl)}!C9O5Y zO>bk!$t>1*+2frrS&D7x!#%oX?ext^xa(}~ulF*pj~|sdz?R|u^K9B+h&1bs&v-zV z!k!qo$hgZZ;J zd|an?qqBadh0V=3sk)Jlj@=D2n3MV3H**Y79X~Ug_^}4HVVuevmOdvXJTN zW#u%9?YMRzs{Q0!D}*G4$J(LAiCbJ%;s!ua_!kO{{n`WQ-xx>(n3W-^vANzRN1$`? znF9tBNp?^4Js#0Vs&dukNys~H{&JTs#~?j7P{X*9G@oNSeC<`UHL{uKrnk5W=}AG= zR!lLj+zcdI=~uD02l6AuUgi65@ZP*MkIE*FIg2G&5wsL&Zk>j z{bx*=`d?MEPrT>^tg3gKe}&wQ@}~idul0{wxz*BzL!S+5YdLiBXha`O=?YG&e+BmS zj1w)bS|=$(WTBig-jmWT@~*b`V@9v$>EdG?Z#%`KHE%7FQ$`E~9J8%~eykn97BWy# zi0kq{YBwdhW@a4xIum~voUew#x5ksrBM%0%M}DKvY|pR zwtOjf=daRd+NDBGO*gh2DYY}?K3-;)Jtyw9tO3pBxD&N5=dHV!p+^752(T3aCoAqx zL>BZZ7x2)ucgWM%9h{$&Ha3=WYAmt#7Jtr&bEk|8^;lyb3!#bC<)T5g9%hD5PwVr2 z;IaqaE6`=G{_@yceUT?xH|A+dJ^&(qer$5Nhbr71IlU z9=l2Zr?NXv{dOE*F`|o+Vy-$6coHDn&3?~S=W+&;;}@6j^LSf2Q=4n4*b1gpbE>>t zQf*zfrD;tmV=TSW)!xkNdt7D>e;L(md{pk7@)FUjug7k)Cf%1%t*lOe;A*+r-&69D z$MTW<0L!jIOV8$VM$c!FxEBY}{MMTI=82`{)8n1tIMknPI}N0ZfetWPF0Va}npp%q z&WWnLxXN?guGP2kMIcIWs`6rR4%eExwbajUo_dPrCax@O7l>(yhTbD?0~3T-C;Mie z96wdPw~MSZIg(xuN%~!RY%uxZjFnDcXNE;x;Hht)zvo$$UAM9rb2F@Yp&1<1(Vz4& zmFs-{0$ocp>l-hQu1b{Jb+X)|l0B)65suz@0E3j1Lkd$#?saDm*wfT&E6SSosXL5) zSb8e?m3CPx7$V1|^gYMD+Q|8#p>{<_ZPIe;1Fk)OZ&qREh2yeDWplIwlV27vp~m{! zT)rlycf(HPM}$7FzKe=Ci3cGhxB7o2cOJdE_QUnGJXw9$Qupaqi@9m>M7LLJA&iS&_i$hDPf=9BustUL#|3{i(g*L`ip%DlvMgT zPDKfCyp$-bTP`k;dj-#yVC5Cvj1hTW_Grp}3wWbOL(^{`X)@_~dmyQ<({tk2l`eKa zjhNwrOV@VD2RVL?e-|CES+Rgd9Q)H`&8=p0A0Q1f2*oj#A@%T9-Pd$CEL#)g1-5ayuIq<$hxBF zzgJ3FoV>|7Sm)e)YhS$G=l2U(Y4mOHYCzkeY`#&$N*|tZQfmzns};2i*er3r)Y@(I zONC*d)qqEMr0T@3?q1NNrXUrpWy3B;6^|gY&o0TqHuuEkUSFQczPt00{wm2Xz=kWW15>X4{x z_nom=D=+~8K*{BQ#b|C$+&gE1&TA3nlfC6mvjYl(H-z4DHh<#KRZ|z@HNxq}yro5L zd+>o(tZY|_jnSev;ugcBdPy21-@(z`|3~L;Y@c+bN2P2{MubnN#|mT+bG&=pa6EoD zSl&10AQsVpFn{QDwT=IDA!->nKUV{wb~Q>qmuO?h=*dU;0pJ#3fSIh*+B4AIOL$GV zSCl`4Qq3n|^`B>GR^in{(a9S9)Mz^-vbIIv(HIZHM6C+`XTbSdjiiZ<4wfUQva~8+ z6!Fy$vw>rx=ody^o@qLDX3+;9+AK@!@<-i+!pC13?-(W^6im)e8& z>6LKm)Ei90tgL6Gn3RIy8wc{C($*>Z*}L$tjga$-QI`QgGilLB>C3;R*hMZ^i*GpW zf7f*Tj?3z=6dpzvY?L#*){Pv|lxhV3yoS5F_t$#fAjgRvDF$jHkLA<$r&^DBm3L(L zE}1V>Gklag;JKsyv6b9&mW^~CX;)Ray3@69om}fBg6BmDo*G zk2!)3cWb=7wBzlZRX2YnyMBruq^@*pkBUSHb++N()BiQw>PE2y!B@_|&WeD;k8Mn> zfUAR)jkt(y>6X)`5Vi*P`C;PM0<*s-9zR4i`qOJI4H4uF!}e+k1ZEf07R`NvNPK!v zy2T%cW%aUIg(%$^V0)_23N-5-@H1ss6#e5LNbVNQ6}*Icga(XW7f~>==nu-_NEE^7 z0XGZurUQ7GD>00+15I8ML`CaA2>sz?#NM-!SQwvbyyNw>_vb&|S<|_AGj1PXu&6s$ z>!!b81N*$Nf+xHE1q=S@5d?eLMlh7%$o0T~{LP@!+&|xmbY0WGx?1p`*-LBou<;1; ziiBV5nGJ;Qp2$_HoGN2`TKU{|n zr+Tv=+U{Mi1dunh_kIq;s=l*xGHFI<$D>f~JfVK7$~8sJ26VXRokgvqx^M?T2>;ci6EbPLtSb;M zl)DagqS~Qwr~9s4TH7kmY^a|m2o-Oz@h@WGxToQ~N~y*p2D56@BtNI3Bj#nY z8bUD#+`nybr(@qCW?fY9xI@#WvRM6sxQ(taJQD5iI+#@yf4seHS!?fkPl+#`7M?d) z-!jWAU#iuD$T$Pzl7k<0veC_L?5+XBx>0fIUy^2}pDqueH)kNQP8)nH1&_wHx^f_P zC-Gfb!2TJa;gCI1RwEVa@?)7fnv~cA7cAIVWF*989E7U91ILd;HJJ+L_Upq^(n#Cp z{mjHn_}W^{lW}jPhNjknci}URIcuwsYHNYB$`M8HoK7a3&pdx`eM7ik@Q=3>+~Ky+ z6OU(aM__vO<-9sbe`6s6DMwK0C?HEwT)?=(($0I8K*YHJJ%=Vy$x$J8dc zc4Urrjvi%ibE@}zXRuHe*BR zN(&VA(jN_>cYWm~_t3&gBCAX+wau%QT3WCDpq}d@0UAh&&P%7_h;ra79ZBzZxt2Ww z%(x0$aBWuv$V8v-@~|blp}NyA>qTQ_jx)(GK~Ce|X!{Rrm^mJv(bffzc0|T7rqmK| zhB4%1ZPgo}na?g2^uXUmCK6kqtv{|~wsE88ohOo32Fo#j;(oVwXV#>r*8U9B5}_cJ zEe)Ql)_VQ=FsXBTvkvVOmd<(7+%x0SzFw{}W32(?_8J<5?F$VyXtI`D-CiuGd{LLb z`j(`mef%OVKKl=!H4J$|*E@x{#|caAat{Ei@(ml;X83z0&SYQOXBAU73Iv32ZYW{V z;?^Iuc+ePqa;e6lg9}2xN5{k^EM9-smpQfb9O743SQj4kV4uBNqzw{7(lq(W-pbCw zl{{LuRHv~Ue3}K)+pj0Kz7FW6$XuA&2KVOdfZn&!RW4&VC(#4i0?K{Fl+Qr$@F@O9 zfAluwV(Uj_#VxD0e;qacGE|zWov^Vy$12(@PrHIT_`-2n`*CW!vB%1A&A&-;F^&=x zYnL(0Dn{C}tW>HGB9sKTmr&LK45`rTTX(wf$M zTUw8K33oBHB$ZD_0s^91&F^M??W&hTlZA#$E?u&PN9v6S4)DY@O(u_i)3dD-L*k{n zIuD94l;stUk>c^{Li3j+I;>qG9s}A=q3msTrdex-xtClYpJ1P?$ub_?@>(t=UZI9} zW#{O)Bvx$(RWEaAT?O$phO@W5cc0+OSPwaMoZ>Vd*0?2{-GqMVSb;X3`z6MY=)3>= z5LfngouABE!WWa+Qdl)6$|dz`wca-&S!XW ze3^KDtLmOf&ztuX)kpJNr=Uo0?xXK#6+SLhX-rgXMcJi`ZJ>6Xn9sPMiLa@_tOLs} znL|xkI$j8i)2p7PsumfNe8|&XWIo5o_>2La@Q|3T+LK~`iHXE&JY>y0Pah*$TOR#o zTG-#^5M$%R?TAri6Y&hY)&YuUQ}n%Uo%QtIP_CvLmmVQA4|oM$51;aA%s#6zYz*A@ zbMwL5D{NTX6u)RGi?qpEH?V0GPCb-My{~SPC(fTV1X3Pd?T)}KJJPe%QZ8TTVk|ZX zT0ekOx_agUnbUB33@u3FZ`_v=`rZGi+_hr@Mv(~-L$x4QMTdbPHhc)q@i=tf3oSs&2DY*xlRVr6F`nw z!ts1{xFS+}3XWTu{rbamG>l!p%BI8n$n>5hBoPibb2L1n+co=}G%g#TeR)AgqoAEB zjaqurB508(KCdD6!Zg;m*YAjfh_raGZ)O_?-#jMnzLg`XLf1n!nB$;H76i3Dh$~Aw|D9XZ|;|n@hjH zMqOKlTtc(f8#?z7VI-mju=;H)CmXMe7fVOBW#zhWZg$zhWm#6HxFU?oP#i(5^^0ot)~uk~KTs;-BgtLS_2s^2Ke2AHRXc zq~Kude11{bw%_V7-y5#y;}Cr9DP!(_sAv>C_jtd(b${Z5B#0;*b*zVsCaI224vaae zPPA>$e5;Kv$G>7#GrlFoXr<BI+hXGoULFW(8PgFFT06r+G4!WgwaJC-O-eK1 zW3>Q?WJB4DHun5el&^+vKOPx(yv|vf2;?kv;i;^P7@kxKz_$gc`)#5Z3(ykrC&x2V zC)-?|nCDJ_n5;3h%IMeplr)9$uP)xdkE0JeP+Tk%=(rzFSDh5jB}O!oF@e z@BSwLpjfhvhzV-Wjajn!alYD`HUXy^#DX_n7da~0B8(7Ari=N za$-DPcOcEIbxYPm(;$i1B)H)!gA70FUb(G5SteCoU-H2>t3RaH`W+7g11BU+BM;2m zc#{Mmv)tROlj6BE9l|k;VsB@QpKG#+1<9wF2|No)=sKV`YK>zjLZLUd ztf>8rz$%T3=vDNW;}^to_uIN|nf#brvSi8H@xj&6p5<-q9!nnUsZr(cRdq!U>g?K8 zUH`K8d)w|Q2UzI7~Xm00$7E8FCT`h)5n&%8hVB~~vm6tp4) zpM7ug*y!m~E)$niN3_?jRZZ&r~i=`CwK4L$EF{lvdk$1+pc8p>Bj7BqZi|lRjQ9k*%EJ={RP)RHV6cDz$FxulNtuH@VV@HG~Z^omqXF^q7%YsT(7v)blt%{gxxc&b$ zIoEi%&FG^f)t<+L9U5fu3Td#87}=zYa3w07{R|WhF?uNpvi+qej)&MGyp0S3r_`nI z&SfmWtLk6-bGixBySi4$I{;ut73A&bAe>A9w_JU5*$zFo`5OC)@FYOaW8`_=zJ@1s zQ%8(~z4I^UC*Q6cl!R6HMPePZs zT^eZ*aFG?qrFBXXP(ru!<8mweu4&%#?wTeYYc?WEO8XfR)PY?5S11&dl{WHi#C*cM z%ejvqLX=RxO)eHf_O!xOxOv1*mJ2=r17w|4cF-MqCCdb5Rr{z{4mhj{Xs}d`xlPq| znFZLrY54$?1*uxqjs>!V-(U?4eR)drtFjpVqpEIIMQ;|Z-mBVNBc{T3(1l}5 z+L-+4cS4u?E?8wLSz~>XyBXYr@lqKP`vJ_)Yq&6mSi#>Mj4?WpQ^^J9kfxFjW74>K z>H+lb-iJHWi}%rmKvQoh_Kq`DEsi+UX04R>9K%Cr*juc;r4q~p6$Q1==t}qQJ#Kh1 zShzy}4}0Go)b!f@8Bhd4u^>f}VntEFLKCH^*hN4I9fBymDosiN3s?{=C?G8qrGzFW z^sb1C5I}0^B2q$A2mu0Q&m-J--`~#e?9S|OXJ>Zy{&(+iA^Dc`obzetHVY8k*+y(R z`*iIa;{1U8{}S!9*KWfXu^m*$+ zn2>{-W53GPSa#{1K_1cp`nIYY2;{xFCJONV)A8pFpYcoRHyJU3P}CgP@q6D1 ztWIXBHk|(l4fj8KrVbuh!+_u+P~47r+yF0cC3y1rD5j?!l5-@2n$*=(~qVzKHP0a zhnd%5c#9M+)9|=SP=m2DcyVy~<@9wrck#ia{*+f>bzBC_OZV~6&+xY{4vM>sNRQ70 z$eO8CYBL!0yjXJif7uD4w>yI-)D~dYYy(y6OWWfhC>>9SO$Cd(B1q6vrk~+}zXeE+ zt_b<{@n=QdnfMfV>~M#L3G{(cA6nt^FH&&^I8#2a^L7B5@YOimK-DD>+?Oo(8&3Sw zr9$uwAh~NZru<7E%ea_K#@4z2>;uP{)Hhwxm8yoEsn>vZGal zOU=}kD%Y?{W!H}+<==42N53A(a6Xx>e;b8V>gU;y5nrOL6sXDT=Uji*Mzq4oWd;~? z4x~&{WLgb*+WcXWj~>k4OU)h*sXd>KHV8NMD7abL;YA9`^E2aCDcW8@=__(3Ey@lYzdGH#5vNOMqi|DrhY3*ns$aO484eAn`lE+@R_GCqGn3>zqvt#83 z>Ji^|V?>ja7OQ}S9+k9+e=e_N5pie|F8eJdZBREc7N*Gk`0j)#A6-*v2baX6kBPA2 zdVF(4;PIKNBE?$;aKhFVUQXJpv+}6U#{O&h?8eRi+6wFYTy$k49R~AY_5To>TmDoS zjAa&*AVtD|0UM{h!XzyKgzD1$+%3( zBe3Q#MI-henmkd7)P;`D77o5~l)K-CY&$S54=-RExJ7e;JCtulzAGy13o^P@ z#Tx<&^BRz91NjNYGutLPy^c7guSiz`)Q2u0><2-nrCs9Tg5&_JY9wc^(@A#j5w8!p z7rZXbiXl04eJsV%?T{iN6GpGk*Ab?PAjG6iA5y)?1XHzMcxm^A$F*crjdJG_12gT( zFb7b2E#^DM)>GTon_X&a4js(BX!gWlkUPx|KIh}5R9Ykm5cA0z1RR*VBtSEcU?s@H za4#kstw4N0QjRLsK(+56wHn(XIuRb%D%~aIfMXZFQw~GH^{q#4Iufp_KxfL)Xy(B;B?G}mz08FA~;2n(&WIW zc@Wq4vNl4M(T6cF6?k$84n79J+v_e+#Jj?nS5+VZ6Kx)xbQCD=UGo83_M84aaCOpV zKLH6H;$q#?xTJ8a60kn2g4=_+RXB2$~WH0-MZ znjSjyjok-$kHeAMpmx(RLt^)8-vj+K& zu_;PQfJ4KE`)&}-{}cw!wd`Wp95xw$wU7H^>H?E~;Yp&2ODU&JcjCJ?bG zO|~I{4fw#`r+8@h_JBWSA9)dGZ&u#XEWonp?h`#_J<$A&wrS8LGca9-O3tW=4p#ih@Hp z8e;08XRs9dI}N6}d^#Iz?>Ku9`gx~T16k(HwDDq)Is_0ii<~D0?4|T_MC<5xCh!~9 zR9Y~L&&{uYZ&)yu@2k(^`hG-)`Vlb0Z!QCH2kg_I+<`YNL?f@u)!^x5drt;NpXJuH z^)L96M<(B#U4{~G+sPL1IPITLvt&#&e)nZ3MLoW^2;D}J_IB%(707t%KBuLsG`9P! z4w5=ViDSie)(f@vkI0QK!p)ZjLyen3N#Kom?d>B+VNa-a>&{cnB^Digv=iFCL`RZG zn!;gWDXrPtZYA_q#apG-&|H4XSLY?u1Z8p&h5-C-s%C$JrZ5xwZQJh7_FStE9>*O` z%A)3(Exct*f7BgDK}oG^I+T4iU=uc2MBY)f{@EOmbQ@C1Wj@%W86eO0`wZu|+r5?y zmuh-@ioQXa9QKL>B?o0Jg(QuwxEjjPWS|Y5jR1ZBZax}bn?opS{!GBBo9ZmT!6n-@ z2C!))&4uJO&BvE%7tkXtK@f9T(iLV{7&K)ZlYNtY^HylvPbRaqBb(uKvd~M~J5}*` z?_nTj&PR4k^w^wQ2QWJJNOh~vu{(Hg*7NwK5#Fec09>kXu%*>2MZnf%IFtWY{oWHh ze9x+K**8BAtB$+ImIFa@McI-B*`&eSr*Kk-JAwr&B=(mY6n%;wn)0>z-cU3Vtet&D zu_i*Q?43HHf@Zj6{YkYIcP!TKAmotRg4fAyMNEFJsOl;$y*g07$UZaS);jm6+IAOc zP;a3!7H7r^Y7R3Jr7#$ zg>%tfe=dOcE06{othzzEw$JWZF*u*@RVP;&P7I!KpVBLd z5M&cBiuN~0?#ozza=C}KFPZ!@f z2pXxZbx=TxYHmDJG}ke5d2Mw2%bxVjXlE2)uUBkfzLBwN{8?-pJTHec8qv2gEK9&9 z>9%K?*Vt1_jqEz|>@r~%q+H4f%E=jSWD6-PakmjXKCi~;DwPo2@}D)|xd z&UIn-Ftt59alMd&bVUaA%=8Wzuh3#OR~W1eVCd<(`2Z}+d5DBCa_i!?dF=!Yh*1YO z-hYH@SbXP}4sRNtnDXLuFjq|V)2%!l`S8;Ys98khlCa0n4nZ(jp)*D8X3}$m>ohn{ z#-N|b1103bIl@X8U+2$h*{j?H@(%_8ziGfGrSEhD$zA$Aox4^L=QUa|80}OsG8gxF z$v;TIuFF%=76`E{DRZ^F`|L9yRSC5Hk(g~%zWs7ZoR}KHCat5}43btd*`|I`MqbyX z-6n05b@`;D4Pq_T5^n%O?cjiStrl|27iEAJBpYgL3v|~5<2n<+@-6RG0`^d328O<$ z;??|+9t5ySE)3XhYL)^Z4!0i9MgXgkC|WTbap?NJG$=4YrwlIpGoFS0cV1HNB6CM)PR8bF@|F?m+HqOfD= zgU`{54{1KX{B%W9u8|^B#;<&{h$HVxZ{bHTGN_A8^+O*N!Xaq~aY*AUb@=T$w*L{` zy!@m=PwSzw>Wjt_D;t?2xS~?#kP%-=gu!4CGySh0h4y!_3&02o#Dlk+h5jC1KEHO~ z__vPQ1PpzHMsRZemH)!QF!k?0Rd<|?3|De6c)7lw|7BTm_tD#>4~4J|`$j5s1tDnN zD`B1ojoC=1n};TxihFDlfB}cVTZ35YlDS_IGk^)4(;EaV8j*{LN)mX6hg)#BH4;>@ zud0CpMjhXKqz#7FW^>NsLvG<5oi)lg)>E}IZ31}Tr)0pU$@HSH`m&|%{-v|2QGkhl za^fECaX5|hsIyPj>H4}w%CdD!?L@S)*H8c?*Ea~%P!+vqmx}D8T8)g_(#0cdBul2R z7J&*h4XBR0L#c^H zWbjvhKOEu%=E=@^3d9EhKaz>sQAl$9;cWr+3z4SnQ0X^Mw=dt+-m=p`L582I1B$$3 zy zSP!lPmBLYs#__mLNLlpmwMe^E;RM)^1b%+nqRZT`DlNPzFHq1sXY|-@{dZrR590mT z&4AEyY_K*0gmnCAr>i|BTtW`s9bCw{RnmpGaOzluCwsclP#ZR@UWyVYr7gO?Uhn=} zFG6POs385fSjc{-U*%4qVzvra5_SNLEt@KC=+K@$m=j#Rp@=*L{ku~|)`sUOAPE%7 z*=X;(A5E^yIhZSmHU-U>E&SJ+`qLZs47tUT)yYgCu{sNpwi&n=UwDhid_Wk$l#;2g zH1`qM5T5=5{VQcPV9+)It9BaDqvLd>jfhb80ej4N#c{02DHZm|$g{TgC%hI!41%cS|G9ieik9Pm{Mu1F!jb@AzJir_wbIcFZ z&|-YYW**>Z{j%nteAwOvQQshqWzi7_7x8PLgjWbCfT}&p?)`*brzpk=V8FwyLly*l z3$TVh+?5;n{>L)D{YunytNn7`5w=%j7BvAHZjvhcIG7IFJP z>3$yN93X#mYZYjtOWjP{;7}#}8qblrP`fX2Y_fKLs#^wsCmy`ilA0c_pDyXHChEnl z9jE$upE7mvk$5NwgBxVU<*#tyWv%gXpFcE`E_*jL|<(|4(x>kw%XYDl@@Bx#R?~aj*q_wc|jwoR)cAQ zl@j|%drj1>_MFJh>$y`aJls4+__Dx>vp1_Tav(Vrw_7)t?EeZrJ*GugdADLw;U%x7 z*?u=S{O_taqXnVHB&7{A?{$m?eoBrA85R6lpZ*o;Ia~q4Wsi1)N>)fm@q-;mkh}Lr z@dz=c(<1gcPU6t!O7@QcZ2?FG1c}s0R zam0+~qiwSmtLQdHX-4Jn-etcOFmB}__8v+Wyw{xJyq`vH!<$g}ya2H5yCH&{8hAEm z(+-C_B_jAnebTTAu5PYV+dKQ4cAD-l4_Rs)=P+F<+I;Jgh41ltEO}aKnjN=)74Pnb z7DPwXvB-GQVANk6mLl>WwX2f5)MaDIs$!(IDV?53-@{o9`u#yl&}C2!(wbY$K0ews z-UvAm^b>g3*jM!%MV61rL3Z%faDha1vH*0J2)GV(E3UebyH*Z8BA}Z-X&`prh5rM5{iSZ^|{{+aCGB-UXoLkn=><-4^xG zqJdI(u;d$93=9C&5HN@TB`I(DW$FKdgMy*cOwP9OHm?blNF?hf0LR+Rl4KZ8~z=dm#8YMe=5SQkIutEj1d2jr;aG!sh9=v{VUN1k14bd;25d+(3u65oG+cgd}Vw*2`; zts3sFdKCX;4&f(hdz}ql7l3QZ4OTu?uK&oU!l*D*5sdi&=$t8?5UxiSEcmG<~#!^o~%ovz4Qlfpm! z*-a^@PDsC?Wjkkh#zAk=0AM~KDY7=mv(2rIkkt>04Al*2vFmbzDE8|mgWWIM5*mYmu{B+{3;z5h6&-iWE0mMrIEX8> zJ?1&oAZ9z4wFe=KGyQJVgJ#56+V53ZpnylDsI=^U8Z90yQ}6>+eRj|ar29Q)^`za` z=HjM#*McD#HZA1L6yz5U?2}%sIDv2Yb>`UKH!TcSBmNTHvv4C#%J;0J}k_A6E z+rZ{CD5YJBuD`x*^&BI$rb4HW3~@Ce*CWB|%9-&l!NEd^v$sSV;REUUmEe%J0QF2J zkk|#Sj|1oT+88vQbBA}MCFbNRh(CDUR4@y^^?>Rd~L-mZ?^fkF9JPiBc*$8-KQ?e7Y18$@}~ zB(5cVlby=8YRS1SWKeR%9JBDJH_>kTb88pX5HJmTBp^%_9VhbwyR8b&mT^k`u%siO z+#eSpnFSxr03cJNeM~sb{*Fb!7eAr7OBGMvJAN3}j_K0%x;?0AB|}Oa@>wL~R&!*K7X9i3Ox;^PS7hg%C2F1xl*? zCxdV+KedH~8hd>4OM?k_Z6_VVgeWyqWwy_hnm){ox7)?{5G>`ilE;M+`OX!Jw3TOr z1kB0ngg4)aDsmlnoVeaNU7%h3vH9-c`fc^(Zo#8pk!aSu*1+7VF=73QL9kVf`t8_d zXd-m^x=VF#t&wO?j@31I#&GwPZunKrAIvtZ;Tu>2L_;4EXbnmE_u!s>XBTFG-)R)k zY*#Fqa{;2Mga-Q;2gxGwN#nCho>&_y zJ>lyx2qYkduWB=Carv9fx^cjr;p96g(68k8F|L%%85@7C%u@5w=zdX=4)?K+X*Y0;m{J_-DT9KL?Oxu8Ynd-e@s{+P#fywmnbY)7N4}JFO8OehEggLwr zkDVBco@$R^t7QJgU$gU4>=|TrOfnf)636pnkiZ1)x0ZqLYiNd7hNyeYJvZ;l-&5sg)7r*^7k0E$@Sx)xty_{U+@}WeriYK_ zN7vwL!-nP$h~)%UK|_<~!(5V7)nNo6@CXg*DLiQzu-!XdYN{Z&YLZY!2}5hfy3Mef z+<&iy@6@gzU~Ab<2N<#gbelF15Ese|acgS3k!7WImuzg)**+i}@?hUe8`6FN-1m^H z9l5ljh~^;kh&4X{eX?B6DJ*3AUZ77q$;;pA^)33S%~oPl&FvO-T)n#-e>73u>AGXY zj8NK*R)*<`b7KcK_XB|8+Di0oVLpD4O4hjt&Z-U)_569>9AX_Q{hYA-N0P#Fd!=&c zQPKqtFqe4C#kRxpJS63J{jMfg(6_{yru8Oy^3aZ^GVr%%rF%>P)SU^s5DL~<+zaif zMbiO@IZVN@zhTbv)E$i~M}UswAEfu0j{JOP+L;(EbyO+phdmCLe7kpP6_e>8i(3k_ zQ@yfX*&ZtC#U>l-$ET~+e-44Kp%sSz00BC1QXOjRl8L_{14C@c#lCdz*ZYOh<&V14 z2z1BDdI-9eLh%8Jj&BF-#4s#@_ysV&cJLaHL7i5goJT??e%D2q7iXZ!rl5dRIDJVe zPqioX*}iW$F{yICho`azWWHA5gUEzv=?>su1>Z?LOtg0YzYEewJhG}d3~S)rz|StC zQ!<*2twh!Er`d>R%Eo!y(@Qdkk^0yafBE_h_|3tE=PyM_8kx*>MC9dW#`olilp5HO?DyFWh-*1F zcO59zvxZ|X8}uWEJjWhybwLMX#NiwMMi4y!+qKWgZAEL+!DP4YFLkXy0JOx7*4)#C z*t7gwq>MYUXKY?o{#Duz!TGhCbsT;gDVRnmzfI7C6!GOKu zMlyi@dz*}PByG-`?S%$hHu!hAGL(cU=JeH|+D}4^I{nNZQuV6wmm2{oeZNF5-blUS zLJVt$?dPc6tNe;36uBh?ps@^*1ftx?Dnw(jM1IZ#6OYb;)aD&*sPzQTYQ-0}=v5>!aERKZ!N^Y=|4N2$Zb1h<&_QX20oG#^SUw z7kgjh)Y4Fm993}rH4o?F`ckZTFATBXGK=rTAq@4}FW z44e;Y$~Mfr9>gW35V~x1SJ&^hjJvR=|A)v_&0aK}r$P6Uesd9}+^ud{xlJ3t?A+-q z3DF-#-FXhU$l!JV5(8^eqXFn8*|s;*?h8STm&<6|Z{;@Ae{S#O8C6UgA0Kkq=RqY9 z>Hqy7^uRpD-SBodL72$+-aXf5p)(s~Iu|Y1bWHClzW=EhnDv?3nTOGLr+M*A^f&FVO@AAhIHB|lf)D%<)5s=P-M`6W0P*<|;2|j;qY`kMx;?H%(iJ$T z*A08F$b5Vqw;X24Stq2>#8wL{sS3HCf^)|Tg%Z$Uv84xX#0(CX3-ullVPOrnWyd^- zCbI$>*gmobw&W20iSJQSeGE=h;Uo{`tHs)1M6dLLQP?0X-Jq z=>z0g0fXmuLl%cJ#Bw|G!)17i#sU(q0l!2q0YwW^*P#GW&=#C*-38&osYuu62{YOl zJlsK9SFnwVm)}6Pa1CF7CQ#e*li}7qn=gwUg-uE(P`k2W2&pRSI__rcmL`{Qd7*x> zbFe~q+`?z;%%3UEkQBXS=SXVf4gIq=IX;PO9#v0hhn-M?1Y7%NJbH4xl%>1VPcoWijhOf!3cOk!BKG zIX;**km07xxb!&X;*v#KZ^JHJp65itcWx0STdYk}E+_ASB45QeIR1x~*c(9!bmhRy6C378mz|x_@cWocuSGry_~b#!U_Z(mLqzF!bP7nWjbPEDYNe-2dD} z6qu;*YZNu%tG~kuGKyH~)Io%OSq~1esjPHj^e#p8Q-be7r$$T+lr3YT^e zdf79;J`tF?$>CWxMXq~$eum}JsAh=l<@YM(kA3rHS!H;q2h7#~5&%oCLqH5>~d|U-a6sD1V=OLf_eGz25KiKzdZsQz0svMnkXe>Jbd`pGNJ3ZP^P%A z5o{Yo*z1AMdl&KkJ(z4=8_x8}i0;5GWwTd@ zM;?|9f35+UVM_#ul}=dW!^JCAXpa3g|Kg8PGD10<)aXO3_hb zS3f0SAjqwl{+9y?IiMq{v(+%)u}=l-d^NX6V^WCVJp6;OQL|D)dYNNTxwAu&)ie|K zf3zm9VDy%)8x+9Nh!7C^EC|1TEOdH>r8?@_?L&I!Jti$)g?3W!)JMk2?& zL-BZDit z_34?im&Ga(pWO`^E1K-xd_ZEH45u*Ky2@xA!%dEvgRYuL?3Y0xuz8cTm>zjTpqm){9!u8vEo1VhQit{ELs2xR# zwNZ*k-(Sw?#vAcYSNB&_YGn>k@J0%{0Uh>PAZFBsc9uu_i|F!~b(DlFDGw>I6~=e^ z*oq)I#tu+0#A3I;ya8cM$bImUQE+BrZhLD#@qzXkXH&3|kxJ%&9oL>O{4+B)@>Y0H0wTAxGyz$&NIt^5H28&_N3xku?;O-NqDg!)r3K82)SXj*(c zw*Wd8x0aO{Akn$_|8&K%7}~PO(WwObxtI1_&kfm_J)BF*%TKj$7)MO-3#3my!@oFn zKoJ`Is}9`H56&?S=F1Y8E12LTK^EsDan|1uirR$|9noO}85688c^3(dOo;~OzqB7e zLPrih>t_uHJ;reW15z3x@wf;Exms|ss%yC3WiTeb^Km22$sw2U(ETXQNHA+DZSWial%@{ayPQsAEWT*s!BqDK09hajruEP zdgHD5xVx!36+2&eDB4t>^iW!nr*OlUPAor&75A>?FWe!JvyG>PCUv&9+G| z_=NNs&r4SN0Zjfzut6%ijA$YI6EZmf)^#KTL@ub8MJbAq?3e}P39C7PZX+5umU+qbI8b2Q%li+YI@5=n4sqQMvsZBBTM2%%|RC zXdoFzl2v*A`WB`wD13xak^x%5_?KdD)OLE5|XEY!A12y_$ zc6?HKG`@1G=(O&2oO}0vCs*oLT?1B#qeeF_`iQQ}Zk3SG3^b42v`T znNAweA3q|uAmP|}Apo!zMg(Fw7(F%*xDRIi1Ff7-mwfV|Pw2lHy`K!MSaErB>VV^{ zE>I%%%);)%t}%#okL+(*nPAKP4!~Dxgkj%pvD1m%4t5e&dW!fwBpA8m!Sxb;N0@@c zKNYf-k$X>oS_BJlP-})Ucfj%oYO66Y~)_g||mj9q<{a>l*r@{zwA(>Fi(`^WP!Z`BFifOrpx$3U{3DUdXIXt{Z^92aKej+~yYa`utP z27Q;nBDw>9pXek1BS#~*Vm1bCQPP1#M-;^b`5L(ZiOVF*t;K9S=^WVv309t4;pVMIzT7Yb> z*tq_8@UZB^tI|nEBxY-&y4ZQ%vj<+alztI0VDSZM{POQnaAKmW3DBrH5FYI8({Noc zo_^y}FWm9m+~>Ad-xOBI$PB@`z<{1}Ux)G4_$_wap>TMw+aiiiq{423HU!5Y(W#Ng z@H==o?WnK?<+vs%Mwms~_mL~S=BS2uxpvrvz`8|*`~ZRV4KXVy6BPw$NL1^2VN{a1 zhw#0rBZs3lHoF+&W#HahwHkY-p(+X$ZZMq%!_my4sm@5~#dv@|Y<#}mfWQ(PKCcQeoIPv3EqlK^j$J&ZSb@b2ltz%hox z7@jUcn$h3KaVHuQ2N&@e%u9>j?S`4#JLfOpqe$dAcrYc9qZf2fDY&vyWb;)Vr`th> zG^s->4Yom{WZ%3AR!lF5kA~&HptGY$ou}`~**;4Vc-fWPV@}?;1?am4bX!F*pNzq~ z;P#pf-k2=tijn?)?8Q0ZKK@nZS0iJsw)@^h=+`*cHL&ZfVjqt#m9{#P2pNrSphzF$ z{zHmH>j#kiiUII>S7-w8aJpbmdpQ{eAAJ|CMr-HfP5vg{H z`uqhAUG$(k?y3MY$e58}5a9f}Ko#9s-GRX@2yWAxYSzIO+>3h9Iv2Pz;~^WJ^UpcQ4Y7=xhZ_U)B@Cg$+FeL1WP_F<2%5$a}d_56BPh zw$}HTHtO+M!t3|RkJY-01*2k{gS+tAkcFZQh^gdz<_i>MhS49B)?({|dde>e0Vthp z^?NS9a;!m_2X_Z8>+YfEsHWi(7TIcbzcGsvg~iRFN@+<(>*F<0X34!gx9d`Wcq>NbCpcjpPflLXE>%h1 z&<-ZNR6umnEA?=%77l(>c;9-H9|D2Gk$hP4M9b>?BQMbQ%?>#CSaDCC>nBcAO_pxa zI-A~jwnKK}t7;5}quXJ2@fCOO;*xmSjEsT?8TP0i-?t%sCsKnUOayz(#%Bsxz@i!T z(pvwszfdq1>FJ-Q`$u6>n*GjG%GV|1oiUlFk#7^5^A-?qUZqm*yG;J!lehPd)-12lhZE;4(R*ekA7-xG!oQDJ*`()=y^q|9>o;8c+!=eCoj!1peVj8#y_q;HKAdq1Yr9bxZv4)k7G}qZ0~AO3;_Sx~?Y zbaj>bwj`cdZ)QJTin!%_Au!`(SKbnL&inBjJKD)_k+xyxfMJAgIDwMuASa`MCvBVX zdW`SP{)%d8YsTB)=#h%lhA4&McBuM!uCPF;48XBGyk)_LpPI8^$m&1Kmg?i)E9HHa0{Oh z$j4C|ywgGV?+Ux}3NNgcajO8Rwu@?sODxwfi6)V!v|hV{oQneQ&epI*UrhzfMm^rT ziX8Q;0*2uE&6B!Xm3<}1dJZ}e1LzYv&u_0)tcS9hr_5zv7K?x6A;-W!&WH}G>ltLI zKnt(p~m0MsN8X?%8JIX)Sz0tZCd*ox&ZRLus@%-=}z$QrN?D8|aj$ z!uz1MDhIxD2R&_9sVaMu!t>UOBJQ7O7>`ks+<5WSmDXo^V@DTmsx7Ieu-^2mQV=Nn z`SMaHUa7q0E)!1hE^nkdwBKwvMp!81)jjaGrQV9UeJnA@p>Zvit)4T51@lFH@~3yh zfyn#&F)fjD;fEX{lF!~b>54tzFzqP7$gMcOWt(^#T9RmJmIdni`Esu{g+d}ePa(fl z#!V42Tk`^!yc-I&isMxb^01Tr^oN+8JQp~9r@FOI)2WxfA}|+ZIrg8k%ryXPjX)&D z44hF#qU}D9FN8`)q4=Wl+~~v^!NM5}ri`;*whXoArT81H86RHIxjch-)Q>~q5r0X7 zf6Nyi^%I)}WYd()ttVm+ovcYIc}VEmig|uw^~AX!QwRpV{%IsI**X)>tHRi?+wk!h z!eBMKfUfpyb4uv2i4UY;jw^|5nHAL&)92mB=p$2lYQqkHkeC9MRN zeb+Iv^JbZR4}G3fRy}mc645YPgZc7`(5XHajn&lJA6L+AO(Y@rS-k%R_ig~KWMkRa;9Vnu7| zqjL=MgUc;kqH{Za*Gv)``MH<_J9Bp!%>RgExbPWT4D@(SskMH%ihD3!bvk?cg{oUri$lSZnM*G?{X{nfg-NYEkazow-VpQ`1 z{pM0QnoSGK|*Acn7cYR0c`Hnr(ot!?{|OajZ)!oGuDoRQmdd+>?tp)#Db8 zAxXlv+?VC<1t%6v-)Wy#nFiofZgi2Edml9aMvF2H2eN(@XeOm@)Q>;rQbvqky!C=| zbgo`UF-&H&^qmLEGZrW~GY{+7pRDc-O<32hSNd+V_;<=I;vVbAYP3fC>2@G~yWL7r ze$qpjt9H4Z_}q|3aK69=qq-ODWz79T(u}WWSi519cDm+FU$(gp?3c@AWJ$wtjj)0e z7;Dmf+CV4LTe=TXjcVb2J1IjB*lp|f2bJ?QBm?p*&^t!#r<#QuiTZxdpRcRhdx>!P9H zxqCKIZ<|T}aw!V>)P~Z}dA}!fgsfX*8&o~EV>TK)A9Mz}R#Cm~G`)v>AG#1OweCcB z7UyqoHKoF1w9g!|PKKQ5^{p3AS4n&OByMaMIXV{{Ztb_}GPY6wikQ~@K|YP^L6$v9 zJZaw3K{FBhWt20}$1Nrf(sAL^tx{4;0+$)}y)D}@J6n}~gfP9^XHE2Qx1IMI1+$Fw z>ha7CkLFQsTn?AsKk28+KE3S}3ipvqj=?=FZ5v80y=SE+pLCIBJ@qO!^pq9$yiHQJ zzdzaup_zN~WLAg}1YDZLRrGRNMcvX-i2}_nJ-_sum2S8QYnrt!ZTtjuw<*#AL64&_ zy_FQP;msW7rsJ?>3vw9S}QxR_|eSiixh6G%4e| z_ZdbxD8UP=VPoe2_a_-G{p-^x;$|0S5?k+mKBv-5Eh{BBHkues^YZK27IiJ(@p_&} zz!mBbj0mRkS&+KcJ|IWA-5U0V^=^_^XiqNdL5Jp_6GHKpTce&fmHq^29CwcEAE>K` zUr_9lXu*58Y6XZvh4n^zLERdU%ojG7YA_VB5>n8UV6_8gl*2FUl*GxDU4d!Xh{4^J z*e^x$yy}RDv4*YX9saF&4#|hrcNXODk3uI}xA)vl+hF06Ylwg4;+Ps;tKirYDoQhM z(U33-@UWPdIv6Kzv0i2d_ffhp=9De=2-Vk+C5QEd ze%R(754jO^W1Zgv>LDY(`V0>2ljKvjuy-l_o&S}b`6pJS=ZzZX>)mELVh`zF&z?VN z;{#;7fZX-;o}QszqkHre;>{Mz?SZA5k7CSZob^i*7fZkE{ z$+9ND?bRKVmfndybHY0+`d61GCJi6{|HYH>B-npx6L#mdro19jtfgK91%yrC07`)DYfRxw6ETdoKx(i;YpSpWeo_yPUYH_ zBg_oP_;N)s1vBp7!FpNk4yZI6JM9&~Mbf$IU{0#3znjaQihpKSdoqS6QlTIjQwYQT zxHqk!X|)eg{`;ReQN6K9t%d~r6ML$ta{i)UkA?MCq0cF&)Oh&jXM{YAZJix6Fh6XA zyN~Q-$5aL=yFO7m7pQ|40jp3|6M`zX{Ft}49J2_K!Ub>4O{V#n+!pbG zE4J3FgF>*L8+iXDRqBZ;$!tEyk`r{-`Hz-Ff>wPzMq%D{I84neWqsL1*741D~5i1%>LVHNmHhg?6*E z7kL<3{HBG?2eYX^`1z2HVMe3O|A`~W96ac;C$zM9nClhyAG`s#$Yq<^@KWDoeG>qM z?_wy*=~9OCI}h+MlQM$>G$TP$7=EBT0*_@mmi7-DMA->&iyQK6dP80!PWAQhZXfKoQRI$4}@ zZ$D-de?+mX#d|8+bNm+Le2?%5jBXbAp3+bY!|Q7>qQh+bl^=E9$Gh^!34aamT>#JI zc-RHHjmTT^n`*Wx7tNy%Vk7o-mBE;nXG|)ue0wzRv^Dw;xH4mE4T zx<}XGKf@}=L8n^ZhRJ9CbGu+%3(m3bs#QV(lVh|5UR-fG?4 zS2j6Yu6NR9dX)`!6yA8I?7M}X*rzlfp1z9u_;CG^eaYcFLY;11NpkpPE%D%;k3^CtFaZ5Mi`ivXObgi-i zAhfhottqijC)`MnXV-rDB?N^!(awC@fHQ`r?nT-znZ%kO)((BG85%0oMjM+jJx5Y# zOLA-TXdUM({2u+u(ESY>R%~4rIMy?~mW2ZTt^VzO&W!tWE;fX>i%H@Q(t2AYV&K$U7acF+mXsEIwqxk# zJE9*+N~4H#EYWLIV06ZNo#3p1Utct7nX8}uZ0hICkU7;f;h@zPfAJ>UM+(!5Fa`Lo z9*A_h4y!xP7DIAG`O>8=S5gwQIlY_wKa?3CvP+!TJbG$N`5eFys{53;TuzgpHv8zq z(#Zb`^sz-3&9GZjc3wUzgp89yZPaz8#JGp+1^EP6e{D#>cJVF-s2yUmU9y+}Xr{k3 zY&gdF!#ff;Hg~ue_u*}GzusM6@vrxWFGYy2%aPdraaFlrm-a5D@OYJYW9RxbZS@hq zB zkL9G^ zjGLEXI2OPVhq8YS;BYiLl!_0Yxh4Knb5Hi{VwA`&V!u$-p=PS3Pal1&^`4|n2^6zttSTOt{y=%kkWWVW@ zgt`2=#@?f#nJj(->zOQ);J;rTR1QLsHaquf(%ukDwp^sj+@lK?ri~mGYD%$XJnc{SvHNC=4bPwp`H1CeAHIjI<{&zGfZIE)lyD&Z;sy z#aI&<)}VBrXAV|0wW{47XNK4?-}1brB<}Oo4mhQbtya)(EM6_7;LMRa0eYhiTH*(! zXU5jtx7rAsO)5N6yUz*s6~R;Iy+phtsrx>LE3`R7uUcHmt3|;@QzbPxI(&;!HMTVDtYVDI zPAvw1*~FjfOBK~>;j`KdY2GDEeE5NL&CQ%8v=c@M(8n6t7z?vez`x}(c5I6V`!26F z8?S)Wh2xjl?`x#c*;wPa3y~j1v=}yQwVaQeb*_~hqxoHI{!R>Q34M!lA+XQqR?$HZ zS#;lJy{)iU37`qISxnk7%O z*IQqYeIZSuuV$vTuD#+{5EPX_~Ee@961Oft?Kxe28nVrEo_bT+f}`%yn%?wy_ha ztL={%lG!mM!>JlOO7SmyCaPq2@M1)ZS3gT=8mfgT$NYN&LUhY5iINn;9R+Po&y%%s zQ%E#duEP);vyr~7C3aj!q#RDg?eZ+!7WCu1zfl~9Kc<|G^w?$c*kO}m3DbKvBkP@ac({fJbq&T)_nue^EeFRjZj(wh zje0uGK0xff+8hF!!rP+&!R8DSe3TsHuJz^kAEMq`zh@@6ZhprH&UYf7qla6$u4Tl# zuayozTj9%!@vE(H;@&6S;(eL7&iYv_{#lQ4kB>fY8)uBn^=rU*u#UZ3xS(0m-G1l@ zw8#&aN~`7ae8*c=hbhV>K#kZF;##&UTp&KzEqMvu*7(_hA^CGTga!^lzwZidFXy@Pe z8QF@(H;Xp=!&>^q+cd*cHySJ~U8q(b=}Qgq2S1;y*kw{iBR>482IDMAs%c(p4}P-u zq@4QQndH(lJ2AWEo(its8Oe-$Cu~|;_Re3l23#y%^_JKpDRGzk?p30PhIvJel~)kI zqv?@^byLv|KnE%b4*1iMpBmU+-RIiMVlKdm6NCRh%taMK4dUbZZKlz8vSjv1~3;KYa`=>E|i!8ZhH`i$d`e?GX#+Gw=K5;@$>!#TkjAue&e0khw50^irL0 z*>viOeMgbESM*FIuk}`IJYAtB-{W9INP+E3)2!#eRy4f@XKpp`sLoi9$;y4Q!XE7X zgk{vNWtL&FgB2Y)R4veeR7(xE6B$O8qNxlKH=WyBu@u^b^KDDMcfRhgX^%2zJm1~- z^ZOPz4X2>$&EdNzRkDk0{Nfo5IV?@bspYQf<>1V`To0%-oZGNRRJ*spFHYsI>1{9b zTN3`nxz)3k*&~VTAWrK|s7CnY1s5>(M5c>4Gxwx6tdGsm$s18_LA{Z0yXZfbv-gcy z@b)pckpo10+Ck5Nu?@}IfNZXrxc2E7_MIi`kJSr1#8)Y;^ESLWldvoLnMC66fj{53 z=Yv<`C5V*&k{%XE4}UBq&>n44cqOhV(sF~wJjF~|P z8!oEqDD)g*9i@p4!nR?k9!IDW6+d_JUV={P7$hQG46kEYoL3velJo#{MY(sg_I8jV zUoO!1=>UxlQ{xRa>8IQy_*$A-3zTHbkf#82J-wWprsqnOGsx?;T+x_$K6g&HUTot<35>!xkc^T-|T`T>f6bu!Lrk9&OYV?{i+3cI7+K}NVD zY*lhqct^+7>##B~rd*Qn6WsvE3uyGm!R zRS>iKq*TTjbXV_mzFC1iTMa&afYP$fMkeYGDG~j#6TOOmSoDywkudtJa#9kKy3mbf z;1#Z#^a?Ri`);IOjB+i2sMhpa3B=x-6?KzK`0RB?F`_vFZqI@yDgr}fb~q^}WnMgQ zTgYjSQyC{m&H>oT3nfH;BkC5tV25cDb=ceB2x2#O9Y{oj_M#ez=y0gN)m({EyTI2i zflo@Yod0rkm8MaWx{e0jQuNA_=+Ij`x# zyGRqn{=Ly6P8Pzca*Dmc$!2~jsgJez151r6B7GfdEeM6lJC$<-rR6zO5m-O;nT6n` zi^k~l!JhgiCiw~{_av%jp->wG_W+HrdFE6m5+VeeO8YiF>RXw3~t z6d(Vh#CVbOBRQHzrf^r8!Y^7Rv#m<)iSHRQIY)&KJI>r7@^JcZpo9=1uU(uE`6@!y zmRdKrFL_F;6izGvtnBm*91Hy(TWWO53oHVq8#meFXF2liOX`$o3&ObY95GMHO`ZK( zv)U$&UVd?$Lep6&LW^waz=#J{BU7TmHL`0 zbBIJRY!Jxx0q3lWK85dU*vpp-(rOHQSN_7hG^9P23GTi3pQ9A`gB4%z-wGu>`ubgM zw_6S^q#%b%+Ro?(YfNfXB$y9;z>vA)f#sxZ&mBEU`UlR&borblVTeF9Dchi90wGTC z*veX;vdbh~aw4bLv?3NZiV+8)Z}Og13ULK>J&JQ>vPY_pLWOD(Yzp^GA-v{dC% zFsKddLyuGIjxr;X%KJ zYY%t-w(fo}{BTy%2@q^-dAt`tz;TyuG@hM2=Bvd^mbb%yYOU}=mYHv~;QYc$h6$IG z^;i>I@x(o0)abUr@)E23Cj)DNrO?4$p}Ez@FF}-5Yx%+yEa4ms@`A=QxXE`T8CrWFZ@^MVTkPB7=O!Xa` ztxg!0;KH4@p6zQ?I#XwTVhX=7Z`x7z8HAgJnqSJC-Rhh|XmHK`*|lo>+dXjDzcR(f z#Q{Pvj|2CS*?3jzqj|DGDRM5kr6<)}_TyN;8c9Y9TBsbbYB9_Op$Ezh0Q+OmW9S#4 z*^8pVQ+?#V3uiU7i1*fe2mpuYKJ@(cZ#;2a+Tv00g=ZHeBpz4}i{4Aq%ZL0PSqJ-) zOX9;^b()6EuSe>Z;usQvTEgpW2K`gd5i%9Z>Pg=dP0H z&W#@yFkhHr*u!DC<&5u58`d9v^kz!6>sxYI{J{socT*;C=AekHXEmd+2W}ep9N^Rq zn2)I0l^Ei|0=n);e@?KvQ_V+}BH}EE;5bJj{`u)4ShA~=6?_5Nj7aG|f=+N;hkTjr zjs5dA;aUgY4W-rHlWW!~lKrxJUSgZ~=*S36MN*;LL6nDh98L(O;;)E!WWE)i=Scwi z9q>^JBBx2}JlCM8xubY8&lN?oCc#m=9 z;xnqCCKbkq-L9A!ViVz(U{gXTXb(@!1ia4b&}L4%dIs=%ykb8SJt@7MMxlXQ`i^0V ziK7>o2YX866Mqa z(JuK6lS`T+taF&v%J#kG-qcxuPf9@%q&Pm4SFV!5>{=S_!uvf)66dj*Mm})9)_|{) z%bXm2?yu0a`xSKhzxgV9`qaVl1a#+|Ck|PUv?U=Phf_cU)*)d-U9RFU@XJG8a%6fU zUK^SW(FdSF_0wr6^r4{Hii1iP4~t!Qif3MQ{tZGu*5btQEQWJC-sY%N{$2GoecYO; z$`RaO@FWidX660QP9RiMDdB=Lt+ex=zF*r%c0=BSiNaxuE&c-f+4 z-u?$YJ?~H$64%zFs6<_RUe_E%pP^f=6W(Uk7=N>M{s~;;XLw4>3d~OST{xVXU6$7j z*x-}Ao$f@Rd|Rb8Ak~VV9f#}fg9Og^0-Ye$r_1qBA+>SS!>}0RRPbvdV$ZGSFnF&* zw>AKrsTy}8;S~a@oPV!kCu->j1(_J+*B{;xa)~Zn6+Y(%#3}Ya8wK;CqVhnz%14A% zDd04_sSJUFwwfjMW`Wdcd>(Wq)k;cma${;+e2-b_wbgDeol}Kt+afBem4?gG%OE1@ zf|x6xj)ysp!;q5~&NH$2`Z2veyvi_Y9<)gzz|LGu8zE&eqlaKP?xHzDCy*#NjTJ0bXLCGDzXho<~N>qVXHpz1$Q z_zCh2cik$-OVAH(QuZ;T>IWh==8BYm24j*q7{!#JB~Bb- zct~WYx{Tw?_olaKB9U@^Lfm4`>9bdW#u+NTs%17%O8!s+n3rQVy?nW0lmY+=9Cz+^ zEhd=KfMSXsLf&xykQOu$*!@gMeruL6VY` zoC`rf36eo_hMucD|G96xd%NH3?l;D}J^G$8&M2wcwQKLS*P3&FVXkL4m1L<7Fde{P zFjR8auie66_QF5+uv6@V7x&1PO?VNvm%eSUYGY#WbjQ{hqj<;uzNL-5rI``Cqp_`> znT@poFP5KI^c=gXz5RVVaXvn)e~#d_vAxIlNyniX&a(ghbuBv#h9L(1Pa@~=_!S28 zN>1+DWi{v5)BSs2v8ZocoMk?7-8*6b5sD9ANCQ4L%$^J&U#;nPcyRl+8k*@vEL*UAxV9O-}wb5fwcqc28;5UeiglZzfB*Y<0jfOg{A`{4U1( zmbt+k9r_ohmW_lRy}pXX97eD53K$CXdWsW61~1-6XfdSldg9kgJ#Mb^L;(LWj}?EA0n0bRC)LanT#gQxWy%A7?%^ zw786hY38nG;m_#P!)^P(?P@=}Q{|Id?y;G6I^O;KarfV!%pUGNcHO3IS))voGQs-t zvgP}xaJH#PT<|UMUzg>%Drq}LHS4CdMj=1FJ^m{Yk^N?BDa(BZjv)=2N z#ad>Yl$;N}Tn=*sl8O@Z)n>neqriD z+cnzMT)mIK>OQUw1;ls+F$G7gY%R8=!|BgAiBG-BR83b}cL~f*yz2XFwIouIGj6bl z;r!j}CBHvYb}(7Id13PULG4zw?D7N*T3lVcJKT75=?u*L7}u5QsKMl-e!&if&~QD+ zZ}z=r=|US?+dm#3%8D;&;QjdK+p_;^ma~CKzkP>H#Ix1kY`bf++Lrk>3-^S&Pv|$S zZI!~W43BvDE&FYX5BYH{*adub8IlaC(=mE>s=ZB?E!K1Gr>`IvHtk+%9Ly1?9qU?) z(r#^w)XlkyiZ%!H`kBJPIDvr^n=)9|l~fCT)YZ z9j3Z-7GP|JT{n%QZtxbTw7>N|_T#O0p7hl& znp%GI*>{lP=K7SuZggJ!-IsD)h2L#&7+Y2^zQzTMMSDmUOsS#SVjRHp!F0tD7Msq_ zv@pK&){W6AO)cXv6T(b>9c%UgLC>jQsC`-F`~9n_c4N&=xm6G9j<0?k^zF0$X!UAt z{h&bO^NXsAE^}q8Tmnsks-cA!S9)+U9u-x}7q?$kcs1qJvF-f!t%s}BWn|vn9NguC zuw=QiGX|?mrY|pT2cBYPT6EU8(c~G?`D)r2_loGvfH}{q(HK`AW(yG}@#Ric+4V?^ zyt0+sW!)J5a)`YNbo;zvB6&F!!)eOfjD=yIx_eAl&cQr!DW0p?Enk2o{wv|`fXi5p zk2fsFuIq(-vll-nP+X-J8+l$IV^gj*(N}0QT0qdxwtV|(Uzag&$#{}V?wC;kk4Ev% z#uPVPpG$d&(|}95?L=yX23pbebeg)@&oln9%P@n)e7(4Hi;PAelCx&GZ_X5k@s+P7 z*+MALxcbRBTr2I{@%599z3`mv*cYn~T^w27?Hk>y-Wy0 z>+SsdNa&bxxQ=zP4BP2PkB&R`oFEUQNvqvJ(I_m!z_Zx4Q$=?4`FZP#ZzeH9uA4)} zwVQ|M)FtBG=E_6{WlJ@jzCWltb2V_Td^7i5^^qHDRrM70wK947Wn3K!0*XS{0)MZh zM(AgGTC>8+tcQz^WXn!;_6Q)@H3<>kNWkx}JHo(ujzWCTw4dxeYar5a?tS1pLgD>(bCs zC-4mr1{bBStrg9dO#N!vR#lb^+w;LW7v+<3%h}?w=2am`7n=L1m;~E#+Y1eRVa30q zx%p`GZn*nMt^IiH&Ng3?lV8iUUVJpZzxGPD^Ui!!;1tJF;N}^r!Tj=hmH_VuA4#R?gC_1y6dTDnamACisX zm#ULKQwqU&?67FJfPM%}aJ+4cfF2H%R|cRXX^WWwR+47bR2VXoAU zT0WXG%o?(Xc|!aBy5bM6XR;m}6-%yuw@aC_$kxs+-kt`lTM50fa;yp-Xm z`+4;$&G^w(+7p#MYMs*e4?k4Gh21V_5h4xkZB5EFwzwKdal*gyEEfaEcNQI*?{tZ3 zWGaegYo67(V=X7H#Qd!C4Y7}bbx~Z>NJ7rS|Jh-_L|L%y+IorC_LM=8lnI#>jdksL zm~N{S1nDowKI!Sw|5U|=u$^oTG%j(R5x>PR!Z}x!QN1ekc_i&~ZLj>hv~nGboC1wd zb-SN|0?ipdUW)2WGKWQBt&WaDI>%#OscD>#h3sxj<%P6a$%HsKgba5_G6e9|%2N{b-DW6oKfC<%bBxa6eR>OR;_Jo_od>NZY7*}|5WH_m&$djt3$eP)@=#CWI6f5qe5~M^99(sXub&-aN zne$(w-~V_-`D=dP@UjP_kZ3*O+OGl2OnTT4MuC^?(Mm`VL!n(U^`~=J(f#oRz zUp$|b(^Dv?cu_!bk@&FKd?Bx`?hsbkAP~0-calcV>TLMB>=0GWz4A3nNmxG|dTfKf zZ%1C<5MJ;1*D8;p9SnF2(cu?XGG=3Xu6QhNeM^X-DV?Y~@BC93GH;DD`^rNw9Q(#?&N#lr-4lEBq1W83(wjtarzVA^ z<}O*h`+{6@h&q`-5JX3p1v3TPw)qzEH#iuTvGKJV)hQjN26MP_BCofH;$NyT)&Mh&2KR8d3 zPz6g@N}G1&sL_>}lh^w_H)j@c!F+sWOSe!ySj5#neCB*8fRJyRJhW_W-5x)mL!srh zvpyz4>87Ahipi1d@a-ex(u!m|2mu8^BKjPep(WBcZ)AS z-WO!ASIS01S~DfWCB0Wi_0HJ1>uSG~ogt&ZVc9QkQ#pmdxY`DjDKz~VqwB6=X{Vt4 zt22q<{`~qdi3XX}Q-^M?)0!Gk-YOR#c)Map8^4((o=IorKIA+j7tpQmo+J@ASG6ELQcz4Fd^S5~&ovys2*U~+*wey`hfzN|Pz#15!7 z!ghzYgPuFp&}lY)a-|8}WmS3V-Y$1qNb8|Q$zc4vRhYWQ*Z4M5o8(7AS)%k>cVFn9 z{n#$gJ<3=yY)FE^t8qbj5awt7uuXlz#%oi@6{>Q(>$G%IDzO5VJ#PogLRs$I!bNAr zS!6hpOJ&_XP;okvDW;CKZ<8T*lu1fSqGg$2aM4$e#^>7ec_<&#q&zl8*ts9Iu0e7h zU09&{C?`Uf7A#K|SF=f%SoK$^!b{J*AxY+#RqnXq@$Vm2Him?sK06&JYE?R4yS}vx z(~PEs!oX{LS*$bI|1IDK^OViwF)gsTS7C^fbj+W4wT+_sskz_XZFw>b70#1xo)Arh zH{$A?K77c&Zx>zSsq0l2W4rS0-q;Cp{~y9!^n6gRe76!KT^wg+i(k&Co120kjjYh5 zIVU4BS~%!=!7F}fks%LfZ^tLNx*c`cx@00vcY-N(~zf}0Jj=zh+r>+J8~A3qyPL1r{vOmQ#SNZ?95 zLpt{Ji}8AfIq9vSDFS;*rHrwL-(-cN@V6B0<#o5g_xW0clReo~g-oh$9=JPqRQl=r zxdV7d3i`c<1(=3=h1A@C;l7u(ZI>DVKsB?x_S8yk(1&4;zji2KGLIobpd^)k&P7!{ zvtY30;U03~&D)A)aRsaM0KWoP?1CLwXlaUMoCxFL+v{WTW0kzMN2lA8WvPdUb{QU@ z|2&&dGc4p&6avNnIjj7dW5xCS)b`b59C2fF+qwj)h_*J}FARE{hoIEhV)VlbBvY^ezmv7PT^h0tKRwoZLGdwVlnna%*XN% zF$+&Nf@Oa*UAB7?N8>GBVBu%@HnQ`sr9nf{U{VuPKjDD_o%*{Vhhe^JJ!{*mgCX79 z(t8~hWN2xBxX;(nuW#N~@H^W7gd`!A268EP*GH#08SY0(3vVK;*-d0~A{aQ{WL#nC zJCGP|{pI8EAS3L5P zL)WX4gnXo9oJm)0Qt(vCzTm7QW|jz5c3B6uDdrYiKgcO)&{cUxcUcjq?unJsCOb9U zflz3jS-|geoaTPkZ*jCn*Q7goE%IVp;vIX^=^`KtC6GH2elvcj`Xdwr?6vCQSH znARf{5FDTS=bFq!hB5Bi#sCLD|4NGE9Myr7IVyV@c+^Xqu`bui zjOLkS4e35zm#CxQGm@yJU@>aM&z~ht2e>NeYwk4Y;nZvRn5vfRhd=gP(iI+S=JyS( zjy-pZZ{QkPnZrC(x{gKAg+?Derfr%iL2`xWdW~f!Dx%8u$W9~V-)rBYn5`#=`$b4= zgfhD8(xx9raE7C-b=Ia0meZtkU|t_p=`J2g+2(3{YNy!Vuhc%FqIp%jdGyX$PjEvK ze5m<2o1z$Z#E-(uXvm*|ncw6|klf|y7#gwKHg1#e!TY7*p%k&S)=Vuo3bd%*iwXM5 z8GYokg@gv#JsR&?0|{&QouN&-nX8Sj-CHk{k8X-BHVV{@yU5U)+G_@nF~nyURTK?U z>`SKd_iuBaUTc)5AMnXj@yq=@A6@5mc<{wqHkC|3%QXe_qr9w;Mzb+Zj z;#Jz*=2N@qzsh5>mp4oGg468l!HXEv2V9s-0S|y2lVxl)-L)$E=@SOIU&z&-unhHIjJ3p#nMKg zrP@aBC5r*m=4$-mn~BBBwm%^`@x)ahwGTZ;0VX!|tD8W*1=}8P28c{!EmCz*Kn=>L zLY0Bp+>f(a3P3R=Z-JpTGmW#W(6W`vCnR?W%_Ssa_N1o~SQp+1gTwJ>q`5{(aRJSsf zDb7q?xc{TF{>Y7UZ6n+8rL8c;GHiI^MS{wL=OryBh2AWG;Rq2dwgftcD@1bTcIe>p z&|=$n19wgY^suh6B!*X15N&zQeluY5rOU6V(}$#Xq4rNf086~c12=q2nywSd#5nSd zkOx&pX@rcI{#+5#$F%-(Vhfs#m=_AZtwt{G)SfZ@%sejA^ki0n(e&GlC2fGu3k(K| zK{MS<*Pg1YS>%*VX00#B0+Y0O?U)zvD|SNKYWtPWs3;x_)0t z?~eikM7EeZ#_EHXgJq%7yC7$!4E*bJf&k~ov-i0)$kWIOYAB zH@yW(H|ztm*G58>f3@hirw@IkYY{6+lqn=+91Z+%vfLMEH0ZI~w0b-&cD|f`?jHU! z_NOBLtV(7f%$gAUK4)H)u0p8pM^~~kg}DYIGI|5#N7i(!XN(SZn8qXx@#+s7olpFw zUzZ^q#8o}p|1souvDq0ZT38CY);2?)!vf6e58J3~pZukw^6gcqvXoe%dYgya&!_aw z>V$c5Sh-WoGnWFl>Giru5>A=ZpEWGg1csVH#ojphInN_J1uqGvHvGUP;JWW5y_^Rm z+Bin9W0~)oM~!K%N1A~lX~6!A(MM=+U+Xfk?|e4Y+eq!Ew7Bxets^6kf90tH}Zy6;%jAJegX5iCA= zCQlUe`{SSA=yM(p>S#^8x_Gb*RHXr-@EgDmX(kgKfL45zcL;qHsBFcH0j}w|oe(?v z8^70t1ZhG4ri~O7^HzpPZJ4?Owo={Q+3o-mGbUWu_A4Iozfh$X4P>KlV1{*LC^303 zfnV^(Jg8;tFpkg*@4k%-DtSYTJ`UuY;R=&KRX}p@^#hr>QKy0~jT`Tq#VwcwO>epa zan^yYV+oCewFN8%Z7Sej_MFm%`sC6)c(5Wa{Xm*Ui91e5B5oo$9?3Nr<3&KqzjV!i zR6ZP@!S*U(dR{qiFk8O8T(Z5=*KviC=M}I81%q^$m5Gki1vZ0HK=E3!Um<*Ufb3Swn!|l`4PXsWOWh~_)K!PXgGKY;#tuB_1{0M zI)LSiMzq(~?tt4|>^(hnsQ0_^$X;(qqJk|;7l2PqN1{Ya>t=`Y#ni$9S8nAE8sS0r zMSB381UI0-F&M+w=CmX~(2%V3I}I+i-{4CHaK|SJpfB@Q<_P=Sdo6Kh5SWz_!&9b7 zd|smT?92bSOR#>$Ki)ozLX#7i`mfplnILD}5>@-ZA+0gh&%l33A@ zpVXE)ei9De3ZAW~*j~{o*f^9BSF%;%c&0biH2&{1=waS{k z2?6r$ssD6-+d1kttkbm9v!>OlHpbDky$m3xB^~^}S!b`*gS0V=Ce6Eij|! zJUz?@L$mYB#Wm{`rPfEx;4+jUm%9l4&Qar<_-@yth}6c(@5Oe}fV3Bi0teyluCFrE zB|R_z;=q`?m$-N*QbN;EHGpKe33yhPwjM(d;AX>@3M;mGoVY{AnA72HcX437Iv^f1 zmBH%sh=3;6`8-l4)bp?&P3CV*Uo=;f^})LAz~;GjXXi~$LASQp zVJQf?9Y}>eU1FAFkJ2S9gGcFW0aoi65wF$3JgwUjh;*h{K1zs ze}JZ1+F?}gbnGi&sNz`;cl&$uW$h-B^3_J4>ppc&dVnKD;^m$iAKs7C@3yp%cH9BX ztuEDG-1d-1#Y9?+tY&nZ?>t7vk1{6Pefg+&cjQ6X5n^6>yTVHU(e2Y!*@l`UPf}aMr zfty;~xVGF6Df5+QUWV%%s_#h+)+#d$pSs~*^{95Q*pmzZ+mogD%0har0On=z;hm@B zo*po~D4Co^E4Wxb@37vpWTQN14+}!7vz4ZfRRIgH@KKUEKp&cLj<&s`5=*UDaa*p} zQjKVKo@O|o<1BLl?ou{`E0F&;@XO6%K{Qv6{WKWw$;ahl^b8;FwWTwA;{VTm`_GS^ z3(t~Udg{rPlMuC{x%>Tjb3wsWFJO9$rQwum?YOBDOF#UzX3axU z&)|ZvzJw#(a_+EW(AnCzNJnX(qw80X(axjDPhz@n-~F*0VU?;+6jJ2WERk26lDK%A zWkr5u1Rtacu9;$KDaah_dquI2Ii^j7bf7g|IUZY5;anc%U=fdi#$F@)XTjKQ|C_}b zcLQ@99`q4)m_2(*o`7L_3Z5v9q5m#z|T9_(DX7{I=+EJvDz((fq!~&Ooc* zSptH`;zE;R@8RJMp|m?fcM_cg({lN^YW<&Gxnj>j-PZ4_Mw?zu#!#V&o6>mWRB95m zS0^aP$?q*+7As7uZWEvZ9M09&F~@v!iE-7#Exf4hwow~emnkq(U0ojzxhTA$FD-4N za!OokFxwRszEMwzBhtTup@E0f(eJC2zkGh=I7e+o zK$H$n_aj3H@1wUFL#{s_k}r@kV1Mc0jG}%8PHxod=VhD^sIyeEuiGw*)+xkl%^k)u z)=n|s8K~G<&XYJH^9I*;${9j#)9@+bW%1xltX{=>#C9ujA<9_l~K%W81D<2sG@v<$;5yOoH-0V-y*3JY5%P zk+3Vca2IQ$pW`D?-hJV%804bGbpTm8t}cV+9*br(cj=#IXnb7hwK&}aXR@0sT~u;~ zWU6Y>{(Xvp)}*?~p9-N=v1^_XPCB^rN?&S+BIPN**7D16R#rYDBjKmSrk}~3fRvT~ zB5TaZv2Wb^LSoG|CY7hh)IF%A3k<5BXGIF0G)d&)+@h%Rp=y}A$jjNspoiVpk4l_9 zsTtSZ0g=jRb9PU?kY$xakj`ArpMW)K&!G!VpN4D^G{DIzd)9;w2+bhbXdV%LU+3KI zjpZ9WO|DZp5qtWefIkt!_N_ zl^NsyUuh6Ww0}chx^TEjxbGa%)r68{m={fDDTFDpK!5-Ds!C$1C0k(E3B8vP#D$0T zc_9biE}scriV2{tTLk)9ZNHoda-?E4NOD^Mo*_;NQWgI50q&9cVHB#j+{?j@pj*c9 zaB45T?(@^aW@k?p1-4u+9cw!-+Pw*Ne-nt9&&g_xMyEuL7PVtNKL{H$MIi=_=GL4 z9`|E(@4JOzcu!h)5TDF#l1zs=Uqa>$M^uv02rRT0Mbh+0EE;jSeh?9!fbMrXiC=7bXilxhCSwHP3d zrplOFvnEIn!m(D@>)3(;GsL`v{^Rf&%Lk=_o1aP zRlCmJ&9~z}zn)}qe~YcqZ2T#`{&jLT_xswmUr%))%Mu}5r_kCAFoE15$)#3N_t69g zRp9I3IvaO3=i-|s78_Z{a{wAo0Ewdj$k)adYCOAy!cY5{1K(~{>~7E5Luvg>z7!TE zKe-RWJV>tXK=|Qt)$KIwWAa1ICk&tXA&(Fc*-sI&e~uP39*ER5#F0|5i1J(E+hmAV z$N^r`G8HjcB8%dUP)2s~FG5OEf<>me6EIc;5G3%i=0R-{5_Y&oJQhOLo!sX-4qfWm z5wWh5nZ?o85OG^R83Jg|CRjnb6SIr~T1}=X9u9O6WVFq@H_gxy?-4SUL9rk;F;UaQ%B4j{*-2Ud|xA*sQ zwV!-Ypf`joF~uRAK=Zlc@SSg{zC{L8VlkDg14vWm!0u-l$k1Skg)i<>T8qH)WG(or6n*yZD;Ex z?tXc04@TE9IhiE+2f$mn=pmPx34yZr2yt1p)-HkFl?Mu94!s@ZA?QG)eRblzL7#)f4z~BVui&Gnh-K4jRA=@rAxXlF_^7W z{Ky5UJXpXREACLA4je@vLw|1ZGBVh%QCTh!Zk=GQf*Al+ zuB4_I>8|YG6T@h_6!4u;3zTh*0%MH!uu_{k!K-Qy6DF0>Br>sN8f8UQ!_}`gP4;izFt4l-2Dh%P<6mT<(8+ZcdO9eEY{8}d`~2}7rEe% z8^yhR&4Z8yTs%%*LuOm1*aPuNzS?I6KK{w5NE7l=fl_$O8i$d?ny zuQoOYp#c$@if`+zt-vuFp)5_(YTKK#|HUcBJ^!&8U4W7wCMKSycIWX2IpU)EU*o?BjU+cB0*j|_!awl`3!2rz^ zIsr8kTTb0YQW?kxP`Nqc-qO4JI1 zhS-+?^I{b-ND_vW zj!gquXwrk4RYFv9;R{ET)GX+I(T^)Ea*H{6;F(W=lqcU`;=r6(G7laiWu!5nz|;}H zvy?vRfEMHicxA&K`yB|n9t>UnHN)26;4lN)-VEtK>R#YI{X!@M7QzgP0)wEGGnCf1 z+Ge1F$y=B+YuCK0;yz-8_;U;@JSSib7)L#>8KUZJbP*9Kfc|L}1y#f$K-irjWNYT> zs%RDg#Owf#`W5%o$=BHk*PjTU{i3)Ilicn;!_Ps05HUz@gkVR(5F+*~(tQWsVm`4& ziE_L++^L$Wr)n8|gS4PHTATpPrdsyKxjDx2og6j4?5fb??n{41WmM>y@XhsqTs z-m?4r>ljxl?APmolnit)TYHK5#G6KqE9}_1za>1C32p zM5x#)__i4k<(bGsgxHXb^~`WS13pr~X}b49&xp!^Y-6d;cTH7)^HQU4hXodVA{p)~ zNQ-=@*gi=S$mj}`buSc6%t3Dz0)Nt{#X9!d1EvjJ@t~i&UAm77PNST)Yg!QacQuEY zN)tZ5+AkJL3JgbZYs)izA_&WK=v_oPcE9Xx<1qECh&*I8UV?)S;vN9G$KZioF+flW zrFw2s3uJpJe#d9Rh_qEY?EpGpHOvWY-epRCT@p zhd{1MC~YLbQhui^xB4d(weqK5-~W8ZfYvngZ!Xd#LeEhS2>{wqLD`Pekq{vFM~T~y zKSzX}gy+DxC_?>~l~BXpkG3gaw@o!5^Mj()lDa*Kc{WI7AX406Qmsr;ZB}}(L0}90 zWdr;D`T+MA-(j`BAm%77+B}Q%AAQ-`-b@EG{$B?40%Uc=@er)m z#hRh?HOnpjRsZnLlb1-Gm~CgiJPzVhtPL%prsok7o5bM}6CrH&evX8|w}K|(h*yxM z5Kk=lJJBnsZAy22hns%=IG3Uz$zqmPSps{dU?4*Qnsp|iHf&feKzaf)i2AyzJS9rT z{5B9`m?s>eg!=O7Cony)tBNTO+5k0Q%GyRJ!pnC222X(+^%zPS)DSZ5t7&A_@-Cph zAyCm`KR{>~gcV#G2W+2Hu0$YjbA+y~-0T%yZq`i%^l1aP)`mmYi>s$A>yA6=mcWCT z-E?>uo~DXQ1c~W?vIDI(3nZ0X*YaDLozP6hB{Xx>YEw;_cn0kP>f_=mD>J(`tB$P+ zmzVXACj>Ds?e->`MkINd?)3=Eob>4&7HzIf^fYxXZ6G0d$8Xj{8vTGPQmOor@+B^YeflAW|?E@t2 zaPk~|wO;$FZXTk~c}z#s5X3cyDUBQj>pwsK>NDzEg6Z}Ofeq+!N#N$^zl_9MUr&{F z!pqMU>?jK)2pXkd*h#uD08Db@frAQ%vEMH#thL?pJx+`}H^uX52lA8k^t}y_8N#pr zj|a};J;@KCKVIK^>R&;tB$_yojhG?Kuyo8pOgt2!_P|_?{U%D}^>3Q?dZXH;7ZQIv zXrYm)LO><}_%Bcs^mB>nw^i-HibR4cvYW!ISJLU`fIc)VHCHA10rW{Y^1tALR!V`A zDEnW(r5CdwLw=Cq`3{&$)2;hUQX$g6|NBGQYjvG6EMS4j)XdXI1&AVq1BWtAa&%r! zKTu|+epl|9NDAPUN2x^{45?nL+t9*R_H_zc5VCcRd%#gMxGf6Grj*`ko3T?*#PgNO z_+8Os?JZ=hh_$~lxVH$>DY3&3=DY{%F7bAIU$$#vfCMPdbebEO08FObp(xTXTHI1p zp`uQ%1eTrQk$iATxD8TgIIkxtP1BMD#?9+(gD_mlu$YId3!XrhNv{G#p|~Tomj(>V znILl`kREl=}BCQ1e^t2w- z^BlS#fH6%$g={7y(_{y>{*O*J3njpNae+UH2P)cCr#!H2RVWj__POj&{1af$+_D(- z+;VKUvLpEcB~Q$*5B2eADWLuk&z24u>;%#!iHQQimom+K>5bVh98+ZnT1^xI?{>Oxrp5jzF+TX>{@@^2vK_wEGPE~J2y zvZ?FOrbNT+z==^f>jBq;yfZC)U!WA$n_ETm2U8ohVrBem5Y0I%Rte{nDb#C-2Pl~i zF4Po|%*AxRSgLJ#86OZkRHrzAa z1g1KUEicI6sQ_2rv>78`%o5yiI@%ooxEN#7oD=BqS1n|e8018mTnAW~O0s9)NJSdz z8_fad#{{VRuP<33LLD)K$P^*X0ApH1VweMVm{Qh5#K&r-6cD?Yf%r)(op&PaJAf(&WuWkHPJ&qy z5uC{lHBU|e?K@9qa1kAN<}oNMKyVvt14`&XhN;tg&6LQz(Px*q4`2Y%{6^hdX^4zD z4%oyDJx@^Wj3qVDP8S&Y9A1(i&`S{yRcJVJNFy@J^$mI?82o;RD6$doq${9S*}scF z223;1+GfD^E<2l|qfAvNk6@6^L=tLdL}DyEI}_g*fcFtnD?V^PR8d$C979%;8Ylo@ zdY_aikiG*gKMmHEKtJe{2o=dgTyu^FYD;x zpS?-;v_loyJ@St95;~u$G0Bh72m}2PUvy92!s&2NU>Vc-x4%QZ(Qz1Bv=apUC;x_5 z&l~kFg{-_I`nJG$CrWwW&Y$rScbNa5W9rXS{zisFG^hxh5>0}@R*3L`TZ`6x8=WWN zoc74MN}?MLmRSdKpa5GPw*P}Z`t+tSkK>W&2%Vvx&Iz!3WUXeQ4=!bQ8SSYIK7Jt$ z85I#h^>=7BEzq{aFhI}nD^DoxT=R%w7_ZGEjB$qswhq)}48bSdHIKNW@Y?A401;un zPy%ps>^G`PIdr!e24Qon_)?xg^VMmoSY#CQBbH4W|kCUiLo zy6~i|Awwzd^9D{`9*N@Gu%J8MHSk{eM4&>))Th9hLJ6tz&zROIk^Y$_?FTA-5YX=g zJE*%<$u1(>m-SDfZ=HEsqJe?!FH+5bnrccMymf;oY`4S-HlF|lj;Qi3yDoHG!qLfN zm{~B=nt@8D1Xixgam!A?)-`)Q*n!E}LZ~B}MO*7hE0I{Bxr@ZL2{`@+f@m9{X)O+- zHLnCKmSvv|-4CeH)u2}sPVIdw;|2!tR;g&bK_%m#0jCd2HCNfC%D( z&{n~$Aq6H(L;EaBy^6f=znlyhF(y9O5^Hp|H#r>usDM!fd2j1SjiPQJLvNc(ll(y9 zTQ)}aRQN=9P9K@pY4rY?WSF_xZ{EqrO{^O1ZfJ%!yat}#B?{WetsdnvqeEWF)wxPQ zOB`!-E`*n*5U(u`;;H~BBL`hPZsoJ!)|LY$#F%x{=A|5`oG***bx93k9+yvqCxdh- zYp}85p(iN~>{VFA*mLL_WU4J#Rw6^R{=Y8K@HX(q9WV~hiQo)Yx5P5ofeoY(^%|zB zF9xEdQnHW=F}Sso?x{RXGDsuK@&`J2zf}11&|isS(53(#Ks806*v}d9Ozx6V6jBoZ zxg|Z)O0WxG^mT)1s~6f(?En$i617;J{%>ZcJJMqjF^U*w!Q#d@@0+7hXUq~X!^*7= zDCm2i&xDi!lMC4yAtl^Si#+}(N}eM50fcfmz_;bAb>~5njANf7WJ84zjl; z>+*(2fqQ`Si#TyPFm|27=z_A?fy@Y}_n6fq+X3PSOV6Pv(dtiH35kmjxh|25aeWI~ zltX=hblM$q*F8q4IghyUe}*0>$+~pNYwcE0-qOszm6?L38$>kGCOEM>{ygmGI5U4?eIpT8iZdWlU0^|Ru8~eP$NwRGL zt@GdI9y}PE^Bc>9sY?HkD6|Zmpc1#z&q*VU-SZT4{}VKgI_!P|8js9=UTk z5k1cYMVK5#0Px3VE1DE{i0K92j^XDdLHl2bgIq8<{~KLws|h;Y={V<;i2a{{{Z6h! zNHx$#s{u_qXq}e``3_sFaIT3H3lhfvCT`l&|M=IUYU1U?5aWu*QH)FiJqbHH`OVLm z-~QaW8Ff{}ug0iG$X!D3d+)#c2a})Dh~TZ%)N|-!;ITPEOhKkk-(y1L8;`Iw{Ox-n zdyhH9CMAJ{y9ro_a(l?B6@XjxL`dPE`6k^=x&@Mb3ecS59#DKC(K5<#iWs?!J5HV^ zfw-en0y?~{&=24dU_PJ`@z%(e-sgP>nR<~5gZzzCWnAb2Tg?l4F#PuLbRY<%4`dU^ zFGnMQ3WGevad0t$U23Xb|0|u?fN1pohKUObq~{YQch-g{TD*4bq}UP% zkwz#EopGo+%pS7nuL@}77t?e8`4rh&mi1v*DJb~w^)EL?&1{YXGS=6v@Z)Qwuz ziw>+@>j3IY1&mPA3lp$xROX1^_kJLciM+J|aeXlW3w3#LpC*ymntTHdCg9a`C_xsN z0GeSk;u?y=KhyG#SpelP6>Ua^Hiw7u+zczecn|`GaP27p@&goVj6+2$2UHNAWSOpaAvJDjT-|)Nr@!c( z86a82!0y1F2kF3>j{LR(J5k;$7jaCTJJLuiuO5;VM1B~ttpMu^w330CM%P=LEli7H zg@O*P#^!#{r(Wn?Je`%%k>{w-eH&=lMxq3757R@anoth!vBkHKzI&aTSCqe4Dpfu=iN3A z-N^rE_q_YBi~ob!{y(=_;s3%1_oj)=8)YML=w@jL4{O9R-TR+9^Ko^Zuvdfq=l!QU z-H4T+Vd+F87k~yL#!0w#zas(_BT#rm_tW^ z;aBwYE^KZXHH{#*x3IziA^`n&Rb)XF7E_lBUM}cEgx%Z15O)9_X)*P8EAbs&KzB89 z>A1Du1ilXa@i+G+;N1Sr!#P1|UxXzaq7Q~Wat{z)Vz9p?7pCsY|BJ9XyiG(x-X(vm z*8kp-V`XyWIiROgGe8w~?E+>Tc6UXP5E5Uu>pB!u(P%fpk~bW1+n7KLD^3imemelZ z42}nu7rft~2D1xk9wYJNi}0~7pnTQTTbcjr0yI8&34QiYM*pv1^KbGK=bw7{KmX>U z@BsfSOZ=zb6}DFw10FV!G{jY)zBklsK$KR}U)#!~F3vJdV$_3nT{@y}YlhfIC|vA` zdJEb*l&BM+Zb@X}gbn(@+fAf9hSfmuuz{Um?7Ow@rGiO9f@mfNdP!Mi(A^Hzt{smk zc_R_EY(Mfsw%bkU@5)$!c{CHA;246f>$?XtC;#=aOr63c8We1%3@@W2|}=G-o5~6`1!dtFEl1F5DPI zN{QA?GY>(k9fl$l_%eQdwq!*zZ(77FF2o!=VHkyvH_w9Oa1+pyRi6&K-6j*BA2XXg z%bByv#UGC6!|&^j?7qQ#H9%y2X<2_y?aP&?;heT9Ah4L0O35H~0D76-#bI*_&Im^E zLQA3cU|AP0uuZZ0T96NGs^m}FQJNZfcUKETh5hhBx~{RgVkEQ*6LCh{$oU$Fns+D; z-`oeizOuz` z_to}Ff%VP|_#qfYRAz&GmU}Bxt#(H556F~`$@&vuKgj&eHT>TRxcrwvCK5$xK0Z&4 zOWy;-&_d=7KSD;6I0%M?SfT?B=pDB%5<^mAdZn!fgG)5pdVm{)HuNGIC<-jby2xd8 z@XSySIEX`j&EDOUjIKlGaR0ph*FCkq6uW4T1fl^N(W~IHs->g~e92=w^YF9s7<^VrYPv#7R)s1|;JzH!f?D#9-n{wX(!{Uo>=?_Q3+{?i6;L>@=66Op1S}eEye$N3ni$iLR zEBJI}il`IBYId4YOi`YLhGF*&Bqaei#{(pvQ6{*Mhz?FbTB=g57~%z-2Lj7n8{FK) z9Y~y7m0`DNJZzDp8RAJ2mUwpnn6U*F0p(=oeLl`jEu6#kw^EQ z;3KUNWoIO(9l~c0>_;psu@eJ!0BA=BH)WFOso0x-A5uX*0B(7~Y?`)aN5!(EdgTU7 zYbk*3Tgc=J4{Wq%FO1B&8lMH|rijmlW;Q%*+M0qK;gYZ#))$bPcpNP9Vwh6k;%r9; zfxeGJDn+p5O3$QMAdv7|;<5pnj z=U=>KsQI=RX4c|9d#r%IyCM!J?%0fr8sE`bL##ye2F#^cqB99tps`VhRFVVOJ`?>x1h6LY ztOMbl-2gl9cx5Y{Q&Tu_gq+>K&d(|#L)a6xPg!)N!8;tEDkE*xxV9R8=FS`tqs#9o z=8dk3W^Eu@lGwRoyE0BK0M`2v*wWY}s`P3(XcQ?(OT+S9X|z>;U8|CVoco$oG9ocB znE6UAMP1+F|;Mtn>r5pVicEEQUqEhc7;S3ijz_Z8Oz_*qbc6 zXIPUnmI5y0A>{o+eAC!|Kj>zsu{rojP9cP8VLFtu zc1oeFS(`bem^76M*^0!>Fm0r?h>{Sd82eIYMj@rLWXUo}*+RC6@Ad5c`6IqRx!um` z+&X4nujli+9@pb>zZCj0LnzI(-rFia7RxJ{4fXR6`+ZzNClSxFA0KX&=k>7H9}&C& zZ@uqV@d(for?zp{_rmC^Eo`GBUMIJ}Ol*aqUePYj$nXL#n(1kDLHCiV8N*BoMQgU2 zU_kJ0J26gTNLV#d_nycQ!X~))$;mfxY=UtSUJ9ak#KMG$)}5TQ>GgOqizQGQkt5i= z=uf=joZ60DM|Wjm>Hgu6VDLCtmxyUG<~`*|Hs?D@0d8s?@$IsA;7FN197K|7LQ-07 zb^CHWs~@F)D>qV4SJ^u*!K3l5PiPJEsIssXQSBPxg=(k%hu=Hztv2aM_9Lo&rk$?j z%v#h=geep(!Ap0ilRo?rfslskt3+0)i(AB`*<*Wy0t!<21% zNayN~ZG0+pA}5hduah(GBHs#wSNaf=+yz~T=9f;qwNccXAH9c5QL4z_OPwaic%GmE~y&E@Gxdtdj7%xg~2o+KXEly{%iAL zzQGi17VBMn^fg?B)2Lk?>p<;@Iqsyjf`wK`Vmc}AKx)YZobbBJF#xzjw(1K4iD~*B z$!NyW!xEQ?K}h>O`z5*;qt0si^6Rq;_{bkrxV^LotKcVf6Lqp_Es&S$%{N-6GQp<- z@AOKhAMMobt^8H#*ZKkG^yaChytA(S?kDME)LG3kNpm+1Ow43KZ3IAO23@lV0ja}h zFCdlr{y5kG^wzt}TNfb`&yVJD@o2BB$kF}K@ZMAe6y_>Z&r8J zcRq3)yVZ6&*RVCuVkXexBYV?c95IkCwbA%XbCpH81n-KK-31i>jTW<)QIIjRbdwyj zcfdx%WTjNzCIwM~MT))~%b zCWi};N?^GzksQk5xaHYd*dh96_Wk9$5 zxy@v{;>N{;MINCUi-CoFOs}Jbt1M@{%{e|Cd=yTVFg959a#bDcFrClEn6jnNl7BB& zfZ6;Tyf}ZEE2aJ(3YMqlqoqy#m;y@jAK?onKj>9Yq+Z27)Zz9~m8<42(d-KBmVzy> z*b)h?IX@9p_){$L7n|WW`pf#?%8cO;zbW-J!y&K=ZG2RpC-O7h--9lK8byV*fnaSysUntdW?u*cqqq$W2ZNLt`ehTc(PMJLThwZ31$vasjBjRI z{OtC;Qpf_FO_e4ta|F$rS<-WgV4RO(-apc%(UIck-0y867$qUgzxC+Yz{YhtOQ$$h z&0u=Jp*b4&;NRx$s$yc`THlIHwhf&dn~RKQ-B5Vb{f={7_>*9I`|K9(&eI2&;V+>S z8u|o-@UX{Z8(tOFEFP~cY)qRK?NEdFJahP?{N@1=ir{im6Qtv|P!HVg*K%T(;xejW4|z5FDKuV@Q%H~W2ygYp@?`?a`*FM>={$- z=nGI4b&cuyGHI&(>AUfLF;Zu*ITxpQAC*x zqHq-+Dp)$?Z(!oS(MB0G;*3QCFi8D)nnI|zzp#d@&)KD^{X78^w4J+ zA;x<}I2l1&qF2G7m2ek2v+TK|zF7GoI|DJp&$6bUXZ@%L`t3>1mh#e;a%6Ayj(hEy z+IsoBw~BdNEw&ZhmMPpKleP_pG)vl|@`kz4`7j(3L9eS{t$EK56@62KKx`%Ypmylj z=$Dsg!9(JR=27+VExVnBe6SBREBRq?Z?rg1LZi`i8`b4#d4XC_8+^WQ-+?bu6$T!P zXSz_LFZ_nh7Mn~dlk;%2+j zcSO2xJwivV^nlS$8@O4@>+_`z47mdrmZ6S%->HN&fdi{#qT^V68=?ASa@qrf=09-=|^-JX7c(Ad)rzm7I| z*2qC$pcc;U&;ZMa(!vYyBht@AGJ7Q3^nv7{?v5f--x|=jb!t&^(WGeXD`++j3-F<5 zuP6gSd%=*o^{B2gwNn%bdASd^-%7w-Gn{}NWs|8Gb6*9#>W+~g`U9crd3Fyz>Lf@P zNQ$k6X+%wF=+&*7)L`e9n^!&<6a8!X0#DwL1bQ%TE74%Pl(54<1cyVt0er*Mm{Em7Ma+aV}<-x!s1XG($DYEoYbGSHc#+Nn-r*(q;G_@?Jm-H8lGaZkEo&qR@L6Z)d(*MpN9! zK?wp12t(|H=6*|%VXrq{Kw8k(GpB>SYUbs7N;4_Pkl%6Xj|mtXrY_abFB?hUM(M3x zjgdfQ*;x4ypSFSTc-V?RhqYfJF0Qx0uNrYoyO!B3`$sO-aE)_X`7j&(z66 zSLW^@=n0&WA4@^WB2~l&Klw+$>fx342tRD@&OyZASlmWdW+t3LH+T451nh-uX3jtZ zFU_Y}3AR%h6dxil06pQ;aJW^J+;Oh}VM}k9_P;^Tbnkxf9xC-2l2?$4QLiR0on}^O zL{5dFb9k<&vY(D%RH5r+t@JC;B^S1RMmr`^{geHIoT-+z0@-3`4eiw3_IXl?s;5IE z`y|;xM>{kvv8igK!7=^SqR0E~4%*=;}(8fLIpGZJj_( z3HZeROoaUbr}f9-P^+NBxu2jXy=OE`Tg=*B|25yMdh+KQz6^f{;{kHPB$$9|MTECP z8g~J(Vih;?^*gSka>}b1ZkNRK%7~YhCvq3iT+?XD#D&wwILmJ#l52QG)r?$_1_N7l56=K1 zd)dCZ_SvIhOL50Ai2+L$mpK0T;e9hGjM%&=hUBQK9+0?Lp{AWy2fzRcr(^3hgkrY(eov^LH{}+@o$OzW z`w(RFTY}fhR$0W2TXxyTd?WAS3ew5v2?{k>EmJNlj{KXTK*dh0ajTKnOx`2kB> z=_~UlV&LJ=`^V5b_d@Q`=U8=J?ivA&TtY=vEb&~cwuT^L`c30Q(Pkv#ET{1{&dG^h zDaV`QoQ#_+HfwB@CoJ7dNhtQqAiQRz*a49cyLb-Hp!Whpb(qCIltX)36n&;{-ks@Q zco6z6-v?xqjs!x*J(oIY6n6J$jBjt%rvqe*lM+?lAR(>GYe3glBZ^FJgTKpC2oiO6XOSmy)SLa>CeS?8O4V zIawF)OFyPb`1oVz#&muzvhlB)eG!B-M2(KzVkkwuP_B4$mc7T;rbCMO2l-J$jMLf! zskjwnrm3hrgEqOG?_s`^m{a79F(TJP00`DM^sprHJN2ML7a-mD(GRAu; ze&4VWyu20aoo8k{V;^}*EHe&_hSfDvaQ+u;`X#oH9tK~nde@1#gzp4_;J-}w_T&0` zjM>dXbqv-fN*66*D5Y|{#-I2oeKWRS^hTjD34SX#shbz1`MjAgD?z9q+N(PLG}tDs ziaK#(G^Q@blvyd>=NYy_+c$awt`Zb+cnq3=Zs2=%BANMIJo#(Ud&Rct9&5yNa&aeFMTHfqX0Zs3`Bc@xLMfblV4*&L+5#J=xH<7qilJ5&bG2ycyG z6KaioAJdeR`s-e{Z%J$bp~p2*xl9eCOZh--?Jn7qSYa*V(l{V*nvKXadydI&|IU%f zZZAkwX18Bo0W0KD=ys<9UXvf2M2OOq;mOBhR|rHS-=V|_Ux(MyGFz{7e9*sK=x0iJ zYGRnC{vS=TyTf}c6w}|y-N=(^USg);pmmwu&m;;MF0a51WtANp&(bwaCt6qHwd@Ox zVZuQCD#B2o1?}wYPK<6IBVoKfg>fnkZy<4s!Ln9yZHyJRDY-voSGZ+ejyJ)#9#Sbf z$mAa)uW`8=?x3A$3)q6O66pg>f|82K(41%ot1Z#8Q|4*W{P5(T-18G_JYb2zcJ7+I z&mcV#S)Eb0ukaB0jH>qKQyj6dfR)cIOY)u$SXN@vQ9L{;F0@|Br2d;nW8oJEEFWFI zNSH6x*v0kNj!;n0J^}jYju7~Ui=F$Y{LItW1cJrrZBiJypzoTw*xTEvL0yQbMrp-n z`75&I0(c(&f-{wM;{<5-mM)F6@+6ype0R72TUZ&#-NZ@u|6_$3>Wkfz?X;LoVeUTa zR>QZnXJsW>xKNGxQYq=&q=2K2;RAICn%y|6hpqDo8Agf6>aI`Ye}_Y~+~GFk{>c2R zHIucU>;(BIYh_HYO=d`+GHUqdDBXBVvBJKOR? zRUmDds`w>cF-~ALbE|!4hDz0lWzkPK$nND_w5W1SX!IT<{`9olbn!G={|cCos!C6nNEdtUPv9+Lle?saOfd zJz|aNWXEP4DezM-+F+oRK%npSyH2oM!eR}DI`r8!ZNQRf*nUD*V5p7yWvt&2kyr=e z0>}}&p}5_LJBaJ_ArcHaQ&t6{zT;?<9CcqlQKJd$8Q=I=C}98NPb>8mg;+pJ^NHun z<}I$E8OCG2d?L3#MOWST$L`^@Xl>+ldk}L1`SgqLKsTs^QYAAKTvUX61YAWs|M_lP zyBG<-=Cn;eZZD0yQX+X3d6s_Jo1Jiv;DSFKwwnzW{tYIe+^J9*u-ub ze%xPST$=XRtS$j(>eQ@7STXU%L-C%86a!s$KuWWxMWRsvg zH|+5HAX3}|k%#P?LY#rioVm*RraD#~0&d=pUle7oNG|0xI6`$o|Gp_tM6kWK!3p9OFpjr)y_YD(+(EG|F17He7V*vDkbV$MTHJ zID(tCaOjgsiYEW34Fc*P%bm6kYYiRf({jZ$>fMeR5h(di?Aa#UO-`3TL$1COyrku& zpI9TX)w)?xql{l}%+g_q?XnZWr}942w9AK`=H{wbb~}HzY`cS}DviBD6wWcBqhS*n zW>05_Ddv_kWQBdG`8ylrMVG^REK`|g8U6UO-`7KsK@yUy3bQo)R%i;0C9HJ8T%Q@O1X4N&5k&%#wK?y-L$7NZG zCP_1n8cW-Ys!LX8q>sgawa10oJN)Eavm(0&T-?zr#5NFWp4KiCKm%I!Uzt===5SEU znOvgx>*=u!%lLH-T3Z;(v07e%T6&CxZz8`5zVEc`^&>u0m0zWVY?tQw%b5D%3`plM z#xu{UWzkk+%mV(i4ha=aQ8xa=`6H7)awgz>9g&;1c%?r=y{i-6m~(CYT%k%QuGm)R z(SSPc$pE{$`FotXon;(0_grJ)29GEjMYV(BCzXDmYPh+q%Az>g?tDBgVTwR`cZ~VU zU0#&SN5@WYFB?Qa=TiB@G2zTf1&^CVVii7NCcBapu)l8fmC)NC(HsP*@|~(Zo~LcZ8NPf{1B5!vv3A^%Ex8(0T4qSw;D_r{gXFMDUEB zAcz}$n@J=U_yrPn6La^HL~XE>;fbtArP}=|&y2)oIRkqnmdD^=-<3j}#Ao#DYpBuB z=u^cB#>ZjCnD_O<5{l+5r0ug2aT=Yoke4B1xR+x5l&P$z?M~0Sed9gtp`zahTmr3% zxF4_aSrWyxPa^6WKYl{;to{;VbS*+gwWt)>{p#ZXVT*YqC}Z3}j@d=YhN`_R9l7M+ zv;=cvPl)CzciW}Y4lbohNBgu1T_rnY!z8;!yGndcPQI1dtkAOK(fskPhV?vaUADs% z$Gekh?;c_wml9z)`a-iaALE5G+iikW7aBmj}tU2@a^LAow%sT7y(>wq^{JQ+=!&G zY`V_jF&@9v>8L+k?Ikr6nkQeghY=+zV=c+1N@C1vY;FhZf?5D=O;5+E6mek;bykD@ z*xOP+Rs_SVAqFtme3$Hu8!m_{iYGPLha7bn`kd#V>)a@<LEE?R7$*3QmvPk-JLarrsLCw=Fox9TAqKuRl23N zxgjm9#NS22-C>Qji(G$1e;Fb533f{BaydNXvzu8Ugd4xM zZnn=O?9t%w6=Ch@NtHw;&U(iflEe^4x6(dS$+|$TfcE7vLEXsk7H@y(sEBp1Yy%fB zN&cmCpgrT_0UXgI-~%_7a;vKDaOh;TjQqQ1NBy$zT<-ZLW%;6_Zb#~xDy>I>B{&87 z@yXha-56r`DNE<_d4YOOZ7=GqY6smfx*ua6C$Q!bg=lD)n=wLS_9@BNSzhr}xWh0U z06nA8!v{{fS^dke{Za{ak&uFJl-c`;HF8LSuEqVsuR)(5mo?@T^~|(4#f>Fp-G~?0 z7c=0lG|%Xo?)hLRugi5gCNae!a^X%r>aNAQ3R4D=Rd@YusfIca^6v*>S5E0H zWPGbZCeHOjTtn+$Dsk0*>(?;vi<{HfF4efEV8PQc7IKrQlqHyIa;NEfB^&aWp%8|;a z9?;8Ww1HTEEk6}>2yIX=DODyB3jR+s^LK(D&E$DFC>&kDM(`d4=tBM|sMHIS>X-^B z7_EX8q7ZMOSRP?ZBXHoVcwP_h%sqYD$i`4`BIq1FwA1l4s;uv8c-6r8B4eWpztq>^ zrPjZMwDM($8aW`qQpL*(v|1Q>BI;3!K0g(HFCE0W`8u>3tWD1!$Qpl_Xp~Vi(52+H zL2=!c*ad;)(%pW#o3cFKLvN-t5d)C3rX)zjIO*%NUXFtT6NX7QJQJYR+CUNeypm>i z99Ej{`j19A4!vLQ5|og*A4$wM zE`hkitnf+lF>Pg4sxo8#RosEwS!uKkw<7s$q=tHGMELdB<>SqqmN1;AbM8F=>$TD$ zQngd@|H#a3ND7^dBsqGI@i>e=i|>V<);2XmNsA1q&iD~i-*@W&E&o+L!zOs; zRnVU}p>c=8XPKTJl6@@{@ zQ5&2kGLgI$mXG>PGko-{XT~ou{i(gQo(0*0h$^{LMV`&GpV2NJm?r=rLnJBGUQlAu zb=^fnne=zmPPG0+-QClKixxHJGqMNl9M&lVBXnU3gbwu;Wns;YZu)8Nf>E$5%G(%S zJ>2OQSpUG)lh6s+CDbWLCi? z*-^N^p8@Muy_AqQ8}*;V=oP{VBQY`Ao=MQ+i(pIq#m%wnwuH*=ki}SNVbLI<#c4@4 z>5_zQ>*<~tzY|I@kgC^64Y;x1iOX^%84ZXi!g8^8D}g#f_r{SBKDrw$AJd+vO`HgO zpD~=~XTA(zK_}!Jx_-HK_`7tQ7w8`9-S0tAa0ai;xoq=QVK5A789}s(um$AheH|*@ zvqIP(1zoA{;~6nnW_+}2&*j%04ZNuROEM!Oh3hoj9RUGGO+@EBHueEU z`GtGDfctT_lL9izQ)CHSzwBk~azQj#@ns+1R@(qucH=EvO}MCwGPKI2MjB$YiJ-^H})Z$qE*pK=S?;Kp}AJC#WhU8q2D!nZU>x%mVyVHxCviT{Ugs$Njw@x3RAe06hB%nm;)(ye*Tw?xE`}i#?dE|Qs*t?%I5X-VMv?=G_c*a^-{cb*2NtqCfT9>!k_cU ztk4`k;Y^7AW*3`Adr{Mm1vy=C#NWxpJj{_?SaGyLk>@CS%Rz4Z~*dQM{SboP;&Ie^0-NNlnQMm{ojZ zX=;aWWZBomNOj8EUqJ*RQYMVI$a_L1*_78{gl_R`Iq=J&1iU{VVQ7-1GxZm+my~dIx@Fp#a1mKKucV)P9$KoWaQl!R*Cp2Xig)6Q z91ZP*;9?if1cveUo=69NOMU9Rd{X}|0ja~V5$hvBmG0?8qn)XRNDx^m2KRB7p5+xU zhb{zRM_YgnK^gH~r1Eb1U89aPG!>f5L`=x38D@aPGJ?L2!Iy9nCNPI-TUjor*QWm{ zb{ch-A?^S5n(9NW)}H-zs@OjLIepDQ);O)x;tFo>OREhMM?%s5@ICx8GLbk>&!eGp z;~swlvhC8i%)_-CF4n>|cw4hGj{M?OT}j*^?~Q|v!pBi$5NgY&YG7i_d$dGqzNmS$X ztfXpbR3Ct1lB}va3F~RBPNQG35%fMm^+S zUx=k2;=>Hu|5ETnd-^^Crst^QOYJ6v9y|%7>L%Wrx><9Fo&b2?OJ8S-x8UGcBnBM@ zcEQcQHl3crt(gDdOpbQ zyr=rG{B9wPuIL`ONT+P#s-Vf-G(M^*y*pl;H!KqbzLde)T=AATdmn|xqqFf*J?5Bp zE=Pzgy2I^OW*cP$=BU@VMEI#Kp#=idLeuDIF`FwXIkaTQ6`Di->j@+N>V9jS(A|X7 z9MnXhjRo1aul6s)E~WXfFECj#edQ&L@g7C%qyB;{3N@;=j7^~l$imnwICM>ES_Akbrimnn7Bx+UwRk3At9ia{_ zQhr~oqp^C6^oYA?q+~bUUtb>-(Jdj0wAK6@j|RB$ZFOeqPl0K&^mn*kM=t)2osFz? z_iuC3HRW#^{(;R+()+w@2*KcN8SJe4W9{4rO^=Zx7&uA~p0N=_b>aYJv&L@v2PWR7 zfR|`Iq{nF%I%)cUpOh7a{#433he|$sw?m7eoC}6s^#w?o*YlQS`s=h3OX{pb6kZ)W z^J>-fXl2oyoM|j)G_1%pnKLt4D}pK9GwHUB%y92L<&S%tLl0QJ$aD|5V38%qTVJ=Y zmY`XTPl%4^lpM2{kb2b*U6*5bUL1+uzE4a7y23XEdUbp9Jbc864zlOtKkURHhaWP$ z30Tmgl`TMIHG)95u5CAavdSP)wV;?Fo5M5*!hECZYYRd_JMDA68y;pc`)Z{*R zUMz>@`n;Bg*+SDUv;!p@Ew% z9K3H=@+%iZif{}bW|^M!LR_Lgamqhh&FAVUX+YVoEGIST^^q4pjq|SLwjbMn+?uJf zrTafsV~*Lss~f)mc{cIs4sqOO`#I4Ei=i#w4d)*0n%nlN^!@a5M8vO${=&37Uv5X& zIyC;5>U%8oX=dh+k(1E}u_@OI99ss5c5M^(56kq-J9O6kL8;62k?R^SOV6HrVE#6s zUt#?q;KH2luPJ>H^0RlJvZ&STT9a-5{yQ)VNfX`LVwd{rSrg6F-e$Gjm8JYg}ey3Zo zuWx3V^pUX(5bV9n3|g9>yz75&L)5e0w?T(JjURc-hIN&YGrZKc>9tvI2;moXlQ;js zU(p<;e+<(uW@4oiIsM?qlYN~mQqJ5nr!S&EGyWty)|mdj+FMs|IRQkn>{XgiM|Vxu zZwWtkd3$+3$Rj~F9(*xoD4UMU>U(xQta@6wTl3hi@BEmd{H_6CA+NmEACt?&7N&lO z7#Wocn%P zezYy$tBr3c+cBq1PT%wM%)bL6tNA{8%7vU3)1IF2J$EJ|nZ6x1IUIJJc)Yc=(s%LP zKxxNM@8tuQXSHWqb6t;)m1RswTwi?bFgaMDn!&eCS-SVcQ~qmfrnb2oNBezhy-{Da zBzbUcBP{p)vB^a92mCF$UNJMKO_x{A&jDueP>n52ZNT&hf?2OfK{K%9qJv2f=@R}$@+%FTA1*Lx~R`9-J4Rb{h|T5pN{C4wBa+MzKcE)S51~c_oNq z=D&oMSJ6X8AUIeQq+NSz`pxS3p?9hNhL#->b)|SAjFnHOV{C_JVdu*ewn1wibBsFf z^-=zO;IC|y^em^4ihd`ao`9jHzoN;|-AxUa9U~SwLN+D03nqc)v-4a|CB>|wQKzZq z&m^gG1M!BBN=QWNr&R`e>H^!^Ob!}DjpdTmknz0gi<#I@9SO|GhG$NPzfEKH&k1AZ zjbZ!ulCPfE+VLa*O7Y}(emZpRKt)XGS~d~Ecsg4Nm-mA&JxUw7`URm_gWC9H~#zi41-&82@aeP0Czg*q`4;*=zVP zL&kh6z|5PTv?6Z+3f1N)KI=Ogeew!CXT{jL z+~y@(;ey7}L6sCzH7kH06Bda!Sfyw^ufSkTeU-Zg^o$AEz1&A%6ZV6X;!D(I)^{{{Gk{tr4K@PC;~zW+gN|Nr{WKdft;sMR^J U*Qm4bFT?C#%fl9VX5N?oAKPLa+yDRo diff --git a/benchmarks/results/prepush_smoke_20260423_123335/charts/locomo_comparison.png b/benchmarks/results/prepush_smoke_20260423_123335/charts/locomo_comparison.png deleted file mode 100644 index 7f5f7c91e81ca4dce1506f440423e88b57b953ca..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 43742 zcmc$`by$^ayDvIWRCFqeiXb2eC@M%zK|%y2rKP(?3F&TB1ZfLIQex1lbW05-C5MvI zC7pxZ_rv_wxA$55T6?c^{yFoy<~2az9dA6(egEnn{tx9OPMx4Sfx%!-Nj?x )rV zmxnKt9EBJ6h_-Eb;kOr4w^y=yZ0}@XYmAXKu(y6{W&hOF@Uo+^t(~cr{Vp^~^sOJ!A6)o^%IjJL3vHvSm;3L|sp&^`2e zorCBydOaV3IfGtL$Y4m&>k%dlF}!#mI)^y~ulJrE#}L8mr)pwvc)>i>KXeaX{{Q^* zPlJ5p@0Ao2#e+L`R@(INeur+opC2fjT!_(e8YsvX`W{p5to%Q(&ikUGhK5Gz?s{LRFSTXFM1)aDSEj}nuI)cxQeyAAC7WlLFa=+{ z^*EuAQqbL*f1!NEcGM^2(w*dwPh$C6?r!v(hN-AZro_6MQE}*3D-Stw-S%fX)joBl z-)*77G^?PeF~@44)M2J4OKVX4PvWD+-(RPv5<}$zSqs(8etkC9b(+m8%3j)J-de0Z z-6^>J=iLR3M{jm|Z(H;xUHzc__a&P;GSeTQ?wguzPN&2z6EgEVq|!wEND2I{cNWSC zrLkUngy|@&%CvQdM2S#2H5~_)nf62p!Ia_wEe|R?~U- z<;wP%?NIZs47Hrq&CyW##Vys{$db{JJ01M9neTE`-Z9BCbMHi`pL?BCxy5|R@uz_) z9J>`qhj-5|nUHE?D|^}tw@|6dbC=1P_FAm7;T~1+N~ql{qdu3(m`?MYirZh)>pC3J z`*yw}W9%?Zi3tzaG)wDBSIOA%thZ6~sKR)srJlrmisd7Wy<;TYfZIe!*AcXz`th0e zuIreCS*gQZZ^0k9`V1SoHh~qHU$8&~zglFLOhhVq4B5!#$wiNPZqKQW-d^uD&ne&Q zv#Qd$JsEAMS|-<5XwkH{J+$BB973VzZleL0W7aPbc!9NXzG7qW^9>ug?coKM1Fp~U zMjvz>FD^L1owfM&chk*VU-4OATKSD@eNSaLR$(2oE7p5I2lEUTH&{eF^k}&3;^@4} zpN4BZPIHWLnfQ2(uFYrNu7w-_9u|}4xgAZy?#58YK*Q%IwmV-}g8C}A=e0(;**sdJ z?K>om@66?Q66aXaNXX)Mem#j}HSbRECJto`SDP%EjAbz&s@x^kb8F_rb!}FtYEF)S z&|}kbU#d6nZ-B+6JL-VJFvUSdh!{Rpy@uDT65x1sZw$U*--h3`rF8s8@AsbDtm``a zEG*|=dS1KSgR@4vuoFrIMDL6S-D(_!CsrN|w>kOG8wuL)!`j5sN^V0~jwfG_P~1H2 z$uJgZQNGf8$C%n8UxBdB7#+Aqp=lET8Sd*{zknCVPP5H+rpUWY70#q*^iVd36@HmY z2n-*vYr9J*cyaVpV-EYv0NJS9tZw6BYD}{prTcr^O>I1dPT#M7c<-^Zgfp(%+cN)E z{p|3{5{|p~Nt_3Bg=JT&g0W7i{r4R6{$iV7@qVDB>B{CbYk^)2q(D z&KJLXGq?H>VbA+8>9>V7_;>B~nT(w9wVCb=_n64Syy@hq@Jk-6?Z&6?u0H$)3r%Ys z-X`2*cMZ>{+-DSecbWSYUeBRhrMt3nmAkXVZgSb0ef<*d4F1mZ3jZVCMz|2xD5pV} zmCsOe!gV~>x{VJXJ^5`x#a*XjvF0zr_T_8^l{k-e4FXzJU845%2!unaYR!D4;WHvIis+)XXBG;HHg&S{~Y+}>Ge4T40MF&rXC zyf+4%^jspWe;W9yY;XO=R{Y65Mw-1MzvHvbXhRrl$=F+$UzuN8HP_oE=(?S0pyDYNN4o@iD^Y-l$1SmD`3T)u6?GD7wRu;agg1uv`rPxG@>6_;uCp$v7|F zo@|}6@>bg>R<+w{;z5*eUPN~YGAUbVtnrH*8&3aZlu2*Bf9#TSrJRqWib^I`*;Pn##k` zW^iTNDauS&viS0m{egy&+-pGtr8~^GSoNAO!-w<*%hKs&sit|@uFLq1xxS)?6{xOu zjd(Yx|BXyy^_ahX@p1u*1#10j6wYD7SP~PX3fbGg9+i({J#xyI8@cD7G_e-l)MHP_ zWGHb`y6q7r&HLN5?OIeU#GbsWW~s?j6q_9LN#lN5LSFUQki=3}&*|J>n9*I(Cyt$4M|8wXXyw>M8V^LwN?B zNBey~@wY#{9LeIdG3dsA>f+x??!Vs23w~U95^0*8UrTyN2JfMk54CafC~3Dp6ZzHI z>@v%O*=DG6B{t%ld)o^vtZu_EDV9CBMt?qfCu=5oK}9oK4J#t4fMk-~|G|-`?&vzPJKapax^?EG6Cm%3r2qj7g({;u^o|dE8mOTiyN* z*RXRRD#g(X@Y;8agqp>)RlpU$!4i_!wR#j2QA+HMsXl4+GuMFc z-U;cOd3D72V7Vgf@A=uBs_d@gx8IkJ$hCQ>osUs7KRb+1%H**f`?kbi5nhuuwXFGW z>bK^JxP&{iN!N97Y&xYf9kU~@?@PCP*7^6gW^)KzYALtz*PeGo)|3U{OJ^K1O_u5@ z;(q6aw_;0YouRr-WS3Fs*ekfMwD8zn*H*&+ju-LkDGohCq0J$iy{Rb##ddjAP1ohB z({AwGTw$MOntYrfx1$-G2dix3Hg|ju_IQOJb^#}}w`s)izZm6A>BGD{;d1Z zd06h!8T1hyqj@bJN*e=B>flRhBHmhUc^M$%BIxZ#$b{I1BgVs zKVj3+DYE&t%E|ra=3c*B@qV2zZ#1*GHHr3#NO1`Fr6!+nwNoc@ibn!yOjw}~PeYf= zzFo(n5-xXI%d*6{aukul-$jI=jpkfLNm~gmtK7avcwA)AAOkullZsw=U&W?WNmP zP^R-4*1eL-Nz3YR$eh!*y=rq;YGto`oxc?z&iFGUoixphYTa;K)aM+6%&Kys8}z7u zrLYr|v$}N0u^DGS=rTEOR5c#1wY+^+l?j@4Dw^H_7I!%^Zi-&J~S zhipXlGV9>1s&)&@=^t2#zAEiflpe=z^xMe?&(^!UV~yB#HcD52zc@XS^KoVeAoVl= zg{Dxzk=;^FQhL|8TL=`Em4pdpd;-0kX>4t%&|dKlK}A&q>;BVndbX4wZiOqs{n@*% z0%DAe+o5|Hec#WMv2GLFah_W@VytV9cW2w7{;b9_2f!#SeC^M#F9P2D0JLqxv}My< zaksH>*0%wvcBLt$EZb#fpR;M+cU=GVWD*AjPkW}fVD`$K5!4Cx1>th&_>+8dOl$mm z8&&(VAM`w33Uc`7i$^=yt9BGq@OcRfmD|}nt}5gM9@1gq&~V+b#@P6L0I})?@k>hb zcf+y)BXDJjJ-icd(5!GuYv#EC0M;-ex&@oGcdOiZ?RSGvla_S1p+_YH8sHTQk*h_h zii`I!Mqi2qL*#u?%7?5y^KvvG?~*`Md(~jMJWk@J+*;D6XEIYgS$)pK3d+l3qc0+w zZXQZDd3EM1v@lKzE#j+i@0|IkKE3F;p!JkfW#evz=;TlbM=J?$V!$l)g(i9(Vht#} zM%n3sD;B-^ror_6K6TbTP*mCU+I~v-yh0t^ljfO`{kF&NXUCSg`4_7Xvs5?&JZD{? zrTMDBmGWwq4$qlA3EUu>t#Wgu+RE*dX1U({BjcestT_!CBrOcpFBGFD}ltTn?4(OBGT0@_e%_Tls;{`^ddP+FImNK2hEN7n86vIVn# zT&KK{&TT|0tf_&FO#sD;R?0j#ss0WMaizP~xxe!)bl{AQWh1%LmUy-DO%xm#DmKDp z{Q*^0WEwYxpDU*=1CHXHL(OIK@IMeZfbM@I@DWYfdAx!}p^B#CTDRH_^y%wtXdv$by&K`E|q!an>N16@B~AS!O2vTpbfr>oxsL#smVt$Gb!S4 z4)v_v2LZxE`NUwHRgsQThPA1?suS+lk14fL6# zXg`KxtdBlS-$9E_hYpEK7$t?t)|k%x$E5ercmJ=om&l$qB*I`iSg7sK=EI9-k z0hUQNa%_9+!GUyhgu-A59!H0f1fkR<#1RIe6uVELyx%=V;tY_G2|pFEFrk9*I8 zsd?BuprG>Y98R|QAqYKexeqM}DxPe*FAE2f&!=in;3&(BITn3|X+XJ5Cq6#OR_m60 z=(A`AjLLL#aoBX~M?Aret#SkQ^y`_!M3>U_@^lOA3MhO*!i6~QOC6a&L%)RHy{y4^=rwc3n z!G7$GL^5bG>F#uS0(Ev+7xHUQJnIwwx$9icsLyL!RjhTu7m*pgs3LQLl0x&eVzon> z&)N+dlH!-gEoKf`TkCUCfJVw_XxIw66!{)(LwWCrb({Yv2o(B8>EH)#Tgg%&_O_pj z+r_D|z?ua>N3$bbXkw_#oQ}wbJDC5|Fc1rJ2TPnevF$K`c3BV>oJfxXCT`E~w{60L zn8JF@=1Gi$Dipo44wjtqx4`ELCHZ@REnmB2`;8tgc~jsG&D+o%C^}K|$q~`Urz6?S z9u$xyptIPj-8n1MPFWsVMu`v^#H0B^1M7S%&GxajisT-kzzzGhyBlM775h%f|9|*OW=;lIwtGMO&7Pha)2OUdRaqqlxMzPyyxlH*rF?Q|jHIv%O)1 z^^y;rD75Hiq=^9NScC$Fi|S`rVEYbvx@1?qljFZp<*B$jKX6CwOhbWrN0OxHJm7SB zmaKfmQb0qtVPlOfIog$;KA94i9F()S%9a|aJE21Bn7YwZ=U+7B)K5>7h%v9?Ot&WORZ?U{+i81Jn)or{wU(XX7S&;5f;^)h1@S z(h}Y-u6D|^pWYYtdB=H$S9D6f9RZaEMw2Y6y_Z+R)frd;%Di)@ITdr_pWDyUqE`f@ zC;YU3*7fHFlr@@EXQ@{X^*nY{5h;Be(Z|L-%A{6anLL?Dl|AF`ezo8t19feTQEW9V zU4FHFXba<$MpG-$J>Tct*ADCrFvHDEjIz9RMiL#i@jK<;H1v;@AO5`tXb#=dO6g(n=gZ|lgwJw|M%L$kk|8V+S^Zm zUOD2;?QlF~;%dC1n6XcYK`Y;6^C|TR*ONrZSh)ULv0GP#RWm$y4Xw~E*E8n>ut6)rh$QO2a=cV3C7{4e;MdoOOoM)|`&35kIlHo!&Yt;?`0X;~zF z2}U+2n=J6z1I|r}ql-zZd$Le$A{ZT}JFcScvNqkBqMGAha$JNb(lp}GZg=Zhss_Xv@s?7aY+v1Rbrc86b2^-$yZZty$Iyq{d7{`=84=iw1l0YU&laaFu8#Xy^0BDkj?Nb0~b zmG>54ao+XwR*2sO&?r|l8r^Pk2zB(N;Lr*?-?9^@V=G#ef63YJ1!FAWf~A338`B;w z>yTo>4%hmnq{Mmhs~yP({4+k>Ro4tG;h+-VnJ+ftAA2p*QMJFjkdKJfQHFMdjUOVX zWl^DeL}&HX4nVOq5?eGMy%FmGP7$jMB3}6)*g^75H@`GrvB;5p$26VvPEN}2lB0Sr zlD2`x#$M32{@VYZvGfqmMF2|DbVg34RA(;~(%4pkm1g^zyd|$eS(J1_R$sy z*R~jEXxSf}$U+Pnqn>`2^xO~Ear~9vYnFixXVsc8wizG@G5@omtzMF=J^(92j{mp! zVcW|WR2<2AKnmyfdyaHUf^}!_PsctBy8=C8KeFfnD1&^U995CbCyNq$I}JFfWv`=h z-?6<=6^H2N0ew7UWh;8!1?nLd$l=vflpL3>%mFFzoh(Q7X4Ssm1~>kjQ@FMGF|Y?K z%ZFzTh7=?&g}JUyDVXlU~no$)SQ<*HJ%K z%CCT$eFr%@CES}xl7OaI(*JyWrA)dm-MPL&kR*yN9$?A4qF-Z+})Dh7(I+G`6$mZ_$8>oNq{UE zg>*qz%h0(N8%n?!rJ~M&QNp2hi1hKULJO@A`QgOg?+kudKO4=6ETI$(K4rktAXEimVY2hm}ckrj^j_>+h006p!s#PJXS`D!`!Tj0}W ztW%;3wge2pB@`YG!3h< z$wjt&!KU1%wSa7r%7bxa+v?kEdcwAqFc4Ut`U++y@dI}8T5$`3IB{x=_b;kUmP;C$ zGsb=}AIbrtHf3YTt4aWK(*@vT2RN2*kq1Y^h6;fE@xnpZSuC&-fdhdMBob+_{q3Pn zfUz-y9vcHRY+~M~#YHmshtIj055Sxc?|GDv7BVh7D+N zYI+`FZhUofs5&D9vLGjTV*&v*knh{>>z_9iyL>TNf3z3p5TS4!b4|qb=K{1|Gwn&Z zC2bq<*rl@yFAkZU%OFL@-nFF%fF?}7F+B7?k@_Rww9}pCZJLW9871mCd{Y_#W;*~t zysgc6OH##<$~^Z)S@vSsW1azfCDJ*m105Bl^ki84cl^}Jh{b0 zPz@f&-F05Soks7r`1M#P5TiE}(RQsg_F>8E%>MLg)K7E2I_LDkwq}0mMK7F0vY602 z_l+_1CO}GJY1la0gAzWjE^;g3in45Eq!iDL>YCbv&VQPIw0VR7Qu3h5VQIww4 zI1SF7zsSVJi(P*if53too}-9S|BRRl$X68|4+*J%f-FE&RdFeHvN%+|Wzs7VbmaWi z&k84KMrz~vv6JRhZKvTvyXc^6cr8O5!AeMXlZU?hr7Guhc3)C_zWW=o3*R?5_jSeE z1h=17=2*&Atl96C5Yq{^UtihMuTP{hZ0nd^02JnjLVNwhe_A zzQnT_Ip4=Ek>i}p!@%jNtz6z$H&8Aqnw!DNe)0?)8W(A{;khvp<1}a%$NutFH!FMa zm7vDbndDvTXgciDqD3~Q9t;-(QfXd!H1>hXkRkyup2q@Te*q^>lpNn1A0Qp5xrnK&q~iAfB8dpL68nGqMMf5T<^(wn%w zL>8NqUxFPNKLu@1h0_oj5C?Q>j=HYv%F04}cL#W|OkwjK;lc%=-ZM~}PKOB%<$(1* zC{!n!#;Ge*c_t&^5kJ#x&^(7iEA=S;j^m$*rUyuzjDCvSv__9%k441`+20P1Cjl=w z4eo<3t}(Y985>7KRukqs)9|{_q@T=g6B(i|y0u=Q?VCN6Ux)8BDZ6xH)F zX(TH;6K~Y~fpUXWfH%_OJ&1wIqzzfh7EWNe8;3c8gbEAc#8yH*Ta3 ze;;Yw4~J4Jk0YUxuGheNbX^fHO}(uB_@lAyRl_m1b4n*pGxBE2#R+;Wdz`$*rYZ5q zA0UsuJDDi;g#Q4u3Ri$ zVV;Sp?hL%MU0wac4%U~lQ%dH>g_mOBGuY0jYbiWBr(~35puAacf=fSonpS)82UqEK zc(nWS_Yw->=1b0?yNB?2={W<}LU={X)P5Xbp3q<^kYP5fLPZ-o1)WV0hC+)5Qhj%T z|KhZ&nEYsLACRsmL@tGES*GRx0VLO!c&`=#$*vz?;kE5# zX;rC%|2FJv|2+cqrm-(HzVfc?>Nf+E`WrtOWCB}DZ2}u;!1okI8kC*W5TJJU(Ci?# zuSkFLlx-fzL-`c=`)}dyh`%k6HWj9(`#t!+Vd^}u1Tdll|$Aa1===tXxaz{xF zr7t}*GO?w>)!hWV=CW5aWnJDWcW2z3yl-d5t3@;M8%sb1py`;d9|x=(v!`Z!LB(3( zDn+nR;@JgY?u`{hhR*Y-Ft+4$BDV|l_-K*5pI*(TYBRTT;r!%nFwO|Lf#b&OXJbRn zZ*M|6wFgd73WCZOMFWnbr$U1RzEM_EtN3%0YjRD#nn;P^caTDa0zsygGb3+FPB7L5 zVio|NH$T&dl#g9175(FA3F3^aWW5MAx$qaL^Z4rHxI>3=dGpCpmYTCGfoo;<({D3Q z80~Y~7lUG2;w9pfj-_uoC7by9l~5Xfgu2S`4%(`z!IZWHKqTWhvaV*(k=bavz5suD zv&|L3_fOJ}{w&gHL8Se781V`Hqvv&O+aM{Rnd1eb)mSnmm%yQW@B{CQKYo9G(FwUB z_1U#VK0i#Z!9Rc5>!0Ex=DWx$b#=wwJZv_fGQ_};)zvJ)8R%MDYtGSr%^)^0+TMDWw2_v zTcC}YK09*Z0SIGy2&^(^K{{)XXW_|5hBEyne&m#=pEEbuHRj819yG@Z<)ZR5CE~t9ly~Em4yj z@zNjIrU9kM0E1Q`2pYP)nv}wqj=1Ulqe^@4qtu#Gj&8;B=&9R3FZ*e+SF9?6O&Vzo}00MB1fTJsHNyxtrGDSUo`ywMFyObGKt_pw;ZMgiu=A*(wd{KF(9vcC;o10W!YvU{YoP-9y1fp3gifcNv! z6xdnTIyV$@T}^01D~lt5DzLB;xagXu+7mkgZ^fbCE7uh$0z-TPN2V&SufC*eV%+71 znp!1b=jtRR-!vlX=N^dby2>^Y#R>Um*Kx^c35S9)$5a3e(+M>Y9O& z5x9e?xS7k~q?lRDAiuc0au7iSkCB;G3tYCQ92_ii-MPXlU|FB9#kqfe`sZg}SS%f< zq3ADU#OjqH{dS|Yqc9FGP^xVKqI=`p0y|{ApN6r z3n-4Jw|^BZ#O7ICLE3j{7;$)iqe$8zr%uKV&SRHfyAG7U$#^KgIxrnyDhSzKlOJ;6 zR~~r>8Mg*hMjO$Z~azVfe;f2NRB}EqbZ76LQT}7)zoT-TwG+V^N7Dd7s_?D zbUV?fPmeg{J%!!W)>Z+BFdh5$#uQZW##!8%UzD2tz+bKp9JYb7mot+6&i~F@V_qN>(}uQpup${jC@Ja}REwpd_yJ z-&px|KQ>&G`SkISfW}lGZI|&d7L}*Isb~6KXR~m9y2^JFAf^NbvV^ixQ{|+aj`i0g zD-Q=zz(D{L^ogye1n!Yn`C?8I7hczt(x%>^9Q5`sXAyU!h&f0shUt3kuC0PWGeI~u z1g63&6#B8@TId3iV@?^#!&yu2#NH~Cqb&X$wuU3+hQ*f63X}sL`#BMRkH;4a?XD_q z!QxWfUY|4f$g~Mgj|#Oqz{)ne(f|XZ{Z8WgeGr_^+Gsz6VkH>NW?W?CQ_+>6ML>Ds zLKJqRFOi`?#B`-5H(>i!$q_haCl{FQ`OumtFdXY--F}>EL(n8*;sP<&QWN`sVH+s8 z?+St2WcC3T>khZ&tqJxZ!m1%@_3?}U=*;hIcNy(Z0!Mpd1}zJ{_dmhyy=&>;&5%D# z5R6n6#r-;-#-GVsQ%-_0qO=CMdiYJMXCQqQ*;au`I|F+leYO%6MouQpEGM1UH}Ve< z2ELuZ;bP=v2=0??J%-R%xqz>j#7anH;a%=*p#qdo0j26BHoltdCzLkhcdNZ5D~8$S zBpVDj13B`}FV=$xtWG^(p7?Ga;7{C(f~kNzd`C%X)Gh9TaIH77Ju|T!^_q-Pe9WMQ ztf$_i%yF(ki?-G-r9iPjpEwxg5|tyoM20HxHs0UQ*zLbOM(g~k<40HTtE1FppTxdx z{4@#;Yui^lYoO0XgaUQ8p9+y`3!6%|XWJg=o7(mcR2gMK`QfBjxfZZ9UDPeR7ujLC zvA)wL@PLA36hiY<+_^VJK2EbD8KQ&)!E6f=hJRfeNLno`zr~*XW-a`gP@a&TK(b&7 ze7(OeF@dBtVbr4e4aI$JP%2=ElXnK0xsHKs=RQ=RwhHJ$29U+Y+Q1 zxg`4n>6ga@*lK8ffX^Z zMw*Kr(I$QlCXeO(3^l!SH)$o=j9Af7Ds86vuG$HTC!mT65Etjqj~%GgAnAT!D_^YX zcOQdi(RRtV78PBT;gamth5T}d(p(!}wKbE$ zm1O~8plOJD%An3RuSr91R(|Y1+%r-P;Iyq}wi?d&uJ2@Tez*_2e(x*CyFO>I9qrqhBkdx9@k@CL)tpT)Lz=DR%CS(SN_*-VhBWZvARMl& zxeqm>b=b_Okc=4g!O}${{lw4HoY$jnC`JZ-NiOXFab{maPkC+l7?{BUjL(BppZ}vv z`}HL3=>DF3@_6+?n2tDrasP9`0zv>Ybq%<|=|((b%ZJoiMtHKAWOAa*0Qs6W^8cI_&_CJ^+Jw9LH_PH-U-wgv0}l97!S@$^|6_>b9D=33yN7N zLNS=#dia>~(7k_tK(=#ypDCEW8hy9F0>WHUAbTY*GNBLbQUK3#Nb%?e`qBw_S=BQJ zfyv2uH;7|(B8%lVC~#b0{bJF40{9@ePz-r|c5hN>UlK{!edZvdYJ%z1sby86GRo}* zySx(=zBtfcKi~l)j4J^xJTGAq_YXJA%)8AQIG?$N511=5&{@~ zSGmgyO?6gY1JyJlMUK1RQYItXk7VC8l?+uEA?U$sNX70%>lb2)I;OiwM@8`_lvhfF zEq!3ILzY>;7q)#TXw5>Bc9=@gl1+hZwr&SnK*oc!%1d5gNwi=V=V<7bH}LH6vE&%X z-Wx3t?jb~sx{nHgdmi@q(Se16QIr~TUy(iV!+~u?Xu=AwO($8a1ms0C1$@uV6gp>) zeHJa?4;o7GAAr%ki4J$58c2r<{xmcZxnp$h*0i5X0YW7q*ZMA8INy~ZW5hZQh0@P_ESBtqBlvn*pxF_2M%72Q8b+a_=BrLRf%KdVf*}vG{1EUwR)Daq z>r}ko^4xnRIZ1#Wwg;l=YZ21#_btkWpgNel{@afCfCN!TkkkQH;Bp2b6hcPOaXzsz z_c4?+`!BLQ&rWA6{{7q@9D3{}r~bU1666=^z;H=PBb#t05|;h+>1#4m?aKh)cU0}I z<^T-t-PZ__8^p~Y2Jpk?kcQHHCHy_HWEUD zFk1*;wt^(Fxq=3T$_ui5z_$l#^qyEMMYr3n7A__1ERBH~apqOBT(M0GygM8^HGiLk zN(l5$%@{43S-{zGjg5`VZf9{i%P`@S3@GU4G4G7b&r+l&88eW9t7JMz1&E16OZXIl zeV~`!9As@Pe&wH%2({4chuSbm)-qP*OX88x%hX%OjUf3I=t42tO_hGIV83fgLYh>vy{CnNs~Cd);kFfcq#bji6BOMt6)fm|M%a zg8>ZDfx|ffL_D$ZJGsNW6KIx42keV+B7IXjvd=0f_k|&EV&|Y^5qiUy3b&RTTzuQ8 zA?W+R1Wq*EG7b1u8UgtNqTjHcKFGm>rg01xC4c;M7Uqf?!=i4PVN4FfPM^L8x@-xb zTmmCs_JKouHw+F$`u$tX1hY^FJ82~N^_?DD`8cw1qoy<$ye=#}!JBHgd#ZrXOXL_(+{DrUCCK?-;SP%h>`N`jfFUo^6v_zp4=^`iqX^G zPQ4ez+?1YB^Vx7y?K1$_$?epMXprd-9DR1*FTN3@mWs^~2I&^mJ3*GAMH?(E+}&`N z^R(Wp37YP-X@VpfxxaZvPA`YrRlO6VT>`7AU|+li?Z|P&wqa zf7kD2p69FKY7oZlw?MbCGx-IDN?=I;q<;wg*E~xxh)X)6Auv*Ea23|vWjOuxxrS2b z7KZ|ebY$_SKouJ|$dtoF3T`hL)5uRd++r5-hEv5bDG}={eRg*ZEZIn)w)F8(Sff0} zH*%pi&giFYbG#Fr%cKL#QHO#w!JzDts{nxHEE~u~G7{6-BTehHpV_l_!3715ilHe-ac=c3M6Npte9t zNSESCzf<5;xlsY1iqS4ov;FRbcY(fot={YrID+!z|>KZ5-gSI8t z63N|UcML`+f=641C{T{v9)(f&%k-)|r{O~BtOkG!ec;hbD-KO%+kpy%>$6lA@!XnD z8O_prCql}1<(Qj7i!&sjl{MF)8mAy@TnP2ok5rMqoViHHaOZz8Lbb0Mc$K6Zl_Kx$O%9)uEd&2_2 z>}UKP$XuNI_8|Ombr6xbDCN4verTbdk$Nf6ByT&+_w)G~=Iuvxvhh!euML4takrl6 zHwt|nbl`@13rH*SecA>P8VjwlC?6VRFpt1(W(K28OXaKLhu@gAa>9Hg%An^1q)9n^ zT7cs;J)8e04&ic=MI<@UkQ73`&o7r8OAnlZ-bF9;e_yGcoy8ToC1xceiuk%#>XHa-o_Vo^w5#C%sTewVJb37wcx`Fh(sfMmKL=uwW*A|Z z?3gMP$*zExfaEKC`_&Dw%F@=ha5Q!@mI3!`8UGnOH7o zPCcFe8sv(qYTMLzK{V)KFc-34Y-ong1p=`eX#TVVT2ho|fms;Jq9Rzl3Q?*a=Tkxe zsq(=uOZtzI)`N&9fJqmjZhx1R5p_e_3S{$J0G>82{PkEGQ4!G3AG?Cp+KAt6^V)mo zy8h=Q8kG%2a7Pa1p`82B(CNx;=%agFFzR9t^dcDz>mnK*SBuimf}rl0g-(JDFO5=n zl}vWcoDya|nHn>&!VcgyxDI^Z_3kW1z98klu2=hW0~$&+lC5Tma9nWFp3{)W2ZYt~p~I!1n2&ieh;d^Z9N8iw^aoFR zGY-ZKZ=(?*zU!Kxeb|{GVwwbaf)%-pNSA9d=!7-$u?Gy&t|ZBK_TXJi{GJ0UN43BY zd3K&-%E&JuZAI%iA1;0JwPi|-qZCRO1mpK0%OwXz%5@76(8@YworYIIBA&z{Nvjlo z+m3e=Zd5XIz|p)kpLXEuMOXwGHf3b_Xg=gXhXSUmL3(ABmyyy9S{hDkzj!o6nnK(9 z{$Lw|7<%WP4WRLtP%H#--mR{E0nR%y+YVw~K8%Vc2jA|!PFD-3AEv5d9EKvB%G-!c zvirS8_IYJEq#PH9^ap%wKL~N!f??!Y7JEu*RY_ zC_&?|*_js@x_S)*SsOoy6=*8|(18a~GGYbbiFQGq!9Fn89G=BZulZMuArDYWjFhwr z9txV5o;*>_0riMxzP=Jrt=G6+8uR)cNNT%>fEF%8m_Kp^9QV+F_^xePa_dOzB+ z*K~YX@6{P*-`C-XDu75Q3GDpE>Ei%b(G(izI+G@aUqV^|jO$B1iMA0(sPi_;(e3aS zYQ&IqGy8fC!^O63kag61O2d)N>QZ|$=GB=4W~vl7)m(Ly@nYT-LHATXtp>lf4}xYuM_?W^z5t497AsryS2I&0_xy8 z3xokqriAD_Kxn5?X*Wj_MdY#K=|p^3~n z{G8``C+6IM)2*E8PPL5snO7I>M5ZrR%pGr(<$P16p2k&0tX_}ii84h){|gvA@xYGf zRZ+g(fju!9{{tKU1x1HkF7w?O*)()hkmZBq^fF-+1)`Vd!dltW2@t-Bp&6Nj>~KM^ z=TkRfyeAZuRy;HFu~8Z%YGWDxDX=?jXn=aK;MHq3QF3h`ZZClD4J8ehjL$_vSYB>U_F z8;EC2NL3zhsZz6X!ydh9vul)H-3v8^ec7Wqo|h94{<{N+^eH}-SJ_@MT!oz6`&@c) zM*2v!Tfx9k5=~oDD({usrSlCtE}SwzD}r49Vss9gxXwAL%eC@qyx@{Jh^-sRD0}1= z!MRfib3P>(B~KQF%+;Tni1=V76+Gt%tMB(FGZQ=a#>ACu zKl^mT4e|c;j6(+n*u#F4)xUv>tU(OlZ&IV6VoG9dA(k?9hTI#a#j^_4((QvRzj108 z-&XNfaJbmV4_s~K;jFLRfllI`c9VIcN7rpmDqhWl?2<VS|bs7~@RjLcxXJxq!)&H#81J0V<4X?8i!)iD0QiR!|IYq)}#U}&2dyq6$JF&kG zY=R3WB_(^EOx-x`d4+LrQ~w0Uci8B~C{p~MdakWAMcE=C2$!<#1tugj&cw_Xrt6h^ z)g)=QQKzIj^slq{k)>6%?2U4dEh<)MGPf5pw03@dBz@r!h@95MUZ^!y19#2+X?wdf ziSDUIB0>>6XH z8$1)+BZJTv!)akSzn_b-rq(&zYk%(@FnTgZ8G&J+a~+__Bc76R)% zo4qDy&dQ36Xg1hLj6br!8)D5qaV^N0{6L;U>LUFRighB3q5o0|oERTTFh{OofzcZX zvvw3R%h+MR^JEkb$G)*}|Fhr1a&PyCyZ_Rp*C$KxZ|Fb7UVPiOwpIJYb$7(>QmQ#X zaXB=T;7Mc92@?j&qos$c4FiwTkyVOeb<%zvbJ+Lbsj#K{hbR8;fJ^ostVPTTOXyDW zz4y4Z{}HbK_uo5MEts{&{{@)o;{6m!4&8%42V|ymz(Z5j5CU6;f{4<@;M$%SevyIs z10pONoyAb4hgo;J%1j4VnjaWuCm4FyT(lU)gB~E-izc%>A?y4W83A45+MQR^{owmMbzIGuThH51;p6y{CFa-uhRC*^~(~BsANN!7}*lmn5 z&;&s`h&)O)Ayx2jE_inGimD3j$P%^lM`I-}ck(Kr8efBH!do!_5xOfU05C~|9^aMpkoWQkY3n-od;-8 z2R?(N(|x7o1CnIjTdoB_XFd$eb!WJ2+xaA@g;1o2NN^v#}O}d29TIGH{I=&xbA-1>?>b1iN z@Y?Wgn*q$!xa1oWyOJr36Dj+6FD)SP6b&^^$L;St!QxS19z_tPq3df!B0&P>O%GS% zb=_F?-XZz)F1P|Nt46Qx-O6r5Bu+vs8^TuvcSh~NPwdaX0DDn z48X2TeJ7DPc8co##xDGL;$F_}ghdD(7D~bKpA4#=e}rbu`4(KDzIUR+feA4+H;eKz z?)P7Ukpsgto5{ge9q407yhskBF{ulhv>P5ebix(ZAdHTmNi7{3hax{f^OhCnt)R2( zvwlV$kms0ph3gu#USW@>`8b9{xH6sSH`uXZB8344`lcGw1@A^7EA$qRL0}MnZHimC5!F1Yau!B6S7g zSOpC=TPWNUiwi-wS|320bjLa{{s}TyrbeHb(hG>#)|#R(;LS(uN}xwJeEJOx{P4wM?nkya)R7yg>M8D#WogFT=mAo>1DqLA|?t1r(_+o>p|Q|rB7l2t4C zQ^@ot8ZS;91dFXYh@N@WjCssIK1-#x>dbxJRf_9tw5h4*N5tMdlvE}sg;_YYU-hYV zaEDgqWaNil zPfZ8d31T8%^=@)NSjbg{@nQ%ybj|dm&+S1IaM~Id?;F;`b%oc|r{2qyL$jVJ5C8}3 zIP?1#cvL`t-prS;_5Ay&lMi5m`N+dfoSJVUQ^xN>I)VV0)oS49N zsDSc>PLxrAH>dBE_>rNpS0zNXY-N8G=a$hp@(|8OocxgrI6IkzU#y$m)R`Gr5{_~b z^D3@!#J6%{KX-MMo;xSvWklod-rnHzZt2vep=i{z_QZp|aLy~eC$AlZ>ii=1vzd_H zN%}n@-bB_g!dy(j^aA7Q#6yb^{r*<_it$fiV*$xN`XCs5;H066xeVWhmD&0(Lvo0$ z){RaA=tb+-wV-Bd13x>el{7h~_>VumqE3z^^-kkfx;1j+_eC}YB5h*xB40QhS z`J7dHCtrjdrM{Y{dy+b2C8)73_4*&^RoOdzXw1dI*y8=6(#R9#3{n1vakg?>@7~+0~u?BUEPzin$clNoK`{~=_S&~m#&hAs1djj|xyPSJPN_#Yl z_blWd&vr$M1`L9Zw8VVs$)ihJYHatDExFA*+O;0apP$~2_N||E-5tJF;kks=9f;^m z%x95d>OH!LKLxQrPn+Q%6kpSA-lq`JNZh(P@m&kt4m<1f_ze&YB!OW)qGixpTO%tL zyG-%xzCg%pcHgrSo@a_kzA#V*;_J2D=nqxTgd`Rshp^v&1NyZrBI1=g5TqbnAkt}e zpuk;{x)i{8Gc%h%JK?%L^CdOqmHPf?Z$j(J{qsU$ripP#PLMDrz;BBN)@H>iDRF0& z^6UA;#UAw=4pv6cairgXLv~2%mBp9NYG?}P&7NzLo(;f6$Raw4^^Ae3ztl-9%QoQ`7fRm1#-*jbW{*Qt+bTb$UCPcO7#ZRolyg6 zAgdaZ*rbqUN6KYrk$^aHPm;0?h3qZEnLHSAO^fr$I4~$=b9A_U;GtXwLg>; z-*vhLx2KNxtKR?LSHjfh7TAN6I7k<@@Gxa2o1g!x_{t!=JIfAeC%!oN+lhn@_DBzu zfF+brp3v50=f{gqG2b(_^&h#EigNh<2MmFEAGTiek0nCs?o z&L;-tYFC5z)eI;$x-TioZb#9O6^0j{6M37MZZDkdu%Z2g$3X%@y5P_hT0QzFilf2@08u&TA^fLDmqDIQoF06HVG*Rd&tGpq0 zM0bqg4YsaQJcQ=sixTt+d6NvFxoKiwucFTy`zZ7DjxNP1F+&rFa<)Sq5Lr%A6$wc! zk>t9__dnWu^JuE~|6hDl8l;d6A%rL@oXm4YY$0Pwrf4#hOv%g`P_Yn&%NKzUF+WUTX%ie_qV?1pL0&_={>z(&*x)$0`fScH+Wse z7l=+2^$nb@kw|;hx7ea8Lg?R8B3ZujN67D*5=vh={NXv!!rHE0tfbqh@922KV)H|| zJr-B(ghDlcD6NLl)!EG@FK%x6z+YN_%q6Oo`?^#%$A~feWNcEnR^`4#irCh<<*S`y_8zqGby66Av+`fv0sx=eBS<$U;dM>!@<9-mi)SPqP}NB0kchF z@h6;wht!re<$M}B-?X^?twqr}3)C!t48nper9a4d{*8Xp8RN7%X6|D<9d7s+Hg4M` z|I<&ynumV@5ygyea#US#;8w!XiLNr2kL)a0%F%g$#~f@A&bD^9&O{Mq?L}*IHRML& zg5AZ=)@-Y+U3-?|6J`0uG4NKQ*dZxn2wi>zmNsazm{QCx2bMtTDAjGZ-KGb1#@_h1 z315AoYXlPPh4sOAfD|`!9j@9&)hA!r0sPE- zB~JM1kFNYv8enf$Z3Kl>r%sTf~puEg@qw4Ilpdzh2lX{OJD;GxJ{{DfYjAJRh8b z|NFac>p?24GXiOncK;06@p=Hykl$D6R9Z1o2oql9!IMN~ULY?@r{j-vn%MEbpg7vN z(oz*uu#YE30)p8N2$v)Xafn~3r0&a?(0rwXX6P?WnMAR%*6{JEZl7vSSjvQUJo`QBPQ9=&)@HG(W)$lfaGuz00wL^x^Yegu)2S&)ql zk+2g(J$3`d+!^rOK|S$7_Vro$Doyv>YH~FGpbKzjDtFj3(e(U-YuhMa#^Li*i2eYB z**?EFptDIsv%d`EX@|lZ=lO2bQ!?)9g42ZZzgSE%4!!amG!*FNvHVsZFL^`kwg7`$;FSa@%M;|LV#Hf;Lx+rnDzf`}kH6 zm}KYY<>jqRVO9|jh^!TF$R4SNbj36f1@p;l49%$o(Nv~x6f2wq|KR+?MuZo(Z9?5q z91@Rf-L#HgM=OY_qu3ZIr98SS!tP7jMuV;~nVR7Px81XoBv}wyho0P6%N?vf)jvT| zQA__N#k*`{&Pz*ayYjO-bGOx!?deC&KWrQSrH9+s+Uy8HeG>V(lxbyyb>aX$4J}`q z){kj@2A%OYi_~KjnHz4x_gqTH*IOi*c@}eR1xuDCP%ot zOAIGNg_BH1!8Qc{B(Py>145EVUxT$n4tmYwC7xoz8#Ro3VB=3TExIH@{B{OKrr2zO z>^vR=kX(^a0Mc4M#24fK5U|YnQi30-dRbD(4Q|}CC?Ty2_>RRx$TuF42zz{i>mH3x zCaP0K>vi^7sF9jpe*Fwx*39K2ROXEupU2IGpR_}S;0|#^M@Wa2*UKp^|gh;v=w8s;GuvmXIQq-;gL{+bWm{NIMn% z^#uOYxhe{?o&E)UH=LpVumt^el_)pH+b|JuZoJKHAwVT7_b7h3|GS*$yDPE(<&?c@ zWj@Y>pU-&%4tNL?;X!p>`c(=4OmdSO8v&uelKu&xIYvjQlIus4Z~_&ao`lG zHR!1j>NN{Gc~M6dNrxg#L%?7$DmMYCiCWO```NWTuDL#1tJe!%QlV}n3x3Omb2_8X zXrD@M>;VB@GvowvjcTBA=AXDb z)x!_`@zn&nCxkpZOriqfcage!4nAzVO1&= zoDPebTz4?T&S=!~a&I`s+5;ED{8R|_2OKJbbBxyKc98PT!s?X<3>qe>z3~2moDu3A z-sC|>j)(c5ub?Nu&N^NMkq{ZM=Xbo)?R;WBBagD2f4ogK07!(YM7CwlI?;r>=q3cv zR$^SpDH>hvyf%R6$j;jMiF%T7LD%|uDen_~u>s=jlmnf^j67hT51){*4<@L$6g|1- zjhSIUP#xL@^hY+x`D{VOp^JO9#c$xK(jb*UqItCw3F}HL6cP`TQSEAows^R; zjB}6e#$TnX#6UQ@-WzQgXpV!+Y)j2FBJ7YQtU#bEH4q=eOgTG4h4=~~S>I35JPLDk z2w{)mS7C({;bV6JWjZ>3?sJm?M_k>Uec zM`p%oA%3T-e*0+xZkk0se{a!sEi=h%-^GF7kZ_o`5*?bg5yIumCPTPL93-Jxh@fr= zPdURX%D03|E%FZGfyUe5$?|`k{haU=gaKrL=9(H+?T~ZDL^_cI48NkTEzk+aVa2%( zq!TNcp@Z~q2MfUVy!`%AV1rfFzY#C-^c3|z};IQlPNk!Vba za~=J3U=EV}Py@dTvarPxsABU^?)n?gkH04mAx^DF2JWaK63CLwJmIEkMb<8NT~GiX zl(@%8-Un30zVGmZJ-uLd1zGIZYsdPdFrfhM4$l&aP1IMeqw`0yj9L$4y8GI9xaaO5 z@`WY|#f?2!7bU^fL4jBMKi+^)u&YpGa$t2l=y&k%a!ike+EVTZJkGclH=MNs66POH z1&x*D$}sf9=|ebRg6h9TJv+qIxBewuYWo~+B~r7LuiCErE!H`>2{}2lbz8U#PI|@86hu zgYMBZAkf`&{_KG86aInLC4v`9)UO;&jAY=$-$#i-juSG5#I9@Te8H6;wVP(Asn7SE zI#lY|1)}(Khy*+5>h%WH=SV`BRvMI<10Im=hbw9B8YbFfII~W%4en6>*(rzD`(tTW zj;ekX21EuT*q*oxc9=OcQ*`If&FBV?<_UEYY;n8OoL|wtJd@)U8dT< zrFlWGCy&AfDOPpTr_w#0!j1P`sbxxVQC7Luz0#?R0m|-~QpuxpAkyS{!4by zn`b-q+n!17`>tv_?AXmaj zuC4iMrxk4_i1t;w{h%(EdSYmL($xJ3HcS(#&#yxKRPcrnaGE{$KK+4J7WLI{cjDQq z{AmmA8kQxiSiL>)HtO8G06`7!=iz?Hox66tAiD|FB(1RJF&q@ev`=)i&haK!-+=KX*%3E0oQKS4;9>j=jLzFnXr4d&+Voz|3-+j9LG#B@7xF~>MT_A^9{Ac%81=y{3otrYXlWW{ddQt z5P`be(*BQ4lzZeOP$E?@A>h$R$6}y;zHDMk_d_+s_0RHPt(1L^bZ&3HY>PG!^ z13LjF`EC%qQVjeeKBXACiY=|}>~Kp=Q6e-93W<=}gIUDT=ICNdVf{CZ%tBd;a@{w? z9Q|!mZ0%IHfe?y{qC)S5U@R3)#fLUEN?BeB)&nwzNkTAM!tgZ&6rj+G<0VO-G&vi@ zO}h)Pywd__#%n5 zs9cc(3>2_0D+Ao}AUHP6squxo`v5>m0uMou*-lw-vt^)YYZsVk+h!l)hgw+!Hgp|O z(t%~j^2&JQDF*xLZVa|soC2dnTrjX^Etl2lda*P{5CY**tp(nM(!Bow0WoKyE z3JNBBQ_MFrzX&?Ef}&q{Z=S750j$O&8GohTsecB*Izwn@EF(=9#FEr<@guNDXfq5P zrd9k88>?Ya0m#y%+-W1n69LjJ_BH>vZFT4%hGg1{ghl3aLAbr%;palT%%eQ5HN!`8 z?`CuPu?@Spo6eM2_$_D`SpGp$1NNV2s6!rP z@}b$c1rdDq=ute5S^d^82t54|CtAlX4n{jXHBmYPVZ;HyuB$<jiC+DvTVa?l$cM@c@leniu2Z#US`4;e>MOZ^NZ zsL!fUYdVp7XaewU<27$1x?jLnFHT|4_VvZ(Gv#>*4B)H;fDgDGy%6#TbTr?2!M(%y z)Mg6)`Yb@_S#Cc?t^*{)vS_{qB=ux{iUmZYwxS#nv67Pz7Kmc;Q=W}LKu|j&I5nrq ziy(WEK%+DnBe6PysNx>uA)q==Kr5e+F+|-?vr{(ur$8YqH~bnMI94){F-nEvFaQr{ z?V%~Y_-hk<*c}Xo?<$*z85em@KR}Sf>ZaUs-$^cLpzX_zSrh+msISj> z+d{1y+B>WU|7^^`e+CfzHzNq^%71c^@}FBG{pWo9r=lm=!1*uP{HvRCV{=#S93dlt zUKJm=z2a0fRPV&Um@3zXj-C2@=fC|W4IjSf|5erazn&6KBf{sGj@|9hnzX|zilZv2 zqZa_^%K%^SiBG?~035=ZnP3ylL|CV&l6FfPS_DyFY`FTFTIV|h4>S1@V#Z+=a--%c z-~c;RJIw;TNZq~XhFb(8AR0XSg*&tiNL23YZiZ%sDv1ZIN*8AdQNcaX_)7mPwVHc) zJ$28xnsScW6tmRjb;=aWs-)2 zAaHopKr>?fsANRyii)Iy&@ zBcxn+yeiB3>pP%()BOl#(??N}{tWHjb(L5+bt4U2Ley}J180ENsP21UKO2VTMa!;e zy`a|rZ3gg4$Oai)hJ$=r>-L!TX-FP9l@0%$3d&$iD5DATtUc|y!Yd$ikjw~@{J<^X+!L2;S$%gvBE$3?%|D6u~ z)0obFow;XGv0zr9;+@ecroLoGfz#`38ETt7Nn^z<3CTEqyVW+C>=^%6lm**x^iIku zj^UeG1%u~l7?!&g9VXQm<;Bm#`(+OO9x6?4JKM)Ua}TP-)*_5*Z{H9Wd24wV#7wBvj;+ab^{0Hs=~EBLdIW zN*Xd$X$O*psSWCgLFzO3M_vibJD8?p`)6GweEgTFu>YMg=szrt)CKpEJg^`d7%_n3 zY1%+s_5j>HLhLy2NGOnA^^t`qMl~Z`k2NM5+E&cKtZmMV=+lR*>-|*|F;vJ96eewO ze;BV}AXhvT_KDnUz~!7-EP=YDqZe`GJs3zLVYLAwxklPCK?eiV5jc~C?1kweb`z@r zs{Ix~*#o#VMF8Kg8^z*Z$I|f)s<`Y~N5X~g9}8yh@fgk#4sGi$R87Q0>$FF4kxl%Hf4Ev8% zv|jXFx#f#94biu<3RRe!&!tMWO@sh>mI{s+U7D|gp6@g|c22)eK) zu!;fpIQ6XvQ0u|uAq4&$8QX1WRUb9sb76}r670q>2eUpI@BbOGTM{37aI6qMcZod0 zsz*<8{=&J4cU9PFEZlZeL$7hT3!~OvL8|ip#DvwRovJ@Vn*Lc zg8^zd|Ar@2yxNDRI;&b&PDmMFzx(%F*mYI9;f}1rp;!YAU@Dp7-d4-V2jrEVZxdYT zQsy?v>Wf!DG{}SAX~pbA_Yy=49Ky)F#qOzwejwYEaDd7RB>?qLyAZ_2yzk^iYF<8K zx1#Hi(C>*m5rEIY&bAp{qTo`pL;fOlU~I_)B23iO~3m>LK03025&Nw1no^G z$OV*;W|b3Ukoe#Z5how2=#u~p!76K-mj`$Cl|bT(9`j5%2)PwKmPKf2G72trF;~^} zvAGSMPSgMWA9^B@mZR!OP*?(D+6l4W36*R-6b_bdRm%R z`)E3|T*vCz8)O&(bSPZuh8xHVu=&L4A$O3C)+dXVNsG{+5Ja(EX6-YJOGy)$hp7`Q zFmO3}RKIs?y9%wURoLL1DBX3|8Zlp-Yl$V;+8!tDWT|6xQMVy)kcGxnH zKX4wh;XBTst8LAd4g={UZH%h7hiO8Q2bU^wrh6g+nFR0(tPynY#bEDv)J zMqn^9DxggKqPKCn2Tgx`)k4i*gMndwBwp9P_{Lv9UJup3&6?g|^>r=J;)jvqlm~A^ zALHcIJs!VHQOVD3mJ+l2L@x_mnOeHk#7^Zp69GY#I|FY`v_*l1IYm{7cVnQ?-h|yB5Y#FZ16&mgwQqEOGz^i`Iy&g#IEbhIYU5t1}vq51qhh^lzVLb0G0?`QZb4| zl(xSZYK>Ei(r|O#_Zz;sih%%&a$p0_Fv~Wb01;7>-^0-QsI2yIm3pj`fv4DFNj< z3tyJu!yDSh)PlM&T#v%Rl04455SwAQ*QZ=VoFl+urFRg(n!MSKkC|NDNP8bL?FvXu zG3=Oh1bWD$G=u|YenII1&UUZkDLl*s+G2wx8oqZgIg+?4)+2NxR~=4ii9b@-YPrd# zwP&FQ$B)P{0FkXU^U#lK$$iRRwZ*#WHfRm=S{V<5b0V0r`gh;P_~?ZILEr0(u=37W z3LRG|BIqeFH`Rz;HbxR2rJv15ogwOWcx9p~QDI1=eBoB4C7Zx-?%uV%EN$}DVxjtE zeFJ*h{zJZi9yL)WZa%q#2HW&bjpvz{xSzpD&NRS1a%**`2-+({GhxQprcU#^T)^cP z6N>7R2U@a~);?};8>@c*@Ksdva+r#}MkTx7D+(Z(BCBSW+rOcGvK1Yderzh>s%ZtV z>Y<-dRMXq3kfem|eONqhef&7=X&?9Bs!Dnr#F^1*Igj?aFHi>Pe*!JhGq6f$je$*b zpBJ3SdK}&L37b2AgJl8PrDu z4Y+x{$pbw3Jk&mJi!)gJec2ttZGdsfD{{Qa^TzXXAi;YEx+u{KUnp@!a8IqrE$tc9 zrm+b4o^O00BX$PuhbUQicD5?mW0@etuWTvm+Z7C2k-Hd5d>&X?%SD%|L+&8HXh8F^ zR`Pr=l)n2$fs+4%UCgyV*;qJ3rx~N4=KCwI`3KtyE z(TP3nM%0<-<=ap%)M4yIXC^ zU~@2w$uIrZ*^TLP8P>}vv>Cw>W(OU>>uym{#O)uN`}Kg5$nq0NMqK&AZTwyT9=ox( zgy8-`=Q60wM5G9zDP@=`i8YbS+#I@}LT2J#d;tic2or2d($IITC}WR7Wyi)2-S<~J z{~q8zmvl{|95rKi?cIg2_8Teq?I5tQn}12&E;cx@1;x8-B?=-7)^>>ohoTmCEy=33m5e)1g<6%C^(iaa?#`UJy>ma{nj$NBj%bwgQQDCV( z!eh=O&=W4V#y=ZgtCimT3kC$FzFBtkzDCk}vM?a-OT4A>@vb6dV|rXSSur@xxq76uI4gzc4OjY&#=WgxW-N$$g2y2%l1!-8en&*dqSr`-tY$9}odl=f(2Xi!W zHO}_Alu344YWFiG*QHp`L?o08`wB%p8G``7=C&z{KL%h^4lDcCiqQp4ZM>II-V0A` zlg?p&!YzJ(EKnSU*DkRb&Z(`t!!g3eubKCG^`&$Wo$8NCE=ftNDV4#6WY-lvCYHH1 zxOL>R&H#YxVysaUd8onGT zuiL=z-p}J!2~kT$QEk2_fj39!or-^GxX1kCXaV`>8pY;DlQn{q5{q6n7r3W9cJ;X^ z`QCd2wQ3upErF%5ILct&Fc}^JZLv3#M{xREy-HhZord-%-eyaXRC;t(LP@E7DvTtS z24WzyB*8UGvAgiclW`sWBug%}<5!a|ncMCZ+1AUnEsH7P^y9tjZCmSU-nPEm$^DUu zNsnQtpyAVKRVS^8r%X~VCj4RhooFTR6{*rmsJFK_`T5LK=DEAQWZ4fsa#8|_VDyQX z+`RUC=j8b>&Az`cye$Q-P&IoU=|rv4{R!;G0z}hEC*HhY zRZ3EV7anZ{WPLmPR-?z0_qmyStb?j8b#!zjHx1IJ?wI&?kS;o>F_U%v8ceE7_%U->&)GT(_$(=?gZ!vZdg>(@Z7of_87y1Lzwf(JppR^Ho84v6`cjCDsfa>w{Z8~GL0oaNw^ZTbd&CF^N1-nPTew^McrqUD)6 z_N&7r%j*)cDrL>R&SlNbwgNY|dYVq@F;(hYNA8SK{HZA}!MpF;P{o6V7*1tTLOMNZ zqjOF|!>Mtnyj&n|TKdVjPy0>=@}2{+OxUg(*CKy<<%686DvWF{MnShf*wCE*YAHh8 zLostTgZra}5>+Fp;dlDnv?8dZZmA0>grNlB%lwt}j^jZ z252<0zua3|Z&!k}tDsN$X_~#p9MRq$l4io2HTGn zFG^ewRR2CpZnJsG$W3RHqQQ6@wm8`<{rtOQSA@9*gq$|9MenC4u9vp|v>zv=n)!bR z_k$Dn?gM%r-sI)DG&{BZf+@#3=rk+SMSc}$GicB6iOaNkS$*;D=!}--Nw#1HOa8@p z&o|G8QzzOU%DMDy66@$&d3fTdlb>|-=doUeI+gxXK_BVi0xgx3X%S1#9qtZap9eJC zJ}@fLSrJ|&)iULC%H|)9cAWlMS1j@(bOOcCpFoYVz}@q_&fbkY)4cwX(b-n32=#Eb znq*?RQ(AZpUx$wsDM~8}0#|BV0u+}NT4ro}-vvz#?K8}%c4xcIJ<(UlEnua8-%}=C zBpM1oWd)dxA<&y}b>&ue%>E0DtFYs+6g;qeG`R7Orh!x{ujuN%rTGb7zl@jLjduys z+-pD6t!AawRLFcZeUsmyOTdL+9=7X3f_SoMck3tg$;a@Cfn?X@YHN7_odBsulP#`l zh;gF%)j?u%b&dD5gU>g`+m-h4s0q%<607vP+oQxNE7flF1Rk&#e4gCMCC@NqBhI4q zGJ9C4+_1y^R0H4VZzpmXcxd887#e-aqol_e7;%%n9BTQ!3g88QoVigpgHCIeJWjeA zg$l(S-PVb{^O>_102&smrk&t5_7a>Prcpn!cj@NUS^peW&P_2!f`{L)FH}6JX2AG% z<(zm_0I^m0W^uI{n?sSUl|V3kB#5Wft#5zYYu~X`sWLyNvF&qykB1p-hZ>zt$ayj{# zXAHC^M}2OZlTk8331Z*Wj~$(c$rE6AsFu8@OS#JXWSO`oF&SEaC0LD^vyF>&?5UXX zj>v9<{FlxH^{+Pd9Ib$W(c)iKv{661A#Y*3B{^#+@u;WTR80`g>wt822^9K3xc^fJ zNDRz$n<9&Z5E$BL^sj+52?bU<1DhJ>iaK_^a1?F713=tGhgg+C0J$^T(VJu zJayHrH5R43ZG7ZqHLsgD0{AdsZg%cWyn2qHu|lw$xKI4aE!Ahf3c~K8%zS2sot9>gA|n)~wkMy-uizL^=H@!y5!f@_$-6cg!m;5<5PgS;cKpwT zdxExet@=+}?Mh%mxq7;k(*s#;+S_M!s;_SM2X?|@h*HGh{+RQjdN&$TxJUHuruu3M zHPgxoYwD8Gg+>`y)M!xeS=69IIv~5%{KW-SI^DZAApTaLO6}1*zf^#@C@A?(# z>#dU+J5py+@9J2&XJ)%Uyy zuhppc!E5`!l%z#2ukSdosD6ZHgO0pDwdNb|Z%Zk;@+Gh(UW_EV&v7MXZ1_kZlR9&tzQ ztdt{u$g+jcu3Gi6hilx`{naPS@e6Ragpmi1!{x5zRugZ7otjeh2$4JNu>4gv^?G*M zdx9Ar4g-gW5;n&qt%mlpmCavzP}z5TghAxh&bVrVuy%s@?sCHe&y)*eM}A=(?U|=m zhXr?yh7A#Y-@SG+kG5+domjf)k{9>03j`Nz9p8bVw~z+<=M~$KLfP|0rNUSm9>I-r zK}WfGp44>ih-v6udbqE==bUa>=Rn#})4bqaBOh&o78gfoNzr>P-VfU5+HLN~S={Fm zw|xSXX5pTig=g1_yU6;}ms-Ck`uYQtervE2(2%G#!)b4VTkzi|>;2-k*~ETJ z!Nuv-gz1~j`vq;^szzm@#-%CM-2M>5jcQ(>On6Ha+08ugkT@Ayy@=2e;113()IN}P zvYyq^IXWpjxSTlQEo}eg+4-Wuln7n23x#~bFRYO_lA#-5wb~2BWI3oFwhA8Dj?=q8 zm!$GX9p*%c@p*yY>ZsF>{W@RG<3&`XZyMcuntC13hw97er0c{5Cx(c+a@56NgKF{| zLd$^s7pON^++ArnwO7laI3ubk28W*t&(36O?$U}FDk?xEGL`EvHU zirMQM{Kc}$Nr1Q3edxVMAub&hZ#iL_(PA@UUZG5?!_E5$3yj3()RvK@8>TzlBV1)v+T|;O(5J= zw(;hhp=s2$@(qbp=WqOBTi;Ye7wbPu*iEB3z+-atuio(RP`B()Rx*F-aHS3z`iN+> zj!u^AtPpdKg|mxA)`dyB481zIYC!xNTdUSLgI7BbKsx;zc&;XfQ|r~b0Ynm8QrQDz zxWi8}YM;-PT9-^`A65vG<>S|YJ*^$R`jY{A)+v@+o7rwg zo{1Tc4*)I1uK+AwpNZJInBm^)^_Ce=8F`fzUxRhUuK2trDKvC$F8`A(bfEpqVA8d(fGZ`Sv}${JZ+6u zokTK#S1gokPL`|x2B8!Sa=r42uXO2pV0?ywy-g6*5JkUxp9gMT1s3@NI)SQvpEN+X zwl)@Dv2vb!73TQae|dX+K-&4{vXL;_FYcuMbfMS%9cv^;M;sjkKgB_BAVqS^n!$6i z1t&jXqG!!F3y76_hpT;7cJ{sx$tpk-d*F6X!eJ<*b7GQ!VZQ@%pwOauH?gA+xS#zD zT+;jd#HJK>M`D&ljS8+3PRUi<6!In^a4q$dOP*QDGbFJWR&Q)(>2t9rx26Zd9;0yO zfuya$RB`mm#X3|%jV_`P`7==B{e_sdn*!pipimZJw&eZ_n2j#t7jo~=#oD3w0DK#M z1`MLV5HsL=0lcHMubm+p9My1WB$>Ve@p};TxGz7=xdS!Jue;q55Y47kD?gasu$w09O+QJo*=EE<j~ggS`^YqoSZIK zfTyt51`Gjx#yqDu+q&UrzJsGT;{fYg@3-!?Ybj75PCW#jg9bQDNst#NtERZ<#y&Mie_vt@Kz$Gp*F z+Q1Qp4HB2(hw86t3~cD|Hww}FL2|N;Y{gi}mWpg>O~1&LPeqtElg1TEp!Z^p<<$N= ztyy$k^6(xh;5HZJbSyHR1&kf9`fZ35-}#qRP{ps((Z_y##=S;vegmSG`^rz2N8c78 zaWf9O1b$*PN}t?`hBgG~YJP8zHmbWh+aPn`L^Rd3Zv_T1FuHkw0nz4^_g;S4{kZ3q+|#Lt~u)yZDc7#NcXJw z6BpiJEBo%Xbxl|vp!O~AWqybq7?E8IhvmnzhBOYMk}pvBF!7FY@9OzGbq70_nH47g zIlVpNm|aZ%cJ~BZoYzEj+6n9Prh=`Gj(ydkQ!|hZxg%V0X|iyiOEt4b_4xh|c;h?3 z#&oMoXLFr`P}ZTMb=-`(@DQx6WvZ;&a&ujfc;C{C11X`p`_VUJwKPj^>5L+^x#WQD zx`s5hmLPvGO~B#J#f&X3)$<_cyP3utdgP<2H9S{NMxRs9M(1pnIAp#qMpmF?o}{uS zVST{B>&6Y6VrXw{f^YDgd}6keAg?n z2ddsD#fJlRe73ur4Gm4iROkxp%K{g;pv@;&=i>Z5zWMfD@w;hmXKD3JBig+Ly4z(O z;ePu9m1t__#@D{_X3oU#VoE~EPqBgZ9+J-Z+vyH8=2{W$T=e^#-qXft9Zp-{bDzzL zJg2pMGhrnr%~+7yb^#1QdcqQn~=`^9jv?H zSve!vf1P{vp~+;6tS>-=VVylvp{s%f59OTAr*F6JO^V>lSzFkD*2x`6KsJ>_S~lL~ zb}Q*<-`~%Z)mc2FLyrbC)al9%+%=sz1!t?Xnc)(@ga&Os2u=$_k8ZyLwvI1jU!Dsb z6J}_8BqJKzZM3lZ%e^A}mF|c1>4|p36A>hr$3n*5FnAX{!Z6j(yxRH65RBE_{$MG+ z!@K8DpvB+bu~&ZHwa7dq9@faMOW)#tJK~|MwPy6E5UA#F&Av@M@z}}CKYSgViP4U+C|Y2H7MQRjahwr?^1nt;dFoD&I? zb3h`KbZy8IJ?*nfQr;VztAf@U@5TUk?dZFL9Zwhz+hweQX-r*FSb{XTt%F;gXDybg*9tQgKF!2!+e#BLOp4;Qlm+iIJplYsBpk@M zc3i8Kk}~B5X}$D7OotDSAt!nk>c+jdb{-P4_6;kP^l=1z8G}QgZ#qZy;aiv6GIV;u zscD*S>wUy#0k*~`&I$!pUiXx|t}$$_6J_t-L!6s|PW9M#U$~<7KYVLgPpN&splXoq z`r|u5Rqr5l^Q1|kwfdJKp$muhibRnnKLdX7W#K^f;pen^A~vvdeF5%<%hKq|)8p@% z37iwHQT@=`85>QtRE4HiKvZU!^C}#!czIT9YFMC!fOnyNg4WuXss4FxjaLve|4i9u zNo~d|M<%Pie-6#4^j zpe#IT%XwmdX6fzDo;hC_*v#&AJUxdchcP{dK1#c8 zFtED`nt9W_u>@!$a6=Yb`vrM?BHp8Cx`TDl5!%MM3BXhZ2yeWDm?UR^Z$8Bn@c z5krT|otZG*p%YkDr1OfDPisULT3#QKE;V25Ar-&nY;{h0}MYy zR%5E^GVB?-a|%hxCXFtbX~mq%e!$e|I@Cj{B5we%)mJJ9wF(=AR-DW|B1rbTX_72b z`j*VuC)GKyXp;162Kp9OG?l+P0qDHI6_z&2bbQ4Ql(k&~Aj>S17cww{U{6aMvyZdr zKW=c;hVGG3EjAJnUu~~pCwzJn#Al`$=fOrxbj$x zPifuZz~9bFD#HrzA_dE6ZDmhPWZmd81R;r|QCM0=l~Jyq@sSvJ&)a%#NBm-74)Ou$ z+0rRV(K!Y=_XXArq;yWGEXhi=2lDvM{#ZA4Hz{}4)f4n%;XX3}X1MZxJQPN1IUc5* zm(93iK3)DCgi|udkO#CFGIp^L~{Wm=OJh8sn&30h} zo6Aa}?yOB{^2H8s*2=RigXby3b<`hO^KHufyWXjGe2p zzvcN&lv6ziPL4vM*jCJ@Yc3T&O*=d_!dZLVp!gB%5jgJx*PiM1kHT+^4F2VPXGhu| z+NrqRJPeNfGz&0Gt9E=qD6_3E;KCFLDcKU-@2qv@v<#7PkF219Xf0hd&qg zZ&>2eY|SLW;ckfKvf8b&r^nHE=E3lcS~BPb?};2sX7UX9SP`|JFH{cixI#V$*qz4s zo^2Z75A@aBo7mS-qcL(=%Bio~M70SxPPUBfX1?%-l^XgOp~dJlJH0C@ag<#?XTZzH zOMHb{csY;B-5OF|co@SW^%J2a?PC{N(7hf-Uk>~7F>fq;?UfwEIzZw9z6TWB(NX`Rqwxo>nDC4UELk=x2;4Jh*i7X2($B@xX8-9-pyl5kSkT;ZCi z!_Z8ckeW+~g$p{orz0Rpph^ibE6o+BmH7Tw`+2hIciWPJ2ps~F41<6B2km9&lJvY- zsR4aps;xMQ-e8YrN%#h~+hc!CuZ+nXn7_A<9Z_M+vh~+A-(p3~k<7MT9@Q~Tcb&Q^vct4q9v52ckDTpq_VYJdCkOl(fs_$3FFzt*ar#KQ6xW{WUdhYxpWpio*J(Kl3ynmNHF!b% z0Kc3jt4y4jC6D1JGR^y11}^z($DCq0wv+es`lYU0e# zNUVh{w#N5`+mz0dmj`7k?lmSUDR}h+Y~-g&up}64KGJ+UZI)LcAP#o;aXEtz1Go~#w{gRMU!E&hzZ0s^>&WwU9ksvZ0 zPQSn5kiKmra{;9D)LP$uwN_l(Dx`xgM9l=~X9BtA_J> z`VM(C9>{L>SZ|WEj;K%CDSyv#037#xmHy}z`?-$a5kM=6 z{5ux;Mw^_!hpSomifh2kI_viv$Jsu=CmP=*YH9uzlo-TcSFzHrNi?X6lkT@|t{Ng- z_PRL_C7kIJ5FDSgdSdg8*eLcN_g{ywKPNT+pglfEWtD%!;w=0c6qWt&4;qWWl?b7* z?y*(6Vn#`2w;|5pk|#^WeRXqR|%ILS&jg6k2Y%(#OsmL;Th9JeGzRg5))4 z7aLBoK6`Id6)e5Q$adyp8NjtQMOhm_2FqeNbqs2*EMvxxN4Nm7)H^pcpmW0L4tsG8ilnMDYD3i@MX7X`XSt`Ivu?B;9b zyqZ#3tsz3`DWoATIWXDFy;z2Ih;^gqU26^v*j}H5%ji5#cKrlqEzLe0IpqbOCMsm> zC+xX=XC+l0IPqisdd0Gzkakbm*BMODBax` zA{i&FV4+F9AJ}gOGMKXt?PH+Y{{nhhR;6|E2hBN2KElw@lqEwx()Ji%+~qXIKe2w> zLyS(-I#2gVkZ32)QA;B<-}*pIrFb4kHsCC(1D55A(SBfxR(DF+xcLqE5iX)UO#6JY zg8P;4$E?Mhee>Zre&hB{D;iYq5 zGPFDWC-3GX+x?ze=EA|`+}W&3jBbBq2I*c?KCoMV0i*SH6##vfzW}$=cT2hIy#OUgLf8A2F3h{I2R#ZR z@D3myn(8LvFWu-x{iQ*6WvCY5f)M*p;O}&R{r*+}kI_74*f^<~z$*<_fnX&@b3UZo z!gdZZH3qtS@@{rc1kFBiM);`gHNfiOn5nbB(NOLH4CHB`&2dh+I;$1=Bfg55JH*5U z2grw7c}&L`ulc6EhJRt-Opiy-ho?B$-vJuUFVTCBOV5701E)p#B{wZUe7Rr-!WI%s6Yc(cj@NN1sS)+M`gE(a3*9Qt0O)C{z`*l-OMr z=a}C^B<^x+yNwI-3|GFqmOI13V_}nifqOOOR#t}6HEF!Wri4XLinL$YN2Ta~w=O*% z^B!phZ9@jN9sfD?Ije?BuGi5q6>>qC*|rKR@vfb$b8o`gLNF{j-{yBPgQ%(*yMXyIn|o^uo$m-laF)?-Dy{o=;{{ z|At&CQT(JwMR0KNcx$x!;S-d!ckkZK5y0bknB<}u7-hmjLRggFL`O%%hq&kZFA|r9 z(fVu5r+drL7?*+ix91soc{L7`Qe8cC?A&Q}b@lDIi?=MVlAOBy?#W=0)xfgrsZ*zP zdjwn-zy7R!&d9*P&>tT_FY*2>bG-XUf5jEmoM2%u&zYg};)90|H>|s}Yvzk(sA5vu z!dX+D7e)y&J~oW#Z322@v4xXSzsh}dboAqz)o?{=jF1O^zw2 zpE-TnpzvK!<%2-x3;ZsNNjsI!YePd(7X;MxNnA z^&GuI=L%N!KZ$y&aIV&JRVhhQc*2-Q2yC&#qk* zQT%7Hl$%XGTZf{>*3RzOz^BK-LZ<8UzaBa|I#R4*^eSC@m!{g-yH{X_Jk)nzc!@Q_ z(wj7gJjAHyykB)k&vewAUx?*LcWlxKx%0AFbZ1hmNd+^>_5Mhd$kN%!&4s>wY4SDT zS7^^_u)NSwx7FX8FUG327G;(;o!93Ru#4DRh5r4;^6^4sMSe8GJE1w4%PvckC(E4| zKGx@Z5oVY%J$F(#^sA`UK8pE1!atn3h*?|8FzQH^r&)y;(QqGH{WE~~U{%dHx7xy4 z{en?GZqV>ThBZw8hs8D{q}SB5W9mJ++WLO?6(oAZ!u;^*8yL`LqetVq)bNC|9+s_b zWc@t+{{6dJ+Z-l4z3jHLo127o_{-zxnZCMFg0&?>R;QGt1CasDgl}FyVL{xo+}+vc z4^XI8YKs$Wy{itZ=}Vk|i{Xm@l+%bu50y@2Gq1~{ac^q8r#Z*2*Y0L~J^v@`q5cZ_ zmN53#Mw&a0vb%S)aRJ*0zEMX+pg*);PbiZQ3} z3fc@;xb)5rmA9g|=f9k4ok$a@Kcmc>g6qoGy+fTpRAPHiSY=v0MfO7FP<=h)@AZYT zA90*$PnEB!xF_RZ19QvJ_$Yj?anqT~YS+2y9Rh~*dbWP`xE`lbuKh(yG%`+ZjrEhNzm0H!pFx)=)p^&1v1odrCZMycQCw1?)TU`_|vkwsa`u9 z5+wCFr!Orn*P8R_#ot}8E-H(|cV~4rrZ1%ubCahTBUrM!_4yL``_rC;F+C|xCodyxog92^1Ght@v18tpB^c` z%bro!_b?e=?K6vi`DdV*^LKxt1;GOoFM+nKbX{4V6BDiQAwmV5MhtntiwahhPLXBg z#wl%KVd1xp{uj6#D%z5yf-`p(GB4kIbs|sW72%7nl{%e{G47OA=K|P=y!x>(BwRlZ zd9zCLp>Zl99x5LU`x|R(@9rwvNP(eS5t^&s^$f8B^Lx|hXvY1IjI0J%^{Xm|C0@GZ z@ay06A@$1_ym|8`o5XvnNmtXBFgX&guxFQ)`BLTM-{)6?fpd*M7B4C)%GQl;4q1?YRYxGZ~N_@tpjHv=|qH%Jv>)uP{Ky8gzrz){LYNyC0#Zx{rceX{B z4d=MVK09<=%Q7~_b5_p8;ZHx$Z#jMD&h7hxDh4id!-5!-N?cUNlLuKgU1^G$xbV}n z4^b!vm4n{!f4ZDwD7mawmD`FkcYV_D81HOtWE~bv5sp(;FkzWiP{B}l()4C%n>w?G zDBxA6KWe3Fer#5qXkK2u#Jf-ZUf#q-0Qn+C|DrSF`f{%*p5J$JxP^9Hh8Fp|{ zwovTtpAEv+Qd^>xZ(5mP)U#MV`$-J}8uQIaZnMm8<=C^znAD?ZnZ$Z;IsVqD)I(oW z%_5~&&MJ`AeV{6nSMsPkGZJgD%1`ux|8~mh9B{>X{A!iwJMkG$`Y$SDuA@m+-rljO zjY)o@rdev2>_S6+Sj*^GgAidl@#?_m>hItVGsAwA*cw&(nDJe z-9$$VvwWObQHkak; zK=1i ze3Ge}T7rX4`ba8-Lo=I7;1-f(!Z}?-^Y;&EdH)NI&t|hN<{SjaS;^igZ6<) zvGZqMj$Ga;nb#v1d6Bw7lRNHh+ zvMV?;HBQxQ;>u}hF#;}1q}2RL@x>NB9L=^FsyZ+|&}m6psV6C~i@=2`&(D+hrp42( zC}f=oHEWA&icWe)7aVc;L!ejH$mE=uPPVOc-p0^T{hr${v7bqdZ4z za>`9xFY2bQjd(3O|S;Ul3b=-h1w|=2=fS%LAd)X-`BCsb$1yHs*)%%lg9XqN)dBW%t`6xg|`$ zMI@9bVY0E%%^kY^sx7kE*i8s2I?A2quFf&u(B(v-hAFg=n4LN(qG9lk?P%S#w%05S z3cB+Z6&1tNeQr~+PKGy02cs`Da};$&JcSU$zoY&p9T#+;mSei@kumAQIxat&)CQAH zCoVoD9MwzJ^4B!qK#zYVt5gxz?-ph3bhccmy=kqtURI|9$%N6$9kLxtZN(vHRSO}D z9Ri;7ExK?8o!u3?=u5Wuu`B#tG=ye&y)4r2IXge~3(Z4YbF};}@+{jm1u@=;N~%$l zJu8}K9Rp#vS6064khkh_HlDk$${wWHF0!F_Rb_=`a*ic;ji0=6!HI*TtF15{t3Pg? z_#<5*;hA1WS!5ZXwqX?IJ@h4rE;DFpIRbM`1vtywi_F7fi2aHkUdqSys)}v!JY`y~7u~FU- zcbp@XmNkTlGGYbMgyk7RDULXcsV0|arQ9eTlZ_K9cbHMKkBEk(=%9V9;ZnJwIT}OU zGtceJ{Nmrd6RszR6AW=jn}q*mb{fz1gv;Es5U!N9#KgzP^DQflHPtuLv_aSxM!~K% zJ=K{a*Gcp)g8%+yq~&a1Yk zXf^p!@hM4%T&0tXzEvnCsM>NByC{clQ_U*wk}dh-b1GbTcS~7+K)cv_29Igbwt%#2 z_LWtZwtc>FQxY2AJ*j4s5~gAv+i&WNdtiR-ao52!IJAnl8{}&q72r8Mx2H`#J*mwr zL|3>IwY)s$Yf1RODj7AWuVsJ!DCo92J)2~Hcxax>q?zG4$*Jx(NTAa1eLs2W*7TT4 zOCzKMGu|t(D9VoGAT(w!O8KmTy)+v9V&aPyWV}zCWtVmSD2wZbP}s!k0XmarlS(k4M`kjTh|4Bl}Aa$N61ss zM5F{W6{lL&RHjq4&^#91vOfH*B{yG&hgIPEGs27##r@v2-yhIg)4%^xnw3`dVYaz$ zZmG>hl zb>Fay_TNoSTUh978$632;uiHYr0J;tn)#%J)?BlaFLF(8~RFj={Lii-Ec0lF1E$ZRsbinvX`rH%M!JU5>(gUl_=qulXN2RpSY zUxs#WSrx9*YkQ@~Zt~lOrS;IGcT*q4F3$x-mGa4iu@T%cVX>^|REG4k{FEb3X9s^2 z6({Gy;6ot1_sGT;D)q=7%FSpgEIXC%#k}8Ffw~cdHz!djS~A74y!7h|Cr7=Orxmr; zNpkskJX3Hu-E<{J>K#B^UPvHbGaxOk>6XfQ@%w&RBq$W>Cl&#!M4o*#J9bkwP2sM) zhliD}F8qeJb`S45`aUXMO)fj(a#)%a>(=)6%0Xx%3Z>_VSZao=8;cX2+~{oGa<%s~ z!W_44hC{$i%=DpE=X*XqR^EVQZ{fYZCz4cDRCV9KKX`TG!lKrM^^H$S5bor;wa`em6U)|z zH;G5$h1}fhh~-IE%hH-$;o#ulvY(W7K`_f}HOi?EW~^)Xf9uW85J}-}(*X~6zz8Qp zMyyy=z4ZN#Qnn7?wmT#_HkF=#uRKa*Ko!GhFMe6jH7P!vO|xTEC}kxBbAA77gKG!9 zZJ+%5e5lgF-ac3*T}gA0S-Z%RY!%^8;IO1?dPlU*bclompA2>j>!Fk)n;s4b-woDI zIB-RPvd&z1H8)s#7LVk9aM1p8g-)|*QzfG&7OC+*ocYHH&q~?f9+haNd+2A@PrWnM zCE>IzzdYeHo;H2|Z^}#1%!smWA@}umoY1Mu0_`nb>B^01-@LuOA8C`MKo%Cpb6*LK z-#DqQUfaH|E2dOi$>3?rc~|N4Hhc=-{rh)b?7H=Ksy_X&eyw-DSzC5E)*+pLk6eueWwPpQz}JD_ST=uqzt0Eh)X=Dnwo*Ur|} z0Z!dA2E>WBUE(occ2)noJe*G+^v<~sfFm-`J|P)gk>@tSH#RnQYq?WC-|>&u5&;~G zRO5T<3CKb_lBCXh2<$(^;F^@_nE=3)q#(_2-G!%A=sk$OCFi-j?X*OIxKtWqjFlyb z9-APD115n@u~}Kyt?}Ij`bxL8Fu=xSlBGi#lv8B$jT%Wy9sfklG6?{fLR5VcCL1f9ER9dSVe88k(cWkH5v4+iD+-6!MjMw{>z&{{7H*?A;(Z8vggrJ zw0T!h2fa01Z?ai@;(*@P$AQZzRM8%ef%MUTX%8X8ygyLA-H8u1&te&$%y_kS zt4gZulbEz*WB!gP5r=8w6Z@C_#{t%Vp;!9vNB;-$f)oczr{qb(`ue(L|EI_GSgb1W zi&r483j(~np)Awq6ixH|uP@2QO+i8YXmhDJfl@fVkL%95c1voojo}oI|@v*2k7LbD-QQCsVul0R^WX8-Q$~qA!m%0a`GD zWI>oVfLhhsiF;llr;k_Ji6lX_qI7;Eo&Zrc{O8F z?YW%eEeMEwhD0ldq=Ny&=J%r z8llYajAXs{RD5i+iA;+}s%;$1u^GwXKpThLahetDO%bgFlxX_sU4Te-qY9BM431F_ zDOqAWBp6s;T+A=?Hz%U>9GOnbt_D=D(`dKyZ-BAHKx&VxFk#pujnbYcfU!55Ma{VQP3`KRUL$KKSpX(n)`LNA4;B>i+znZ3HcZF?$qzd`d6wDkBygP0R^2T@emHYF~yC;-^xCYRjnF-*| zvK}nj{;VOYWYeo~{xqRm<=(e7sP*=4=Qxe{#o|-@Bjzg)vqkm17M4dHuu;#DKQKAE z*ip(!l5~X@J@1Pu=7ZgS50wKQ0qpUQ7An3oJ>C`jKfNBtK!##9FVZuqdrf9H-A=dU zie=4(XprCnxZ^kWwGbqJsdC>)f|*Cwytuf?Me+s^tB|+nImLU0oMv?v*4BF3vCK(ZITMJy7MEv~ni;x~$6urISPr~tGc2Gt^0y;T$1;$kSXo2fKEB?QaF1j=m&tzeTbWFr>)(|D@zI3? zfI1+D^ z(c_hWzD07|EsWkz*U0BvyMR_eBv{o>_IB@wk2_PyL5iX4qnIe(1oEdbofPu~jcQL1prGE8 zvU3BEfC#~Csx~J5u$TTeLWSr<)I7a>oWPISwcmZ*E+A6607SMs;MZKP z%hQzMed)@~Lmw3mdBY2ZgN7}lxYBt+6qvTP{*}GN7xGe*<09lW)B?^5#We+!pwV&N zss{YEBJc)DC}iuAUxe?!bc8(mM;j3Inaqs7zR=VrfK41Msz*FG*+th=8Dzp_!5!(B z|4W-spoD=m6Fe#j4V{4KQ?UpW=3-96!=0;DMT+fs0x~6n3q~#Lqa`OONQ^-%B6{zh z$qEj8gVi1)l1b}&aN}NChzb!oUg3!|8=67K6*(&!&29F81>{qDISo_FXU#!Jp$hO7 zZI*U1g}(ksk82iSMR*g?(=dPzg<_2bC@P&&yZdt((I$15ovl)9qBKO0KoW7Bf-L~L zMFzs|B;aNc^kHZSgjL8P8ASAf%WKEIDfC+SI~rj<`Hu-Pf^KTUU6y4K*_6JcyfIp+ zKlS2?Jq=|`B;=Bm}w(tGMV z(ue|QoDM3;(iwyVTpvoGLtb(Xkfg#Fh3svWXnu;wKOa#V?^u;G9^J@U-MdCK;|xW zQ|!OBRAei04X|4{3Ok#7AWB?b1R!5+?FRan97y!uwJ3w))PQvNbFt;`!O{YUx#6L~ zLOy;XBBCEfq$);~7wi2`1W7U6Ca5BCL?2a)=`6jpbN6eeyJSv02 zJH>Wc<;c|B9E}N0p>Au*CKGYZtCN>-*R8=kjsE$=4oMecFsUI5c9KWvf3OBtFN_e= z4ZRi?5D;*6BK^WU)mUKdIpaME>9SEbrH0++1o;5WqY`r0m`lV_byB?i`_Kg9n%&Ln zT{nwh?+8Gi?~-_oc!I0DxEOixby`o@NDVxm!I;|)+u37dfRFS|Am!rB$K z&#cS%zJgTHZ_y7AAOE}FerluoMFEIjp=W|g_JJk`w78cyHm2f76#ndAf_g4X{CCi6 zYS>Zq<(}JR2M->co-*(7%hW9BFO%NCPUNH65`{OAhav78i!<#xvXNYbUo$i5%v})> zU}s_eW{C{uw@#w&pm7ZG?u)(%vfQ?c$S1cI+OWUN&CCA@c_odBeuC{l&c{)ya?wnrq@?)|zZ^yod1QDi z*a6fWotjE52K^}hpDdf|u2It_0AdkI@ zFxrm7Aa_0g_f@YPLlqb|ebgorm-rJ5YP4b{=KXJK9$8YL)F7E?TfSphcLcL}f<{PZ zP8nnb#ag%jUUHSna^*9){AY{xjPi^Va2 z;bzbQF&ra!0SWQ?D&2~^a`g3ebaWi1Q!j&L zx#Pp#1AT=S;h(Gqy2Die7Gj*?i7rIlMCKzhDIoQx10{4f9??uEUU>Gmvl-@5?P7;K zL3rHNR%|uE1U@{_Y1l=gn3Unf-}{Uh;sv*6(mpa{0?|0D&JmBGL11#nj&EtLUj z5(9{BCef_Z1qB7c5WU@pnujY0-t+J$*ak~gx&4lyRV?VE^fv!|%Eo5y2p%924j@Lt z5L4-827nzaux1cBeNtp6LcxIf#_}@B(5*VI-pXp^ZoC=Pi}2m=nt5JYd0jpeJ;nK% zVR`2G(E{{+6iVbXDGE#Sb>3+6<*QfC2%-WG;O+GppD0Q(xnnSD^ZWua1ybKtTo|HT z?u;6kZMw8)8$V|7l>=I?E=?=E#T?Jdz%L)~(=;*QKk=kSbqYrrt2?=qzty*5V zVfdmeLsbS!Ino+;l?!&nFJEg1j!X>;V|w_}M(XG46WJF4qSOp3=Y3t7ScyAssoWPQ z5LVY9WY2LUqdI4(08M@-JomcG&Tn7dmjdUe&iN<13k7~?YUH>3@6puXXC3!6EEjI zj29@SRLwVjM?{IYH=Uy;egFN6os1H{aPXj}4w8yVegNw?*&HeZ?v>G~;awo2O8`1C z*=D4QY!w!I@HrMGF{qg^LgtdUn9whui%at8O;z^YGt%oMlrIUfc2Dfc!vjvcFh4J_ z)@Z4t1*wI=Eo!22k*CC5pn1m(fOhL64a_Rr1k+8`vz}=;b{&fKQ8qD^&l4G(BvZ5t zQzYYLMdP@t7-4i;y*mHEB%pj(In>fFe|sx&f#$UQ&gMkEd;{mjf9eBJ-X-))AWGVo zeIOI|e?lPYnjmplVatWF$`ewfK0lxt3G%gU|57M+U^r(Lg$)u4Dsy2F^DwK(9e z+sL^sRDf0m$%o%*M1Y9$RjCPW;UC0od4jIx5WJKDX;C)vr~nPWi^530!!Ua}t|%Lz zEN|6=-tdJ8710gFzZs`rY$1uIKt>H(M z)-a%q$F94nbLa;N2?~ei*N+4f^A$*A?HG**(-IaLZg$}|?|3f1cK_9*ni0azbjZF` zCjA*-vYy`?F1xQHM^BU(dY&xQRm6AUUwXkiMzkZUQdIu#SKIVX0(YtXT9vblap>Wy z&PuDpN${szA`lGTQ>YvX?A8q_{hN>iW%ne-cXV8!jg zF!%6}L8VWK?O4N%NFKxv2$hb|>Yiy<5RVmWrGgRK!x*&+&5;B_D)7q7VMkof*~OJ` z&mazu!%Yj;94rcHuMZAYGdKEgvnVtB2lG^OhL5Oa27}kC&&X!p<>$C}cqRPih#^MSYZZe2LD64Jy zAeyaoX*{%{Ra?rpV|a1;$34wr*W{9tk~cPgTx|ssfmoZ~Lpg#<+S{2sMuV&PxA246 zk?(d_k0X4?@akGJD*eaSSLagHY@|)VKImqVn z$sJlm&hk`^)a-Z|_g(RdyzW|Z;vzve6UXm1d2^MA~#rI9Fq5ZGwO@*#E_GKN!RX5cN;Sl#{qi#BkVSK z;-jRb6o_Z}6|Yoit~xVVx@3VZzWe+z*=OMiP^Sk#Le>K7)!~|m3}jb^H@1}XjaS+q zYOT-9+d4W1)_xIx`~3OyOx?mO-Dq}=wAahHff1@57MI0J=93q1ceXkG>7{tM>uV%| zOCC9Y;NuG~>1=@w2KCvByln6RlKqc}fCUJsvl>8)a7N!wV*w-gXLKOVQB?{brZqc! zJP6d4(l9giG~+H>$oo=tM1BHRfJrvu^b{duEGrAN19=-O^cFyJeKMip&y^?rJ;1Jh z-~hw;f)Q}$o%mp8r;t0vHci`ZYYcwT+VAi<{xJf! zvOWQ=TAzFuETA97d=RM`t?%&)Pf+U{FXfi#9?~$6K88AQ;6NffCoSzy6Bj-^jX`TG z583KP;TSpFB*VsEpka~WqTrAUr#gJ-(Nnnx!h~FDlU)LBts>LUoPtEfI6EdB|gl{Hlb`MBg}_W zuOi_7PFS0+aY}%aoF`a)gOfzGV93w(e6mI_EPiEG&lw33dM+VxB$z{oG^FZzK7!7LzyU!+!dlGHIdNL}O|$3YnPvta*BawGApgQ{>}0fYt~E2DACy1@0s;!>+`E zAb;B9TL(*QZL@P&3m|afg4&#>*gM5B!d`l-vqP9-zsrJJNGTI$@%NuU69sx6CcQt2 z7Vqa|bJ?uQHcf`=X6u~{_(HNOwcXG${uuSWk+Gua#$?XvjPtDT$A!u8XjGq)A0FH^>$OX z1DKJIe z+FiXV+suwKY8J8}yOB{q-VW48MLB z?6Z(1;gomQ5)Q!>M~?rLqIFO8AVM|x7AGMZ1gKLUWD&sd^_Q{29epnn(ZUBD2LD7$ z22DU)j)R(M^w+Q2qp-vf`4n6r830=3$9sw)N@87a#3I$cb9tNm$wo#hwo|)f>KZ3{ zz@5pB^s=90OcgSEp{%M5fS$V!A%UF`BHu+SR)9!ngmh{YDu$5+1&N`)2GZ|UP3M1E zT3EOd?gf<}j|WTgjK2olh3u6Mu;_3=BP2C6DAmcqnCb7e=lZ65`=gn~`qSWEf(TVY zL9K#sGl#Z()^8Fd0m3q0kxg&U0*e0(6%`xMgCWwPtiF|%mBU3)b1?%L4V9p)%CO56 zb)`N;&l8YVcI4;_Q(t`b>Qw`QHO;OigoPe(vmhK6(*mS4r`h+IKp_vp>-cz@{YE$t z0dno#=WkEZ#J~GkDcSw;QrQn>&oIw>4J|%2o>xJ;9t`-*eLf>Z#Aru5)bNRY6uJ}w~F1G)c@97;Dk21%A``3C}o(k^lL=~TfP}9=ZL*(gB#gVb z*q&m#z_AslDR9Xus=Jfu!$qeyB~&kjXX1ha)M+8c-r7be;T|kLDpy`eYPe~ zHAhbn^E7eK7aHk8zV1!7*YX-RpKzc0g23&O6~I@+ip|=rSJ=R~sX?I<15m=bW%3FN z3Yj_2!o$PsSInQ0)%TBIftE;}4EBZ(+J?;5Y>eq@p?D#w$}jz)GPDU~!*=*?c9+F* zUyf@W!CnyzG7S=%@W6-TDs3iAD*p)ZRvfGkzhKlTCkhpd#{V_Gk{r& zS|l*lX+0MTS>O@hCOPy>7A(K%Zk6j1*4OKBmP4iX5}c5!Ccf}0gV2s|6Ruj=-_pUf z)08tLt9(_0mwwD^>;5W@5C`JwAT(^iSxz(sOCYwlH_yoROu#(ARxH3*kNt+^w~l*H z=~a`dJiw$>*~hF@C2PCzHAwM6j1XtzgE%bRFyt1%X|Uz-Ya`+mh_0Jn96gJ$^-2k% zhlVaHv=ZEa*5ANka$u>@>A;fob{0D>0!@qXqaIXeIyJ{xTYJiQqd&`z z*tf1=m%5kc6G%+QqLvv!1&oVYjvhNgY(c5W2FPLmz8r-&+$g|I-9MsDbjq=HnVyM5 zuY&TA5LCY}B4E6Pr71;`n&WtOUEs-!HzkGP>iIKBNygss|Ne#+K{}|CC%;4&78U|w zX1)O}ERqQ$gaH)Ld;H0Vx_@?PJ3$fZ)Nd>4?>F-&njsyP3tx3qxY%z1ikIXt)1g#g zDhCxQK=e(7!`=ntE?euiF)dX3Jwv`KoAju+eJo&0qXwqo#ym3i>+fp z&!O=>PVgEiHiKgCI5opw71UFUux0d|f2{WlSpAd@bC>Ak{;qe(x_bnq`jZdfJ;6IFNC{IfVWfuAd?wnsNL+xk*QHk_p{lAdREo5`YJOOq{n}N55BGd z98OMrF^Hj?R{_D6fu+xI^QMab*!fE!66wg=K{U9atZKc4;f5xksZtzE{Tjtyp8Vv; zZj}L8K*>izVkp?H;9;d~I7cj$(|;Ki0v+tTTpL*^<-G5Mpw;If{JO264ZmlgfD}B4 zTm3Q^?K~uTM+G^eoc{+sQ-nsY!BaM^Pm=z-u=zsiZ3ShxZs2GM!GD?Ry69pkhB=#0 zTBfXGB3X2Mb#aXI?KpUdJPsQ`t(7op?6&syonzNAoH}N$Q7huOj8_JVmAwC&m}}vi z5{lCE4qzi*-f{dL0Q`zn!@CQNn)$|y0OsR8E(;u?*xj%nFL>}L$@Qg@5`+A$?KJJI zig(m}V$U*9oHepKA;L2prTs8ckXMXYDU;Kx-s#Sq09*hY9k{H zi4x<-tFx7oEpMhJ48)?av#nR6WiuFa3l=1f%P2j@m#RFLw3m6-j zB{pRjv^a787RZPZy!;>O5uqRyGQ;dIrOM3Xc+PDG55kEcdNU3dm6F#mmF3^lY%8ZcxXf&! z&K}Nqtw#zn*efAcQH^zUk&fQgIQ!9~M+0Tf?}kNoGRaiAyD)B&WF}i0I8}HK*`B!L#|}aH?WODeO~Be`i-|I>Z_+}YxNkOV`58~in+(P8R*6eq(0Z0 z1@BRrww!s?D_&IuLoOjkU(|IWcPQqxTbtDt49Q}L8o(N>06lGB!M$Nhmx6`_lgEGZk+HNZFn9&Qb+H}D_C8d z?k8dGZzQB5BKq?mF=05_EF;G#Yuo2I>4+Ci+%8ccv{3!k&R34scaUSom1HjA6;JSJ@+VGM7l8P_^zhZO3nQ0Y8b?F zV$N9*aYzDqk3KHMxMh1InYa#6)%zN$pKHK`O>>&1%0_+VTkO4QCki9%fEX_t$$CFe6VV_u(QQxS7@ zAJ*4<9b2*Pv{_y{5`5S%x+%xrE9}oekyL`=?ji3nasMuv>Z${E#Sgw5F=6eqpX`+B zB+L)c&si*P)qA1yi`wuC6B$eMze~Sn`8-^_e)C(vTJVc^ZA?c=;^p?_mIzZ09J4-X z{1MPJJY&F@C&enoBk_{GxFYv)2|(~~4>tp0)H=|m@f2*Gi=Pu1)ze-@fA~Lyw_fTB zImhzRQO;VWJR=D$!|pV9Xr_(#ho47gL%%X;^Q>&$V1Bg{8R(<|l%h`b0dS1}5J|CL z`!(J~J@NgIw%zWqpw!qT#aJ0ZWr>F*;Mc^rSP9PEjj1S?q1H>gUz^T0F4GYGD~`ep z!qoaVl~hz4APaE~IwiwRM8iA3_}m66Rg849XDDR=d`)dkQ0$6ehn62z{kamJRbcn; z5f`Jug4zzVZ1pVv^y%ip?t)QO{pH~*v~rn8$6eCdktL|k41QV1gmQ2y7Wi)89Kz|A z_TtrMOq!?!qfyGYAlAc`urC?PkoDK{b6<&!<5uZPoaJiY;)R8qSHY3vD7;yek^%s? zlh7UpiSKf8ZP;bu?d9NMiIZ$rTIYk+Uc#@Pxn;9fuh=^}7P)@&uBvocSI-n<*?2sB zrO#tMBxZfHET+^>TNtD*Tg~}dQep3W+hE1~0b8LkGg4ix_nz{GC~P%gte5kbL09K) zNQN?DQFZ-JHe0yA_(b5u#EC2~{URZzh5s1CkzSd0RxM3g2sV=TJ6mbha(HOPkznb? zqMnzZ)E4kVfUG%D;?%@s>pIPs??YcY2g!MPuU;M22cU?7T8ar}*_)fN3@EAmYQo{9 zY*kFyWfCTE#CBlg1;gaVHUIp>t%G$R$V4KUg)F7MAlK3-_55H^{qjBfqFT!~bD}h| zfcN+juAUyG_dWV{+vqCzu?jU0F9@sD!!8X-^>>*G@fg`QtE4laOE{m)sr@V1GufiA zfN*nfx7g#$zsk~0z_cDMb%YT8h71?chmVJ&SJ&3Ukv$U-9t1XA>K-|bQF|JF+s4zR z7=gGG&9#cl5NsRMVO2RD^hmAIvOUl9?8k?Hs?Yo@9>fkH_Pbm;x%M02 zwR$pgaLEZwn#XN^e7x^Ud>YwvzbCsQEtWH@thj&S>k7 z>&E2hy}!hx(w(J8X0lnzojN|)*E#8xDieP<=nj8#R`l)h;h}PytT=cBTd6a_-Y|VcnvWP9#LqdU@Zb%bwa&n3k@+gC_SKM{=Hw)Ai2Y{FYcT~*rcP|}m%M%3n ze0?eG3kEoTGo&JnK(Ce+H$auqN)c>EB?H@V7^%iVcI*!CapIDINYyk}b+R>%rLQ8t z1&blFTdxsHA1R5X>MhqMK9DWxj#0#9uY24OY4Ex&TjsQ*Fx(ody;!NPAbRg|3&`ci z2O~SV48Jinh2F_2J!F8z@tFL{An*8K%E+SFtR1tnt>o70WE=nZ0^pf4LFNdwi)xW3 zXTM6)vnkUzgr_eI;sfFJ3j*&?Opom9Q+{LeC6qZ-r+CmYt0H`i10_@t6LQiJv=_qdx<5_|Y>Q z!xOCHy}QxBjFRt1le!j}V$GlYcvgyN!wd}1uiiR*v)1CqNn`5M1|9~pbt%KUR~&O{ zqbEW}7JDU~xlQ%?BGMit%j{Yhs(=oD&#SFwcYMPXI;S)-Wk>2VL{_fM2?g67uZhTS z(-M1!v#g=kR;T*`m3pNDWp28})(4v^gtMEzv`@XNh-WZ%za_=HZiy7?)p5=;HhlNk zhuWYc|DUM4q5Y9tP|%4;r|YPQXyF5n{}uXF07V-uFJ|nM4WE++{i-}R8dMolp`8(S z^HI;e$+mc#;kL#`3IFZ3H@OJG#;5?*C|oZDzap0ix&<@7MER)PU#h7zp5Bs+fnv-GH8Yo zEedH>;DX1SYV1 zuFbTu?X9hE0}z!n0O91Gv2Gs1+$IGyoDu(ZpePF!CDks;5Z~ zdcz|E(ArKj1??XnvP;e2&Wv#zvYbOQz{{uF`&R)W!B1c@>hziFV@Fvb?k`%`f`W%B zX;>_EL`1aka|qp9u5;*T|IXvT1S%DZ9!%4nOe^)XFRw!33|Th`sGm%b>X@*{R)TQ( z{+0kJThKn__w<7}&3nC(nG3#n9WJ5fLGd8A{82%{uV23s!q0PRWj`-mfv=o-h((Yq za^y8VD7M0L0)6~14~)N4$xxBvoNA3Wvn`LZLd0@;5_ zr`h|w*XZaHki83?1(lJ`HX@lFuGD6F5JPG5ECAq3yJYzNXbl|)3A|a2Mwf=iTT{Q# ze)jU_{#Gx^_ewm;Ix~hrLRJ@HCMv(*L2C_mr1%=-C+uoe z)2DTn#NSh0m@~_`|CXYxFdlF`oiYchFiw3AWB^x2U`mxDp^dG`K0K7{fDWvR_ei4& z>@#CG^vo75Q}@kIMzTDt6+pM)P+QVK;DsM(i0}OX&i+e_5B~=dx;GujTXlc`{yjcg z|29bB(_=OLMPnIGAYDRUm^axhp{c8vH@|hO!l2k3A)-m)4mdPMh(Us&%t^NgmXO4q2O~R`<;hDv zSxK^i0zp03o?oVscbTWv*qUk!Y~wEs#*s@)1q$>nCt4RSaygElK3e7_+@JqNdF)U6 zO2*c;a89ZCxOnNl(K)H7F`CKdrDwQ+x&nH{d}W(j*j21*9?z=UOcDI7bTC<^HbyB& z{Qd<#;|~W9>)fb*)t{Y)#;tq?Y|qFNs6=sBl5AszQ16q)LxBN1{ z*jqN(CN)DnhXX`$2px+v$N}Xx|3TOkEaTiYHCa*`AcPs_5@|*`rxW2 z06N4b=z&6gxr#Lg5$&G4svt6lDO#ESG3Iv=w#J05RYCzpjP8MQM;TDLC&R98$Zn}p zza}qFlx*0WHbHokgx-9wSma{b0>YCrv5FS80OO{e%?G-EbGLK7Xs8?HC#+k1SzY0Jm^x%$)r#Y57#ag5^gPir!AzX9SkGC&g;Bg2@lKjO$-{`?v zi065yFbol|*ze!VOOxyaI9mH5TxifOUH(d#HAK}bh{zoZ>y7kmjxE5xl?HnM2Xk*7Q04lqi%vvZ zK|x9-6i`A!B&1PLN~A%Ugi3>SH)0}4he!)ZOqxZv3P>v*!XyRh?l|LPt$pvgd;j)6 z``ml}xbq)aESZxzzxR9JF`n_nHuxKT2Rd9D?3k*bq>1R*vvM2cVY>ojqruDcjhY~M zzxvYW9WjdrMso9F)~h5IDAyjHIgYrYE#~-vO8h`AE?&=QCI2U)O@(>oTPHmdD)y_@ zds~8UeB={6m*rkGwr{NKiIm(M@R=A0b^o*7;LAFm7{;L_4ppp8HX=w3e073}^I*^^ zPa}$e$&ykvKh5?gI1xFjo1-{mK@9LiZCn=s=8ZD z)?yUqIS%>SPJuXs{MuBI{md3;xzBP!s+~iCot{_L?p)HX!npGF*=exGx|OG<_d+S* z_GhW@CY(dxcK88$>XJG+>27fcXH#|F&r;8IrzIzUs=p0=)O_099)>>89(c;f`d?cW zfUJ{)#!mp53?RT5TuJ^rGCm>?>o*_6ez3`4sg;|Uj~lwbLO_El--d(i6mcKkXsffhucsDJv7+t@b+4$Hzl*7IktugtfBmb{V>_o8R8#|0 zC!3~pD%PS_+KHY@{SrG-y^_QsGeSZ~be+wj>RM?2^4AtP>8I)ux3uFDdkk#U@~{^g124;c4Z)!2X3!wJEVBg0@z6H7srV z3wFk+wXql|g6jCBv@9u&ilL>M3frR-wa?!7SpATu7k6d6)OW-M1SCp=H&b1i3NEW$ zS~tHOVb3sda1J^Cn0ZuuQTMOMx!!QAOaj&R5^ls;@dy0Rq=bY#Z86!~wQom^S4@Pa zOGW$6j|L8EkzUJE`_py&kp@xj7{B!J52(iQty^#=m>2q6no!G`*c?_W>7ed+iqB(A z_Y~WncAq+T9%Nv;^dr?=ID&pgqYV91Pi7sYMwQ5U#bcQ-dkXj){4E(boDu(*P8p{p z;ctV-f3Gxs`@%!zP*xT!XSiSth$se6sy#*(CZuaa9%tI>`zVobZXi5bgb@wSh_~;+ z5AvR$xc3vO0*4noa^NRYNhCb*&J5=xc-+$fQ*AN}c9MgDlfTC#k?bO;UJcDvSR<;~ewo5@)6nAB<8z7k z8JeN=%t>IQtl)n(fC#$Mpou|*7|`GK4@1Rp9Z`CNG`>Y-xhiFw0|bKLHm?I3xpb+o z;5qZi08hgjHvB1Q)m5wAo!r2%7CEyKnmm*$|NIdrmGM>HPg`ZB?(~HcZV`-~`~Pb~ z#1gX8a-v|k#bAu~CucE&I~|lfbWaX}s+9&EOriIo`^epVU^EwG&4bsR5zv{t${kGC zE81fP6+q}=1oyu%;l{>B;6D_Fzq=YIi(zKya5w%8-I1qi<^~h^Xy8CR5=A3tD$Nn1 zJlBT}Q!Z8@2N3~D7?jyQ0h+-`dC;^y&R}#e(Vy0l^#A99*bx z&eiS=U4TVx;y|5O=qm7=>qw8=%hU6zQwhu(Q=*|xQ~4sH2j9neK||nr|8HW5c@cF> zpWZ%GT1a z(FF=NgU@BIM$N3(A0|c{UhkYcX9E==u!Eu=#pfV*0gR&&ZxMujiukEopUp5i6my&n z@!af%djI=5gz7CR`U#H_zP2@zE^&mv*^V9?BPjG-)I=Dro*dEPoC+p$&7u=`i`Qa> z96q1)B=n-JfF4s(Q&Vfh)Y)@?vGkuece13u7hz=DOp_fdP4>=HKzS1A%pf8nvOAj+ zyul=Jiv-OfVy`vv9slgbR0eoiSO1gx5Wf5WiOFBo)Tugf>VFp;9GnUn5W0GLNNI%_ zWw`Bam}j>jS|da~g}`ozrP@?;_&hJ-uLhbouk2Q2G94TFe{;sJeNY?ztnJkt&RM-M zQmsaw#Zuee?{v_$eCuQE)zu`N zvohGW1v-nXVGmGc&jxwvjxy`4L4N!j1!_aI@EP^w5$KyZVRH*oabuB>_)-4^q+l$F zie?j*B?a8(gQYf!8%rZ-*c2&kmWkjM{97|t)uvV_WhR!!%@OZLUuNcJM;v;AXCL&p z=KDK=tNiaBVr0-!7I$31!NVL@>dO@+1XNs7&A#y{-#51oo{~qc-k?L*?c3nst0LI5 zM?Zoe2R=T23&3le>j)e!#$IEwMXyr7lGyBUL75+ z6*r27BVyCE26lLavhApsz6^x12;l5GUOr91^r74b6?03~|0|usq&zuD+ViDU7-W0js3^!s-JpGf?YG-n z(5`lLbnMB4D;DzT%%1f-&lNd?!z#_lwJ?!4(TH{dL7d;1g@PTP0wTPDQMmoe*d4ls znAfr~fI}QTNJX>bB#f1@w`=WpmIqbb%NBp|QwUi-;L30|kQ+vcjJAE9u*-sdE?+iz#CToiWsSE-DABpXLsOBJ`E%>k3d zgNq#1;`;*=Cu#<;0BkM1cjW%GBCFRDq1LVeP#7a*Rk$#+$y>##GKBWD?C1B!x5 zH`RH1xL3=@%O)mtH{;=HicUz3`uUoVd-c=xXZ6;vwMC4-^KsY|~ zE)OVZYhb!Gr`*{iBK69_;=%7zc*L__y(IvkSB0XKdkx``=&EqCTo(obrukrD8hsJK zU@c)^+QnMz)@Rzno=e@Ao8O{Kd4^CFypNpxU88dto5K#b<0~#=Z5|(s3kSyjO9h0u zK}(F<^)xqJgTXTE;>+S?2Ja$SjYoaD_xO1&W*Ox|zsTJW+Sj+}x1>p~axi`9KQQ=b zrX5|d{16QwGIb{_UIwLZQq@RFrQ{88I7StV`r?Xow-Fzw? z7`(pmm-x92IKnq>e05p+c7&dK$*$HC%^-bV;_~A0{v?UbsF-Ohr$0s3IouJEtCL_G z6llwBSLi}TP&xJVy7L5MGmG}Axq}tam=tE|l7ez56^lICk`FuYd3~m9>H*S}cVA27Lfa?Yrdw#_PfA$3w>3>& z)>7*h18et=u;&GS5Nf!&&&n7zt5tl@Kc6zaZ5&`cl10huDE$rfL%l?bH*|`7U93Ig z3KhGq^xoEd9u_0_{M52s%I9{Y(YQq8PKIIN%pISqlLPIBaiTYF>YcA|PspabSNl3g z*MPPVeyDZba{uM_1a^D+F?U#f??c`2VcP%5bsp#dV)9kbWEuwGlHN3t#(DE5&3$$Y4xb@A=-tj~h_I$PH344A$ z7ZCk;3(P#*_JKd&T!ol2xqm5!?iEprOaUmq1}5UN;P$*3{i&e0#0rg>*Fa@_kx{Vd z&1A3|gx;uo5;B|cWdVC4zxKr43jtmB|7XwzqrMI^l$Z>EsR*B3gA>fI6;}k6FkSQ9_$jlu3)JJyGxY`M&z(DGH808Z z{<`aAeJfehTZV_7OhK711zQ(a=g(5El$8IQ*T6}0Rn}6JpDrBXc1?*wokVh!T^pj?D-40M3`Dujmc|?AQC{J9rR9S z=M(l74=8*C!79D-&FrA56p;64VsCAIVY2DU@5N}Y z+Jl1+H`Pq+oE5E#bH`s{n}&`$4Gy%@KSX41sV&bBI)tLhzD_+c`*3CXIuUK6A~~se zEHd3(4e;<^{N^%W3&ckg0RFj|I`2{^n8%u92b8gwF{_s9;q6e4$Gnh)QGV@|i0!(jm&K{`{6R^?bLxb%3k%t$XD{DgDQ-LtAnJYh zlN=Pen5Hwc!wPSFEJSPQt7}1NPa;3ZXYEc-n&#<$;C}aDxWi(eOkvVyU`CXWzSG`4^rk_xTNUhEv=dEV&@lZ229A1Y2^>a3wChIEuyZgWN9 z#M?bIe(VqU@H{QZi{WIR=lU|Y9y|95R-Z4J2tDo zp<-DgzI^!tbLPZAI5ej;24DUVPz3k9sZdSRCyf$?=8<){H_#yqFKI7d8~B4X(Lmgj z7qcy%BTf;ob6)zj`@RnZ`HWWpgOAeBeN=E)Y2f@Nu<-kd^O1PB$B6MWSIsr{GwO*2 zJ*gHJG}Isj(|S0&#a?@`7SOvIOUb~H_$V^{i@@fz4UjoNyw&oKg~V`Vjhjoo-=KF* z`Bi$r!A!#}jT%YAz{6Cig~&@0Z!&14`V}^>tihD`_vRdW8k(q&rdbLUOH#j}G(lwE zzn^D)MOV8Q^H~k4`?!7-c48lgLR;2eotuO|Hh1r;M*FCxg$i{7@iMg5>DN~YJKNh8 z5%M9pmI%L9GZ@I~)Vub%&-ZiGtGw<*Ew9h${Kt1*Iw4|g7^cp1H9^N2tl@?n*B~zh z)uoqq$$-R{5~ep>tsl-~TZX%Gv<#{*3tD7kqpPpBY{7xjTwavke*7?6`~|a%V+2~z z=6iLe3sy%Agrrt1{#?U6yg9(b$dJAHuC?U1VU5DEw^ud zx2m5Sz`y&zSc?P;q1mKmP+9ap9|07P1+*537z1zy3K^Lun#lFppG-t&&+>O(6a*DF zW}oyT-&+L#pwWc76*(j)mY2B^llgGB%R=_jowHy!r)fS?QHIYuVe93>z?3eJ5(S5YCX8CsD>*Q>h2ST-)n}6$39z(xrUKkCH=V+)pQP&y zO@EI;BS0GGus5qlOU>mMdsed`%B(WT^=pga*V|eiVL{-u+*~%6fXX+Hb6t8>tr~&( z$AVv*-l2ya3%y$Jk60*sUDHHHnLJ#T`K31{iSz@^A6rlgT)bC(d~bp4gp(X1K15X;ZQgK9-#lLe*{1Krx&hU z_?gng`(#fyb0F!3k!zk`nGrY3a#R+}*`du4AaKWSgZSqsB#(fT5)k2$2)aCfJHh+9%kd;QbaV zrAr3ji*JE0Q5Bbyj!R^1#{nl7;AJK1DsqW^il_3hA@Td9&8tO+h)ehRAZ^HfLCltb zJe2yP8rEYgFcLS{p|Wa)*#~lKBepaA7Ck2!kFa>0WHU;Tq>8(4|45}<0CG+Pm0ra! z06f0K=;{WjT3lOA;m@d}IT>X$B1%RRhV8y_gH8G6*0^RV7Q;Qq-?u2p$=Qb=j=r=1 z44iZ(K!HPn*72d=3noN}xDNpy5Q~w%JcB$mxET)YkG&v64A6P)r(_WZ*lro1Y6{Tg zZi>&jLgdGWc(gRCV5qH6$ zoo>1e#9+wS4hhLXDHY={fWfBFff`6!(!l3H7^R-9?~3@*0C{~xhsSd#J%bmNOE)O4 zJ#dv;266@CpT~&u6!pC#r*cEv>}S4a#LBY9jaPmky6T?9(HTwX zZJ|tW8hCS%w2C^+$)uXG1Hhv#oncr5=Z<%z5flLgqFJ3^G99R`oCN^c-B_Er*XopC z0<9A&?{6?`b(0O#>36)wzqazzFg7j}CO&_rAwcVfwN}wtSn7;he_y^$Pp{NR^`7>d zdy2bb;o|oD9o2-UpkY%7B!D8%Gz$WA+!yNLaA^f+i}Gt!J+q{D_xQ4IS^2O);B3;rlcFO7#z1`&D zgJQFvcT2aRWR8RhPcsxBNXlvH(IpY*8(PHhR!RRE+(zVbb!``sZ8GuhXWwh<)R5g~ z|5<$D3<*h5MGn{yfq(kYBL?`~QLX<+DQNJ7|3?Bc>MoVIY(GRWE1-!aIsmTwEa(J0 zhv)w<3=w^Amr?hU8IMkAOz2e>r#5fa7_mLl)^YiBJsLOnj}>Io@`9s0o6cVu&_c>X ztU)Tc#6z!t7Gt`K;N$>TBaWqDNGJymys0pJ;{`|6fc^cQows0}h4{t&Z|-u+5xlT# zWV8b9zY??<4-jP&5=aXH3WxOW&bzW5i6Vvt;y(1^tSPTa!>&1D8`k7z z$k5FUA|;J`=`wWd?fnMwURZh63R`QokXsmaRt;r$=Rd46!O}S%uK&O^@?bZS`k2dw zO@k-B2cx2LpOlvMIgZw+h2zdmOWtQITB=2br58FD*&5c|JG6<*$@12cMXVXhFiL{@XSYSSC?Uzq`&Cv1>(BD;RW6=DOAG5Q!_YT;n@XpW# zHeh4uZ+{J73|2twLV;n25I@skl_7Hyx~^h~WrN&6CXA^MFeDHq9Ac}_B>@a&1PDL$ zMEqi*VV<|;VP=-L=*hT3PfuS6Fb^F$>@^)0=09me_)(2Q8)Wco$9Cv)@e>~=8FWuN$aTI zF^)TYOo!yG*lXbc)w*W!3}9=|F@BRh;3V3 zsq|KE%=LT3Q@zLD2e+jyN*1pAEQn;kD2C&y8OSnI@h+o3hXKE00TDzEqIG@a1w1@~ zXeVHC)2(Gjxj?HO1&3E`DM)nX0DQcTFxjnF;nS;Rb%L?UerKqg7r>B~)*s{gRFhN)K_s*FPj4f)<*hcqf!Ci)M>qvazp z0pUeOZo{89Egd<&U0{ndnkrjPbFO}UK1d9v%Wy~SVi5=}(W^_}&u^(UDH$slAVB9h zmI08POl=1U5{!nnClTfT%`8FaAgcZ^R!SwfaDwHJ+<|Z@KY=^&!1>;5IgG|-wiZTdpA3I1VA|{gDL#x{ShNy6!#K8m`WI(n)SQHJhFu~w7sYvjyfXCH zAB4YLK{@^_(R9{&1Wv$#f8L#rMyrg_rE+`M>VvV_v82kO`%(WKw^?o@umFtOKSn2i zu!uDEpCit0Toi$Ez)~Zok>HGy6Lk>SV>!Bu)&W5l{mVL&E40`KE8Zg9JCWyHeSfWc zgRRh|lR-64r5m!oFy%S1eXWDt=~G9C9%~EO+@ff%qnv2!qmt3-0djr;Y+w7OsbvRK zIALTHjz#FP+Pz^)kuFDZJL;L%*6l89-)_=P@Pjt~T(QaU=d06(6tR!zk@>B{B!;Zh`5 zgXi|&d)}*M_KUGrp_x>^c@}om+V5-)ml`%UpQI>L(yBfOj6sq(G_N0y*D>qrXpoLGDoRWk|^RJ-w`inb+p}-=p5c--Zd+yX!3Ah z{EOu)hU>PwVJ|5=;BVWxL;V;&iEjabvz=J}KoPLF+N)(C2LZi^Js0A6B;>pR#Gli% z+I~QFJkZSb`*=GQgfSJ?FzI){KMrU>EA&4N4GkKN5B5}4U&g*KlUvv1HSOcuo!?P` zsaILVjYSA$rynSlxqFvhVLvF|c&p9)ruX3%okS#1Z;Xe3pH(f^wPVw%N=;C4#g6Jt zEKJ={lrXa@h%|s^-Kli^KOwJ(#V)_S8IIWOxE}*kZt4xKLmGO96uhI3n*%e$nO#Ob zfvjl)=08#Erk&rX6{u5X3&^{Y?A8SxogoqK?meWVNNRi#tGSw!K+x*Y)2RE7tY7}= zk8JH*Y+y-+Si}Q^kJ0B~^$p-qx`?X^V6R~?ww8m<6|3XbBYi1LVGX7aj#^^~rcERS zbB*wX#O-ihx3zl%2F0JyRTkk7F5kNAY(0Dx7-r;xrGKgBehPM6-h2gQ&+D3biS6@r z+9f8Kz+>wS((|RybhzRC3ai+>1bi2RmIDLtLicCUOV!hd_LN2)*?$UsmbnJ=w_NO; z6xp-59m?db?f)u$KEqrL>B-2 z@o**CY(~~%@Z=@sOuwYAR0WT&)OWtD_bwzbYMj5}`}XX4O3Hkf=|RcRT*7y2lFI6y z8IO~hdbz6QLr3=y`fX}mW+aG+US0@)P?K0(crB*f;gRy z?3YfRE5TIl`+5(!?lLt zIoQJC(ZuqfTXe!yk&|*djlY-|A6thN(kQ4>8Ok{}d1ai)b6}NA-#q0DU*I8mf$3!S z58q1P{OAi`80R8cRlcx#0cgd`&I=nN^6+K7|8JG{+oz|IANul{EG^J#mdutr8&*iy`zgB3%}@PSFIR&D~e)O`fM)~Ed=|5ld*w>BVhIbf|E1*nk7Oq@eD^h^Q( zv55ffHn4UI5=1xwF^B>(90mRUcI~khAjWHXu!}XhF2K-oH{+AA~oW!J+P1K9IP@=5j*Zladvt!q*?ypJVvMHL-CO*(|O*-r}@cOJ`_uJMh&q zSX=+&ykC;DHN`}xbEp$}z(2qBr-{B~q|!OQ(Eu9q)aynlfPNSkMrHK>y0d&V%$1|k z@R2{ul%wBzg)tuB$02%;g_}sEDnTElyv#o@fz5UpTAD?Oarq16-EV@mY!O@Ht*>Mf z0|&e7IWWo|$F6Vo8#L|IKnVAEq7xV15y+Gu!!JrK9t(r_Y$1F^17IEb=XnbzuV(0l z?;fOpG$7fe^T2TroH^hzjT-6Qyas&PbV@dzwmE(0ayoI(0dL)^9|>;RrC#g37cTLx z;fk!c-8yu|&EywhX;h**7sHwe2&|P zO06wCwigP*sG~~o-+y$qb?k1GusZssP4D^4!K>-?;eJibUfO0^brGGg%*BICjVI8;?0-1y+1biZ2$64pi74>3Y~q*`f>0o_pP14OpoK z0gyD=$qhnyN$lv>{!w_SW~KWy#Sy!fo>OG;v%xM%yrG%&#_PLK7t8a+j{pU4mfAZT z_%QS+^$>1nR6eA<(fhtTgO4AK*Qpo?`OSNla0OyvD0%yGNCZT~3;+JfkY7e_@Z2LhLC3$EZAlrD)eZ0u9F#ZK1O2C!?EY&R+v z>Xvr|*>xmF-2kkUoACIcBfnI1#|8VsYq5i#?Az_FvFEtno_C)eS_W*+*;Nu8JLZZk zl}j8fcMDZGKC_%`sdl`Q`Q`a#ee==0v*18Cchyr<z9T>t94T zb=3N<^)-8Q9P@xOVM}2|0pTN&BlfMFC6KS<(*YQN4wzInlcRddrDJ?T>m^j2V@q&k z+`)nx?w&04t36E{5;IW5&oKbkg=c-e4E%=G2>>`wAq^on2ExgMFN z{(jZL_F(#Kp6K|%v%w;C`;Zg)hRy!=2bB`jZEMV<%od{O(uWW6P8+yy*zh6sb zlce+DDGq>Uvn8qp8rt)du2vL%0{T`In)%$kbuQD9TCZPt)_Y}L%HXUR63@1(@;`5M z5gfdRi^zc+=g~C@v{#2(tG+6YxL^)5%_<|dy!dosZ`I_d#@H{IAur-H)`L}lBJ{kb z{cEb9?Ui;f{HTB7zgOBGf2-RDU1G^FGr2f~oh@~bAJV)$3?N(6p!RLE+vNQYu#4`= z;3He>7x@cpmzNC;f#K*3pXV^hoF~Oh;N@?6Yo?szCsM&512|SpLAhSs_$S*AnC6O0 zJluwM@+Uxqzc)A4kGTh4dsJ9g{JD1Qg*2VT=G~3BPSkpip|wHa0pU&&F54m-L27m6 z*5Y7;EoqmJVz&ZC!C&G^qVw(IJ(5xe)EC$+sI@TWUAspQ;w z64d_jD1O9UO{7!$#&$5V=6aa$Rwq@?7o0C@M5i2K)E2zH`pa!?Kxk&F7{uDHUiGK& zNE%$LR`BjkcqYuz;N$2666i&Ea*=Mp9Kr60e`{G{Z1ec9YE5k)aQ97*05)Gbp2jHGFu|jOC4#fuNdd zb8=(Z%40kAX6tK#jWGy@5nf*=;g|~8E)N~{q~_Ey0!GyOo=<2n*(HHC$Sd47B0}Lu zY^LV{mE#9)II*bn?2r0Vp5Tl8Ex0j=XTHy2>B!*Z%OS6H`_bLq>OOsC8yn=LDb*rF z|20P7vD=Vd?fsb9cn@#Td+yu*W<>Lra%bhUwv19Sly+r@@%2NmmRWSUXG{*dQn;f@ zgF$)Fr@jDLS>(w2dT8Dhr00!>{OQ{aT8(<1z&W#+N$|)$+?fm(00ErsVXfr31t4C$ zB$&7m!hznZ<^K~FP;wcw`{tA{80t=}7y_2PIm|7-&)i%qQ@n9OHUjItyPMjSVAn`X zzkzXadvThMJGv)#zLcUV`2D(_({3a=3-`v*D|q$0CxuA5dd&m2T97AY9q!HN-K4(+ z83^y`F1bju6V_xdB^0iJTf}VfDnxEEcA}y zkNim9y`>CB{9gh~$s-1}dFArPWXQ~KsxsCoXXb$^v zet}j+Bm2=$>&nHFl~usr3hsQx7A=*!6hW2Lk|`n=j)ozti8UvKGVKB)0^uN!(Xvh8}l}O$o`0gkYn*mXPFcGX-4}EiRD>^23PN&eoGO@jdb}P1XT%$jtD>25|Li5OUXEl_`yGK( zS5^~TrG!)W7-Ov}x>AQb`p5_n`Vz2-8m{rGB;e2>Q%3u0>?7`OHxtx~oatgkojz9! z+LxJb9sXq$YI-WcW65n)sPx{vv$IC`@A1h?0yQmw+gib2UBZvi9GZBwez>fXzk=7d zxa6_eh_){Fd&Nb_;DeP49&k~s{i`8>YGC5t`eC_nso%Styvq9A@{;#9L=WHMzt1c5 zH+WF=#OCbEgKuo%*R{7U3uWKW8FKb&dN`_qj80TPE9>SAg?u*C(1C#um=U-o?vxT; z`BSLJ_ff)-*xhBsf4HTq@V{;a%+vSe{J-vVRyV6=yt{`no-a+`Qjjf$;4G7UmuH2W zXy@VmTY^J1Qca)iMdeyQ#HP}%h$fb^TdM_?oH> zs+D4>rdf7&05;SCrvcRsez-QNf+znp#AFeo$ws}lM@}>3Eqy!r+0&~?weQP^fe!e7 zr!ah*ntXEp!1JAJIN6=yJCsV`$>sf@ekfaY&IeLiyM&R$meGQagl-efA4*7fT^=X( zrSfdqtE25LVfVmo@DwmVk0S}sR!mAK@oYb6spy3a*-+@=@mc&+vidS)i={8#1P_f$ zH7%_CoIG#?OYIjOvWqZ`MuphlEphD3ixRZdZ>D_ zS`dTK)1PxYtA?3}WANk2g`Wg6U8YrQi(r<+Uh(@Mr1E>m2WyrOExC7G) z2^`^2PVxocG^hY>m~a1b2(}t5WEnINfhz!y5E|NRsDUC=rOKs>`DegF)C5vy<=z~| z3MgJ%kOzQ37kaf73t>xvlN@b5SQrV?@A z$T&;O`wfnCQyA$LLs>BNUU1Y5Hq8<+?s&v{LH?%?vU1>I%T9Y0wF7}5`&P@N)gS@1 z7~k&{Sr+w}&nZ3dFkV_5ip2|AOOVq7MuAPTlP&jsgqh_OrxG`;s+0QSv zi9Ap4k{cj$d3c2IsW70B@DA+n*67Yy$8TvcFuEBe2FVv&FX#91Iu%-W|NfW05f?Ub z>R|T0YgO>HVM6Rat#4>kLL0o;S8S6@N4F5dZHGrlSVulsUg?vjp{y&bnydi($8B8Z!M_lB`p5PeHKhHU3mMx zfZ6zFRBHByKosF55&7XUsYCdM$k9fMMsnu8W4gw$3k{FL+_?tFd_ zbKr9t4=3f`Xi*VtU~lc4kGDt4>{75)?CPw8ooAV0$C5X^ai_7~HLQz9DdZ+kR4QL1 zxpX@C1q{TPCd7QIWtzkF1v9l0?9y;4_oyM1*Yl+i2=s7e*Uuqq(nhs|8j0ER3&vBG z3Q21@#(*^mDiyCy5ycaQBBB;!GhSs?`ScQ-i!S9FHS}xpGgSHvRmu1=OiW^@*8g5; zC^j~D75FGOhrvLzVK~ANa6oS~tOm6Na3O&86SP{D8iyEbz(ZCp=GK=!-;ey3d)t5& z3XQJj|2uZi7+iy&b>of-#IaFvRm=1r9+f1Xay3@E^Tv zpj3AMQ2-24wgYa2%K?vvTvDe|aFm(@V6n1L1&sh2pt>A*Lp71xpj|GkaGD>2JD*}w z2)Ja05V7?Y)QomtcEM`+8FZ-|T%549*BziC9|iWMB>>SSzz=qHPO8}cXI*Eq!Al?v z0s&w(RBm)BSiyT1fPpHOK0CV(xP01M3+eq2s~~k$0LG@+-w0o=lSUnX3_?q+0IebF zCxRUZd8n#=R)+hqSL^TpL@_oPeK?^Uz<)C`oovF$|M^eAtAB}T{bxUX1~>MR7U8SA z2mH`CWetF)3ov60(wrW_TsfqzLw1Sclq>_d?mStpS`PyIW^<@0Eo_aPkn(+I0wWZnjp%k-0VZ=fb_xp422cy`Z|21gJbnCtbq3J3`UL&GQf2= ziq?0})X3A1m!S7OjdbB+uu+Q(3b}8X?HmEIVhARzHZE8%!Fq|hHr`hQ>jcu(Ky<9% z+g`RoOi^qwh#CU4XFqC@EDE5I@z75T%yx6L88c1y05w+|kVjq>_FB*?fnDIk5RX zTLi;QG~6MG=ahiwj_p2d#fPha)OErFZrbrX6MROJ@f{xR?LE|Gm~Q6zZT{(Q0!U_d z*t=7BKAR#4&v&PhLh)0g*OoTf5%5sjum3`tS1WAIh}M9_&KPdzscaA)Y2@fcN-+CU z#6iax1wl?ClYzGuax0e#YrOV#Aim1*8Iid20uA*XihK%OT-L@6;4O;%aM>C(-9OYa z@G2Gq3~TEfg%aEkRF~nHXfU9py{IVZ%Sy$(1EpoR4M_N{VBr@de>d#`(-j&?PrmzR z0q?I3QR~`1uU>7uM1K$t%nP^iBPfiALYZV1z(E~h`m8pcw*68y73a{gnk|917a^AY z*dP_4JBkC`kg@r=yqZ1s%PVRtO6%vmnios4)&P(e!QiCbGLc{#mzEL-J|J`Q95R{S z9pnUs8@|2fXm5UHl7RH`N^|M8aRM@WWs88+MBoxj)ES#;KfJ zsV89ANCcI@<%AGi$pwnzwdzr;Vgx(X&!#XM=&& zpiS?h7WE3!bLR-cmn0J? z`NS6EK66pYQ86{?YDnmL)ag%fAx8@gn{25c8!ukhI#lfutF+Cn~oYW|nLrFt3dDeIK5;c-N z1o?A`V$dTc^rq`1>)7@jl2>ZF@hmHwGpQIRe{b*`bB*LwK`9%)Eev`Qc$blBBK(5v zSB94b^ApUk#Sb>_WcdFjejrV~{Lyy)s|Lf%A?K7&js8O;Q&^s^J z*TP4l0~&L&OOGP_Q@T*ADpqt}Y&+@Jb)g&!^$l*2J-z^c?1uYu7#a~2-^2MPN9zls z&$SA@3fum>sV#&^I^QUyyxHh~CcievlWSt4Q-(Sh5B?$x{Z~J}$>1n+zRr5{35?+d zfI_zt1{mZ4A2HjJbj0BwIXHH<4x`AS|)tBV)U_am!#%+GNX z{n&5jm&bB-O|KF|$X*Q~j?PQvGdfTwinSjBKXk*&7ia>^B zX!#SmA6tedftyNA$`~BV@3zZ7#+s3)xcLV~KE6Tpn;Vi4i+}{b*9kH0JF!O^l0#Cc zbFwMwQVM!&j#oJAnML4@{?&^_FE7$c?4bfL=qAm7)x?X_y7r|;zvFr(~A}d$?M!54jveN;4r9a}NM5z6mm@>cS&Cv!_ zyj_Nhvr%A-(f_VUBD0yRYXu3FBp-~uQtyCA(CRQ$rF}M#)hFixANN-VTlx1A+VQ#F zX=K{QI9d2f#FKN|mi%WW9lNIlKQMC?Eq@B&T5FU9Y1{j4A$F#u3cVq&%QxGm7Y)S z!hM_DC>iS4Y?CicZ-kSDcJ;jD3)RG4iI_2&McWTNcSgc%_#-})jIwi&@Wz(uOecs7 z&7UYQdpMoxe`E`0=YUNOKLZ)?Q7@p*MlsRS*0>Pow`>Le=SN;Z>-XHJhxgKWE)cgx zwC_`j^RBoH|p1k4yX0hr|W7Gi?4! zgm?eGgp75#T8Is>4QedX0@j=8J=*A@GqZK~wCATiZ-r607VMo~LdXFmd{La#_D0D& zE;jFJ3z~wy5jLLc#Zjm3=oH^?1E=>t0i>aR*c?gU!h;?~-LtRw{?x2@h*Zm#UK`A- zT6;&mOgcw=N_Ibh*;c|-q+@x%hvuR?Z0Ap(SeTu32r#cApVQHkaDO0 zm5*vhRMKyui!OYm=Fe1`4D_Zzw|qES_u$@sxM2aVa;gaO+kvt=*(MeZ@;z*Nf#k9q-fv{0g!MYtrhpuR^P|<32Fiev zmyEKo#NNlLr8e;VDZV>JVRQd;ZeqWlOML%1ybu9A9Jls6eD{&6RbO11wi=7YJ+m=R<#s2xF6rGGhnF7pZvCPEVZJ8LGPJQfcopIoG!7)z_=(L|C5MuU zim8azNj5@d=nK%++3LLietzmn2$PA@lbc<~hRL27FE$_HJ%Mlm*WE>r2)v zL2)0WGj6%6B87X?IXnubV#`yKa{*^cGKD@ghQPpjp9{rFW2XNp?5agaXv7F{z3x+R zBBNJU|Kvx--uvAU#0BC?KmA^kUwoLpK^~$KibHttUcLW5$ij2da_ojW(j%>%I=i<< z!9#lIO`PTg+*zhj8gG#C1lhC`-;~~y-KURm9((#ClcUboY%c;&$ZUj8TKgHdFcX$e6Rz(a!N8%om!^=aW<% znujU2OE5a6y=s)KFITrk<4Ed_&2Qt6ym9i;<#+MxrybNKlpIP=N^J3CeQ`bMj6}PZ zdbkqx#-29yK-Rs>RsF>@JkVN_kz1Jh3432R5hw5WyJkojvZq=pq&yY(6f{2KeHn5g zr=G(YZmnt4W2n?-x&Og+5dz<17+@`19!tZz^LpP+|3QuQ8SG zoRU4}d-G!;IpCSoswSPdo-TAHSO%@@wu?GuVA~jdS$X?`${(Efcd`t#g?%e#;WXWC zxa9z2t+~m2y2Ut?E#H%kP$AB#uYOB@dYtG?R|QbPD50W!G}mIkG<$UF$7Iw)G91Y< zK4#NHLdB20&GOX4S(vouwVcw?IrO6AY7U8qV(ts8IwXKNuM8s)JohZatWhWl|lgDbI-mQ!3>;J52yasR?<6GJKZn(jqOI2{~7fh4w)P8!!j z6LcU}AY#M5lkxiwl3g1tBrId^EI<(A9gFLq8w}Cn^M>!|j{gYD^()}KSK50@O<{ZS zF6!YNfJ=6xwLp89KAuMWGT9;2ffYE~$-rB91LsQViXmukJy}7ApA1L&UMF}TkxBGH z`nk`8&Q=hovO?*8w4VRu>zAh4oq@dF*`F!NfPKCib)9^#41-Q7s6f^d4HFsaunNny zpqqYaT{iyuhqh__^zt8vPBB5qV-)wXnco5?G!X-mDO76(RiFe?E;N-%-1J+03L2Ya zWI}JCw9w?_-AK&n^hYeCbA13PJ~ZBYfpwYj>;XnDJem$r^C*+zVo1QGW60cbDOxb`Tz!y zAn;BE@Kcqc3p%XcgGTx_z3bR>Wf(3Wh>6&aNlFONghPhW#Vq~miq0oBFt#)D-~=MB z#JKoxX2}U?ARk`cGI9r$E%^<-`*(>QP#;l4w)7A4<3k|IQ)WS+2`ivv$xyGpSth&I zjP!OA!2MnN0XmPvn#W(jcxmDXg_7v`Ebt;Cz{R@Hb4nO^ZXAG1)*BG;TxJKTv(_4q zb|X2L=(cYUKZMZNKGsx*l<~e~$YR=I)&axOKlOVgTxE=W;AC3643XsC`{QpI_V&vm z7Vx@WhAgpSzS=|;>S6Hc8@p<1P#`WEN<-^^cYk|XX(Q2TXJuS`*$i9bWd`S`Hx~ZX z!!O>@izkEdYR5I@)f=cin_P1nQvq&%-55kS?~uB~D!U8o^vzQB_NX#+AzyL60x9X& z44WOE08T0?%az%~Ov^CFCyYFxxADmaBcEhwNSa(9EZc}}fp+ROD7mO>qU3|{q~UX1 zOCln)C6`BQaf(|NYg7|ajv(h^qzyP%fa=7#Z4#&eyd?!3;S!HT#;Vj*n3RiNc%X`^#=5)x~hJG*A z#f!K&ZbR?Vo}AU9e^q%urZJ@6HRtNoO zr;jURyZ}Mh{@$_$={Erb#u#Qb#`AA=0RCO+%?4zzxpWa|Hkf3vv}NA&gbrXUsn8wq zLYXYMBK6w!^C-9t&(^L(7cM%!ThFeukRz#6tabkhEI273b#2c;K--Yo){DhEsIJ#z zUfDS=E%`*OFYd?i!F=4qbvV8T;z7=h(o-AJTPg(6!&@~7bZ8o_p=L0?P>@3{yAP6M zTKBokbWf3lmeUZ^^R4JLzrl~o_tH+Oi%&V?IlP6g?v;zc!A7@|Hx?k&yKq6s5dWzj z@PYrKC^$0J6nK%Jy~B@Te#1z?St0YJP; z`5N5H!#huH289MsqXP&6sU_zKsn#bd7)cwcdiT_mlNfbCn(c? zfaT5C8BsmZBd)BQB^Lo1Pk&qC&i(fipg7(P@El1XZ8wG~cDYdw0sh$kDDO(+q1@Z> zgBfEVkqC!59Ub9>L?khzEay~+7!gfPj%4j1yJ1?TMZ4WtvPGhrJ+8m4UG+#AV5cC zP_~jzlN=L8VCPs53=;D8i(miYH_NYZUO3)$y4=!T03$|vkDn0{E zvlB=?B_m5Ca}4!DrUkf+%HHRinio$zoIMsZ-+^`#pPEMG0)hEP;}KWd_#*}EC|95U zd^{*?bg4+Ra8xo7JCjcAkeH-lY%c&0vL-$&1WH1bK$a&!xv&sviQw? z|A?0gZF{M@Z#EqagIvfxs7m2EY~rY|t8vOsnDE8t20%|6mY%b(Q`bgRnkv@-N`f+US%loW8nu~{h(;*aY1#5JOZHIY$ z*Mf1>c1YRL)$P$}XSM|h!Aon!XIEgy`ATn-MKtt)Q4=!O5N#K(zg|qkj!hmpCG=n;rsbOoH^5--S;L3 zNm_tAg_x7>!XhM*ig@>#eTAJ?e_Z|OxNk=+A%?L-Hm(xm9!M^69hfgWa>w{wObJDD zf*n+=%Oab;Asl8q07MDsp(B?jcWglRWqzh)m$3hg%Qh0gAG{Hol9 zM&z%@Bk9G60D2NdHwS?Q=z@w5CNq3coK^z*!SpUnLb{IG^3a{wYSS6fEwo{n4Pgnr zjKxuj{Kax{i|Z6ujw^o5qY-#Iwb;FgqbT}e`4`CIq6Q`kZO%WW(**p!b5s2rP20XX)`a7KTm26fwloCr7E@EXV zUn@*#L9;~DzPXs8SwutK=tbqsTb&6M3$oRzZ6*voBity=Tk#0(a6Si2d5?u)*`8L0 zn`~N26~PWW2Q%Ofk9;a)gp+8_=;jfNeX9Vp1%@m%{&e%g*o9VAF<4rK-wT4~ONlT$N054z93VHDo=*to-Hq9h@9~rtyLkrB= zXyf?X_$}%OBs};}Y>~X<;F2#OsefGI=z`#ye$%(ZLq6u=G7aiRV^0v=ifGWfrxf%P zw5DL4M>H)sms#VMrWpi*i(a8)?y?7>5_0gUIp( zoB%6UFDeib_(rp>KXOy|K(5NdLrcPUPbXm5)DVg2{|LaK*C(GSj>f^gF+DNn0^{t;-{6ne-dV7IGDg!f8DNUnOG5kEMvop%nzLD|51~^AqauUDNIO8Ud(o z^)*>1EY%NG#~d3Ny}0T!&$1&ADajpdgr>&n@5*FrN2)bNBJD*7JBbs( z3OvO=aNW*eS&awPdbPXH4QGU{yhrT~;l2TIrK>%&DukU~PQfgy*a5XwyX}T&S{zd9 zB`}x}JFf}o75wfSiywerA2GkD)G|rdC&o80n2hFvU>~*C=e}f=Lc{48o!hV~;&`L} z3gs6LUtcVLWKB!ic#id;f5-I=+A2Zd)`^_5qeAw4<9Vz-IoF?W5I#j=cPPpIqH*QZ zn5kW`@nGGUkmVFrSF8U zc&19>bY`la;76n0Xf9P=Kjz#ud&JfK19E0tYmt*C%ma^-VE=)X_odyMJb>kS0#zFY z=i?dW)Z79yu@_63wL}$h3AWy4(*QSi%MT_>xUF&?{HC()_3i}|aYD>V zN&TdQbo?95%y2qeD@ll>Bz8rHw3ihz%ZrfpJQRDR&nYEDBERUvCFW=CH2arTjHC7N zXsyvb<5z(R=@p5K@6T~_Uk`Z)33P20g5-Q+BzB_>hzulakL1OP9^-mDkq(9GWnNm% zCGvPz@#p7l1lJ3ku_Iv=tTa7C^yQ@H5-v{$&D@Eo;V{XT%Prd;{dVTC_f(~_2sTl~ z?N--W35~A@QY@-$yNcP&VIOz(xc2PGNK^=)_~UJN#MkW0W=e(cjzxg1zgN=w;<9rW z*{HG^7!4h03E({l(_%OWjF%kZ95!E+ z;AuV5*Co01`TyDbJJ=*EOw3C$x2XM9#x1LAIx2oDw&sWU1BMu)T(MiJI-?V-0 z6Q#}?;aV4Z8yAC5SyN606FM}>=heON#k_*kJGPvb$-8(7^4LfyCRnaxlfw01xlQeW z4wXW2Li=&3i8u*U6=Z*qO7N&_QxdaPfd-~Sq4OJ$IQ35g`E>PcOKBvnD%lgxBwZME z8rX4Lo!M{Y4QjKG(@igr@{4Xf&`nn$aF!U0FPI}fdot;9r0cB4*@k>j54a~Z<=c5V zRFkPiL)=w+*(Ys-3w@2VZdzPrakxF+Y+{H!%BjgG`M<;%kME}OqG8lp~&^2He zx>TU-_IBn4;3I(Wp0g8Sw?g+it3Ltf73*FA@XwzP?`&*!>a8VvM!fm69ot@;3yjJ7 zJ@8XW2^O)fX|O%Te-`kFZriT-$%H^pOFcVugSM~)Hkd{k76Tq8Q zD`2+#JvH!}YL^U!OmCxs$Z&@mJg_QM1!NXS#L+K;`JtGX17`e~oj9rDBq@D$Fpe4XqADdRWI|*PA zxTS z34`V4q1b099d#M8?VIBFXS{p}N6QdVToUj5-2;oMotgUaJ7lXgJ?~_I5ZxL!QNAx{ z2iF&A{a%9vuQeEI6x1y#iL|Qx$H$iYKUu8d?f>n&{{Q*srkZuU?1AjbyS|Qv=+6FX LWw-yaC58Skgq<$1 diff --git a/benchmarks/results/prepush_smoke_20260423_123335/charts/membench_comparison.png b/benchmarks/results/prepush_smoke_20260423_123335/charts/membench_comparison.png deleted file mode 100644 index 30a6b2575088134a38bfbd7efd972152367f5e81..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 33079 zcmeFZ2T;`Ow=LMBpaOzQXp~?&h-45YNK!cJQ52M%5fBlHP0pYalxQMI6eZ`JGc>KB zfMm%*L}HU;1G7K8|Noo0^KQNSs-|jcs^(OkYDM}NzOeV&YpwmgyeEJ22<1si3tx;IzGwwT+3T zg%F>h0G}w&>BqLVRyJ4o`OW|R8+?}5#{4x}cFk~=!&d)j*kCYBk?8+@x9pz3!(co= z-TM2QvSZZDz~S3+EU-(VOkP`Gc1nu8wO14zfd) zZnD!|8L|VVcCv&Z);e;m_R&Oz*cjHUwCGVGr|K z>DH6($kLO}H~VpMrl4DQB-&w+)4D$`t1_ai=t(D6M~V_hdc|t*NShr0k-SIWxZ0OC zCX-@is@%70uS}=YO?9S)_4Xzy#0r@8WM5=G)#uiu@9Vuf+m|%)Bl-TP?1ctqUw{5e zQ4(pXC4nuhtlMR-gkxLsaqEZHPd9^G^G(`2f|{eneND3}gh>m8Q~qXk2T!>km6zK1 zxtxAd#3CvvICeV5W$t}5bL@C6X+nDUurHhQ<4yVW{I=VZ8V$s6SUuNcjdjeH<;Q}g zHh*^{QXDxMPi%*`XQq*uub%(?DL9d+b?dx*6jQUr(pOu(h46m2Xvfi)BX1R?I9Hq( z-v5fRT;5(A^37|$!p+Jo{mC(QL1=3xuf>mL?dMwsA6AEc!TiVH&(0Q&_%J69T_eed zjbF*B@YrmRt&K3vd|&CGmRb6w$6?5=4qljXV?)PjJY-Vjc$Q9@YR0l;@8eH_wLgqw zSl>a-I@cbXeFTax{Dq{yl%KVdK*0)j+GU-V35J?Jwp z;w>gtZc#qzObyxL+v$Dy_AaIS#(3DX6s06aTjF3vy<4hEnyLVdzhc&9leYM0nL1^m zj%ITO8^0p4q_wJDo#CD^J$JFHmKVp)QC_kAeQ$KZZmNTmNy^pn%CFS4%-iN%CRMEV zJ#h~P%blny-{0BUV*4gycS*)I z?pw2JQ)2f0IKpw_Ilt)^$+dpP*lTlku-?NqqaD{re5q75O%jd=oA&0L)OhSH#Xe(S z*&{-+x?-$aJANA1S8f33RgZm+dZ+-nnyxNNFAUvuyj z{nuFOs$rNqz7@@zr`(p$W(hs|8g5(pCMZj{!nwwC-~QVY1lS@)pC{V0_3NHZ#ClZL zk5PHVjuAh`6K&=;+*dl)sN8-fMOinV?ORNe-ZptUHoD|r72?fWwAjdvEr&(TK3JE# z3FGf~1LD#yr?dFiVGUEd zZ%!uJM(=EIVJ}(qi<$?UUtoK}Ky$H6+veLd6;+Ld;UMw33+wRfN?NB}W^$f|@@oAm zB6wclJ7rb9kNne(z}x1V>tn%GL06`dNlp|rEX7yFsq~WEwpJ%~v&!a5pYd0ba)*6S zxnE!sHdC)@fDP4Z^Y=kE#|OQ-uD@a!iXA})`t-~&*s@z`fdv=q}DgJ{}lKjR| ztHyVnR6A5UaV~-ST}~6xVYSKY*9KiC)?T*6{bd|&r}`S>Fh5*FPRAjCXM#z@ zLc0b+jYc+H2-|PdBFXN~p;_=Gq>`@BEblI5_~U`w;p z17!|1N(z!H^RNKoTev7}Ja)H+sD$B53kUZxN|vbDgiToplU?SLSw`!?~2NIacAsd^`;aLEk(m;HRcL5(^^P+0*sJ zHdyWvWCQEeV|Jqx$GZ7vSQf;Vr#iU{Mmp8>c-NQP6(AVGI-_KB9De%juE@ZpQ|=+l zl3XM=3kci2{4v}+zeb7YV}y2m7#T2?>!beHYnLWsT=?9=+;=u6mRp|2$@(&~z-&#r zX!gT3b^aO-CNvNg%XQ7ch<4x$7k8K=+hVc)d}c#{yV8JM z7bmFQq!JNA{}28skA;?bK_}(;JK^^Xv0*Bk@?@J7B!qdgK3J@F1 zd8EJZ0pFbc)y!@m? z{lWINgRJic$JSkO+J$oiW&GI+N%GM$VSGAmi&~ZyDi8a=N6{?FljBL3k#@DEFM_ewxokCJ_8+ zQS8O`aJUmj9Ua--Cj9*kck#F*{{9oqZnThOKV6SWiZ`Ys%mnDd^Lo!Q6Ep&FL5(fqryY3OIBY63Gw3`F^*^VlFYB6n;7+D0R24ij0YEk z=^nm&m{HKx$WP2~mrscKBz|)pPnTr71in@KJLiZV zBtnA$(|1(Y#@y&*?>$_fADMIumR$XZ&iCD#!$*z0NBSWus<)aTZVVVii^yMDT<-nGaLTK5|>@!42~>uzC; ztCE@>4xBO4KApJ4*-`Vr7GWP>$l%)Q?`nnY!v4Yr@xK(Eo`a|y_2_W0&~d_v2py z9kbq|-*M$yEXx;=Th#GumfD(eU3aKIb~AqNYplnP6G~g{l?!(81c|dH_X>EPE{yuA zL?9?XWj~N>)YQQ)syc-~EuhK>$mOg#_f8q_^n{9|i%G27b+=Rd9$aUK$|IE9yDpe% zhcpy#rGtI+uc-qPb`Sfp$Jo+sf@_Wo=?l=xw~cE;(R~$?uc9H z`2!hWiHhkD%OC-?EutJ5YS_cbeSJ*SccF|?FTtBI&RaO^$DMg-$bHLvefT+TEyC*^ z-r}nK4MTP;4mb$Adc)zhYibd-ePO;WR2bC*WLGf{N)(HyM-T{Br9b6C!lm>Xkarlz zx`l>PDg1pbZ0rDQh*Or})h~C=76x62-s5*&Z<57L5SktBfXjhqHb0M==7 zqv?quZdu|p*|L?CIDmWlPRI8*ir$i2Zk44SC*0xj7yYH(49Qz_${qLU7ZVwy^dlz{ zpX6$f;l_CFP(tm1iQM!c^Yd0w8w(@9F~Jy*oelP&jrT6AfWlNxN@rh^^`X8+6l5BF zl9s`hUx9hWus2rIizm{&KSo-Va+Z@8o2Uk)RVByI(SmpI>(b@-JL5l2rM>tQq_6B)fTF}6KWPccZzy@qs2q_ zccph4KaKMHE5SG|Hz9MZqQ6J=+PJ40YjvJYkw{P;NsC)}&fDGcR>vLKdt&i>Q( z&K7ZLErKe3v=T?=-^j@E<;9r9pk24lMqvZdF_=xV&DwS>P=rzJynNxYG~-W9f9W|- zCQRD+uWk35W+f1RLEfqOe8Rq|?l4ExtJ3YAmR*v9BP6tT$@&&f)vXockaQH3ejD$d z$B5@R?5_vU28~V>57_s40}Ma29Xe_`MfBKpNmUJ0`LM8iVZCo?M|EV445M*BFP9mE zxu;DE-csLi7=C&%5p$+lJ0h|o|1X&E%w=Dux`;YjX$f)@c5hqUKn{UTYcxkN+yquO zq$IDNL7O&_QM%t{jnyMco=Sew`N+Y^#_iC2;0+m463nZG=!G( z6cIT4{!xRgOlf!5pnA-faQylDrs6znnjj?O8x4iOO3ks1vaL+V8(N839^0nQ42&KM z3DHI6cpQnZ^!G=_B0^&o(?u+8Ju0*4*L55Q>`ZDPPu$M4P$Dy3UELts);F-(u~JM)-2G2)WRo6;P#<5Q+`J~mJW)F_&bi4)7;K5L|5 zz2fD(GL`V&5DKB$4m&X)tCHL0M!Zrj>Tc`6FC_SVQOBL=#!t|5pQF9<`-AIsJ)Gdz zfKIC%Zd|`ekSw-JZh>vgCXmh$Dl<-fr<}>e$V7;#@&0Qk%J7-Fja(^Brs)CVD;-L5 z5}BwlX~}|OchZF+c)Bx9ZF6SSfkR~S&h?X*9^acvWR`w=M5JH9uKk-^m|@Ylw%ur` zmW9S>{F$6w{+-ORiT=d9k+!=FHbGU-nYd8(b-h9Q0ERI`^BT)hV6c)et@Ya|h)yMc zYRmPIJcwzGq`ZoO|NfW6l@OHx`n`wY|IOe37l+iDK0}fHcxAeq*5=pO!vNt@35YR# zbw-*rFPjYS87^k?$XiDiQ%v?^WT@O}iL2}cy3xBlqt_dUFlv~R#;PxGIn?Zx&`&Si z%+M;LVfe=6iT?J#&AOq7L0=3>`tafO*GLimiv4nlP`}092s}CI=JXdL={&j4`?yK( zl5ExitJhgpZJ#GSw?0}|1xvsiNE$B~Up_Vc4}1Jx>Z^^V317>~jq~0fS2rl8g^ZiO zR>AccQq}aNZVc^>!BvJuC_=~Usu}{BeSMBF`-A{nP?Do8q^K{UtU_o$we<+l8Z+ygR=_50YJle@Z?}>3b~a_TdX2-X2;Zy#=OWLGK0hWz==+U4qU1S`3`*V3WoRvpM~I z9hZ7^j~9J)Sdcn!M3?&J#_khFO*Gb??Di$tZMu%Up4#+ zhCN)u$yyZFKjI8!qp#S>j5P$xuFVb90gH9)*SE-`o))u?(XmOpL?IH3v1{HMn@sy3(6tPn2lH|JJ3u-*~lYCW2yT^-=|_ekxYXEb|yVR zQ{}x8q8D;r{@n`9<2`XIm(^WSft;_X~ICGg;CF>pFcABs_u2HNIqIXv+sLlORHUcT@E zVEj0UCi;-M6xL4eFZm5@AuSZUu9Otr8phGtU(QE*0mKv|$EoQ$8KFR33n1x)s;<*G z{>e08C9J&k_A=r+vu6wY5=|@NUL%0?{%JWQ)jhhiT2$rjzP;&bij?!X^LAh*crkE8m=OW~5s0Z%mV-u1Wh;i06xm7u~hp!|;L zzd|?Q{o)*mgClS~sxVz=6Ut1+@iJm16@o96yB5VGK32cp8~-Xg0(srrUTzE(!*E{@ zl>Zl}DQMZE$OwoOH7gph=d|US>(a73Qq*rPCkmtowuh19yW;{L+n$qiQGg?e=+JhH zBOzyp4`>cs(&+D@Dv#9D0WFyj4@?FC1u_-(82HcE(<#JreL;0?l?Sy3%1p{DB0z`% zMRkx~Y^WGFw%F+=cqj1MBWg^fh-FOHqrb5a;Qc(wxyB5zZGcfEy-iEz`8936T~_MU zSO&gQ@+%sgKXiYM4*{P$ivwu zAr-tF(d#I+o9-yA9nn*U;=~GAJ8!s7;zWM20sF{!m|hI+?nfs12ELm6)T~rxKi|l; zxln8R!?m2JGq7EdP4N~tRX&j+ZQ{eL{&l;9thVOn#Fm%kb9 zE8DulIzKi?mbu>=aFtsEIN}ABtVjTppg~;s4!jd1H-^VNbXe5smU8jOi#|c$0UL*& zf1v;6JvkFx_jH}LiVMCI@Z!|WFP=589}y^6g~V~+0rhQGeP*YC;J z=S=q}BFwLD*Qv5(c&1@C$Hn4$lM0nQa1r0CzrOjC>*(9^j9W-|O4;u&3flf|?bt5T zuk$fc^uPYYzR$dUYd@{Tz0pSlBv?B9N;LftnSfX~sgczgaXV-K4#i*A2o z75jH6>sm`}CImcoNLJlC4k44_qQd(09MU7Z|8b{WsF;(c@*p`0N7;30T(6B+H+KLb zC21T;#>*gn7{6}elYj92!?ouEPdbvxAN>rZv+YMk{|d?Q^j^)Qv4$ju!&gVmDsrOz zKVA1{G|{dyX;0)m%2x@b`Unufv`x1y7>B9&%LN7$a1fEHP@%R%1lpF}+cQUI0jdhk zN3+_4?+>74qFy~WL8}`V)2O7zs%|miC$!su$xw)u8Y%3vkQ;bV1RF^J+tGI7$j#+7#|B7c(>ML7`(h; z4y07v&0w}T;IvG%T<26m_JEM50c=P{C{ZisfpO;~H^^z^^RWZ6@K)R~1o&v8g(f4~ zWT~tEK%g|oxGsL7ztAvM#njIm$b^wv{CqB@mxzk+btpvZAXKr&I8UX}kMh1`x-GR( zPp6-jQAjyP2v(SgfSg0Q|KO36^q~nLx2ctZLgL?_(sI|B1%Bxu|mrF@ye?sw}M=@i*p zsn+DDdD4b{#c=gNqonj6ZF<2D@k4%+rn1E)g&AeK16?o8!mR*z-T;j z@0%Z~tFtJZEmV%xN`DfNrS6U0UkwkLDI-((l8z&B;>7-T03K!aZU*z%wEa~;r@PKG z#V)|mF#xe&V%4dl<}E%~{HecBRb_8cdXh`nbv{+uOpySrssA-x5Gl=aBibX4#mVx~ zS5gczib2=O`ez-B2yM=4|4vT?=A=me5JZph6%RkN4P=^Pum>3Gb3*7*{F53;#PlsV z$l6{YarrXT0B+WJvW~@DR3n=Eb!y#u0Dr}D8$G6h>sDx#;2!ezdpV%;sRv)=xLzc{ z?)?m4qnwVfT;Cu}^zzPZ(Ga6XEc0{XD_I)O14yLHf;cZKdwc~hxr ziGuWDKTAM2NUk@o8$GrMN8h~#DV)D^mEk_ikfr`;K&Ag1{6wzAf-PFTw^RPv0-=a& z_D4Qw>JpX6$l$jG@<5)5lc@Dpk;^3=|ATAW0n;w;01uXM!kQax%*a6`FdQ!; z{P1ZK8jb|zN9OtGw%S&h+(hl(YxqA8K%WO|)r(K3H2$U7DrhMJz^k0?GyWFAfHWLR zHwy&&PKsM0_z(U8syj!m0Yip0*=ZJ8L=Q>>tAD)6vhc?v6P`-6!+@E{`dkRenWt4a zRGRE6yoKlJ(_1T3`mj^pYoI)f zts^)NaKyS^48&v8_HeFs$qnj@S*25{wp9@9s6az7cD@UkqpIbEanvy=M(IoKXHPZ+ zu`(kf5fzHd?#thAhRKO1i*^D&qvV<=y))#ru-LSOaZo>wDsY6(HRoF6Q0*Oh@Nfu8 zT#KfahI;2Ri>91-jWhe0E@d%qU^*1>kF zxnO#X>M^pb@KaCf-#_w<1-iM}1u0I$-~S)B!rUXzH|^$APE+N*6E1LuhaV+}-sMK_ zEU)oSz~ygyQ61yMpb~NzLx3>416d)7))YVkHRw%uP7UsoT6TFlo%CE6@YyKPL2>B| z8Qcfz z{|KxD8ecFbC`Mb(^yG|yfrADuz!r(6JBR|TgLHHpX|Szj4n$=fuz(z)T3cvy=ANhU z$PqC@DZh;Ky{ZvewPCQNFlpA(g9@_QUA^?F&ak;Iw0iE{yLS}n=h;im;uGBj{uxlP z-#4F1!$?C#Wc^qYE|$%MA9P<}!0%QPzkj;<7&conEaBgS6)s3tPa&WpgalrRJ7axF zw?i(LIJnD5{9AcSz^+`mM1OP2^qd_r{;;AKi4^0mRP5 zU|j&tAIT;t$hk@5{2qq^dh@O9fXEhYg~XEyUC=?7#cb#L#iDGQ#YTW!Zq_?z5jjOWi(1fG$WU>fk5xY18 z5siIE#DlsPVGAp8gCrc~qHuv=aFa6AqH22;(dQj$0vr?%k+EhE&e2Mfva8GYv-4ti zTp(!a&A6O|nw@gWz>|?^Sh3P2C&>)_X7?oz8586ZAu9tj!$lpDLF4kQe;4Z~X?Q8@ z1pf-yH};NGsqLN#Fr8FT)$7a@R>IoShovZAGlpOg#K1_~X70@5vn;l)*Vx(UsH%~_ zprfJ~fS5iMt#`~J|I6=H=1u%nQLFybtFVFdS?U^@m>AuSZeL@tgQ26Z?*pBx3A@z; zaQieXvmR2dUT#GJb84~$M0Zt(xj8UH&aecl+rXJ2O$oeKm@e*p+xSjd6@VRbpu--$ zHV@ObzSUc}+H1;hr46pI_ClbmGlr2ZCU+KG8h84x*w3gVTNh~)aU2;Qz)mktKl&D- zg3#16q?3ODxI_+>zsqZz5ZkXG^Fat;Rf4+h8C<36b0HFGGO|GbY?P$zgA!}0WeliL z&V_!oAQ3>Kg3Yb=a2q6WDjHxLZD+o4#Fkg%N^H;zvi}lZup9BlGCJwzxB)$>(03z< zMdN2N3cwjzFv2y%HP5ldgJ8vZSGg;O^qF31smQi|08l2eJ&Atn(Y*fsH_3;Ozeq^B z54Ti&Z+5Fo-7xU%79!-hV<5Wlm_aFZy(=1ma~ve65hw_F+Y&^Y#pWSRdLvo@A>5SQ ztZXF5s2p<#2sr{^>04Hd%0k6_E!BpdW%K+qpm@Uy|9JSe7{bdvIFEvcN z(%*9(!m5?8!77mWR&&oW#<#zp2G4!Y5dVmt`v9%zuqSzQ5d=1Gz{w)YmsK7y4lmL- zIoXWIC785+xaPbIMR}ub^SUhOT^{p5A?jVA;r$0x;jKTv7qQf(bTkM*K9ncita1It z*v`bWqNsJ!Xb&#Og>S{-fZjCyH*=PD)2MBYu zL1&#Pt!7V$N%i91)K>d(7sGYc0=*7-SyLp{YQps z>t#}E_YHsi>8u{QN-`9UppzwBpvcXUdAqBCDL!C5;0^_y#*|dm&Zg;W{+}P;t&VYN zvoaZ{gIi%`to-BiTl?<`lD4jJKNCG%8!WvO9%>l&b=_dZU|jQ-b?4ih(zJD(fVZmk zCycR=K=0$R)}EnY*z&E=j+{gMX{|?^u14 zST_rubY1RpNr$YQB14mIallMU>sTUpq2Qng_2wS6N3i&w<7>Duq0(#fRiz$3CRh3> zGnH>HtWVdmDpx8j!bmhR-jw%_t2>NzC2qaAU+BW!>uS5#-6!QvhdB~1Huta=$KxQFZs6t(vEc;XTVL=DZUcSi4&W>u_e zQihDC7t5V(R=cNoRZPFu>pG2+>Kpab3}u%Ir@1Q$t!eA4J(3O@Dy9t|aNq+<#ZU)B+&N&B@6vhJTcjlc{a_HyFRy*FY^x#W4ZaR9GUO`sEuuXA=c9d<5R%%r>5;tx zxP@k}wDdRDG3s&Q<6qRsJ*kd)C@&D~fpy`}&H(U`BJIG^v4pf*cekh!sOy4meQ_Xt z$L{2y`1pg-=9#D%0qmoZ&POdEca2ouzp-r_i>c@8n_H~hkLY>v1tdX)ON>f z@uFjvnYmUbWYrKvC0W1?mm=xQV*O9*`F`+UDUJ8T4rcxj7Kr5>anzq*zDZE_<-~Mf zk>yljd8evYLJoCbp*eG?xB3e^JcC;NL_TN|dm#?xOOzm~0VTLQLtfU}*fvy~0;c`k zxF~L>p=S;utBTzC#(;fH2U{cnH7$8~K!TavhSmR1yDe6}#-`nY@bv!JjChV$esNC$ z5*}!gdobe7mDMsm;0^1LzP%YiIJE(2u=36vqF@^PcGhMKWh6V!_-^m+7G~YS#dyMt zOHs2)h}yZiL77<##8Ox(p`rW?g@vG%rfEh2qo}Ztvzm^bihD`6x)*)fVcHWH|BB}J zagd|KG(4C?sdXU4aIGI4D_5eR3g%4LI&K_=yYU#3Fy&FhkO7p?{61QgjE7#j_D_Fd zy0%pf`J~1XCV|s|Ve634lUmq=zB3U8Ex36e$qH%*#+j#k1-u>qdly{HvZ~75>WtCZ zVf&E*r3Z@h#!E*ZS&RlD)xzXg1k)UB>f;9|0TSVO@aztmvQ3XpjOS!7bT{JXD=zBpG4e>n;M%M%EW(l?JIPyajL zXV0F||HKd5fboe2NS`?}^Wc#ozOtF)0Mkw&tef8gON`ZrhV`T_IOLYi{0;`Zi~dt@ zhl3`S)XL@KA(&hSOnDs&R6|mLA?SZIIKZI#!mbiwasC~*VFv?apXx3SG=h>NB7T!7 zt1nQvApfxAOb$VDrLX$w{v`rnK8F#AU|-Z)q5%kRnAiEz(gHKpr|4#plPXvVp_Lkg z?8uade}H8_JjR&%4l)ZIUwc`jmrg^1aI{!s=-L2vk(wiTbZS-9bQtD}$AYXawN%dR zEoDZ5pV>)i%SySdXRNnB2Azvv5Lr+(hb|c5%a$(+5|28Fgv}KW3r7o(UdZGu@c(dHS25GWBl? z2`^iMz=QQ75d;~n*UBe4}Uu;aF?tIzega5NlFK__$qL(_${ZX_M zz|RRcEW3Y;k<;iTdeV1_uU7%iEBEG^plT!*bm^CUa$0(WPr5Us2ge+}sM10oZXl}> zP?gYuA~m@k=gW3m=%=cWdC`Dyg*4RO>=wa!N5YJ&MW71trg)v;9fzv!lKkiY(d={K z5!fsbAugZJW(Fuu{i%+W>xGEolmJf{!)AYw*st(Q_QTLWQ498b#_-cWfi2|^o6ffw zxKiN(ObollkXhchHg#8|p1>N?duptiz`2}LT*gy<@X6*C{>W!czDvEUv+l?M3`~wK&FSfkk=;NeUC6R&LNc{ z;-b-Q_gGZQP@lYX!FIOyasaEiumljkaUd^EZUY6)7iNDCx7T2$JiNh4Q0(tt9>Qdc zu2xx!&{97*!DQXj9wXFLA!hT-t3`TOg4Y4Wi4JJ}$c;P@m%7`m$^RTmF7)94gH}p^ zWF&%q;F+)n3&IOpfJ(n^5x$6Dv77q&U2PxPRrGJv>mUZS{mp7*2lNmnChgF^)9~Y~ z6w5O|guHI$3dLEKhL{N2!h;rI%bs7YI>P$J1HB?bHpyy0Rffa?yW}^8B z#5i-rXYOSgNfyw<_u7+kBth1(7X_zLyO7C`T%%;LxI)fv#~<52}l8NAfSAK$l*ZZ9o5V(V5mI?E*)YnQsgF2 z*iU!yy7?@KJq-C1edqy5xg(#G;=O!Gu}Q7J0qN36IxinW-X784AD%1rQH&P-Y z?(D5ua!U}531oX&i!vE}tRC^hW??$^hmqNYa*_r_U@JthZfbzQD5Q^alTzZHTN6as z!GF@?=hx@zz$oyhwg!j^@4Y50pw7Kb1^+^=WO9<0)V({4VO3tJ_(zRvoUa0A18RCu z3s%qWGb+AAF#K+%&F3S{-UPjTI)Eg^-nJ@_UHqmL>i(K^0^MH{I-(>IAC+<>SYq*? z9>^56fKXLOaUNTJLyCu-Kp{#r2K$6dVv7;Ol`rjb2s82z^J>+g;)cd zeYXz{RQ7nNmJh;(oPUcW^Tos_pd!W3SZ<<0azTsK*3TmRlLGZ?4iNd0{HkDOx;`ze zZ{_vT`L`^@ieLpji#x2uk&TI11qRRWU;$Gs#%9@lyWAb`!pW!W03!S(n+OQJBplL( z)&Wh$q_P6L8=JaFska2lOfJq?Uyx@86bQ<{*C0*NUk$xZJw2KipyKZ>i9}RH<#oOh zyb##=Qfzm#yGoJ4Abr#SscE`8k6C{)ZiBM(78fu+j(`YzcOu}a;U!QaY+YUL z5!fyU23EE!`A<5*8-?kFBBO;2o-e-SswK!08<1Fa)D^Bw#^+18uFs#HC>{0Vt_2mQ z4iV1uhj%|Z27#MUh?ig!`j^iW$j!fNsrc^@1>^m@zj=GyPrnh+>C9e`kgdKQMzw2A zz)zJj4dXqt9m#dmy?EGmt_!Y{+GBU#rT#_=@;%j@XW5SHVOpp5S2MLTWc@-;kHwaJ z$8{{ge&>`(%Vy9y-!;?qVPlo}z*yoi#gyKas={qY9yCyzcVjPtUgb0x*T4bKN*cPC z@aUWJ5%u0x(LwEI(#BF9D$7C*RdjC|s11u+3_jk*l)f&wco zKYzpD3|Q_R193w;eHNE&%c08ag@kE5--tla(bHG9fX*7<{{?@n58HxG;{=T; z=2S2d5MN^~zk^3k1T57-xif=;1Qw;lC281HRBo$+GXX7Jsx%EQ>abd6L;vW>x4(lq zi@Xc}iqB8QO7H*=AC%wtqghrDR)NNfAHNLL+v^tbIlXc!^IPU$3OMHU+E%r-494|u zA#0~D`%600zkyKU%ayWc57O~6>>g?RIak1g(xmd2qU?n=%2s>@ODlcqt}KC2CT{kHHhUyxup5rP&7LogvKC8l{kF8wOr(D}lIL-qBf_ zDtP!1Z1bz+{tPzA*Ydj6)ajVNsg6bgOUGG(jamPT!`_KcO8@gm zo*7qU+5~A7%cwpCCY4Kqa_LiL$b)Aa4Vr>eiR)^U-(L8AaN>nfcUPv4IAY)Rq3XF_ zN#{*nkI^Iyu0r>>m$QB$*3f68E>;A9^jFdNqprZ&STIAro+pCnNCS*(r-6>R zLuHShs85g>*jw#Gf{5&2zLBvQU*qGn4S4J13NdJ-3gKYoaVCz-B>-RiwYfP8ks+H_ zW6#ZrQKtZnEP^&1)lSMQg&_QSBN^|T6?0u77-_BEtES-R@*yw$W9~0>5~`jW6S4s0 zv=0nfgJ{P1K5#wxQ_$_%ARz|AEtypiwJ&ArY9sTE4+LkooJ@~~6UOlmEWjqD4Z(w`EM^q^tD9{>R;eX-KmcmhOdWJk=V;&SVy3FT3*N1@aPGSgC&3xZc7Tw zA3e@-9j zUbCi6{*#P%(~&|A*WOu%A9EACL{Tz~a`}~MUeQD<%(&%eeFc|vo;8U4bjU9N8U=Fc zhLmPu1o}n~oDy{P53w)#BBj1sm1h*P^_z?uWvzer*b5=y@v4~m_fk2<%il5vM86U@ zyYFQ<5d?@^!7?i+J7rYAMlQM=iRgP@B>GzJSdIiUEdJuQzwjHs!d0wFp6}3h8=G(Z{2jSop6+di{|qZHoVO1P%=#VU8+&NL zL0aOHR?c} z_DSSM71llWGPhZY$plD1(fG}D&CE6l7WeA8I6N13QkBC@iOQzyJ~~B1=kv9V#^*;(EfbHfWH?A)C@Q?C{LR1v9pB&;60}*AbX2C8_&ur4 zRAG>=u;<|?@)$ypWD&K?>JL7%))d*-D?{#UvOU|aO0})}b#;acRYzC*+z6z?K`l$R zhVr%hlpDg+x%%#IB$t)YG_N0d6JB9{XYrD}WnP8h3Eu`u7VQ|@O;E0#q*#-~a%WO`LQ z|JbrG;59hdbPd+Ac72Y|$!n_~!wr4XcC2q`x-U;+{miB-Rv0HGRV&6_H&%Ymch+0h zuJ)N7E1dq$>pH(+%$d%ud#w1ceZz||NEVy1?to2Fq4&JFe2CLzqd;WnFz`bu^Y*IO z|Kyn4IYf&-{^cL_3mY*WcFL(!Tf~mowjuJk(_hzDit@MH`78NyJ=QFkK_{OAnrlM= z>DjUs&UQ--?t+-8jXvVY_axnarZ?yEOizl;5A3?KL|n395}qyJq1;Y?(v^N<)2yhI zY+5U`SK^X++Xa%uY?#2x#N?!9W7vOS^;yj<-Ko5nXLN_?TBzu%-YbHN@pZl6^ALB2 zvbgP#*_iUa_02xRI(f;W+YIi&@~g8l?XP~kZfco;nI9Lqk#tzrplARDZ(>jsm5VAk zM1^Ug(&EBteWF-V!ibu#&QLzl=`3{hfD=V97_R#!Z!PMlD%^slguFd&vK1RMU7p#Q zAHrpM|9Cri*rv*jKfq<%!?>GQ;_yH6ncPJGtjhO=RT2iaJyrujCLe^3SbX!y(JCLg|?s>Xli%%P?~wvPvb6{}rkCQ+4{%$dwPPGbos64gC{mxD~O zZmTNaGppDG_pi52Tl(ajay_27Bf-1OML&?*u~Cw_UUrM-RYWQJyF@pcWtC&bw-?1;E`NTl%L-Hr9U`PNqBrLo=d zpQ+n(&W|6j&osT%5A4)pd@pKfLRJaZ(vaec&L?S!Yy2GYG#;d!;lEkxbY0z2G8M-E zxOsRIrAkDR3cJjKPHeettVUXi?uG|+Y>L% z%U>HC{Zbm9MAt1oL8Tr@rV350S8NXz>{ycUjbHz7Ud}&fxd15^c`9j#rMZISb5T0; z6YU_!a!(q(2CN zUM%?U-7u_{HmaUe@1GXj+mA7#Z%t`4;0`%iMns2~trnc(xJO5T(amb|PCHQtJ3i zUY~J8uH$ei%mly7NBCVT-1R%m2pcOx{;_+@!DhC zepeQ5rMSCNK2jBda`(<1uR8Tb^FNoVxT_cT)P zkPp)-NW88sX6iz^+FnlwkgDqI;B%@4yDQU;0H$N8@$X_RF}I0NZe$3UkK>hqIBhFy zo*(puqJqQQL+~LB_+}h|B;a1IL|1;GpLz%52z*-N8frm-BNTphLspeQMphGWuUpap z7r@`XVKDxHKJ6ex4b_KzNIL+|q%2ZCa$TcbXU4<0mo10L_g?o-yYmY z%kEGEctPcq^0|=dY$fXpSOZ7ryHbU83{N@nV|`{7 zV$4o%WdXMm=dw1paqH@+sOJI^aRzX3N^#U2BE2z;dLf4}#$Ur<4^DSy=}k%DtY7^G zV=iaA*jUNv*BY$0)wN<6JEf!Hd0?0Jg*+_<&dt=X=4K!RrSv7L-xb1ZcBrEJyD^TB}Mf8Xqc zj9D+QXF|u5RkPT{&u#~x?9fpToCjQips54FJFK?)`g;kd#j|Pq{ab{~&u)!$KWnrg zyBc!ec#t+^;D9Gr+!vw|v-}~?1T}uy6Sr00FdFzVkt_p|QUc1*5 zk)k*Q_mX_Sn*N`sG)@+wne${&fT1_U_V!#BaRw3=+am>KlJdS{glw-J#VQ%0G2Y&;NPXo2Cg}YliB~p>w7dexn-emRrBNYqy zYI^6jIh~){AgFvf=!Yz*;UH_go<+?~OcIV(nscDXkq@H4iQ&->2|w@`v=3L*nj}Rt zLc8)&)T%^3)d{C!p^oP}k;1&+e!D<36;|vKhh>LExfi#F!Vbn?G2_FB z4-a0u+@CaLx&hQo^H&a_*mwJ`pY-%oGp^W@c;spHpf%n>!WcOBL2{*wOX*s`|0jV8 zzn67XUjI+4tN)5o|6deu|3wn}@6!AK|JPxU{D1dwy#MzfJ`JGGIvDF}!R&VmY21i; z^Z^5|FYrks=BU-Z4*bZq;OV3T9a8DXUDQVH$snIV;R#ZiE7I!MfkKxB41^B(dknIu z`_}rxSRLwzU<0umbt@`AQbL;518-#$9Z%_<^|5SF?0qNOlYBvG|2JrPQirTNK=tI+PXsXdAw%tB(2OW~0lIW% zgK@_joRsPPy?e?8w=#x_gM@ftY{?K@xDL#nP2&;B;Br!KK2T)vP^5XG#&5{};Sj?` zq4`M$`Xp;X!>ClB*aEj|+s3FUYLzuWveh3`C9-3KXTb}Yw0ClQT{{oFr^x`@jG@!I zc&w|CD_5n1Hv=651P*X%I1BAG>a>l6BNR-wk&~{r3iJ^#f4L*z2DWK`E_6 zkg!5j)dsg;!jn@l>6nL^R8}n9mPkF(dE5;`Vk@{<@}+`WupAu-ebr`F?VLo`(`?kB z)L8(2kjOzmTa3L$mU^R7NHhz-WTMzB;`4?|2uKg*eN5(n8#$|yE?tM#Cer>&Z6}W- z>qo}`YUjkHKEyCd!t;ZDgk}VKdvCve{5P~}YdGV7*ut8du>w~-|B78DOu+IWXab6B zFBv?&*K|6Hy#%8I24S#}?R8=oAV!pp;squGb`Q33oxpS)QR*RlIAxYdz0zxPSM3Kah=;?CA+2 zWq4aQ3d+dNb8#<*NP6m=0|&yE3qct4(^8Cltco;!e(y+j!%=>1{U#_5me5=@0t5;F zRr|2SHEWmeDL&6i0ss&D#2egMbLV>nlePZKaF;m4FMrGWarP-@p~vmT=g+|NPq}p7 zna2$2v$Q7fBj$Vr=7n=f}#IDjM}^U|i^ zv3bVES90fp8Lh1QNQxf>q)lC@{}6nVLvg=*LV3YKHDyA`1L}< zBnfqBW>oJSD2Wo?Pi2>D*t_dXGpsV)Irce*Z@@b$73_a}E2!3y!a#V6&JHHMV1nX} zsUEFQE^TXk>36%QcqgQ`^l1WZz_)MZ{ee0NcXTa4c3-VwT)!To@8N*MXRr% zYThl<_=iGGkbFIr_Y!Fmquec%-BD)Ud-R0}P&Q~yqF){90;6YXi*XsMATcu68$Lqc z0&S46(3MJusS|BPj%E88DcxP2Qx2JUOrkRhUJ-V@sX(PrKltf=w8%yAOZ3xydWBR!G^60tIXamK6 z&%$gsAJgn-1IJM7>HUs=LjmJCRgC^Qu3`nKNtF$u!o*|&3ZYATA(TB~UXAass8^j$ z-#X9Eh->330GcKi7@Ha_ho*Fqq6DLVT3vTBHE+-P%L6s9(aR?80OJa5NGOe4opvUPt^wJNR_~^xOH6YG%0+GkMF`tNfbp#@t z&|S99D0MAd-Dv$Wk_L+#!)Y&%ZVWMSa^|;6$F8KK- z`W1r+-5rb*l$PF**87sf4r#23!2iVx0%JZJLci8SKb!Yffsik8uBQKJhhWU@M<-?l*)ep zlql?E3|XX0`&<$1E@YO2E*AW`tv0KFu@(G$5SZ0Ui!*Jg{)R|_b*`Wef&LSF7Yi?6E`UKlwdV@^7i#8|Q>d7ga|@lfEx1L^nk3`g%*~^4PC< z$I$GEEU&RCAendbOL#LO@->3N!H+7+$q5X{Z6Z#z$A9=8ld$gVKt|PG05>=ofkB2T z$ov`}cfk_11g1oYXW|V#x~)9FTUaFEjtNbDa|nLZjbwO2@YES~<)#NSX?8vD4vVJg zS>n|u;8hb5wRmLuzHg@@OavHy%K79yBN^dC4=SFnzTEQNqYnnD9 zQ#QEMEk3Ww$mhQw?LAZkYq96@$w!7{nGn$2BAeLbo0q?un_Etsh-wz%%If_2^=_v~ z*PzfL%SSd8$$dk8f1BVDp;!Ch^cVq_l_?&`t-qDw+XjFUe0mFOjX zpHZq5{v8PWSO1Fl_#fK(#~QM^s{$wr(eSn+-jgke4fxl-Smld8Wf1ieTPyQ`B~z(g;|5(_p?Z#1c^tes7;FbS^R72tAv+z7Brk2l=lw5!JtQIzH=J z4|*yw5OQ;>TYp~p>wZ*8WAhOxgC+RcDB=7I&{blfQ(ltl|M14q>jRPY$PJL7{b@yq zY$$?)n-|=??HPAi!?0pi0Cb43|FZ)*<6hDZnX1OD1`55@I#0O8w$44_m(?$|egf&;-hyzSZjF+hjlYL3wF1!V_9OsU$BVpvF zvM(qseU!hpZ80S$*dUqorQk9ewWo=1i}Ic#wS%x=@!f^7u`uYRuZ;+spl zjBLi3>*ORmNP12)N2^Hl(X%7ben0x7D0L-`P+liUq}sBPf%YgJ>b!WC@JDKr zEt~;5YE~SeUo&vgHJj2rs`amO1DEz~0B1 zE}1ktR?x`vI=O6xVKBiSLEtZywlQKCmSzDQCbl`&!ja=RDyDmngmN;}&vkorQ7gHcbr5JhGnkqgr zyzJviXH3qOrdaCZx<$L-(KON?12kYa1tO zUXX4>CG4?9_$XZ*aKt)BB&&Hn`4xV6b63XpD#*gAk8(8UI7kMPd?tbIL4qR@Yf1L* za)@=-e!!QPKO`N4F42wCfElutmvlaTUeY4nliNN2aVWVx_Mj2;MHcfLoN`!EM$sKS z@hU4K%-?1;(kz=rukHcx=vgxp_rU0+Va(@%);}8t`FE<+m4GImt3z{` zY0!EEDxG{-^$X!WbI8(&{3C)@{b>z{yXl-n2O9AKv-KTEei$u^?CYpS%UtYM2T`sB zi7QMyYcj-#6&t-L7vHI+YbK@AyEGyN=jyU21g89BI{abf^7V=;CJ)7OAH`}>Ps-R;K>Z`RYkrGe3{J#wm zqk4xSpfW}4_T`l;oX&LGbBHv}<4NCDC-eZy0-j(;=i=FVQ(N*M##jGSXzQY2%C%36 zAGyaq`EC}wRrou#=nx zT7KGd=XKHQT!o&g0wDqFfMeZAtv+V6!hR3b&tc{T-*cCt-q#oa^oa61CUfJHFx%-PjP=AwZZqH8 zYX-skh7gD6PdgYtX>a+J>FCh(q7K6^JhiH~P@;eg1f*!ls`-lLLm8~RH1W?4HDrF< z!p9hI?bu!Ka|wL4BW}4hwwi~P=<(d#h#w^P;lCSjU4&l!9lX6-o5ENwk%V^XY1?Eo znOMYV&}^);o?=j`-O?-75$1n5Ck*LCCROxsm?_yvP24W$-qP`X5OwQeJo}fV7sDCf zCj+lbUp?MPP5jctb!x?K%tamyk2Q_GMY#8>J@d7fB%06(kCqg(OHg>a7~_;4SWYpF zu=HLfuxw*14(6^#Vw3w?hdys9!)USy)w|-z8lnPnjchALI+~ zgSm+u;kUU*fN`|BVYg{Upe*P#xbMh<+x+@ObeXBcMT{O15ZOX?=lQyjBRW*%TTG3} z=Ni^yKefO3I&yn%9cF!vq&PAEhGY6L?imk%2QkP}#QWZ0Cp~lWpaqFTVfK*LBX-df zd*Lxmcn3l4TXSO9@vvD|)~UOJ73YzUBRFi49vK0;UT`ARKGU7Jh-~&~!$n%>$o2UP zACnUxZJ7`xEAD@k13qwG_|CEU1uZa(V8?D`384R4d@I?sW{p?$cqS=;a_`OViY}Nw zjw9a=GBWFV4@hA7Xiwz!6J*Rv-!cN@*GBlK@Uw~*6bY}vWmvXlI%6_5Mz3My2oDn; z7BE9ui4?}<pwypTg?4n$hYM57MwIPG3gs@Dn);XTSvGNJJVtT;k(FAPvmfdsDfX5IL@y z4AZ<9(lxCvq7qyLzwn5=z(`XZ;>H*Id= zLMiwhZw#Xc@OC6lg9#Px=ig|k16HZsiuwzn=V5D{4%rDWde-`l#Gvgg+C?OfRS-qe z%$hwOk?d(R3^|PFS$b3kGk-FCEla>XG3fco`p;lbu&$rw5NMK>yJt}-tj2@iCSu@Z zse4bM-)k{+&_I3J#u1C5&(lOAMDi{LS5g+z%ErzSiXu&vc@|UWBul>^H z4XEe!rTe)UmB*t!L6@x|+XDjWSmA9hrs@tQbij)(D0gFP*klKy=fUj-6tlzh&A@ zpCQYcn+Baej|Zq{6Hp#YTsS<6*_sj>89$Z@=R2Y6WILq}6Q&!T>5zuH?ts*iC^D8< zNSb0y%R|8@_FRE4i%)%W#EF(ZXe*<-lK~?(yXpUNCy&xaate#H3bE9v3^p5%!LBbQhcS!!=r zfm~<`nz~xr2ZUaWlpt^_q;jo^nurQPqt=rg3290?rA_*t0%Wks-3iMzODEfe z4a^ziwbL?LjZft5rrbH}1IB^Dv4ISJwM{sx_YJ<>#d!kJZ|S%#lt@9cU^)M~DXhXA z(~YvL%JL2-rg%tdL^y5VC3csbwKS{X%UOJ!9^#&N*pxKK&@i%8hC*<7P=R-Yz;hy^ z5$QTrI)^;wELv&|BJa-C{Z%er5!{7m>%`!`!HW#w!8dXZ#~Bsz1)yXLsWTtoBu?7L zor9rq<-1wQzY%Q})$uN>Ew?F}Z@qYqe}u}TZY=P~_l4mfPs|D2c)uRy)>)XLGU0Yr z@i*Q>z_|D2Q^GlSCnXaH16As4^i7?tygCd!x;$__8>Vm-~6{qa^qHk)Kod(N!0#>2^o9O z94`9s6yy6!gQMigM6LQXb6GuJfM;?<&!9@8*ijXUjTPI3c!{JI^5w!wS@N6Rxq}bF zrNij1|fL9l5;yUZYfgMxV%49~{$n>*H16|XD7jiZbgL+P9+==b(bp>3a; zu#rGFpMH7p(iQ8=$Qy(9G-ZRO;v3=x0%jSC9g_Jry?hAtuqCHB2E+IKpYbSYN!wx9 z`R2l9RPE@!>ICGLMY>!sV6lI4zKAn;H_t@8k~4q8ce_l9FhoV*Jp|{1WOh9}!dRfA zGIw|IPv4r;tWqA&Z(FU^dFJwo>w8KEa@W4*2sY6a_rL!?_$bG&Bj##w4ZCX9~fY4c7;x$=#r9DOKXe`&RBpNre z3s4`to2bXyeC;C^yDzqx@~xd`yAua&arA}!WShCeqnMl1M$?TiEOQa1!BsZ-QPBKf z)5w|S*pV3)#lx^)E7dI?*5ucEI)bS#G)@UJ0gJ0)bQBXp;L1%%SEWfq*g;YsThH+i z^7+3M8s=hjq_`0q<_TrBF2_kQ_Wf97q`hb)YkW{FZ3Y#6U2O$&Zi3)M80!~_Ji?y| zcc$$}5%%%bB@U8xWT*j}gdmo0uq>Pc-E0PkAPNJVx%Jt1=i0hkUk*nL#)W{Woe81E zvmxT(f5v5Oz3mBSHvm4!?x%owI*9UkjPe;(17B{@7Ga*OPHds ziRn##-9aA1b4Y*FHcaG#SiVIy+45tKWE{v%SITMngpsBwjlzho@g)qSER4-ARgf_mY&ECC4TeR}jJdrs583-~} z&j)~_tIslCr5|Q9SDG{7tKIV>)Kr9SoLe@Ccr_H2@F-bOE)a7!mFK&sn802K#2Zl` zVyd$4>4;LLbi#zV&@ggNuhB7o|6uPz z)%!&5(-nPaZf+w|G%nya-zY}KmaOj+@tG&e)_ZoFl(G09;Zt-|9RboVK;)Tdn zmZPvC;k|6$$ic2atC`+j3;A}OU(1;*^j6~Li#uUc9`hy#DY}==ox3iuqTn-ko(4D8 zGPiKo0YRIXIXQPEZ9?UenvAm#iydD!mT6daL~axlKXCJPnuF0xcYWgzOxU&~B{jTG zSE2{TGI{SuW7ny^-cR1I`QB;X$A{q0xcLkr`YN(qtRd4;UQhbhpElt;GmNJRZWDh-Pe$3w kFf%Xy@e%(" - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "example_count": 0, - "measured": false, - "metadata": {}, - "metrics": {}, - "mode": "raw", - "notes": [ - "limit_per_category=1", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": null, - "reference_kind": "historical", - "runtime_seconds": null, - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "blocked", - "system_name": "mempalace" -} \ No newline at end of file diff --git a/benchmarks/results/prepush_smoke_20260423_123335/runs/convomem_new_brainctl_cmd.json b/benchmarks/results/prepush_smoke_20260423_123335/runs/convomem_new_brainctl_cmd.json deleted file mode 100644 index 9e74c7e..0000000 --- a/benchmarks/results/prepush_smoke_20260423_123335/runs/convomem_new_brainctl_cmd.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "artifacts": { - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\convomem_new_brainctl_cmd.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" - }, - "benchmark": "convomem", - "caveats": [ - "Blocked while loading ConvoMem evidence data: HTTP Error 404: Not Found" - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "example_count": 0, - "measured": true, - "metadata": {}, - "metrics": {}, - "mode": "cmd", - "notes": [ - "limit_per_category=1", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": null, - "reference_kind": "measured", - "rows": [], - "runtime_seconds": null, - "series_name": "new_brainctl", - "source_path": null, - "status": "blocked", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/prepush_smoke_20260423_123335/runs/convomem_old_brainctl_cmd.json b/benchmarks/results/prepush_smoke_20260423_123335/runs/convomem_old_brainctl_cmd.json deleted file mode 100644 index f239bdc..0000000 --- a/benchmarks/results/prepush_smoke_20260423_123335/runs/convomem_old_brainctl_cmd.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\convomem_old_brainctl_cmd.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" - }, - "benchmark": "convomem", - "caveats": [ - "Blocked while loading ConvoMem evidence data: " - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "example_count": 0, - "measured": false, - "metadata": {}, - "metrics": {}, - "mode": "cmd", - "notes": [ - "limit_per_category=1", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": null, - "reference_kind": "historical", - "runtime_seconds": null, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "blocked", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/prepush_smoke_20260423_123335/runs/locomo_mempalace_raw_session.json b/benchmarks/results/prepush_smoke_20260423_123335/runs/locomo_mempalace_raw_session.json deleted file mode 100644 index 749316d..0000000 --- a/benchmarks/results/prepush_smoke_20260423_123335/runs/locomo_mempalace_raw_session.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\locomo_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\locomo_mempalace_raw_session.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" - }, - "benchmark": "locomo", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "measured": false, - "metadata": {}, - "metrics": { - "avg_recall": 0.6028, - "cat_1_recall": 0.5899, - "cat_2_recall": 0.6921, - "cat_3_recall": 0.4604, - "cat_4_recall": 0.5809, - "cat_5_recall": 0.6188, - "perfect_rate": 0.5534, - "top_k": 10, - "zero_rate": 0.3499 - }, - "mode": "raw_session", - "notes": [ - "granularity=session", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": 0.6028, - "reference_kind": "historical", - "runtime_seconds": 2106.411, - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "mempalace" -} \ No newline at end of file diff --git a/benchmarks/results/prepush_smoke_20260423_123335/runs/locomo_new_brainctl_cmd_session.json b/benchmarks/results/prepush_smoke_20260423_123335/runs/locomo_new_brainctl_cmd_session.json deleted file mode 100644 index e12e43a..0000000 --- a/benchmarks/results/prepush_smoke_20260423_123335/runs/locomo_new_brainctl_cmd_session.json +++ /dev/null @@ -1,4455 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\locomo_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\locomo_new_brainctl_cmd_session.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" - }, - "benchmark": "locomo", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 199, - "measured": true, - "metadata": {}, - "metrics": { - "avg_recall": 0.9413, - "cat_1_recall": 0.8641, - "cat_2_recall": 1.0, - "cat_3_recall": 0.7436, - "cat_4_recall": 0.9571, - "cat_5_recall": 0.9787, - "perfect_rate": 0.9196, - "top_k": 10, - "zero_rate": 0.0402 - }, - "mode": "cmd_session", - "notes": [ - "granularity=session", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": 0.9413, - "reference_kind": "measured", - "rows": [ - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_1" - ], - "question": "When did Caroline go to the LGBTQ support group?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_10", - "session_13", - "session_12", - "session_4", - "session_14", - "session_5", - "session_7", - "session_11", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_1" - ], - "question": "When did Melanie paint a sunrise?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_14", - "session_13", - "session_8", - "session_11", - "session_16", - "session_17", - "session_9", - "session_18", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_1" - ], - "question": "What fields would Caroline be likely to pursue in her educaton?", - "recall": 0.0, - "retrieved_ids": [ - "session_7", - "session_13", - "session_18", - "session_10", - "session_4", - "session_14", - "session_9", - "session_17", - "session_5", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_2" - ], - "question": "What did Caroline research?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_1", - "session_2", - "session_8", - "session_14", - "session_13", - "session_18", - "session_15", - "session_5", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1" - ], - "question": "What is Caroline's identity?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_11", - "session_16", - "session_17", - "session_3", - "session_1", - "session_8", - "session_9", - "session_12", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_2" - ], - "question": "When did Melanie run a charity race?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_7", - "session_8", - "session_14", - "session_13", - "session_18", - "session_15", - "session_1", - "session_5", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_2" - ], - "question": "When is Melanie planning on going camping?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_2", - "session_16", - "session_12", - "session_7", - "session_17", - "session_14", - "session_18", - "session_9", - "session_6" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_2", - "session_3" - ], - "question": "What is Caroline's relationship status?", - "recall": 0.0, - "retrieved_ids": [ - "session_16", - "session_1", - "session_8", - "session_9", - "session_12", - "session_7", - "session_5", - "session_14", - "session_18", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_3" - ], - "question": "When did Caroline give a speech at a school?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_13", - "session_18", - "session_5", - "session_14", - "session_10", - "session_7", - "session_8", - "session_19", - "session_2" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_3" - ], - "question": "When did Caroline meet up with her friends, family, and mentors?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_7", - "session_9", - "session_2", - "session_16", - "session_5", - "session_17", - "session_3", - "session_8", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_3" - ], - "question": "How long has Caroline had her current group of friends for?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_7", - "session_1", - "session_4", - "session_12", - "session_13", - "session_16", - "session_3", - "session_6", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_3", - "session_4" - ], - "question": "Where did Caroline move from 4 years ago?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_8", - "session_19", - "session_4", - "session_16", - "session_15", - "session_7", - "session_17", - "session_9", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_4" - ], - "question": "How long ago was Caroline's 18th birthday?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_11", - "session_16", - "session_15", - "session_3", - "session_17", - "session_8", - "session_9", - "session_7", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_4" - ], - "question": "What career path has Caroline decided to persue?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_7", - "session_14", - "session_1", - "session_3", - "session_16", - "session_15", - "session_8", - "session_9", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_3", - "session_4" - ], - "question": "Would Caroline still want to pursue counseling as a career if she hadn't received support growing up?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_7", - "session_17", - "session_1", - "session_18", - "session_6", - "session_2", - "session_10", - "session_5", - "session_3" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_5", - "session_9" - ], - "question": "What activities does Melanie partake in?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_15", - "session_8", - "session_18", - "session_1", - "session_4", - "session_13", - "session_9", - "session_19", - "session_17" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_5" - ], - "question": "When did Melanie sign up for a pottery class?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_16", - "session_14", - "session_10", - "session_12", - "session_17", - "session_8", - "session_18", - "session_11", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_5" - ], - "question": "When is Caroline going to the transgender conference?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_7", - "session_17", - "session_1", - "session_14", - "session_9", - "session_3", - "session_4", - "session_13", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_4", - "session_6", - "session_8" - ], - "question": "Where has Melanie camped?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_18", - "session_9", - "session_6", - "session_2", - "session_4", - "session_16", - "session_8", - "session_1", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_4", - "session_6" - ], - "question": "What do Melanie's kids like?", - "recall": 0.5, - "retrieved_ids": [ - "session_9", - "session_10", - "session_15", - "session_18", - "session_8", - "session_17", - "session_1", - "session_16", - "session_4", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_6" - ], - "question": "When did Melanie go to the museum?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_13", - "session_1", - "session_14", - "session_5", - "session_11", - "session_18", - "session_8", - "session_7", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_6" - ], - "question": "When did Caroline have a picnic?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_8", - "session_14", - "session_13", - "session_18", - "session_15", - "session_1", - "session_5", - "session_11", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_6" - ], - "question": "Would Caroline likely have Dr. Seuss books on her bookshelf?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_18", - "session_10", - "session_6", - "session_17", - "session_4", - "session_14", - "session_9", - "session_5", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_6", - "session_7" - ], - "question": "What books has Melanie read?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_7", - "session_17", - "session_2", - "session_8", - "session_18", - "session_1", - "session_4", - "session_13", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_5", - "session_7" - ], - "question": "What does Melanie do to destress?", - "recall": 0.5, - "retrieved_ids": [ - "session_7", - "session_8", - "session_18", - "session_1", - "session_4", - "session_13", - "session_9", - "session_19", - "session_17", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_7" - ], - "question": "When did Caroline go to the LGBTQ conference?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_7", - "session_14", - "session_1", - "session_13", - "session_11", - "session_10", - "session_9", - "session_4", - "session_3" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_7" - ], - "question": "When did Melanie read the book \"nothing is impossible\"?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_7", - "session_17", - "session_2", - "session_8", - "session_14", - "session_13", - "session_18", - "session_15", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_7" - ], - "question": "Would Caroline pursue writing as a career option?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_1", - "session_4", - "session_18", - "session_8", - "session_9", - "session_12", - "session_5", - "session_14", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_8" - ], - "question": "When did Caroline go to the adoption meeting?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_2", - "session_17", - "session_19", - "session_13", - "session_10", - "session_5", - "session_15", - "session_7", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_8" - ], - "question": "When did Melanie go to the pottery workshop?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_4", - "session_5", - "session_16", - "session_12", - "session_17", - "session_14", - "session_13", - "session_1", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [], - "question": "Would Melanie be considered a member of the LGBTQ community?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_5", - "session_14", - "session_11", - "session_9", - "session_3", - "session_7", - "session_10", - "session_15", - "session_16" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_4" - ], - "question": "When did Melanie go camping in June?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_18", - "session_9", - "session_6", - "session_2", - "session_4", - "session_16", - "session_8", - "session_13", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_3", - "session_5", - "session_8" - ], - "question": "What LGBTQ+ events has Caroline participated in?", - "recall": 0.75, - "retrieved_ids": [ - "session_9", - "session_10", - "session_3", - "session_2", - "session_11", - "session_7", - "session_17", - "session_5", - "session_14", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_5" - ], - "question": "When did Caroline go to a pride parade during the summer?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_10", - "session_11", - "session_5", - "session_18", - "session_16", - "session_12", - "session_2", - "session_15", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_3", - "session_9" - ], - "question": "What events has Caroline participated in to help children?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_9", - "session_10", - "session_3", - "session_2", - "session_11", - "session_7", - "session_17", - "session_19", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_9" - ], - "question": "When did Melanie go camping in July?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_18", - "session_9", - "session_6", - "session_2", - "session_4", - "session_16", - "session_8", - "session_13", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_9" - ], - "question": "When did Caroline join a mentorship program?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_10", - "session_3", - "session_8", - "session_14", - "session_13", - "session_18", - "session_15", - "session_1", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_8" - ], - "question": "What did Melanie paint recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_17", - "session_11", - "session_12", - "session_18", - "session_10", - "session_14", - "session_1", - "session_8", - "session_16" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_3", - "session_6", - "session_8", - "session_9" - ], - "question": "What activities has Melanie done with her family?", - "recall": 0.4, - "retrieved_ids": [ - "session_10", - "session_5", - "session_15", - "session_7", - "session_1", - "session_6", - "session_13", - "session_16", - "session_17", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_5", - "session_9" - ], - "question": "In what ways is Caroline participating in the LGBTQ community?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_11", - "session_9", - "session_15", - "session_14", - "session_3", - "session_7", - "session_10", - "session_16", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_6" - ], - "question": "How many times has Melanie gone to the beach in 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_3", - "session_14", - "session_6", - "session_4", - "session_7", - "session_17", - "session_8", - "session_18", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_10" - ], - "question": "When did Caroline join a new activist group?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_1", - "session_9", - "session_13", - "session_12", - "session_4", - "session_3", - "session_8", - "session_14", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_10" - ], - "question": "Would Melanie be more interested in going to a national park or a theme park?", - "recall": 0.0, - "retrieved_ids": [ - "session_11", - "session_18", - "session_15", - "session_16", - "session_7", - "session_14", - "session_5", - "session_4", - "session_3", - "session_6" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_11", - "session_9" - ], - "question": "What kind of art does Caroline make?", - "recall": 0.5, - "retrieved_ids": [ - "session_2", - "session_4", - "session_3", - "session_5", - "session_6", - "session_11", - "session_10", - "session_15", - "session_14", - "session_16" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_11" - ], - "question": "When is Melanie's daughter's birthday?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_4", - "session_8", - "session_18", - "session_1", - "session_13", - "session_9", - "session_19", - "session_17", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_11" - ], - "question": "When did Caroline attend a pride parade in August?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_5", - "session_11", - "session_10", - "session_8", - "session_9", - "session_14", - "session_12", - "session_17", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [], - "question": "Would Melanie be considered an ally to the transgender community?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_18", - "session_17", - "session_1", - "session_14", - "session_9", - "session_5", - "session_11", - "session_15", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_12", - "session_3" - ], - "question": "Who supports Caroline when she has a negative experience?", - "recall": 0.5, - "retrieved_ids": [ - "session_4", - "session_10", - "session_12", - "session_19", - "session_9", - "session_5", - "session_7", - "session_15", - "session_11", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_12", - "session_5", - "session_8" - ], - "question": "What types of pottery have Melanie and her kids made?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_10", - "session_15", - "session_7", - "session_5", - "session_16", - "session_12", - "session_17", - "session_8", - "session_2" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_12" - ], - "question": "When did Caroline and Melanie go to a pride fesetival together?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_14", - "session_5", - "session_11", - "session_12", - "session_10", - "session_17", - "session_8", - "session_18", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_12" - ], - "question": "What would Caroline's political leaning likely be?", - "recall": 0.0, - "retrieved_ids": [ - "session_18", - "session_4", - "session_14", - "session_9", - "session_17", - "session_5", - "session_10", - "session_13", - "session_7", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_13", - "session_8" - ], - "question": "What has Melanie painted?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_13", - "session_9", - "session_14", - "session_17", - "session_16", - "session_11", - "session_8", - "session_12", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_13", - "session_7" - ], - "question": "What are Melanie's pets' names?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_7", - "session_8", - "session_18", - "session_1", - "session_4", - "session_9", - "session_19", - "session_17", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_13" - ], - "question": "When did Caroline apply to adoption agencies?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_2", - "session_19", - "session_17", - "session_8", - "session_14", - "session_18", - "session_15", - "session_1", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_13" - ], - "question": "When did Caroline draw a self-portrait?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_14", - "session_16", - "session_11", - "session_17", - "session_7", - "session_8", - "session_19", - "session_2", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_14", - "session_8" - ], - "question": "What subject have Caroline and Melanie both painted?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_9", - "session_19", - "session_13", - "session_12", - "session_16", - "session_15", - "session_3", - "session_1", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_14", - "session_4" - ], - "question": "What symbols are important to Caroline?", - "recall": 0.5, - "retrieved_ids": [ - "session_14", - "session_7", - "session_2", - "session_11", - "session_6", - "session_18", - "session_15", - "session_3", - "session_8", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_14" - ], - "question": "When did Caroline encounter people on a hike and have a negative experience?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_4", - "session_16", - "session_8", - "session_14", - "session_11", - "session_5", - "session_15", - "session_18", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_14" - ], - "question": "When did Melanie make a plate in pottery class?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_5", - "session_16", - "session_12", - "session_17", - "session_8", - "session_15", - "session_7", - "session_10", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_12", - "session_14" - ], - "question": "Would Caroline be considered religious?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_18", - "session_1", - "session_8", - "session_9", - "session_7", - "session_5", - "session_14", - "session_4", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_15", - "session_2" - ], - "question": "What instruments does Melanie play?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_6", - "session_2", - "session_3", - "session_8", - "session_18", - "session_1", - "session_4", - "session_13", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_11", - "session_15" - ], - "question": "What musical artists/bands has Melanie seen?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_11", - "session_17", - "session_14", - "session_8", - "session_18", - "session_1", - "session_4", - "session_13", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_15" - ], - "question": "When did Melanie go to the park?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_16", - "session_13", - "session_1", - "session_14", - "session_5", - "session_11", - "session_18", - "session_8", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_15" - ], - "question": "When is Caroline's youth center putting on a talent show?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_9", - "session_11", - "session_14", - "session_5", - "session_19", - "session_12", - "session_10", - "session_16", - "session_17" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_15" - ], - "question": "Would Melanie likely enjoy the song \"The Four Seasons\" by Vivaldi?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_15", - "session_11", - "session_8", - "session_10", - "session_4", - "session_14", - "session_9", - "session_17", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_11", - "session_16" - ], - "question": "What are some changes Caroline has faced during her transition journey?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_16", - "session_3", - "session_8", - "session_7", - "session_18", - "session_11", - "session_14", - "session_9", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_16" - ], - "question": "What does Melanie do with her family on hikes?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_7", - "session_12", - "session_4", - "session_16", - "session_8", - "session_14", - "session_19", - "session_2", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_16" - ], - "question": "When did Caroline go biking with friends?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_6", - "session_3", - "session_8", - "session_19", - "session_2", - "session_12", - "session_4", - "session_17", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_16" - ], - "question": "How long has Melanie been practicing art?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_3", - "session_15", - "session_17", - "session_1", - "session_6", - "session_19", - "session_4", - "session_8", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_13", - "session_16", - "session_7" - ], - "question": "What personality traits might Melanie say Caroline has?", - "recall": 0.6667, - "retrieved_ids": [ - "session_10", - "session_16", - "session_15", - "session_8", - "session_14", - "session_1", - "session_18", - "session_9", - "session_4", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_15", - "session_17" - ], - "question": "What transgender-specific events has Caroline attended?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_15", - "session_9", - "session_17", - "session_3", - "session_1", - "session_14", - "session_5", - "session_10", - "session_2" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_17", - "session_7" - ], - "question": "What book did Melanie read from Caroline's suggestion?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_7", - "session_17", - "session_2", - "session_8", - "session_14", - "session_13", - "session_18", - "session_15", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_17" - ], - "question": "When did Melanie's friend adopt a child?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_2", - "session_17", - "session_19", - "session_8", - "session_6", - "session_16", - "session_3", - "session_12", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_17" - ], - "question": "When did Melanie get hurt?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_8", - "session_5", - "session_13", - "session_1", - "session_16", - "session_18", - "session_14", - "session_15", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_18" - ], - "question": "When did Melanie's family go on a roadtrip?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_10", - "session_7", - "session_17", - "session_4", - "session_3", - "session_13", - "session_2", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_18" - ], - "question": "How many children does Melanie have?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_10", - "session_3", - "session_18", - "session_1", - "session_4", - "session_13", - "session_9", - "session_19", - "session_17" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_18" - ], - "question": "When did Melanie go on a hike after the roadtrip?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_4", - "session_14", - "session_12", - "session_16", - "session_8", - "session_2", - "session_9", - "session_1", - "session_3" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_18" - ], - "question": "Would Melanie go on another roadtrip soon?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_1", - "session_9", - "session_13", - "session_11", - "session_12", - "session_16", - "session_8", - "session_4", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_19", - "session_7" - ], - "question": "What items has Melanie bought?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_4", - "session_8", - "session_18", - "session_1", - "session_13", - "session_9", - "session_17", - "session_15", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_19" - ], - "question": "When did Caroline pass the adoption interview?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_3", - "session_2", - "session_17", - "session_13", - "session_8", - "session_14", - "session_18", - "session_15", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_19" - ], - "question": "When did Melanie buy the figurines?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_8", - "session_14", - "session_13", - "session_18", - "session_15", - "session_1", - "session_5", - "session_11", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_19" - ], - "question": "Would Caroline want to move back to her home country soon?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_19", - "session_18", - "session_8", - "session_1", - "session_4", - "session_7", - "session_10", - "session_2", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What did the charity race raise awareness for?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_7", - "session_3", - "session_8", - "session_14", - "session_13", - "session_18", - "session_15", - "session_1", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What did Melanie realize after the charity race?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_7", - "session_8", - "session_18", - "session_9", - "session_1", - "session_4", - "session_3", - "session_14", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "How does Melanie prioritize self-care?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_9", - "session_13", - "session_4", - "session_1", - "session_7", - "session_11", - "session_17", - "session_19", - "session_16" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What are Caroline's plans for the summer?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_2", - "session_10", - "session_11", - "session_15", - "session_16", - "session_7", - "session_17", - "session_14", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What type of individuals does the adoption agency Caroline is considering support?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_11", - "session_19", - "session_13", - "session_17", - "session_15", - "session_14", - "session_8", - "session_1", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "Why did Caroline choose the adoption agency?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_2", - "session_13", - "session_17", - "session_16", - "session_8", - "session_14", - "session_18", - "session_15", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What is Caroline excited about in the adoption process?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_2", - "session_19", - "session_13", - "session_8", - "session_5", - "session_16", - "session_1", - "session_9", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What does Melanie think about Caroline's decision to adopt?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_2", - "session_17", - "session_19", - "session_8", - "session_12", - "session_4", - "session_5", - "session_16", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "How long have Mel and her husband been married?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_10", - "session_7", - "session_8", - "session_16", - "session_15", - "session_17", - "session_1", - "session_6", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What does Caroline's necklace symbolize?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_14", - "session_1", - "session_8", - "session_9", - "session_12", - "session_7", - "session_5", - "session_18", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What country is Caroline's grandma from?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_3", - "session_1", - "session_8", - "session_9", - "session_12", - "session_7", - "session_5", - "session_14", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What was grandma's gift to Caroline?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_19", - "session_1", - "session_8", - "session_9", - "session_12", - "session_7", - "session_5", - "session_14", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What is Melanie's hand-painted bowl a reminder of?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_8", - "session_5", - "session_12", - "session_16", - "session_11", - "session_14", - "session_1", - "session_13", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What did Melanie and her family do while camping?", - "recall": 0.0, - "retrieved_ids": [ - "session_10", - "session_7", - "session_16", - "session_8", - "session_17", - "session_13", - "session_18", - "session_9", - "session_6", - "session_2" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What kind of counseling and mental health services is Caroline interested in pursuing?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_7", - "session_11", - "session_5", - "session_6", - "session_1", - "session_2", - "session_8", - "session_3", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What workshop did Caroline attend recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_4", - "session_8", - "session_17", - "session_11", - "session_12", - "session_18", - "session_10", - "session_14", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What was discussed in the LGBTQ+ counseling workshop?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_8", - "session_7", - "session_1", - "session_5", - "session_6", - "session_9", - "session_11", - "session_14", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What motivated Caroline to pursue counseling?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_7", - "session_5", - "session_1", - "session_6", - "session_3", - "session_18", - "session_14", - "session_8", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What kind of place does Caroline want to create for people?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_14", - "session_17", - "session_2", - "session_3", - "session_5", - "session_6", - "session_11", - "session_10", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_5" - ], - "question": "Did Melanie make the black and white bowl in the photo?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_16", - "session_12", - "session_4", - "session_6", - "session_13", - "session_3", - "session_8", - "session_14", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_6" - ], - "question": "What kind of books does Caroline have in her library?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_7", - "session_10", - "session_17", - "session_2", - "session_4", - "session_3", - "session_5", - "session_11", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_6" - ], - "question": "What was Melanie's favorite book from her childhood?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_7", - "session_10", - "session_17", - "session_8", - "session_14", - "session_18", - "session_1", - "session_4", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What book did Caroline recommend to Melanie?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_17", - "session_6", - "session_8", - "session_14", - "session_13", - "session_18", - "session_15", - "session_1", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What did Caroline take away from the book \"Becoming Nicole\"?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_13", - "session_6", - "session_17", - "session_2", - "session_1", - "session_14", - "session_5", - "session_4", - "session_3" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What are the new shoes that Melanie got used for?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_17", - "session_6", - "session_15", - "session_9", - "session_4", - "session_1", - "session_16", - "session_19", - "session_2" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What is Melanie's reason for getting into running?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_2", - "session_16", - "session_18", - "session_4", - "session_10", - "session_15", - "session_14", - "session_6", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What does Melanie say running has been great for?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_10", - "session_2", - "session_6", - "session_17", - "session_15", - "session_9", - "session_8", - "session_3", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What did Mel and her kids make during the pottery workshop?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_16", - "session_10", - "session_4", - "session_7", - "session_18", - "session_5", - "session_12", - "session_17", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What kind of pot did Mel and her kids make with clay?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_10", - "session_5", - "session_7", - "session_2", - "session_4", - "session_3", - "session_6", - "session_11", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What creative project do Mel and her kids do together besides pottery?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_12", - "session_14", - "session_10", - "session_7", - "session_5", - "session_16", - "session_17", - "session_15", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What did Mel and her kids paint in their latest project in July 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_10", - "session_7", - "session_12", - "session_15", - "session_14", - "session_17", - "session_1", - "session_13", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What did Caroline see at the council meeting for adoption?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_2", - "session_17", - "session_19", - "session_13", - "session_10", - "session_5", - "session_15", - "session_7", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What do sunflowers represent according to Caroline?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_11", - "session_14", - "session_1", - "session_9", - "session_12", - "session_7", - "session_5", - "session_18", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "Why are flowers important to Melanie?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_14", - "session_7", - "session_2", - "session_11", - "session_6", - "session_18", - "session_15", - "session_3", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_9" - ], - "question": "What inspired Caroline's painting for the art show?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_16", - "session_9", - "session_17", - "session_11", - "session_8", - "session_12", - "session_13", - "session_4", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "How often does Melanie go to the beach with her kids?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_7", - "session_14", - "session_6", - "session_1", - "session_9", - "session_11", - "session_2", - "session_18", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "What did Melanie and her family see during their camping trip last year?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_18", - "session_8", - "session_7", - "session_16", - "session_12", - "session_17", - "session_4", - "session_9", - "session_6" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "How did Melanie feel while watching the meteor shower?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_8", - "session_17", - "session_13", - "session_16", - "session_14", - "session_1", - "session_5", - "session_15", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_11" - ], - "question": "Whose birthday did Melanie celebrate recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_4", - "session_17", - "session_10", - "session_14", - "session_8", - "session_13", - "session_12", - "session_18", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_11" - ], - "question": "Who performed at the concert at Melanie's daughter's birthday?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_4", - "session_8", - "session_18", - "session_1", - "session_13", - "session_9", - "session_19", - "session_17", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_12" - ], - "question": "Why did Melanie choose to use colors and patterns in her pottery project?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_16", - "session_19", - "session_10", - "session_4", - "session_14", - "session_2", - "session_8", - "session_7", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "What pet does Caroline have?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_13", - "session_1", - "session_8", - "session_9", - "session_12", - "session_5", - "session_14", - "session_18", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "What pets does Melanie have?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_13", - "session_8", - "session_18", - "session_1", - "session_4", - "session_9", - "session_19", - "session_17", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "Where did Oliver hide his bone once?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_10", - "session_6", - "session_7", - "session_11", - "session_8", - "session_14", - "session_18", - "session_15", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "What activity did Caroline used to do with her dad?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_10", - "session_5", - "session_15", - "session_7", - "session_8", - "session_14", - "session_18", - "session_1", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_14" - ], - "question": "What did Caroline make for a local church?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_8", - "session_15", - "session_5", - "session_7", - "session_10", - "session_16", - "session_18", - "session_17", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_14" - ], - "question": "What did Caroline find in her neighborhood during her walk?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_7", - "session_10", - "session_16", - "session_18", - "session_8", - "session_19", - "session_9", - "session_11", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "Which song motivates Caroline to be courageous?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_3", - "session_11", - "session_14", - "session_1", - "session_8", - "session_5", - "session_18", - "session_4", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "Which classical musicians does Melanie enjoy listening to?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_15", - "session_8", - "session_18", - "session_10", - "session_1", - "session_4", - "session_13", - "session_9", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "Who is Melanie a fan of in terms of modern music?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_5", - "session_11", - "session_8", - "session_18", - "session_1", - "session_4", - "session_13", - "session_9", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "How long has Melanie been creating art?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_3", - "session_15", - "session_17", - "session_1", - "session_6", - "session_19", - "session_4", - "session_8", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "What precautionary sign did Melanie see at the caf\u00e9?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_5", - "session_10", - "session_15", - "session_14", - "session_1", - "session_11", - "session_18", - "session_8", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What advice does Caroline give for getting started with adoption?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_13", - "session_2", - "session_19", - "session_8", - "session_15", - "session_12", - "session_18", - "session_4", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What setback did Melanie face in October 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_9", - "session_10", - "session_15", - "session_3", - "session_8", - "session_14", - "session_13", - "session_18", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What does Melanie do to keep herself busy during her pottery break?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_8", - "session_16", - "session_10", - "session_14", - "session_2", - "session_7", - "session_18", - "session_5", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What painting did Melanie show to Caroline on October 13, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_13", - "session_8", - "session_17", - "session_16", - "session_11", - "session_9", - "session_1", - "session_15", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What kind of painting did Caroline share with Melanie on October 13, 2023?", - "recall": 0.0, - "retrieved_ids": [ - "session_2", - "session_4", - "session_3", - "session_5", - "session_6", - "session_11", - "session_10", - "session_15", - "session_14", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What was the poetry reading that Caroline attended about?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_13", - "session_6", - "session_7", - "session_2", - "session_4", - "session_15", - "session_3", - "session_12", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What did the posters at the poetry reading say?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_10", - "session_6", - "session_7", - "session_2", - "session_8", - "session_14", - "session_13", - "session_18", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What does Caroline's drawing symbolize for her?", - "recall": 0.0, - "retrieved_ids": [ - "session_14", - "session_10", - "session_16", - "session_7", - "session_1", - "session_8", - "session_9", - "session_12", - "session_5", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "How do Melanie and Caroline describe their journey through life together?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_8", - "session_4", - "session_17", - "session_14", - "session_19", - "session_7", - "session_15", - "session_10", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What happened to Melanie's son on their road trip?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_10", - "session_12", - "session_17", - "session_7", - "session_6", - "session_14", - "session_2", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "How did Melanie's son handle the accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_16", - "session_8", - "session_14", - "session_13", - "session_15", - "session_1", - "session_5", - "session_11", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "How did Melanie feel about her family after the accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_10", - "session_7", - "session_2", - "session_9", - "session_1", - "session_4", - "session_3", - "session_14", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "How did Melanie's children handle the accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_16", - "session_14", - "session_13", - "session_15", - "session_1", - "session_5", - "session_11", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "How did Melanie feel after the accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_2", - "session_9", - "session_1", - "session_4", - "session_3", - "session_14", - "session_13", - "session_8", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What was Melanie's reaction to her children enjoying the Grand Canyon?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_10", - "session_3", - "session_7", - "session_1", - "session_4", - "session_13", - "session_9", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What do Melanie's family give her?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_7", - "session_3", - "session_19", - "session_2", - "session_18", - "session_6", - "session_4", - "session_8", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "How did Melanie feel about her family supporting her?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_7", - "session_8", - "session_17", - "session_16", - "session_3", - "session_2", - "session_15", - "session_18", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What did Melanie do after the road trip to relax?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_17", - "session_1", - "session_10", - "session_14", - "session_12", - "session_15", - "session_2", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_2" - ], - "question": "What did Caroline realize after her charity race?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_7", - "session_10", - "session_8", - "session_18", - "session_9", - "session_1", - "session_4", - "session_3", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_2" - ], - "question": "What are Melanie's plans for the summer with respect to adoption?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_2", - "session_12", - "session_17", - "session_10", - "session_13", - "session_8", - "session_11", - "session_15", - "session_16" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_2" - ], - "question": "What type of individuals does the adoption agency Melanie is considering support?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_11", - "session_19", - "session_13", - "session_17", - "session_15", - "session_14", - "session_8", - "session_1", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_2" - ], - "question": "Why did Melanie choose the adoption agency?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_2", - "session_13", - "session_17", - "session_16", - "session_8", - "session_14", - "session_18", - "session_15", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_2" - ], - "question": "What is Melanie excited about in her adoption process?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_2", - "session_19", - "session_10", - "session_7", - "session_13", - "session_8", - "session_5", - "session_16", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What does Melanie's necklace symbolize?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_14", - "session_8", - "session_18", - "session_1", - "session_13", - "session_9", - "session_19", - "session_17", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What country is Melanie's grandma from?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_3", - "session_8", - "session_18", - "session_1", - "session_13", - "session_9", - "session_19", - "session_17", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What was grandma's gift to Melanie?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_19", - "session_8", - "session_18", - "session_1", - "session_13", - "session_9", - "session_17", - "session_15", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What was grandpa's gift to Caroline?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_4", - "session_1", - "session_8", - "session_9", - "session_12", - "session_7", - "session_5", - "session_14", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What is Caroline's hand-painted bowl a reminder of?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_8", - "session_5", - "session_12", - "session_16", - "session_11", - "session_14", - "session_1", - "session_9", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What did Caroline and her family do while camping?", - "recall": 0.0, - "retrieved_ids": [ - "session_10", - "session_7", - "session_16", - "session_8", - "session_17", - "session_13", - "session_18", - "session_9", - "session_6", - "session_2" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What kind of counseling and mental health services is Melanie interested in pursuing?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_7", - "session_11", - "session_5", - "session_6", - "session_1", - "session_2", - "session_8", - "session_3", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What kind of counseling workshop did Melanie attend recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_13", - "session_8", - "session_5", - "session_6", - "session_7", - "session_1", - "session_11", - "session_10", - "session_17" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What motivated Melanie to pursue counseling?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_7", - "session_5", - "session_1", - "session_6", - "session_3", - "session_18", - "session_14", - "session_8", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What kind of place does Melanie want to create for people?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_14", - "session_17", - "session_2", - "session_3", - "session_5", - "session_6", - "session_11", - "session_10", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_5" - ], - "question": "Did Caroline make the black and white bowl in the photo?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_16", - "session_12", - "session_4", - "session_6", - "session_13", - "session_3", - "session_8", - "session_14", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_7" - ], - "question": "What are the new shoes that Caroline got used for?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_17", - "session_6", - "session_15", - "session_9", - "session_4", - "session_1", - "session_16", - "session_19", - "session_2" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_7" - ], - "question": "What is Caroline's reason for getting into running?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_2", - "session_16", - "session_18", - "session_4", - "session_10", - "session_15", - "session_14", - "session_5", - "session_6" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_7" - ], - "question": "What does Caroline say running has been great for?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_10", - "session_2", - "session_6", - "session_17", - "session_15", - "session_9", - "session_8", - "session_3", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_8" - ], - "question": "What did Melanie see at the council meeting for adoption?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_2", - "session_17", - "session_19", - "session_13", - "session_10", - "session_5", - "session_15", - "session_7", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_9" - ], - "question": "What inspired Melanie's painting for the art show?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_16", - "session_9", - "session_17", - "session_11", - "session_8", - "session_12", - "session_13", - "session_4", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_9" - ], - "question": "What inspired Caroline's sculpture for the art show?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_16", - "session_9", - "session_17", - "session_11", - "session_12", - "session_8", - "session_13", - "session_5", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_10" - ], - "question": "How often does Caroline go to the beach with her kids?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_7", - "session_14", - "session_6", - "session_1", - "session_9", - "session_11", - "session_2", - "session_18", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_10" - ], - "question": "What did Caroline and her family see during their camping trip last year?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_18", - "session_8", - "session_7", - "session_16", - "session_12", - "session_17", - "session_4", - "session_9", - "session_6" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_10" - ], - "question": "How did Caroline feel while watching the meteor shower?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_8", - "session_17", - "session_13", - "session_16", - "session_14", - "session_1", - "session_5", - "session_15", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_12" - ], - "question": "Why did Caroline choose to use colors and patterns in her pottery project?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_16", - "session_19", - "session_10", - "session_4", - "session_14", - "session_2", - "session_8", - "session_7", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_13" - ], - "question": "Is Oscar Melanie's pet?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_7", - "session_8", - "session_18", - "session_1", - "session_4", - "session_9", - "session_19", - "session_17", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_13" - ], - "question": "Where did Oscar hide his bone once?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_10", - "session_6", - "session_11", - "session_8", - "session_14", - "session_18", - "session_15", - "session_1", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_13" - ], - "question": "What activity did Melanie used to do with her dad?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_10", - "session_5", - "session_15", - "session_7", - "session_8", - "session_14", - "session_18", - "session_1", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_14" - ], - "question": "What did Melanie make for a local church?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_8", - "session_15", - "session_5", - "session_7", - "session_10", - "session_16", - "session_18", - "session_17", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_14" - ], - "question": "What did Melanie find in her neighborhood during her walk?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_7", - "session_10", - "session_16", - "session_18", - "session_8", - "session_19", - "session_9", - "session_11", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_15" - ], - "question": "Which song motivates Melanie to be courageous?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_3", - "session_11", - "session_14", - "session_1", - "session_8", - "session_5", - "session_18", - "session_4", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_15" - ], - "question": "What type of instrument does Caroline play?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_6", - "session_14", - "session_2", - "session_3", - "session_1", - "session_8", - "session_9", - "session_12", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_15" - ], - "question": "Which classical musicians does Caroline enjoy listening to?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_15", - "session_8", - "session_18", - "session_10", - "session_1", - "session_9", - "session_12", - "session_7", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_15" - ], - "question": "Who is Caroline a fan of in terms of modern music?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_5", - "session_11", - "session_1", - "session_8", - "session_9", - "session_12", - "session_7", - "session_14", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_16" - ], - "question": "What precautionary sign did Caroline see at the caf\u00e9?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_5", - "session_10", - "session_15", - "session_14", - "session_1", - "session_11", - "session_18", - "session_8", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_17" - ], - "question": "What setback did Caroline face recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_10", - "session_9", - "session_15", - "session_3", - "session_13", - "session_11", - "session_12", - "session_18", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_17" - ], - "question": "What does Caroline do to keep herself busy during her pottery break?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_8", - "session_16", - "session_10", - "session_14", - "session_2", - "session_7", - "session_18", - "session_5", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_17" - ], - "question": "What was the poetry reading that Melanie attended about?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_13", - "session_6", - "session_7", - "session_2", - "session_4", - "session_15", - "session_3", - "session_11", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "What happened to Caroline's son on their road trip?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_10", - "session_12", - "session_17", - "session_7", - "session_6", - "session_14", - "session_2", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "How did Caroline's son handle the accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_16", - "session_8", - "session_14", - "session_13", - "session_15", - "session_1", - "session_5", - "session_11", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "How did Caroline feel about her family after the accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_10", - "session_7", - "session_2", - "session_9", - "session_1", - "session_4", - "session_3", - "session_14", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "How did Caroline's children handle the accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_16", - "session_14", - "session_13", - "session_15", - "session_1", - "session_5", - "session_11", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "How did Caroline feel after the accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_2", - "session_9", - "session_1", - "session_4", - "session_3", - "session_14", - "session_13", - "session_8", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "What was Caroline's reaction to her children enjoying the Grand Canyon?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_10", - "session_3", - "session_7", - "session_1", - "session_9", - "session_12", - "session_5", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "What did Caroline do after the road trip to relax?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_17", - "session_1", - "session_10", - "session_14", - "session_12", - "session_15", - "session_2", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "What does Caroline love most about camping with her family?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_18", - "session_7", - "session_14", - "session_9", - "session_6", - "session_2", - "session_4", - "session_16", - "session_8" - ], - "sample_id": "conv-26" - } - ], - "runtime_seconds": 5.863, - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/prepush_smoke_20260423_123335/runs/locomo_old_brainctl_cmd_session.json b/benchmarks/results/prepush_smoke_20260423_123335/runs/locomo_old_brainctl_cmd_session.json deleted file mode 100644 index 50c7cab..0000000 --- a/benchmarks/results/prepush_smoke_20260423_123335/runs/locomo_old_brainctl_cmd_session.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\locomo_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\locomo_old_brainctl_cmd_session.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" - }, - "benchmark": "locomo", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "measured": false, - "metadata": {}, - "metrics": { - "avg_recall": 0.9217, - "cat_1_recall": 0.7614, - "cat_2_recall": 0.92, - "cat_3_recall": 0.6616, - "cat_4_recall": 0.9738, - "cat_5_recall": 0.9821, - "perfect_rate": 0.8817, - "top_k": 10, - "zero_rate": 0.0438 - }, - "mode": "cmd_session", - "notes": [ - "granularity=session", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": 0.9217, - "reference_kind": "historical", - "runtime_seconds": 445.74, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_mempalace_raw_session.json b/benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_mempalace_raw_session.json deleted file mode 100644 index fe13c4a..0000000 --- a/benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_mempalace_raw_session.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\longmemeval_mempalace_raw_session.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "measured": false, - "metadata": {}, - "metrics": { - "ndcg_at_10": 0.8948, - "ndcg_at_5": 0.893, - "r_at_10": 0.983, - "r_at_5": 0.966 - }, - "mode": "raw_session", - "notes": [ - "top_k=10", - "Runs MemPalace benchmark module raw session retrieval logic directly." - ], - "primary_metric": "r_at_5", - "primary_metric_value": 0.966, - "reference_kind": "historical", - "runtime_seconds": 695.36, - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "mempalace" -} \ No newline at end of file diff --git a/benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_new_brainctl_brain.json b/benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_new_brainctl_brain.json deleted file mode 100644 index 540b000..0000000 --- a/benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_new_brainctl_brain.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\longmemeval_new_brainctl_brain.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 1, - "measured": true, - "metadata": {}, - "metrics": { - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0 - }, - "mode": "brain", - "notes": [ - "top_k=10", - "Legacy 470-question session-level slice." - ], - "primary_metric": "r_at_5", - "primary_metric_value": 1.0, - "reference_kind": "measured", - "rows": [ - { - "answer_session_ids": [ - "answer_280352e9" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e47becba", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_280352e9", - "02bd2b90_3", - "sharegpt_Cr2tc1f_0", - "f6859b48_2", - "ultrachat_214101", - "5dac7cc2_1" - ] - } - ], - "runtime_seconds": 2.792, - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_new_brainctl_cmd.json b/benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_new_brainctl_cmd.json deleted file mode 100644 index 3876df2..0000000 --- a/benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_new_brainctl_cmd.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\longmemeval_new_brainctl_cmd.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 1, - "measured": true, - "metadata": {}, - "metrics": { - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0 - }, - "mode": "cmd", - "notes": [ - "top_k=10", - "Legacy 470-question session-level slice." - ], - "primary_metric": "r_at_5", - "primary_metric_value": 1.0, - "reference_kind": "measured", - "rows": [ - { - "answer_session_ids": [ - "answer_280352e9" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e47becba", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_280352e9", - "02bd2b90_3", - "sharegpt_Cr2tc1f_0", - "f6859b48_2", - "ultrachat_214101", - "5dac7cc2_1" - ] - } - ], - "runtime_seconds": 2.587, - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_old_brainctl_brain.json b/benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_old_brainctl_brain.json deleted file mode 100644 index 65a3126..0000000 --- a/benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_old_brainctl_brain.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\longmemeval_old_brainctl_brain.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "measured": false, - "metadata": {}, - "metrics": { - "ndcg_at_10": 0.9253, - "ndcg_at_5": 0.9204, - "r_at_10": 0.9894, - "r_at_5": 0.9681 - }, - "mode": "brain", - "notes": [ - "top_k=10" - ], - "primary_metric": "r_at_5", - "primary_metric_value": 0.9681, - "reference_kind": "historical", - "runtime_seconds": 85.439, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_old_brainctl_cmd.json b/benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_old_brainctl_cmd.json deleted file mode 100644 index 3112e98..0000000 --- a/benchmarks/results/prepush_smoke_20260423_123335/runs/longmemeval_old_brainctl_cmd.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\longmemeval_old_brainctl_cmd.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "measured": false, - "metadata": {}, - "metrics": { - "ndcg_at_10": 0.9247, - "ndcg_at_5": 0.9206, - "r_at_10": 0.9894, - "r_at_5": 0.9702 - }, - "mode": "cmd", - "notes": [ - "top_k=10" - ], - "primary_metric": "r_at_5", - "primary_metric_value": 0.9702, - "reference_kind": "historical", - "runtime_seconds": 130.863, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/prepush_smoke_20260423_123335/runs/membench_mempalace_raw_turn.json b/benchmarks/results/prepush_smoke_20260423_123335/runs/membench_mempalace_raw_turn.json deleted file mode 100644 index 564a544..0000000 --- a/benchmarks/results/prepush_smoke_20260423_123335/runs/membench_mempalace_raw_turn.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\membench_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\membench_mempalace_raw_turn.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" - }, - "benchmark": "membench", - "caveats": [ - "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 200, - "measured": false, - "metadata": {}, - "metrics": { - "hit_at_5": 0.885, - "simple_hit_at_5": 0.885, - "top_k": 5 - }, - "mode": "raw_turn", - "notes": [ - "FirstAgent slice only", - "turn-level retrieval", - "topic=all" - ], - "primary_metric": "hit_at_5", - "primary_metric_value": 0.885, - "reference_kind": "historical", - "runtime_seconds": 804.35, - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "partial", - "system_name": "mempalace" -} \ No newline at end of file diff --git a/benchmarks/results/prepush_smoke_20260423_123335/runs/membench_new_brainctl_cmd_turn.json b/benchmarks/results/prepush_smoke_20260423_123335/runs/membench_new_brainctl_cmd_turn.json deleted file mode 100644 index 0903d28..0000000 --- a/benchmarks/results/prepush_smoke_20260423_123335/runs/membench_new_brainctl_cmd_turn.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\membench_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\membench_new_brainctl_cmd_turn.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" - }, - "benchmark": "membench", - "caveats": [ - "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 1, - "measured": true, - "metadata": {}, - "metrics": { - "hit_at_5": 1.0, - "simple_hit_at_5": 1.0, - "top_k": 5 - }, - "mode": "cmd_turn", - "notes": [ - "FirstAgent slice only", - "turn-level retrieval", - "topic=all" - ], - "primary_metric": "hit_at_5", - "primary_metric_value": 1.0, - "reference_kind": "measured", - "rows": [ - { - "category": "simple", - "hit_at_k": true, - "question": "What is the name of my niece's company?", - "retrieved_ids": [ - "simple_roles_0|sid=119|g=119|s=5|t=18", - "simple_roles_0|sid=101|g=101|s=5|t=0", - "simple_roles_0|sid=135|g=135|s=6|t=12", - "simple_roles_0|sid=11|g=11|s=0|t=11", - "simple_roles_0|sid=25|g=25|s=1|t=5" - ], - "target_ids": [ - "119" - ], - "tid": 0, - "topic": "roles" - } - ], - "runtime_seconds": 0.558, - "series_name": "new_brainctl", - "source_path": null, - "status": "partial", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/prepush_smoke_20260423_123335/runs/membench_old_brainctl_cmd_turn.json b/benchmarks/results/prepush_smoke_20260423_123335/runs/membench_old_brainctl_cmd_turn.json deleted file mode 100644 index 149a1ca..0000000 --- a/benchmarks/results/prepush_smoke_20260423_123335/runs/membench_old_brainctl_cmd_turn.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\membench_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\membench_old_brainctl_cmd_turn.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" - }, - "benchmark": "membench", - "caveats": [ - "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 200, - "measured": false, - "metadata": {}, - "metrics": { - "hit_at_5": 0.93, - "simple_hit_at_5": 0.93, - "top_k": 5 - }, - "mode": "cmd_turn", - "notes": [ - "FirstAgent slice only", - "turn-level retrieval", - "topic=all" - ], - "primary_metric": "hit_at_5", - "primary_metric_value": 0.93, - "reference_kind": "historical", - "runtime_seconds": 140.592, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "partial", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/prepush_smoke_20260423_123335/summary.csv b/benchmarks/results/prepush_smoke_20260423_123335/summary.csv deleted file mode 100644 index 7aad1f9..0000000 --- a/benchmarks/results/prepush_smoke_20260423_123335/summary.csv +++ /dev/null @@ -1,15 +0,0 @@ -benchmark,series_name,system_name,mode,reference_kind,status,example_count,primary_metric,primary_metric_value,runtime_seconds,dataset_path,source_path,notes,caveats,avg_recall,cat_1_recall,cat_2_recall,cat_3_recall,cat_4_recall,cat_5_recall,hit_at_5,ndcg_at_10,ndcg_at_5,perfect_rate,r_at_10,r_at_5,simple_hit_at_5,top_k,zero_rate -longmemeval,old_brainctl,brainctl,brain,historical,full_same_machine,470,r_at_5,0.9681,85.439,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,top_k=10,,,,,,,,,0.9253,0.9204,,0.9894,0.9681,,, -longmemeval,old_brainctl,brainctl,cmd,historical,full_same_machine,470,r_at_5,0.9702,130.863,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,top_k=10,,,,,,,,,0.9247,0.9206,,0.9894,0.9702,,, -longmemeval,mempalace,mempalace,raw_session,historical,full_same_machine,470,r_at_5,0.966,695.36,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,top_k=10 | Runs MemPalace benchmark module raw session retrieval logic directly.,,,,,,,,,0.8948,0.893,,0.983,0.966,,, -locomo,old_brainctl,brainctl,cmd_session,historical,full_same_machine,1986,avg_recall,0.9217,445.74,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,granularity=session | top_k=10,,0.9217,0.7614,0.92,0.6616,0.9738,0.9821,,,,0.8817,,,,10,0.0438 -locomo,mempalace,mempalace,raw_session,historical,full_same_machine,1986,avg_recall,0.6028,2106.411,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,granularity=session | top_k=10,,0.6028,0.5899,0.6921,0.4604,0.5809,0.6188,,,,0.5534,,,,10,0.3499 -membench,old_brainctl,brainctl,cmd_turn,historical,partial,200,hit_at_5,0.93,140.592,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,FirstAgent slice only | turn-level retrieval | topic=all,MemBench comparison is partial because ThirdAgent and noise-extended slices are not included.,,,,,,,0.93,,,,,,0.93,5, -membench,mempalace,mempalace,raw_turn,historical,partial,200,hit_at_5,0.885,804.35,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,FirstAgent slice only | turn-level retrieval | topic=all,MemBench comparison is partial because ThirdAgent and noise-extended slices are not included.,,,,,,,0.885,,,,,,0.885,5, -convomem,old_brainctl,brainctl,cmd,historical,blocked,0,avg_recall,,,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,limit_per_category=1 | top_k=10,Blocked while loading ConvoMem evidence data: ,,,,,,,,,,,,,,, -convomem,mempalace,mempalace,raw,historical,blocked,0,avg_recall,,,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,limit_per_category=1 | top_k=10,Blocked while loading ConvoMem evidence data: ,,,,,,,,,,,,,,, -longmemeval,new_brainctl,brainctl,brain,measured,full_same_machine,1,r_at_5,1.0,2.792,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,,top_k=10 | Legacy 470-question session-level slice.,,,,,,,,,1.0,1.0,,1.0,1.0,,, -longmemeval,new_brainctl,brainctl,cmd,measured,full_same_machine,1,r_at_5,1.0,2.587,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,,top_k=10 | Legacy 470-question session-level slice.,,,,,,,,,1.0,1.0,,1.0,1.0,,, -locomo,new_brainctl,brainctl,cmd_session,measured,full_same_machine,199,avg_recall,0.9413,5.863,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,,granularity=session | top_k=10,,0.9413,0.8641,1.0,0.7436,0.9571,0.9787,,,,0.9196,,,,10,0.0402 -membench,new_brainctl,brainctl,cmd_turn,measured,partial,1,hit_at_5,1.0,0.558,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,,FirstAgent slice only | turn-level retrieval | topic=all,MemBench comparison is partial because ThirdAgent and noise-extended slices are not included.,,,,,,,1.0,,,,,,1.0,5, -convomem,new_brainctl,brainctl,cmd,measured,blocked,0,avg_recall,,,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache,,limit_per_category=1 | top_k=10,Blocked while loading ConvoMem evidence data: HTTP Error 404: Not Found,,,,,,,,,,,,,,, diff --git a/benchmarks/results/prepush_smoke_20260423_123335/summary.json b/benchmarks/results/prepush_smoke_20260423_123335/summary.json deleted file mode 100644 index 691bf68..0000000 --- a/benchmarks/results/prepush_smoke_20260423_123335/summary.json +++ /dev/null @@ -1,509 +0,0 @@ -{ - "generated_at_utc": "2026-04-23T12:33:49.889512+00:00", - "metadata": { - "argv": [ - "benchmarks\\compare_memory_engines.py", - "--label", - "prepush_smoke", - "--longmemeval-limit", - "1", - "--locomo-limit", - "1", - "--membench-limit", - "1", - "--convomem-limit-per-category", - "1" - ], - "cwd": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl", - "datasets": { - "convomem_cache": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "locomo_data": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "longmemeval_data": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "membench_data": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent" - }, - "generated_at_utc": "2026-04-23T12:33:49.807677+00:00", - "git_commit": "c130fbd47174a0932822db2ef7fe64e18a74d30d", - "historical_summary_mode": "recovered", - "historical_summary_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "machine": "AMD64", - "platform": "Windows-11-10.0.26200-SP0", - "processor": "Intel64 Family 6 Model 189 Stepping 1, GenuineIntel", - "python_version": "3.14.4 (tags/v3.14.4:23116f9, Apr 7 2026, 14:10:54) [MSC v.1944 64 bit (AMD64)]" - }, - "notes": [ - "Historical old-BrainCTL and MemPalace series come from the recovered 2026-04-18 comparison bundle.", - "New BrainCTL series are rerun in the current checked-out repo using the legacy benchmark definitions.", - "MemBench remains intentionally partial because the legacy comparison only covered the FirstAgent slice.", - "ConvoMem remains a coverage/status benchmark here; it has no dedicated comparison chart in the legacy chart pack." - ], - "runs": [ - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\longmemeval_old_brainctl_brain.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "measured": false, - "metadata": {}, - "metrics": { - "ndcg_at_10": 0.9253, - "ndcg_at_5": 0.9204, - "r_at_10": 0.9894, - "r_at_5": 0.9681 - }, - "mode": "brain", - "notes": [ - "top_k=10" - ], - "primary_metric": "r_at_5", - "primary_metric_value": 0.9681, - "reference_kind": "historical", - "runtime_seconds": 85.439, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\longmemeval_old_brainctl_cmd.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "measured": false, - "metadata": {}, - "metrics": { - "ndcg_at_10": 0.9247, - "ndcg_at_5": 0.9206, - "r_at_10": 0.9894, - "r_at_5": 0.9702 - }, - "mode": "cmd", - "notes": [ - "top_k=10" - ], - "primary_metric": "r_at_5", - "primary_metric_value": 0.9702, - "reference_kind": "historical", - "runtime_seconds": 130.863, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\longmemeval_mempalace_raw_session.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "measured": false, - "metadata": {}, - "metrics": { - "ndcg_at_10": 0.8948, - "ndcg_at_5": 0.893, - "r_at_10": 0.983, - "r_at_5": 0.966 - }, - "mode": "raw_session", - "notes": [ - "top_k=10", - "Runs MemPalace benchmark module raw session retrieval logic directly." - ], - "primary_metric": "r_at_5", - "primary_metric_value": 0.966, - "reference_kind": "historical", - "runtime_seconds": 695.36, - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "mempalace" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\locomo_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\locomo_old_brainctl_cmd_session.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" - }, - "benchmark": "locomo", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "measured": false, - "metadata": {}, - "metrics": { - "avg_recall": 0.9217, - "cat_1_recall": 0.7614, - "cat_2_recall": 0.92, - "cat_3_recall": 0.6616, - "cat_4_recall": 0.9738, - "cat_5_recall": 0.9821, - "perfect_rate": 0.8817, - "top_k": 10, - "zero_rate": 0.0438 - }, - "mode": "cmd_session", - "notes": [ - "granularity=session", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": 0.9217, - "reference_kind": "historical", - "runtime_seconds": 445.74, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\locomo_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\locomo_mempalace_raw_session.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" - }, - "benchmark": "locomo", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "measured": false, - "metadata": {}, - "metrics": { - "avg_recall": 0.6028, - "cat_1_recall": 0.5899, - "cat_2_recall": 0.6921, - "cat_3_recall": 0.4604, - "cat_4_recall": 0.5809, - "cat_5_recall": 0.6188, - "perfect_rate": 0.5534, - "top_k": 10, - "zero_rate": 0.3499 - }, - "mode": "raw_session", - "notes": [ - "granularity=session", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": 0.6028, - "reference_kind": "historical", - "runtime_seconds": 2106.411, - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "mempalace" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\membench_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\membench_old_brainctl_cmd_turn.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" - }, - "benchmark": "membench", - "caveats": [ - "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 200, - "measured": false, - "metadata": {}, - "metrics": { - "hit_at_5": 0.93, - "simple_hit_at_5": 0.93, - "top_k": 5 - }, - "mode": "cmd_turn", - "notes": [ - "FirstAgent slice only", - "turn-level retrieval", - "topic=all" - ], - "primary_metric": "hit_at_5", - "primary_metric_value": 0.93, - "reference_kind": "historical", - "runtime_seconds": 140.592, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "partial", - "system_name": "brainctl" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\membench_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\membench_mempalace_raw_turn.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" - }, - "benchmark": "membench", - "caveats": [ - "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 200, - "measured": false, - "metadata": {}, - "metrics": { - "hit_at_5": 0.885, - "simple_hit_at_5": 0.885, - "top_k": 5 - }, - "mode": "raw_turn", - "notes": [ - "FirstAgent slice only", - "turn-level retrieval", - "topic=all" - ], - "primary_metric": "hit_at_5", - "primary_metric_value": 0.885, - "reference_kind": "historical", - "runtime_seconds": 804.35, - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "partial", - "system_name": "mempalace" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\convomem_old_brainctl_cmd.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" - }, - "benchmark": "convomem", - "caveats": [ - "Blocked while loading ConvoMem evidence data: " - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "example_count": 0, - "measured": false, - "metadata": {}, - "metrics": {}, - "mode": "cmd", - "notes": [ - "limit_per_category=1", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": null, - "reference_kind": "historical", - "runtime_seconds": null, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "blocked", - "system_name": "brainctl" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\convomem_mempalace_raw.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" - }, - "benchmark": "convomem", - "caveats": [ - "Blocked while loading ConvoMem evidence data: " - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "example_count": 0, - "measured": false, - "metadata": {}, - "metrics": {}, - "mode": "raw", - "notes": [ - "limit_per_category=1", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": null, - "reference_kind": "historical", - "runtime_seconds": null, - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "blocked", - "system_name": "mempalace" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\longmemeval_new_brainctl_brain.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 1, - "measured": true, - "metadata": {}, - "metrics": { - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0 - }, - "mode": "brain", - "notes": [ - "top_k=10", - "Legacy 470-question session-level slice." - ], - "primary_metric": "r_at_5", - "primary_metric_value": 1.0, - "reference_kind": "measured", - "runtime_seconds": 2.792, - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\longmemeval_new_brainctl_cmd.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 1, - "measured": true, - "metadata": {}, - "metrics": { - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0 - }, - "mode": "cmd", - "notes": [ - "top_k=10", - "Legacy 470-question session-level slice." - ], - "primary_metric": "r_at_5", - "primary_metric_value": 1.0, - "reference_kind": "measured", - "runtime_seconds": 2.587, - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\locomo_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\locomo_new_brainctl_cmd_session.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" - }, - "benchmark": "locomo", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 199, - "measured": true, - "metadata": {}, - "metrics": { - "avg_recall": 0.9413, - "cat_1_recall": 0.8641, - "cat_2_recall": 1.0, - "cat_3_recall": 0.7436, - "cat_4_recall": 0.9571, - "cat_5_recall": 0.9787, - "perfect_rate": 0.9196, - "top_k": 10, - "zero_rate": 0.0402 - }, - "mode": "cmd_session", - "notes": [ - "granularity=session", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": 0.9413, - "reference_kind": "measured", - "runtime_seconds": 5.863, - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\membench_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\membench_new_brainctl_cmd_turn.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" - }, - "benchmark": "membench", - "caveats": [ - "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 1, - "measured": true, - "metadata": {}, - "metrics": { - "hit_at_5": 1.0, - "simple_hit_at_5": 1.0, - "top_k": 5 - }, - "mode": "cmd_turn", - "notes": [ - "FirstAgent slice only", - "turn-level retrieval", - "topic=all" - ], - "primary_metric": "hit_at_5", - "primary_metric_value": 1.0, - "reference_kind": "measured", - "runtime_seconds": 0.558, - "series_name": "new_brainctl", - "source_path": null, - "status": "partial", - "system_name": "brainctl" - }, - { - "artifacts": { - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\runs\\convomem_new_brainctl_cmd.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\prepush_smoke_20260423_123335\\charts\\coverage_status.png" - }, - "benchmark": "convomem", - "caveats": [ - "Blocked while loading ConvoMem evidence data: HTTP Error 404: Not Found" - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "example_count": 0, - "measured": true, - "metadata": {}, - "metrics": {}, - "mode": "cmd", - "notes": [ - "limit_per_category=1", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": null, - "reference_kind": "measured", - "runtime_seconds": null, - "series_name": "new_brainctl", - "source_path": null, - "status": "blocked", - "system_name": "brainctl" - } - ] -} \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/README.md b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/README.md deleted file mode 100644 index 60209ed..0000000 --- a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# Legacy BrainCTL vs MemPalace comparison bundle - -- Current repo commit: `9368f288279ae1d3395e37e89972038b74d66e3a` -- Historical reference source: `C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json` -- Historical source mode: `recovered summary bundle` -- Command: `benchmarks\compare_memory_engines.py --label rlm_srlm_probe_refresh --membench-limit 200` - -## Datasets - -- LongMemEval: `C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json` -- LoCoMo: `C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json` -- MemBench FirstAgent: `C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent` -- ConvoMem cache: `C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache` - -## What is measured now - -- New BrainCTL reruns: LongMemEval `brain` and `cmd`, LoCoMo `cmd_session`, MemBench FirstAgent `cmd_turn`, and ConvoMem `cmd` coverage/status. -- Old BrainCTL and MemPalace are frozen historical reference series loaded from the recovered 2026-04-18 bundle. - -## Blocked or partial runs - -- membench old_brainctl cmd_turn: MemBench comparison is partial because ThirdAgent and noise-extended slices are not included. -- membench mempalace raw_turn: MemBench comparison is partial because ThirdAgent and noise-extended slices are not included. -- convomem old_brainctl cmd: Blocked while loading ConvoMem evidence data: -- convomem mempalace raw: Blocked while loading ConvoMem evidence data: -- membench new_brainctl cmd_turn: MemBench comparison is partial because ThirdAgent and noise-extended slices are not included. -- convomem new_brainctl cmd: changing_evidence: discover failed: HTTP Error 404: Not Found | ConvoMem comparison is partial because it uses a bounded same-machine sample, not the full benchmark. - -## Output files - -- `summary.json` and `summary.csv`: all series in one table. -- `comparison_table.json` and `comparison_table.csv`: long-form metric rows. -- `runs/*.json`: per-run payloads. -- `charts/*.png`: regenerated charts with old BrainCTL, new BrainCTL, and MemPalace together. diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/charts/aggregate_primary_metrics.png b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/charts/aggregate_primary_metrics.png deleted file mode 100644 index 5364ba6d7f748d064a7bef10c7b29d8802e33a5f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 72174 zcmdRWbyQVdyDzq&Ac~5Dl$6rlp`y}Y(6vRn8);Awu|T>(5Co*8r3FN~Vbdk;rX)78 ziTfL_vM(kCZ4z2Jh_|{VDc6fMKqLKfONIQ6k;2puklfHXf$th-j z_>|M*Dct@a@}Q$-Szc)G5_BD#8r>mCobMQ@=(7qNZU7eVI{7K-v=j)rQ8zn zKmK*p`0$xm*MlR3htI7E{`u?2#J7$e{yOFb9@*j3v5Uvs51-5p(;e;*3B@&3R1=WpB}+Te}@4^lWiRRy0A-c9NrRcN>#ktF8P+so`CYd^F#$*K54G&hE2jCt^tP z@Sc(*!@4YUhKML^u*j_Km5)=N|HJs$!8Ij{_3S#V^_@s7_#pVk>MQ+%`@#` z!Y3f$A|;Ghj1i#mK1mqlgHO`h%f=0r_xb8 zY3rj6!HnS*?l_t0=1BRe-+4xD`paWgxle1~+)!0M#;LB{VusB2Y$hhSPF{7!Jzqj6 zngC~x{3H=7g?23KNP#~@3|M-O!&4)S5S(%-4Ned_AdMLlRSt? z`HM9P(S06#-9E}HGW*_4O;tBHw;zLpngpb5=Ge^4TP*qVeubxRYUHP;4WaeWUzBCB z^Zj}oIU?@c-S5`B6vaayCCfzOegsgVI+LVnFEJ^7i1*l*UK}np+Sn7Ci}8QVnDDsH zpCZ_wLRcnCtC;R#;$6d7m8t25;JP8p&ZLGjZC0g@3(bX*Tn4r`hRzA&#}_JG<#^&H zs45*q$2qgOyD=KRi`#frGK$XBY_ZKW@8z6n3RkSynkHY}DX*{cKGibOgPRrXotQO~ zVEj_%^e1t6HMvfS+pzgG|F}VYMLP{+t zXQGaFXlZM8rnaf6$z0fS=$r9-o2kaoN1yH!HhiG*-YW1Gd*!~p78uQMRk2*LtwYYG z?-#+zAfS~*IB!xxsjk}(*E)J9Pj@1?U8s6-51VhuW}pa zkDx~DwgvV!Cjw3{WiRbM{B&QsA&8z{Jy(BvWjflbk-c)~X;F~aJmycl=N@PFdYSWT zlwAb9>}8Yw+~gP$XRhwuFg9(brLtB3buA1=cWt)gd_|>VycjF3q<_fop7iJq!xzW# zn_V+*JwMu6Y-hBpt{3nrzLq^PGwe}{g2iIW%m%zFu!0>A&+m2>GUdErtV~v(vcZWJ zmTJceIb;m(uC|FA_V?!*Wj0-A)4H-VV3w`EZ&5tE$7;@+x4gd2i_s{uq}YM$c&hF- z3FA=3+M{%p%rQpAxa0A{&)B}fck>lVf1tj->`O>)F6JLdD>+c6STDP`v(>U^flP&d z23PESRZ|%28HE@Dn{T-E+ukQ52k}W5&Q`#j9$t0XSR5Wpk`9it5Xa5uJ&~MUH4vPJ z`SxOUvO#b451mZN4X25p@2ABie9z{%TBa+fU*4JT=Z>y(+1qipju&&EA5mrB7EsC5 zFj#fFXWub1gvQvk#fery#ULswuUM=W^JvQbI^7y;CR!ON=%GKin!V9em$Fe*v{e1_ zjOHl&<&CA$a|(giU#gFmK3h5+u-d{usuo`Zqa?)h^TSeFcExt%;uUVA)@yFRe&Cnl z`b!+ltOxQEL5X<*7vqT5uJ^)dUn=xZ_CIU*39*5-juXS zAL136RWeRzJxF*lj4L$jZJoA}sGvMYA#^c@M&e}jMsM#N%5&FXgpta9EpT|kZFBq; zUw|D3ok?1u_^U2}Y96NU>Yl&ban1Z{tm1bn^im-&@N(*GL6qNmVNbD8Cei09tUm~1#( zWYdciXPO;J*>%ohS5~+N{jYK}Jem9h4uBUY*%+yWWfs+zPZF4^@NMA$;I8n zyj&r4q%L>guq8@R1m?<3*@)D(X#a<85|tjiFD7c=8lk6JqI-`|=u!w+e=o~+h)%rs z_GDWWZ&Qr0=&Nl3*R@$)(K<5QLD$M8xyVkRe6n}Hoh#k8Y?nisRhw}Sdgu7m^9&1| z+18xG*st5OY~QPg*3vxDz7gY5yi&Qp+vdjLQ;w?NkPM=eesSs?rTK`Fkca!ZSv$Rc z!-au@aes=H^;BGYQ;eWpGfuNEJ6pbq^qN)qdf56IHo}DUp*b5ON|7r&i=|5%8+y#f zy}kBu5xid!QIM6HbTMFzW}%yThqI_ZjT3GPHzWHwC?pxU%i@2-D~`+H&AOc|s%v}i z1Rh@2LC0e7+Jkm8RVKmf!U(F)VeGjwx%_jEBzOwy^x`0d?J2-#ZjGyo4F`V*I zJPc0$%@gOyZ^068(^2`I-C;5w48!}Gl;*>isnZ86;vO}ZGPfi|g4W8L1) zGuO*b6u(Y0`dDMw5@VwfC(>Wia!!1Q*wX#auiBQ9#!%*0+lkakHWNS9YdXHp2u|Ye zZ6sgX*PSD!53^W1?E8hR;#KlQ2RoyE^w0O?lAA8w z>?zeMdCa?=43n;Mo>k|=P8Nr;A6+~1HMRZE-0u3Bk6*8Cu@sWtG83>;A1N?XEv@v0 zTF!n>6yU>mX8F}$P2ucK5uAES?l`PrB=*?ud|sQmWwyI|!M?4%$-dTtw&~mC_*|^* z?qsl{92NR26b{4fyCn@ZaCn_U$eyEjAR5~yzMtQ} z>VU&8pvBrdIx;o+yBGyC;N4MigT{mkDLMKrx`%ho0!}jV-q^X4hTV%3Q)hLyC%P zDm3yGM2m&r4MSkVGbly2<88-8U!8 zg>J?x`0zdbak}(&9}EgPtzP@!FZ07rN45_xS zum+#fU<)H&`0<*=u`X?}CqtdPx-OCC!y-yPlB?G6d+KP4Rj@YTU6tUMh=IN{eSO>m5LmD+GB>jng>>a`G)9RI>zLly| zR1s#e#DBKeG2^WX74fiaHp(zF5;;$6ub;KyEkn1FYOdw|Truc`;&mr8@7?W}F&Ui? z&zGyRqK14#9rZT?MVxJ=!`Whscd>(3(QS=>BYIe|@b1s@(Mb*%wBfJYyISpbdRtn# zWmIc>LgU7AWmapuBl#weSLuWOH5v80*%bh5yZX6A9>>GG#gg#+uKzW`B*3trtI-#@ z`k&$permeVHxL9^{8gdr^4RTf4uDj6s#XEupac-`Kd7CeTuUZWKD%RL%;&WHBvRD1 za0f6NLrA_)zK4egnm|S~G{N7mdCPCUn9zBI=Lv#Ii(x{dqGk29N*(1BnT4`%X0oZC zFE`Y?Q}yBmt(FzO$9jSN>U7HkU57lw7P*294;n7JX}PVnF2zg?0zAUZ@UF&=^#pQ$ z@$$mMu%G7npaSiMOV`*w?irzRfs9xpO6-Pvb77pMwiSl-T;Qb+v(b^9*mcUh_2M-E zJCEOs>HR$W`mDQXzpKJE=h0bqvetNUs|ufJZX^HhbXE4w4Jg8`k|%%#I!zg4j%Q|c zy;b2gWj;w5&!8B$Q%&7=;n&^p^50dd=A?zclq4v3ua^0?k!@lzy^4jF!=^{C5T0TU z&}KfW+^2H!p>r7-lgWCbN*J4FVRMO==Y7xd>Ju2gj!TiN=lku*SIX{Ptd^V{uX%Mr zyTe)D0pNUYujIy(>30Ye7x=bV1bW?Wq~6@jR6lv-MJsBOfjFS>k0iZd#ouiIK`F9j+Zj}E}`Z%w^V~%wE;_@d|N4xUWaMh;qx)wNH=XtV~5xIO@4SJcJ&z z0W>;|eMRLlo6D&Y=els7qJKskcfaoEOh->X{Rh*bZ#i3uGL<*F(9|05Ew{8>)@C2R zqf)KZV$u5cb*s56>^A4_4)D5J8ibtt8Q9<94P{bh;(iz|>bbkF`i`=f%VxC10V5}T zCR0c&S6}+wiFd6@$|Yf8XTuy%=R8jyT}+U872^`4SLc84+um%)Cp5J(>Y1)`hFZrO zj&SS5%c&;(OsO9HZ5{L#WngnP1-9*|v&KzVRK)@YC?y7G%HHJ9luY#!hoMkq@7m~- zgPANx%GBRnW;Jci!3Lnkx`F13{uLK~aD@@vv8dH;YmF7A?V%btKcK2Cp(UDybw}%@ z3xzrT`6Ugoy>_@9y7{zAN8&xh&fL_mYy0nKhwA8>et45@zKW@Dy{f#f%V?3`ZdU-X z#jX@OvLoY#{QM_d3q_-H$=O(M6$Fh1Z`=!4_2GmIg>TT+bJliu$ok|$TV_IDp?W8AiQ~dRuFb^#lxv`+2>2qCfjTTl z$=(#Dp1FNSHr#WqLz2@&i}1W=zOinmWr5N+(*j-JvsZr%umA38(lQG*glmdYBQ%RF zW1CNU|GP6;c7<#9xF=c0O?QR;tr@D)!Ta5c{9K{|4%csM=GQhs_0mzB8v&G0(r(r^ z1zex3N^@EJFAP<+{L&5zLpnZxRrvC#yIfkw|jxH||F# zWB=&r=*_lI%qEv|lZPAND=jQnVL+nTFT~s%jVBWV3_K(klby)kd`Z1-<`{*$lW<`lNGUf?AWn_1Wu5f%;O1Q7|S|jo9YwISB0#k#dzs-8H zBE#rt=2kgCCAdPBr($C|c$9FPmu?B28Qro_T|sm$App}H$^F^F*LOvC?zq+JOk2^Q z2d#emGPG5VeTv|owu*|1Y2NI%giJI+B)3r*%(*COshZAqbx+pzueuT>0~pftVeGWm zO$#mr^Yo!P%Qi+_r)2|RTt~FbEl4%zTqV5i3AY_`70Z2+f|KDA&KM@%GV1(@(nK2% zQtaNj*H-MIt8Y>V>`;dd(VFm{(qt<2RFn_QC^5d2{XqR(zagkr$=Vc8pFQba1Cw!b zLJHQ)pOxp63bIEBV-S}9ZC%qhwP;WjrqE$k(ebE{=8Ok-B9fJb7 zZ5zO}UBabJ`p_f&H}>z zD$A_TORP*bd+*|Q)#^nSwf40(G?t?^${MB=Ue2jYh!AV-XRva6Awh2Q-G_YMw+1Ue z`#Q<5!S7q|Aqt+Cl(5TMs>7XZD6n?Z#EpLEX`3BD=SzkWilKm)ap2Ry|8?d;jQX6b z>?&qefRSbhmA#T?jsYR2{l(1Fs{C3U;)76W#lKy{S)5so!&l3ICi=Hj(06+ zNBwdfC))}Qnl5uk>PQ(hh6DmdX*bY=>+a>6aN-oa{yM!$XA*;!R?gFx-C5evdA|F& z_&48d=*`d>oxmv63rs_uoSkhP;iXT0;G;($zKW zzTmtv`8r84K91y26{o(hRqmn;`UX3cXc8kDFaHxfZXVeA$n;l#g;3C9FRjee>-!&X z&+$rL5#-=#?~yqC%krE0Ao{mm0k_Zno0R@PqM4s>6D)mMdK$E1qqew6GrWf+j0)HO zfLzggFb;kqM`u+(__pv{jf8+))Clq>e*E(*AII~Gii+suqm%%S-n(+3yA$IdZ_kdH zX^U^FeM>68d6wF+vak5rQrvowpq=4s@^H5M2cLPK#>=C2=$lOqnOKTc;GH0PSq2cu zk~B#C3^#XXJ7__R4g&BRy%AB-+xc0($r;cdJpAyEtTFewNF*!7YH;b-kNu`OlKt*D z{)Gs3`+U-T-?nWX*G>A1CH+n(1+pc$H;|)jfO86`9WPz@EQV zdFPI=u0}FG_4G?OtMiBc!`*lPMfc&k)IhGmTVOAeAW%;(3wGh=YR@Ao{f6MP#h5`6 z$?ZM|ITyB}p0&BImMeq#CKJCC4i3Rd1lnMZukRl+HZuW0e#m2kw5{hD9Z=y zH9dgIEJ{Q`hsV<+@cO{}%gAzthh+PUW0cGs91j3rF*vVIhu(dCLHfkJdC2DiGg5snmi(2kL1@#u@v%}5~!(|7M2fR8>(xN|}}PNbMlHmrH=sZN1u&lwz# z)o6wG@9s2OP)35mmWbM`XS#7iia|b#4kR=F0CAAIryu1TKb(WQIRxMR5=SJBW5w7G z?*r{fHfRb1>5LYxXUI^oo$UVJ?o`JoNk^*Dyp zr+s;+L0HAW!@2B(spJ*zO%#PsTe5+QH{98;HD)%!?-uy;<4}%i4wJk#tfY zs2|~?feG_i40Ipwz4zuK=j}ykJBU7DJ6RuS{N4fdE-v7tB*ySAO$AVt0Wcw(r`x$8J6_vgZ3j2oa$(1W}} z^nQJ;%DWSYv2-X?=;rozl-}Y{@#B+(Wcsb%alKzXcK%3i?X0#W?C0?QaY`XbsnnuBw4DC`C#8e=Osi=oukfdGe$lwNM$v0q6T_u*FA2aayBjR*p zhI(ES95>nkK7{xAvuvI6B-^P*hBYEHSbtm~m2S;`3aHNPlHTDn?|ZVC9@=qvV6+&- z%g?>5swj7Z#U($G@*@dMMW5;D=ot0`3rdD22~y{U$Miad=IR(M*~-RTSDlh*TLh;B z8L#O@Q%}hN$^sRBvxw~5B`4cK)F@D~J23aYyLZ5xm1OoVcq#E&E>iBB8GgJnh{kd~ znPl9C??l`-mtW$dL25HGNuV*SUsgAX7=~_Z{9cv$g@kY07m%JlyZ`J;QIz+xlm((; z?%T#B8^&U2PE%tvwXnbnLRdK9_{i1C8<|OaK)N%>^E$1yG;MDXmP?RdenaUcNM7c^7vV1cwnNaj-t4rv_Xm2MWxlugw|A3r+i#yPzFvNmzw)>fPr)LU(W)I->bw zcuxgodu-J}D=KuQT1d$l&%^5J6j{c?WE5Wk(I}6--1x?|j*12R!Ola+HST@>X-Y=B z>1H(ksRcn-ta+*hZC)izn0_rQI2!ezPpv8k+WOB!(+XBjSBdy7T|OJKX=1?%fZj!w%~m9S?;RkF3jL-S#JIWR8&+qRWc$#T-W-n z5KESZGqy5K4rV`CG2eI3!v*9q?`u~Y)hV%;c3J;@iK)5UbOL}`_s}8$RWMd&U*fv( z*Mb=2nfCVfxPZz&z}Ua|4wdSzSs9t%K1`Q{Fjlp%-{o#yVb>AiR*=-M|4_B&P~m`4 z`DX13XTMTQimGqodh#95Za5nyzUovh1c2w7uw(P~5(vqW#2U=`Rzu(ZbpFKuI|g2< zx)<)8Q=WNSd?A{Js#D-6Hk3p64#^lg^`XSi04l422O&2f-+%Gqg?`s(`Oi!xaMWn) zwR>C!jdWlF1YKs;kb-YDSea}<%vxf?E zG5)Z&G>RhQF{VC2aFIi~;~b^PhsP-jk$5g(e?0p7QI_-G8&Y;QFi(lC ztgP}B+Jq0T_t2S3?aNY#k8(}K>A_#>RBsxZ<6BEF6T*?O2Gu@ef%5t%;8&jq)V<9J zEyLNgO^;e+dZ5Pygm}E-_^sm79HF8_UK4XuV^Ybu0YWc7jjfXLjS46>H&wG@z)!25 z$3Qn>co)a1=UXNRC7!pc$amhQ6^2TmL8|*}F&KsbNJ1YBf|30Y*chG7uOFPZ*I=BR z4?qOoBBR1`lQH&6I&2#vK*oL^zc{F~JYC^pyFZNdjYPp=-|Z2Wz#ApjglT ze<-sc@(50$xi%KsQq8+LXpjHa+KFlZCB}e2vJ+#e@5%WagBca+U}46H9E@?<1$Y^M zAedgN=MHh9-^2VduexX+lb{e5^+#ZfT)+AFa|kduq*bB%QkC1?{Fo0JzA)1WQ&*dxS7(@Q}vOY)~L$n1K^_TK?FbjY0tiV@4YKotFfE znNC$z73IFjKUN;`t(s3DJ;)-XJu)(~GUB=@0v3E#_XXz;7Uk6D zpNU7A8oW1w*C75Z;sz-PvqmOsNYrBK^jx4iJ9Nm(`M0IU!e zErBrAurOGJG`)nd(K6@9Q9LHs!Ke(T^4Po$W-f(kxxpwvXWk+Sm0MQBB|U!?3jAhP zK8pc51XcWaPu{DfpsB-bO*vxp_fWrg+1AZ|JW56>Uo-Rp8q^cDK?7qy5$6^C=nb>s zl7g0H^-C;j99o+ha2e?Je!k-@0!+QMtX8moA9~_k9}qY;D3QUSGzpg}9-RL7$Os=| zN_lU6O}O^-poeg~hr|ELE!}~aULVRnS?w|kFrfjYohbk!pB3|_PMsR?uX*V3yE}*6 zDO)}7KATS2?6TP5yfV7(uSd1$@7eC^ip`2wllkq)e}1K@I)Z(*RbG=M3~96%+`hkMgF>5b+$ z(;CS-c#EHQD)MFc`%qXYEsGUqQ?A8!FToXCnThv|dVL1eBNjlbXH+Q;z#*O!feh1l zhhb~X)$tV|mxB3~M;C+CVUx$Z>?||Svjx7aAFVE`Wjo%+5@H!Lb53+jF)N&H_TVvFTsKm^x^#}t& z&Snx56C+w+w;0GrNaa;t(-5Uhjh=bc^aLbBBLAcp zT+%?89#Wt&%Degq09F@1I|M#P_ZQ`+;{igP+Ru-kNJ3S5*h2NcLM|L5$sAPoo}3Z! zDd=Uxq+o`^BC3(2Oby1r?-2j^vs~Z>dYO6v*akgclz#yZs@%m!yWtj!IQOm`d^6s& zON;^n0%{_+z%`arHdP}#ZK`alH-%-Y=g_0-V4!-01%iV!k?|_ zt`nqD&RcFwHjoH9{0=FOWmKemO(pJ;A3*4}arN(qJMkL4)?d6rLHYQIHngcNaGV$+ z1{4M>2N6_16;vKOMO2>>?_sLJ3;yUM5a5}>JqQLE7mYrnVKWTo&=qp=!vp=nN7;3( z1S)&N9tFSUTJ8pogs%RF$DLJTKPoQTBGNeA@}GXS@m z-Ui`=s71h}3BHmZ@*B1Q#7}Sh0Y}4mdo3;U%ERmlD7R^P$NWUxtxWcxxmOOCI5d@w z9(?H2Pq4}*U9-wa6OR5f0H<_I@_F*YPas0`mUbYM*u_GjkXqp8;8=dy(*fYv-~Y0-e(D;#8I^498{viHaIM5r4kp158Jk$GU335OHQKTlK#g>>)@L@t z9IxqWf3v3PE<8N5F_Y6DKT$)10p^i~<2vMd0SNnnqEJvZFp_4Hy~iBSIyiK24|N6FY1bKlM{juWs+%uvgzE^&i`5Cq`Y zPLpv>*yfkd=H{l)V0vwR)I+H@NA{xM;m zO>!Ptg9wQ}6w~prQzuVGy{(1N+jRtgy2>D{lXw?GBuy~*bU?%r_!l7y)aR-B&f)11 z4?4-6@rM#5ZEd}Lh8<}C*O3`ab|qgE=yp$^RU!=ld{~VvR3+OvDzOkKftn}JQOa%Z z5OnrFFVybn4f?d^;bP-cad}+~U^2hJE(a%YI}I%Xg-!!XdT!hLXL-`|4g;m7mhpyS{PytsUASxsn5 z9~^2rtC6yx8xOA|_IC(7N{2xq<_4Uwos%cP;zxOg^3ouJBFYY&7CqFz_klUk8Pu4R zQmG*w!M|HuK)MPSQU|aFG#JQUQ6L^C(&pAARc_zD&1KR_mxijWaNa>eaip z12GEON)#mQUG<{Cbhvoq!FRc^v;gH}x(>7LG;@79pYT6}(#;MfIRwH}QPgUT9Wj7n z=SRvlAznDqt_BeL#&GGgaKKitf$j)`XCZ6=`J-u(S!Y5_9hGMYgbuXg#68(S><E@CgR+oPos(-g>d^#=)8ETs~^-bS?9TMc>y1d`V!2 z=68mmV@5l`$OR&X0x;33XsZg&s=2pdQr&}p8dc8#eV2u6or&SI95TK$F?&VOHmMoH zgHilemfFzMwMd*GMJLOonq_`x=_{x%j8Mfx1H`s(3<<^lu!7jrbcM!Mi-g_SQv125 zE~1{0Aei3szXIJa?IYT5W@^eHi~{gGl2im@&Z+zh^q>OEY#L8UuF3VRf+C#+dO`yB z85oIX4tc&?WJzXtv3wR9X1x3dCpJXNDlle~&!wg*px+qM;8Pm5tPUZACZ*vooO(a& zYhIm0eX9hF8xeMyEZ6sJe5h|fMEl;LoeHiC=x=6_@mrb?7$J$zh-{$Bxr$D3gMU{+%+vVSS>>#Fz=|3g};TwhJd6Id(F_jBvXKw1x)>cM#U| zs_qaFZQMG+BjTUzX#>-0zG%v0;+vw^%d5icCIoEFvk3fk)Jz zqbKQ8S7A5fFL5Lv+f{gvQBjG8xsi8G1+fxHZXLzD_86F`rA=elIg|O{G{|UM>m6^u z#bT7ELBtF^$#Ln^ZV0a%+M{_OyI3vfDT8rgK5Yk<4w!vmsg-MW;Ti-!0!xoqB5x%o z|8Ti#0x(g((D+ADSkQz87W!C#D1yH1kx@K7(ACJoa38sUbGem9{7FDSNXnE${K>$T!W?I!CfQHqC zwmlq&P#;q^#1QGAt`YJ-|HL{FtuZ(G7S1Q9Ww+{TErruY`N7zny91Y70Vcfs`a-g7 zIMPxQs^v2I_8{fZvS2w}^72fIMaO?xPRJkp=70&qFOnE2cP(=JQ%ky%dlpPLtqn|X zn@HSOe@am%fL?if!rCSF3_fUr1vt;Le47KVMQxhug?*5d>1iORU>))wO9_UFWe|nc ztmz=_@M!oSR1zAMt`(W~$o%xDU@o?u;tD>xI8e|e;t$wf8ZfDVX1U9{KGdkrN>5KD zB8*f_rF%dkZz;qI862GjB~Tg?BrVthh9nrk=`CiRvTul&TUK7)A|$$mOe>g^?8pB_ zNAM-0U;?_sr~H#lF8s#BYM)#o>39`VNfkIPIrizL3S6tw9W$vI5NjXr4uQg9e5xI) z1vgF&lmwH2hd@9MXKCr9Dc1 zlSbJ)<_ewv^De?+TlBgeXyidjfO@VM$4{6TFXH?(j0CKLWlyr!cLScZKk~O&Rlm8M zMh-kjpo;xdKXYO&SA*68|0GWNKj{iiytRdu39UazjW_P%EyT?t8zP>X0fJWg>p^6z zEhB!>w=)bhhm`8SvEjjz1>(ok2h#@c$Nm4zHj43%#nDQ3ByA1ysMHswG&zNPI`AOAbWt5jM`PxHVmyw2mj@tv+q65C1|+Uape@}* zRg~d}vg?!?ADyb)+m?k@eEGK+FchBW4hT5=isX~~3oW9H%OSieXg%iTw$i|esD{on zvCk$!pwHE!#|jyBZVU#q)#sZ7w${# zGTu|rIGZ4BZ7|!OfS@7FE)tA}d?xqc>@J9OCfY?1@JIukLDGgyd3cPu9dD;J-TLtJ zS3?*(iVa{%F!ahOYc)-jPY8guG!S0D;dqVPsg*V}AdzNtR6o-f@vM2>R6v+Vrmx*u z!cd7Q5YLYTD>0ozlm&PD-`&#p-;-UE2sk8yPFpbD0+QULLJRv&lZ#3IP`Lr!)3mFk zW@H|bIY)0;i0 zl}&Q#3tYID8kASA7-s<|FtLlr9&od#0Ya$eOV7sUDt%6l3BfG)FVp??^F7+-;B&tR ze+a9hRc8tXabPi*6X_{hkhX7~c5!VwXue*3uqTKz4bEhG8)CAd0Qida&mx%uP=NaR zdU0D@<&))k$LL_6SRww&>w<{&5}L20eqeIBg&%jPQA( zFjt~wChxtyjMPz+fP|*l%tM{jT67W8H1s!a)GLVtff$J3mae2Z%B3py6B&0qxUEGi-lNdl+Od+J7~+;kU$1yfNyULsdIr%vy(sHD>s) zula9ou==Npd6*dZdxig_ty{kYjfxJY0N(XZYeS17f!nK^S=Q^s%LJePg#rUcp@Th8 z_aLWF3)sMRGiGjg?Ux_uP!YHom?su-wC?jl45Flqrm&yN@Eu4pHtWd#CU}TY9^vs) zlc;eb4CjqGp#NUDPp5diq*hl~gAwWpL`}f#;Kfcf|K4B)WmE6O$&-!k2@(cS*K8pc zC>_C(aPj7&yNJjJvLGotJNql_%+eA&XdFDfpdf}GJwZ?}N##xl`k-!pU5Vv7L8Oog zXxEN+=>*gWNLkqyO6`MXl)tUFR zWq0qlOfC*ns~4N+m=BiS-V^hX%i6Q69pX1DoPU(8L1;2s98ISfSZ#Q2HdC{x5z^ce z>(ywFCfH5%0qPt=IotrB=X&hA6g#hehWyLDC_W1&l*d-GU|r$-XeH`?03|E90Kp)& zzXdxVQp?IC&NCEq0|n{8iHBXzULqzY&Ps8#wzkf1J7AX9u{dw;lG?eAMc4iw38!Qj zBie=GQdOjG^IHyaD3^o4RxwoJJ}8$}1UttN`IkkrQ0@Yq6mi;aOT74~PX;g$ZW>3V zgr{cS#)3_m$g4tVDEaOR%{?7vw$OBy@YN$m^4~MU_l~`iN-w0a7_nM^le{vRF&D(F zsB=@bO4V|-liI6PbI79`*di_#%Iu)t9rIi>FLR zO0>T@sOK7n&Q-DnaZNGF41L?@C!O;?wPA2;biX0!gq=b6SDKaek5m+7nAC@#5K(b@ zNS&!3N(l>+odNC|@p~C|6lfInen56xA;Q4p;UReK100g(Dd<$M*(nL)FGAuy|7ZH8DsXJqEr(rJ6j}y8_q-a- zN12ag%Wb{k10y$>GcWB8(N(eZAFs=&qN{4?bjqA^4X3zX)oDP2bPIesi30YHH_M

I0gaY8-Cg4zxV4zUNRUuUX_JfNuXZkI@Fw|MF+> z7f>1HuVL0xrS0-sjSNk1!+yQ!#u0ua<9rsb^S5f6IDkQ@D9nTMTdXu)EIwMwI zKfeSf6<_b5CZ&2;7L9W6pJR$=y>By>M|r4 zVYg`m42J>4Rsu^Dk!^wn3Ka+@hsTB_tjjLQ7(RA)o$I8Z14})X<8k_X@!hq@cBK#n zIbWFbwf)}_Ict)O3K1MM8WTOY+-f`TC4^U!*%u$HT2Eha=$h_#owh@FlgVga37ElB18jw@A2U9I`eD*CU zBhpABhgBp01E9dB$%f!$sle-oM{A%d$%4j@?D&01nZ1~&+DpV`(J9an$Xyh3PJsCp z(L6s!3?u%4;3CU>`(9aD+2jOexxq63Zjxd^fIjx!%^+S)qGJ1-4~SsS|3+Kgz+pF= zYQLZPWe#5*44WCHA<~l!KOcR(b>#7--La};h%tcwF-1O_0S=vK&@B1`q=KSmvLT6q z1dRS*v3!`dvIEiedpUTi0c$_Wt&rWYQxgH=e9tpfu9~T70azoE$p&e6V3CFkS*%}T zdA|{zKK%jMchoJ-^!Qijs3h!k`dMYawj^x>Ia0 ze~D#+p)I6WHZ&~HNTFex&}{P0CrPfcQmV;U=O-o&e#1VN`GEpiXoRM%{jfVtA8|U& z7l9*!SVZ5Ah4@;k3DN2;7H<}mvmWirXDwMU?PxL_|5x(Bz~7YKe*>B6rHmF-m$k$ z|5??jZ~Nl8)*Fl8OP375(Otf=xH3w#)8eRz{9Ih&kFE?|j26Cs5Wf67Y5pT9^-tjS zpW6j*t%n%nq>51bFRw;0LvrmOsWj@SfBCGxRDuMbhQsYJ+K}`DTY<8kZMstm4l^6@BM&0 zVc4@#cU1&hgC$1-HJ~4$v%?^-8CnXgtQIj~`AqO6SbT4@W>H{gcQ+Z3Z4h`edG5g0 z^FTOrXxOr#9IX(YkL$I_?Q}n$NCT%4RUT4&2xy}A*_q}3U^^=}mC{rai zy+Zbf)Y09ZMPHg@#H1k{NM?xx+R^|t^#*i9o*Bf~KoV?7k^#&EN!T!b-V_H$d?Rd= zxB-?q&x|@%N}QQ!0E9N|QctbH2J2*yK&JOfvZ1*}uZmiYh(r&<_J>|WPZxK{mT$p^ z8zg*c*z7V4+scYT06J!L$j1Wjfy~s2qZ+j${QQ(^jWuW|br$_4ou7%NhQUp21 zmim^ojSQDge!{X&VHY%m=*{?UqB>M5#XVR&DR(3)c315B%RDmWV)oKuSN*8@J}_sJ zYmk9?7{sW^c8OVqXS~?Al#IuWBXdTJmMLY*v_HewgTk?@@462uUtR zHa5bg`x z&k`^0m1My^A-I6lJ{GjVPX-It?B`e5I<#tVuyXO95+MpZ$PAW8n-G!V38I^WgG>i< zO|CO4vx4xd7_)5Zm!5VIA#;4&WSHG~Zzz)u_$GyxfWO1Qf&6_~5`2nl9_ITxrKVX0 zD}wM5`6WeKCq^GMeO8R5MmWrL=Ad+*3YA8U5wJGXE!`gic9rT~jIuydc~A4zrK)j4 z#3BEspGvT%T49lc0+e1B*sH6OL>cY7fjBr8;%hOl+}vE`npQtv>MKMD3}}yLgz=%t zCth{RI2@eM4;DlzU+Bolm-oS5cl605R@RhPJF2RQfk86Mf{o7?vwCwnzvyVf-hNhp zm&{Gf^mfY9sm6hxrm2zg(({&;4inq^KebchTGqv`a!X@liYI@rdrl4YcNs<&=CnW2 zW`4ulUJl^^pSl}R35(IzI-2^%C+w8eTEoj%6MH6Q2EBYld&3@}1~Yw>!8EYT?7wcDYPC$3G9y+8)K_VgV@c;qQf71#&Zk~ee)RB0_#u{ zQ5!T|U^9y{5+Fko$P3JdZE+UclCQd9Q>ZiK0p#Ni7D_YSKLMDu(7w=*C8s&|)OF?_ z?t$jEf@Z6Qew;8xz#CJJLmM0iQjq|){@XXH8PE7c_r7+JB-{1IiaHs-iRrJb@!|C`)3%LUq8mvy(MHKnL^hK3p0&j<0&w}<~1L56ZkEL zY{yQXeM^$8TnEw-ziysN{1Pb1TKR-;u&?&O?3JT0$SdA1KIpuN8NVzLCesJ$ z&Qkkp&_ek11Rz@7vJO7T{W!N1&B^XFmY<*Hsfc?HWCoiLe7zsWvMDcSyn`AHco)eP zjV7Zgal3x+ZK(C^71)x+5xlF;u+vqSNW!@d8W2QG*pT~g6qKuoV(y8$Z!1B7=sJ>@ zgS2Vn$N=PP{kyk>?55wftPfCa1As$Y930q~OUUu+31jlp>)`k{f=R5!M}!*PXE@$n zc&QKq+_m*K_PJuQZeLqnsUke7gkAbd7Bd_7&g%b-gXlEJ$0-70Vyq)@F8Qe zN^GaznADtibal<|)WGrXLy(FKwwKz8yhXAe(9?|;1P`-18@J97!SXx-8%v))c2B>Z zS2&ppw}k+C_@|uxGjLM56SVC$p4|0S7=iTd|0bsCwNo~Vx~#!P6MQ=S`_hVfTL8WL zNvfM_-%{jfI2 zEd}|r7_qItNVrxeBxK_^Ki9?R6LQ-`4`}1R&b(>S+ zs7VB<2A}#p(S?H$Rg1{|aui&n=J2C|CR+(TzqXTa!wnGh;TAa4Z+$>yYy>(fuNhEi zF{q*W;-Ce%-#W}DG#v$Dg7>jEx$Z-IGdsv z9|m%D^zRwijwf{OmS^7|y@n{>(w&_BXM!%9ihv;yEnQ}1vW2~1bpMu^aW{aoV4spU za)ZW(FYkp7&}$8bzp5*VC^%-W%SYu0-L$)$rdgSwl&xM@;trCpTx#Z#6~M0sFe{J^ z#PI3qbGsS3d14H20o^yiZvPfXmku1n5OhY|R^>^q@I$~pf70(AwC(7Rt{`nAyA)T4 z=%s`1B5_Ul%ph1C!uAyhCp@NQ^Fm_JpTe@?!Ze6z_rJv!-WB+l2+^MGRjQaJ{V~sA zOtA*YqqSYd6S$1==7C>C(?vVkC(bN?NnfqiQax9d$wE~f?zy-bInTB^NIdYbJ{@JP z_TJ$0%KpU$_*tdBXSiC_>23$Ns^8$>y2;fX>vAWbPPC`l$%29b6kAc!RVs^nOv zQ$cNp8|6qO0hSGd9o^hlBO$i{j*=z_5Yr)l4$~yJ%+D)|NFme-;zBUw-tS}-pdA)4*qU4bM*kQ-B66LsmHPXUJ>LA3~zf{0Y7me|%LiQQr76VgZ! ztty8QEXheTwKAhNN_B`$A{rFjqTwcp#-Wf7uMbCgT%Fq`ye81@(}pAxyu9%8Dj`|b z2(d7T5zuTP`3AVw3hr75*9UJbON7A-0ta7k#a(iwDFv z7&s8GqQIOP2b9Pm6)|vwU}8iMLQ;ei`;TlHHQygUyluFr)VpHxPByfAoYN}}ia5YH8#qwMti6>rh8{f!KZbedr;D0KBq(lpM zrr}lYK(4!0#n#WYutJ~1P#NBsiAds6A>=kp+u+2*NG+?;Ltg9=&K#1ak$uZRYCgap zG{Y<59$RqRE&`G}R6_PrkRms%o+ZUUd)7_YAs_qIzaALSB=+A zyf-A`x6!V{^SCn>r6l4WZmQy}D@%Flzle0!uAV4k${HbbTqhnP_~&_+?o-RW?pGrbLqz;k>Lx!FLuMfTGHuC0%yYwbsLPLTaH z`*tmo{mo%%bD=zc%k0b&Srl$NZW)W6?mk#$ zV=rNeBX=&D3RV>33?>W%r4mU|Zu87E1WWF1W7rXo6-kOFEZRX3LlA;=5PcFF+Jh|? zOd$C3_Iw0QCj{0!a?2w`V(QE_kmY`;4)+a0_7$=Jd02g3=#SpN#L8Buyeo(fGG&5d znE1-dN@wl4ANJs9{k{8Iue6+6k;_Y-G?c8&#b!2o0{!62gqX%K`Tcd8bF#ftO;K>S zgryJVt~<$(2Mt#YbKHW&H8s?EmMT$rL*?Xjb~Bp|brc%X1#hgRa@^EglCguA^f%j= zCR-Ntb)Fob7Qt z0VQDk^-DHkrUwf4P$i*AK)vKAd1Pvxnf)H&_=Rks|2JoZXQ>LJsj?MrTa)D*Bf)UrrFzH&=)rAIBq|;(lW*(+1Rq0vs{;vrdDlAMiXzyXRdnq|pG4)2c2ft! zzp1rd)rIQ zr*dmM)6Gt8d5ncW`fPr$LG|O4Z@88}(M-oGaJQq1EJ-c3!(cbUcJ|vOx#Bk$_!swg ziGNjj#n|kaPK@)A}+D*Lg>G1XcyvvPt$Q68^Pw3x< zgsY>_8c1$ovd`rY9APFUqn_d8OBC>m*Ceh@{NAsm0aEL?G#U^E6aY2; z9QvN#O(m?5#}rm-LLt?`Znsf1sl-cyOI)m$2iJsH*jeiY&yzXSP?9uB*CzoaOAvke z0_^VPDSusHA*+uy${sDk@}d9JW{BL9?Im%Kqs2D6sA6|aSd!HLqVCPZsovYZVa-Xi zT`EO`C_>3li4-Ln+PlnSDVc{9GAvYvW|bnTkd&|pYng|lk||S`c}ivr$&mT^e4~Bs z>$$J@e(&cv-s5-}?Q&=XrjnGeD&DO!@zjv~7OzKc#Jc>0`#ZkLz>| zl3j9JM+>y#tLoYY-C`6f?pU6ks;^tYrNbw1kq1wDbpp@vXSKTCQQHoaNR{`+A|~e1cqskLV1jQihqjSW`@cEwUf zBnc;cnd=tR=yNC1IUrgVt3X$gkX&C>B&^;1bnk_ig)NG!KZ{j_-uaW+j^HeEP;|4rA%y7=FHav=+7 z-Vr|zw>Os53I1)A(|U92GI0yRDkqE-Ml{HIWPr1%rsb+0h90kO-JOi1u#uhxeq8$e zFk1(}9HnRbweWrnd3S<>{w=~-(ek}h+S!h|v5fb&@A|zL@<-g`ZQI|v#`nZe&Iyxw zm$j5xl%KCC-BY3il=%4Qz|W)SiXLs&if_5A=s&yVZAL?w)r41^wNIw*;#i9YP*Tv0&JFG6j@ErUckaw$BL2`Y;)ojli1N*=`kK3{ zIJP8LX4pjHo`j;=_ltMZiuC=wcAMXneAHm;8R^ZE_D3w!Iy~NSw86w-0v+(UrhCdsH+~P-cIx_yhGvYOVgqGxt(DYFuVLu2Ygc5O-JCban8e5jzDsWbNv1eEfL~NKz-Lq)QdlN`pYLPY`9lmOZ4n7fNb?1Kw?HJT(Fk;{~b&X4{qVb&|blXSjr?KiGcR{Tj z5!&$^DjTs75!cLRqD)qPG>)cN0WD@$oi(JDyfh&M?6Ue1mlL!jyAy#6@F2P7?s4f6 zIh3_CSqzxv^Dsat9i?`GtY$ZJLwDnk&OHyu8bF)P=Q8&tV7APM@?*V|l4)6(ztfV( zpN6P`-x|y{!k$~NiZ9r#b4PdONs?>;4~;J@>j66_aU}CzViZ?B2iwB?2~^>84B<6T zZX%W-@CO|{fg^l^XDnvMW}D-`6?LzP!k%OoG{t+p$LF|@23)J+fXmNfkKBZD=y|`k zwejHzcq251e=x$+)!*dJ{g}lW?wpsiwN2l^Dt|^R$tDC*d#Yev&!#ubnC{PUrn~)1 zPOl^X?X&N3uRCkZ`2)q=dw!mGaE!N@cYeOq0?6}44>}*!o<*HKd$tn&C!~a1k^6Wq zK>9OU%g3EP#HNepr|UuNbnp3=LfmJ@7-2~mQ7$SQs}~IL{0Jl_#+JZgOXzIF`?T<> zigeAzF5)~#&FP2X7mu~JY|C97i<=2n|Nd{R{=MEruiF$GhqSlYxq>z3KP~ohV2Bb) zq|Y7&45Y%%H+?(k67`(Cv#5Y&Y>1TqhSLI%F#6L?8%|qO5am)h2sZYRV8x=-kiay;OplQDgxXqvf%PBA902K4L{0$T@6+3UB36xl4{3mO{AkrEvqchiB zdbd4iTC?Nc_REJqz)NC0K?OJqaeDuoE&Z?Rn6SYQFsfNb7d85p=o;*CQvvt2-fF^6 zx$F7G86;&RAt0XMUOfhk1MX^+*u@fSpAGS%!qUrfO3=lYNmk>ThTLj{)s-ybI5*7% z(-6=F1piI4^aa1D`&*2yk}r%UD6QidO>!Q0_|$&7LB1nt`g6-@lFD5j1Va78eJGvT zMa>oOfE!c9&8(yr2Q&(I^oNe-PY<^HDm@leKnME*pjAIfH_rJJ_C?|~elRqqLfzLo z>iy0@@l`_Mi{2^edl~7e`P^f!iB>;4I1G+~@;>|Jxa_y|bq`-S`!5S~rSg|Xd><=) zZt19fFFIZDnsuvEDa#)YB2zg*-9F&Dh#r)Hnhn-l?%`Ce7hoFzI3R{YmQ z{L@K{C-SC247P){W=|)0kJ81jlv|r-K5|7KUCEHg(|8o$^ zZT6-KmHG9{pEU5rTl3SRe88g}?usI~yEhHtT*n#)^&rw&$_T59Xj_t|LVb&9*LqMo}630qIqx z_u|1I;RG);&x45HO4Q<@_u)6csr$=Oh)C#Hdq11Uq_4I*Z~!UpcI!d3FQNAU*gof| zI;YkyQsb!aDE)=pyY3UuWyDQO-b7ne(dt_hQdFGfd%rhqavuIkZq;vvh9f*~3A8wq z`t|XJb*TqM9PR`WUwS|Iz0bb-2>as)TkH!h&)zR{v`H2XGt&j`O%g3mK!6JYaLji! zWM=Mn8u--HT$AaRNwcb}elm>2oPf3vSY1j%kg+aY_=ZciLO}1_-il2@&eLE09u73? z)sUCHfF-Nd9<-2BQVL{^M7+I2KW$I#5KdWAxjTwOuCg8!(~L$!0;>w2(xZ(aY3_mz z(M+sx7l-S`7Th;x4YP0xEL-ndnRlmLfxB#QRdPM)b}xE^PJT1u_HD~M?@L;{j!=eG zIx17H|5P6P&@-VPqscGGS^ki%twus1g=jg9x33uKkAD_-CTA)89~Fo; z^%x>492n;FIjz$+Rm2!LT#CZ9{s7o4F+qNSO26KzOEA8O=y;&5u&8e%E;J~N1BbX+ zSMuD22d(H9^elWL7L_}z(j4QfRQxG@?-sNsc(tt4^iG?;C~rqh-p%q6cN#GHj&vas zOP$gU3d|4K;}PQSgiBmMoJl-s5sqTPBs!VZUyvw52x>yK@j$`e!~d^Z4Y*7VtcwOa zzl1rXz#_6deO)W^a4Ib#y8wavu3-%}RtnfSEry(AW0dC^-3)`Gsej_FaF|lnjr%b& z?`(zxn?I|bHC+DFNr z*)VNa`$26}G}VMeghYp5ceKg_Q(i|z?WNFEpB&te3{q3fAK??GTa&|5l5ya*j+=Tq z!?>zU7n{Bk``^qSh5qAZ=XZQ8q&e&_0HQ7HS=4z`IO8sK!k>VcZioo;n*U)Pk1;e` z1TN%vQ743!+81QCK7blc$6x2rx52Kdn0X&DWD;|JDMSz?dWe-yr+Y6#G#B`Mt!#67 zHOi}VDEjF-iOX5O)_Ya=-E-MNxxsKe&C$qwe|hu2FbP`Xh_sa)03<@xOe)isJ)r5ZPQ-V z3?16J+3A4o^8mieQj3tRHx9{YKUQXVFM4_T(OK*J2cF*jQwr^0N;g`{R&5>Y_N|-O zee}Wx3O`0(J7fsN01`$&a5*9M5Y}6t6J; zZ044g-8H4}8@-|C(`0(;miNfSv zqAso8<3qI<4`jw63=%_5%drtxq|@ftLE7bCcGI#OS3ms;8P8;!zON*zRM_M2RkcbgrTlrOT6usQ*FrG&B3>>7LP{hz|o-}KXx;X z5FU0)&-%4jf~vawtBH@JVgg3LC8J7=Xl*NPSDHo5yO4=6Wq*ChueLi_HTXfo;H902 zp>X8uP2q1wC9Aha{1Yiz*QvpdD!O|UP2Hpc32b4v*{8odA&YnSAVA|tU=-gtw#3)A zwP2+5V~fJN&ryQ0xv+A)-_ZU(08qS0daj+wLF%XSitr45vAc}UNs@_BkvQbER6(|d zK-8Tbcgu_XY$P#|zMSW6R1e7bH)l&ibj_`m_3h8 zl6GKXW&3ZbD9S&{4$FNg;2-dZ4a8Xlekl{@K?cARD!N_~_>@S!6*Vb->+y3Gfk4w9 zKPfWRWB*VYb6|C zV*iN>p0Q0NgDCTC@lL0Yx7PNH3?hJbZUYYEhrz)cwP+^y5MiF}QHbIm`%L~Oc->>r zL>261Y;?jL{M&rjc9ZBv6TQwOKIJ4m{PJD(-Qs3nKGAqg%3j`H%qgw}-v-~*Si@Ao zDDZwwKJFi?PF4@*_pc!7k*{Z~;g4OHpGYCuLWplpT_{EF7gxTwU)ou#wq>GQC{vZ? z3H39_)ZmvKH#mC>UfX2<&!wdwQx>%o+gF6bQ>AW#1NyEotBlI0854K91@DqTX-Z0t zchU6i$Ff`}na5q7Aczw-gW5IgOg1Ix6WtghWLCi7x4UV^l@EH=HkZd^H7M760LTmH z7VR5(tS*O>S^eurJ1wCZeG$vGO+pGzVyB4gf;z<*;#{{Mx`$_d)3G853SA+@8tjXX z6GxBjhmh&X7`p(j=ksn#a~g~gSUMvyVY&2hPWx&>?N8`&TAMt3|exY z)6OO3bQE$KAh7%**m6STh5}x{U)|>UrLVJb&F=v;Nw#UaLbIqU)x9`73ow{>jk>_0 z*AKk-DwEEKLV+lUpYHxGn`McWa9>(wD09_h5M{2imQh#D!}9q)8c&rOoJoJXkm{(7yuFlf#@cw5p;5Aae0VsC;)K=$XhRJ z)f$6F@*9XR@eo@A65(MYj5mXdAi@-fUMp==lT@6>d5`{d0v^&2@jzgs6RO31n3TPr z-DB!VdSE0Tc|I`aw6kdEOc}qSINRi{o)U5yNg^SC zfFQ|IBpzOxWesC)S#{9z>IaaBa-10b&+hQ@Ex}{ z_;DrpDJ-D(U&Iu9UrcchuH)H&2YaGQ$suxeX`yzrMKF# z>(zn9GM{ar-eL45gJgZ_e=+2cCl4OMl@Js-j{l6!_FblpcoBc&8UJML4aJpId{c@k z7dw=K>fQkOf(Hbj;}}|VBOnpcYc}uqa9v@TkAGKS=eHvv@(y#a>v+;Q?#pvu4^f_? zxB6x2Msr{{BNS@f{xyYv;K$$~aUQ4EwC}6?s4uhdy$5Rbk)$*@$d)pD(c`yjXKEro zVtm{ho@m ze;vj)DtudRh2JMD7I=&2JPE_yM>42RS~g%eh9i?+LBa@l<({F9G+ofV(2}zZ0aPUI zH9+p9K)78dk%162s<%G<>5%5k+3nDLpCtt|ETN(Z7F~GO^0et-xJsja+vg3Y$oUuW zTDUA^RW9Jx-7$2$#7xYTEJSSQc-e1tN+UphGnr{&{Rzw!rH2(j%*jjm&szs(KEYc( zPwNw7E8&H$Qxa(hg@guk_dz|Jc7-X}C-uwm0pa(Fv`goSWLhxRS$bR8_BvOiPN|x3(I?Hl*V+>k)UsdYgH1jhada>`x`TGJ8}r= zYJvdR4oBvj)Z5WDv1h!I#?Tmc=Xb?iesfOyGU=Z(Fb@&v`!1ZY-a1n-^AOA=nH(^; zxSTNsodAhJB5o0K^?o2=61X@3S-!vO+fV_*Z~~JxS^NpO=Y>HH_+r)I$M(vZQo2$6 zdfSbw_HdvRrxL_04-vLTc!a2jpV5!{;|Wj`(o5kXRsfPP^YKnisQfY#r1t(M$8mgJ zn~7!w-Bzw~5QP^{ia*Tg3d97Qo=N)I<$AJ~=>{0rw_>7vX*~!wfy5g2munb&sQmf& zn>eq)LOo6ODkAnJ+`9Ql1pUK|0u4HB^@mmq>syQ-&hfzg{-{q&wvG5`QU9#`rZ8df z-eTBu6Spk7v>l~m-2~wY!az@94@WD*Na?{l@5kTE`Sh=kY>VGUM4Yp=sr&W8?#!J# z_fZSpqc8G_kXVDe`+98i60ZG~xl8Y2ex675AYoZ z<;{n0d_sT5<2wE`gn7$mof?XV8hltp;RyijC@_~*uPJws4PVXJV)1iv(xHDqi#7XaRa@&d-m;y%B@57+ye2VC;)&bwY#}n1g!cJ9FoF}yE!BZ09Gk6nU7 zm!2^EMm}eyZOt`s1s~V#!~s{M^k9v+4GBUZL;lN5(Z<&_glZ-K`0misJD zpYCot7k9w&=4sYolHs9!+9WB`cR3f7ZaCj~iOmLR?fp#o5^|p#iII7%_ ztO>nL*E*+E=zdQ`>+bz=iNr+{eJ+@m$Ld_w$4J7(U3^JLBKQ)d2>!Nstq6&t($@@_ z80a|pD@%P>)yKKfs~YlSEjLuvuNY0SmHRXt9XfoQb(4>2@6XYUSf%yr0(NzZkFE_H zE)y;rm}oUKO)tGEd8x$sOMbu25zF%nu4~7@8Q;j$;oCo4zM)~PCreM-w{BjHPcav7 z!i0H&KXX&B&f6_hHJ6Ld&d{bY%O=$`TpoM3>)OZYvfV*kBw2_lt2}=X)UNFH=quGR z${Jp&Bk@&}#XU!|`EKH<$%n#MnQhsU*qcb`2Ex4Cm)%TlV7S4pw=_0OoLOVpQI~E~ zvY7}K1hp^DK4uvihveFYb3TEuB(|-s7>HVLE}6LB1UAzTC$2{|@vMmEYzKXnG|}$A z+V4DMucBY}O?bKs72&>p@GOfCFgSbIOrgZ3D(q;zMd`PzDRGX?PJuO@DNh1QKlSKI zejApWER9y%yv2nUG&DWM*mLq@>#b+`xa0Ck!nIReO+1#VnPL&h-rmuSvbKW9jcC|3FHCZM3ZP@bv9^X`Vo^*bBBA?K3ECFAW{=43K#5-FutBcNV<&XrK)1 z(e7Y@oX7=AZ@pA%ji$BfOfNWd?bh43Z#mTE&1KfD zpl+VRB`#!={A#w%Lu7l(P4(CcEHS;qeLx6;+m}uh#DJYZgttxkz-aXsg+bAuJAe|B zbQ0B00_n6$L(7;r|FaV;qBso;60bYe7{dC+s7cAZ+sm5&f`XEr;K&s;uJM9%`$^u3 z&1HN({+nY~`5#k@B%+A89Jg1=1KfKMZ@^q*=A1vgJ>OsOVZ%9Pe_hA^V^xbXY{wYN z-*5OAuOExaymPdnUaIg+xw|HJqU*sI2j95nl0sZ|2gki=$dKjhtcqc|PZgmpfnrkD zhz1hL^e7gdHL#vpn_M=2>}zH0nK*n_8=qc`nxa&v0nU9BQN?YYz(0Sbq$Ah(4?%5J zVxh`7T0F!(`o!c-9a*$A7JDn_#Ji|7Hwv7OWV)B;gw=rTEEUY^b_>#5<Cbzw(6yixf#f1Q|kZm%>3k>cZt_w#CidqA>Y*T#H*=k(m!=Z`uC=e zY9;f8e7UP|HR`VDK$+R`B@cGcp6{|~IjgG4yOB-j>AL$ydS`hB$}C0QOE>1}dqx-~ zCltAl{Zi>M(??nZ8AhmV^o|43n~W=2NW2j5dJ-x3`pVL^+q&c)%wXVweX3WLDNjvS z^`VzA%eZC2hzh7uk92nEuG|DJUL5y52=(_V!ucE=>trO&K0aU-u^5>9S|=vNuIm2` z2KO92RqC$6*YHIjs{r={xAFeC53)v*-@muPZzK5bY!aIDLvbO+YCvPf>d~$%L<5WYOkW)|9IbQ!-t7d(+b+xc~uf|K4!+{tRavy2ssr4-xranU2#Vzgw z^{R8sIXB36oT*qeaF_CcPEp2jSIB&mxp57V&vA*G{F#I>UM$+7Eu5|HE9f@?#JMaDmbql~ClFGTAF;sy(Ggl*^8po!^41 zOApnYYX5*ee1P7c*iYIS9ziPp$3J^dPM_IH!u|v9=s;4jHGx4}TdYtp&^BCsK;o={ z33Vf5XC%S{cG(u->}~LPSN%g{EZ;0Virv>#Eh1q~$th)>SspJoCC~TTB)gN1xgCRw zF8gD1gGT&69+(=B zmmMzOspY)2t4l6hdr7?Dta$Y)^RWQ*4uLnuLCgf4(s0|feAVT)IWJCyY;N*1SUetc zrIPOqulJC{)TC5ZieozKP5RdZ{}uKhZCS2XJpGmQV1}ZV+Rs~|CV}y$O4snQitah{ zei0p7CRJbmOOdwDipKI+=|Ln(0Z*ha=GMcSyVk2in5N_&xhXy@Cf@b(>5$Vj*X^N# zhN8vg!(Qs`^1$7HywJYwkS%BsXA_+)q?eE;JH|E>8W-{mh4AK= z{o0uWgo_y|Z9lxjfM1LE(lo&EHvdC!!{wt8+jzKu`+Vttfq0Q2p}_vuKFu*Vb0fE- z(}Sl}&a)|s(Zb^P?PE$XVzjmk_z5kdTR@u|XJ2kx)*otAvy@~-rwkx! z{20-B6g=6x`%4NKs(iBeOZaBOw!cOXzlGx=n|k2)P62DnuF(I=0YtldnIpVOQeA2u z!jhId?6j;FE!*va@rFo4PTs?gM3s}XqAt_a+oWB`E3stam@(UuhkzB$oo;wXxh5bN zz4XN{!96=_j;M7llT<*w4DkRG1Gj zNX^A<%q>utbZk2y{6B~EKPwd^!^P`3(0-3K6swRo37>@PL%FXGdlbmdA$X|*T>ICF zlYJ&mT!O^J0NayDo95Ma5;#gy`S8Cd!eoI!{SA{6`z8Fp)Q=ZW9H#zHfcm5ZdqR`u z^k(=Y3T%(UQ{l<3*FIBN4Dsk>lNXFIS$dP=No374z@{dS} zw0*M%px6C|FCu!+ZwFkX+R|WRC@E`9!3<>!8bV|7%yi7s$3MSV%pc0Ov`-Z#TPpWp>M~i$_sq$fi5wE*l`Um}}^53?JLGB1;y!Y_H& z{dQDsUj;lAuhs?Y|-!9#48#P5ezt8gEa zbIE%T3|OCt+yniAsDXPslPLOGwCA*7KJ`lV$yiDkEP!N3`OOnVw-e?zIUV{HB7|Ul z1?S4P-n1WgNVGjc+YfaDYxJxkNr~8vPGblgTon0?R!)%wMt~k_xjxR2deINZ@L3;#Eq=@@A&5vt z!-z>c&h*UrO{+;7EczVD?=wLpEHNf5PjL`-y|}V&2h4GGs(0ZI+*TZ6D%Sv_bpNPm zC&yZRsDFhlK3#t@vu|M}gfT$+%V7jz-x###{2@e7TOw}&F%_?0FwU5SufLz*OO6U4 z+8&F79`?vaPRai>U$O7*>htTQ9(}QSc^=Zk4XXsXS`%0T07-scmu`E&++@hVND=nl?`O!6mbfGx7vzF)xSdD zxzNdlVtuivw(9lYsfvQnl4PJ_=fSQ7A>E{nh&mPZ$7!Vq+%DUa6RYv~!E8iSzxg@vgpMSdn zOsq6cQZRTkuXw3^j)^5@<{wxt(UZ^q1GDz|d#PPtC#~^~CdAC3s0;|jjV>{E?_Yr` zhc1l{nEKzE83>RNIoo8(V@@$M9$5Y2*6!1}VyyS7V&>)DD|5CA?jcg>h#;~PRttt? z2=W@{7HYGSxB$W@R7JZ@O+uouR%7w+WdO@L1%iPXZYL>M5I)_6a20S<&W@r$iI`+8 zzaY8eJGdzW_Yf1N5NBM}bK~^ybJ0*|-T*YDVUP~V;OGdVzkoh3uXBcG~8X2*c;~a_s3J5(@tVKbv4UbLnXQyC#*g>;r-m!pq^(d|N_k>^9!Lraz8gB_u~~2E%0kVfpWJ8oSHg zl|;|taVJ?mU70|;`EjI*ogP@g?Q+UV4ktT_j%>N{cRhN(9d{gw=K;m}l7n^(i%JWS zd`&7+6?BTp;UpFw6l>4Fe#Gz$%Tqk}nBNV;jnHTBQY#@s{PZBU)jLf76&n0r1$^`A zj#BOjvK#`n|T#-(Fy zOK;+Q&O?PEOmJ(0l#<-S_bNEOZKozFW%e`h^s^kJSpy_j315dFCZ{wT9hkE0H(#1A zB7GKkf8R0A<`aA$q98w@-A4fwDSfi&o&8?)|1a88Gu}VaLHEc?1=AbigeoTU_l=lf zX!!uW(UW{k4gYlqVrv>7Ho@*+MnVRlI#)#K>)@Ic{SJ}*Gq@swj5`%RZ6{ z@-q?v%mQ94YY?1r7YLp@YCj*LR-tpxCl(&Crmcgo4eogS@y8!zZIRezGaPh8CUaj% zZ{IIMAJC%+Kwqs)*K~5~TExtBdY%vgDOQKfUR*h$AC!(*!)MnV7M9ilN zW5Y}+r(+Zt5(%F%kZHUN83rN+ffC2!pJ5`gZ zX?RZN0}&&EV;QsOW!G8XX;59;SGTM*%`VlZEADI|f0>&ek056{-}&arhGl%T%y@NI z^O!c`1VPAASFtAg3M2mTL6@$P6YCh*i;YwpEyHJlzrs4<8V(?};-6a=wH2&hTUj+^ zr0;cGCqoD-Rh{Wu>FpeC8xxaP4u?U;AzT z*~odfhBkeM_H6wg8yT!7*@mHBET1k+^U59_RmQTOx@S2lPs47@e`0Y0oy}nGcxQ&k zy!8J)&b z-24cJvqhX+nV)Qpshr=lEx(pG#65GniAeHCej7#uN}``KgV#^jp`H(bIMjC`Dx#LQ zeht8E))*FzHxju)Do4W+MQx&IkJ5O7N|hK`xnju~SuR<=p0Z!s(??u+(=eEJ&3RO) z_d9^N0##7;v#7EZE*fp<7w{KGKK9lhI{VxRicug-{x-rZQQ(_ z3L%OK|CbriXcCODIolkjTcmHY=^Bk>TO)S4st7Me=O^qeFI|=GvB2iVYy*KIH9T`+ zP{XgV;!L1=d*!Z^G3Q)LHDlzC(MC0NG)&r2D-7dRGM`+xVwu?N7t!*D7855m;WWIH zY?)-Z7LNWDGXYzXF(^eD={JI_*4sd4`f{XIllO2m%-3hb0oPg&rv%;wWsO>iTim@; z-_X7mzx@eqk$ro{I$rabvxsVrDz8Z2dWzx4O)>;A{^!H!WP~EeTRAv5c)N7ZoT3Fs z-yJ%fC78M9r@_3~;~$kE&zPKWjh3vc$!gBBBe+xSexg`|u)D{Q^PVLOz4vgXRo3VBUW`aNsX{e4_jA#LTkg*AMHRje#mh0&FKQP#stsJed#3HEelAy(?z$p)RmrG}$3=8DKO$N4dA$B1Qk6xz z<3#DlnN7tU^NOKBxQq5;g>>ql*#Y#e;sG#zCS8TuG#88<(U-k+OIH5|J$#ue3kiEy zC1{l<&ehfY?g<{`5qR&kDKbc~$sSpt*l?7e@K^b0whFQO2Z)X+P~sMVDgC1K_zIqjyL|7qprsY5OlAmYDv z?}*HzNz3~I4G%Dh8No#RsJ^bcQ!aBn&Sk}n%%$zd_+pD)<0nT^V3WTV%h1}p?Eg2| z3Y$57Sr39%wVFix)MGQeWG%nf<*J^qt^gZO1XhOwPEbyA(Fc8*bPkJh0;F?NrVxD)Z|EW;cJdm_+uv4ePxkfHk zsl4I)$a=BwCKr8uoZyso@PDKlrCFl)^^m0m^aa!(u7cCQRo8G;F+4UGL+;K_19-iA zllbpe{O%foV4?yXZ8M7#rwbK-vv~Zc(!;QOrj`4F7ft_kW2x`PSv0SD0nOEeHn*vD zv;I!K*L}Q_$DK>k-r`W&j_JS4vN3i+F$Pn`YyQ7QI6OlNGprB`%XRKqqRT&inNv5c zc%1(WyMAvn@;q77w@?3Y(jVG3`v`8$G$`1&@+oJO!8#|GjwycTq`yC5au{brp zy)1Op3KAT);E5uPO4q;db=^Bl^ezn>2;PxIB8;+nhpcBFGa*lw(d&HAY!2Fj*55L| znC0H~OQ?r?xTjGNnawsO{K03&)*fAmUrWbvQH@)ViNqh917Oz5qEmbJnU_aABw$0I z=HM26`0Xjs;`;F_uX7&lc*O*XLx)TyhZtju_W!trKB!P?2-cQ2{`yY2Kz599PSNm` zSK;5ab8$R{IW#3fGzFv712zLB3Y;6Ry-?(s9+RHg8+$L?yE-eTt;#x+8JB-w?_5Cq z+1M)@s}efJIxExloa7uo_d46xv?n&uJb$!RTc_G<^?&o%3duT5JsKO$o?;t0Ejj+lxVRYvj!V{@2YALMlP7J_l^&gBcD zp;Ju_0(N1AlRp(J!?JslnwJib0muL0NU5LxnvXWJ?27|=s|_VP6_hY)O(X*Rmkr= z`rUPC&_ABZbU1J^Bz7F~CI;L{ILV$F6+9}L*SpAi3LE)2Ly~m5NaiQ z1Ksi-z}>TpP7d}!E!D*p+6K&@?~PJZ^p)zLqnEPwaEK=+MLv}p92Z!jUAOI$UWSFj zgSRQpuk7vn8fS6->{evNDLbcj$K}_?;fBa}vP;idEjV6X?|R|dV0&FfT_yxFiIj+a z>V1J&V7ahMH`dTI(qm|#{txNMV=f5CSi$9t`9cR-zfbA|`TcrRx`c+uN>Y$i#C^%l zNqIsj=hoWhtSJ=4BaRt$U1v_M zsb^mp1le4a_)mIMk7IKKg>R>6_?1(`>&hbzvj&AB(JuON?GGQHp51s%sdX32l+Qa^ zFX~;ke$Oe1TAQn51y>t_^vkBWR4;FM3iVVq9*DIIgAN~bM8{NTsTMIkyKG!Bw-~32 zz3kv+k6^CC=i)06bmy4%j8pGgggU#l&21}UJ+7lWaWD8Waa*mN^C`Y)Ef+nFQH!7s z4gE`b=ZYSmMw-~2!S8IX6i541PO&@oj-PdtsWKsv-7yrm>IAAzEye6i$SJ*59~NB% zR=lChoSe3L0#7t_zav~0@p!$5NQW8Hzot^LMKSqfd}dK8?aJ=(uRUXRY5MRWRQ z_9=BVc{WE4ObB&!-m#l8K+)x$Gnd7M>hwf48!FKx=Axj(gR=+nsj=INtrVLVrH=Z46}2n@c_d|gi+>N{rdSd!zTqV#Ts zP3*kH@I65-AulW394l)WVe;I8Q|0~%wj&}s^skbEeUq9FuhrUCJO6lkKKe@4aJHY$ zy4rK5Wk;y6F78$IduvIjngG^T4w9}L^kB5$=(hFe3vww{{uaKP?|DQsnL<1ZBcu(` z$)<)cxiKp1(8VZJJK?Q;Xlf?6=>GG$tihCdp-y=b%c6yL{S{OJSWWLeRo1ZWxQRvO zn@tB3w*aNX;9^;`n%R#Fd=nXtsreK=r<5t<2v^rP#>Qi2Jsd+F-+a11uQsl6{g(GM zht6oP9CQ`1qXb+^hd*_1?O8Y1tOMyB%I}jP>LB$5r-Snfu9yg8J-Mk@WMzOkbYe@8 zNHxfa#@P8pItJAL*6NndWfMrrn8!7Z#=3j>Z`^wM@lVA~rx1@`YiGeBwL0nUm-&aJ z$NOYSeAN|7Qh8+1)8&6!3OwHpp}>&M@k<6d{gw61vSr{PK-UnmX7JKQ(=$0x=|av z5X~;LgkIEGr54gJm&n-xzKEwxHPlp|6mG=3_Z*k4c;VQ&fqJ8fFzf*~y?{$@4DWlT zGWjXPfJ3YRCds>kWA?GBPbG_{R61)b!n})?Xj-Cl;Csm4a#eh`1*U?-0W)Fkp^UcXm@hkvkNh8 zu!~hcb8HQeDKedY>pV@^AARVAa3dndQaW|!6-LlvwCNE24I^m3;&+)@W6Fsn3 zTSF!q4>7O~$c3+ij?=Y%BklfER<0b^dlPuA=+2YO`8dm4yuJt|m7YEOxIk&ssvXN) zJ9gi=t-YCOrF-n}Z9Yu;I~SMh?_2eZ`6?7m4*xhue<8GfDoD&ZGsN?%fy`d`DPzZ>gLRvH@0n~=_3E$rQPw_CzD(p z7Wz}f$MX6LHwfo0B8v-?JLT16=NmWT2=Fb${u2F5P4hUxp^9M;4-x-OaM-+_SByL2 z0!6nxo{kF0NU+qY;8y?Q;DE*?0W_$@6aDOQG1Zw}V=OkN3U^tw*&jTWrc~=Yb!>Hp z+$3~|#xkwJjPf|_maCQ?+p+&|YZ}Yf9uY+rgf7KmxfYU@haSz@b?_i z#G>tcX}hj8hvyfk-_y&lQRgCBEK76Bo*mX7fS>i}hF#x(8$L z`=4tB%jSCg+Y9(EM@Cc$A&Rm2=nb^{nt)`eSd?m2g`Nt!#S9j*+XPrr>GEZ6tDPePf+ z{?+LBr@K!1o7R%~iq$SNzhWWl-$_j}M-gOkU9Rw07*#XFBEh_EDH{hg4hZwRUxkt^ zsGE*9*@_dAhoIuXX-4C2JAULS4Yi{+s|7M*-hr1hhmnGUkoz-H)BBNm1^2h!3_BiF zAB|PmoJh?jd$Mjj~sE^~U^JT>~fTY@GGt4bKa-7UFQY zV~p{{-JdU%Ic_`F85z@kPo66lD_SejrS5>VvW?z@jFL{=3CxsJYO3tF`g>8AxG=AEoZ!1IOFsW1z)L_;vr2Mv#sZG&WD0l(aG z#Y*B(wnoIw$qgiP3J76)#MemG*Iy&5&Je||GngygtrXbK1kpaIgADG%9x%)=yR*(z zP<7h$+fursZ^m({)sHcMVw%~qgV;(^r(EXL?v0m@_fziF30OHVO6mJJ<7Jhl_*A5D zC?(48wZlMM*iDO(xG+YttakmV;#Jsvp`pp8aOYZ7({2!4<1!cF@3Uph8XC!(2p!H5^Fh2`L9huis%p5#}7x`JI zGQ~NbWDGKN;I&+fU8r`A-xrz0r1c*gn;h%4#6z)yAqI%n#CpM$i!@dDpDthY^nR)SA`w5=VrDG2wo(_bH<|)H}XPrM}Uj4H+rY41RrF`hi$>UXsaJYCdaYG1_ z>UZiO3#jf=^XJ?hL&f7dCU_tO4iHhB$xkbfZKuY0jwpwz5~% zbENiFvmKYRS|y+sZ}>q`&avL({F&}tU?WV@zDw2{CR zEYIdTO^M(cE-%Dw8Z0NDS!bG$H3c3r z@w~U_gNW|eMM?45wgcd19w3A9la_qU|v*{DU` zz?gF0;F8LdT;;dP?xmjc&JMet-_q ztQf1VxzU?(dyqb8ExPB)@GSaNV#fq-`5q!jC~=*foQja<%pAu=e7*uoJKOwFXZ^G9 ze6DwVuNKc#KCkzx-v8OFT#l=eiR{OQzO9J-&YqV%c|>nVPT7$Ir7eb%T)aUGgwh0r zjW?IsjAwESWwlt_CN(z5%XNpkzu2ye)>vZie$RXdZ<~?fcNzU(Pjanvd_U&t$1}VA z&vryZMh>aEY0)>m4cjg~_0Gk_3BXor?t(L+3jJ5ME2*Ty7OZEm0ah^Z_@iPK~KQIoxY`B`P2jPB!+0~HYm}6}KY33i{XqUUb z>`62K7q`ag(s%dGeL0p5x`pd}p%zLtR||V~llMU2t}|X;Q&DglWq}RhMkfTeed}Ovzfi&rg!KL=_o-d^LG{(hf_z z4S!^gmU@2Zw1m99bz%PC)UhqIpN~0k)U~Zyzh6G1;fLA>rzZx{3EQ3r>%_d4u4`MF z*7L}NcPIC!S7g2W^&QFkd`N$IVp|xSZ_yl%`ugnel2v|{ zp7~P~HYr8`c;;zA+{)Pe-kxObmZdh#r^{>w^##*95RsSjVE9txyGI#SQq@U11lSgH{8lTRul z+jeNa*_dSn&ekfxDSLG6E2sKqm#GbPP^tHvJxubd&~N_5725X`1=>a``0Dl~wM^r| zHQW+bPa@5*DEiW=0k4fV00q8xt%7poxUF`g`nfs1sVvP^hHU8>~8TNR^m2Is>t zHB>F-+Cv5~R}UkYn5j~u?V#%9;5(4-zDUXia-Ul7VQ?}H0l+3 zu2+Jlm?_}u z7fIKC_7e=0RQSs6oKam@R>jp`QbNY?d2Fi^Hq<0mSC!T@y%O0621&ITW*O-v=f4*mpk) zW$Epo!~|^Ezbc`DWDN`Sm%guGHIA==HSW{m467 z*@Um(drJNIX?jN=<R}(%@?<>J=Mam8kd-(@@DCV)Mpj@ zY8om&cxC>`2;BLX+GOWtS8N35I1`e8CNsxcJ6bzsDS9_bl~|z&^pUsy2I;nvhoPq z?e}fEQa6!{&fP3BXsiD7wW7*_mMH6YRQ@vF0p1ynoXqokXZeHEiE~|1_sMZ1p7ICNFT!$n zu0Zhb`H5rM-z3*Yo_}rn+jQ{XMyZXr0PPmcP^b^#3@OyA_ z=ehH@vT*sz-{=#I``wN1NABB!DCWedr-o^Rc%lL5B)}`A2}_v;o*4vrhaOo{anlht zb7FVYMo+i{2Z7%v+4+(B=_$nfdpXDT`v(s7RC~+B&-uccwOD>cI?WsBCH;i-%o6LJ zB8{*uTq*;zw)nY%N0&0W>9v1$-R{lM>vqjZ9bxtuyN#Uv>ao!0WaOPkKil3je4UWl zqYhI$>9ta=@o@A!ufa)4!&R;PbZ-mOi__{24#U)(I+q`XJrmQ{cBARhd&QhAJRTr( zs|6El4To=S8g7GglCyLy%;ByFHr4y6Dgo-pCt?wrN-v7nkn@2j;``gq{rf%7 z?|#nnJm=i!{^xtnw^L($miKjCukAWRVb#_DogzSrp*QKp`c8tP#k!R8ag9DZj}cj4 zBVSGamu;>-o3ziVeI;f(4=BHCLbZhGbWa^Tn{rx=#!=+GS{W%3Z-$LqluS z!llC6l82$V=?bjFr83C9`=uY_i3Om*sgr;-=S^`^5-0jLduzX$E2vln)iTOOWgOi8 zJN!BsIq)cy-o9lk6p#!Xrr;>%COOB5+Oi+>j-;X6UV_r=Y(b)10-KRlK14XOyf}jz zsv()x7REWJV|Kug=wAbj#1`BM2gA5L#}ZEebiN00iRV#oGWJV;vow*`0xb(B@-{F{ zKG<8d^``C$;%d3>)RCmuahJzw*;}xcz8$)CSCYhCed7WP4R^9@bcFIy7^7*|5r*O zE12PGf^`nK(o2c|kP0k2qVJs(#R-O|m0bp9i@y%<<^gcMe`mb^^4jCl$-Iq%HO=uC z7(sxvR2Q(D91Ch)Li6~E%au1Cz+y7y=};?!FoUoyvRVrffcX;^(PrjuXxA+A>y9Pz{aOO zh^%MybKUsT0!vnONf{`^Y$elndP;J*#{EXng5tSk1B8xI=HbQ&tZxR6{y9+BuGR{OBcaBiJwXO#b* zXa@?bJV3NT7&bhxuWrO}|Ms}c&ZoYb!6{S-@A2`jQQLvVs4v&%b==F9ZLB(QM8VeH z0%wzqzWuKNCySdupYSg+xfQZc{mTSMu}# z=_CPaC2CcBF5RR&FM{u79iO3Y2f|4gt$jR$C+U@YCNufC#S?mS!q9>SHCYvWAJBYS z{o?Kxw$KQLE}3VJf)1@6^JsMqlmfQEDHI|^If&~`F;k#Ef|xHi1Q1-(gV(bYK|qP% z1_o`URKssy3h9#9kpB*Z5{05HzH}Hiiw;nk7ypSz`h}DiGHi9!hZELN{Q)tvxU6Lk z(Qg0?goiv8Ti`BS?wEt2Cj$&-k#H`1O;;>sfRhM`jZo10t8$PP2ign_9}(cq8HBH7 z4sAdl*sc1e0Ma{DJPbPWVyJYzpwl3%hQJ00unM^p>t-(h_*z8?bF~_5TfgjBgVRp8 zyzDemaB}c$j%X8mFFCWSwq#&{FjaU=NXlGy9s#{wOMRPrI|^8 zh|N9EGw71-Y@0?TI6I9r*qSuQl8VfrNX)K2D|Z?HB^}-g0T`(*KOLLe(~xC-*&ZYx2b@JUL4Wnh(GJd zgPa6*e7OqRR1&kwX^HwE{bqcNZpu(E1Rt8e8LrC~7ITJjN_3$mi(#8^L(#{%27W;s zTc`{}m0(wBF~%&Sy%rMUirky+IOePGrGpCG0BKIx1e@b7e^v}TbibLIYoA#xREF_^ z&##+D^=9{VZ;xB*kEbE8_h^u1!^1?q)-NbWh`SD5UUHeR6Jk32la_d!AzcFcF4`4P zYUl$k@N3~P|Jg3{x6pF#T=Ipn2_Wc;x&m48(g23}n}Nx?7&J>gXes^~*ai0>aC9B8 zZGQ$gM953|xC9%W01R*^Y7Ys(MC06y1(*d90Fi6=tJPYTZ@%L*gBmoUCrD`@;ur#( z6ZdEf9Py5q1m0M7`K;BkR>;qC_kxe!+qF2E-;4(f`{Y~mxZpj~AzkN@3BiE)EKH67 zC>@Y?Ls`XkE4}&zjM`D}gJ@It@zy$mahb=RXVbZTRF>^E<{KZcZQq@T)I2|Z8n>JY z5uyU!}BXlpWM{?{~cRdZu?U8>LU<_{VAlQ05CWZxxxb!_n4Rj<%v7 zK{ZmcXhB(^%z{cIHaLI^fJm9n5v%mg^V=`vA%n#UOWeoM42#ypDRGEe%wiW$8J@T_ zi6BIxU@~(zs0d9;(m6755j+)E-N)rRD@Ng{k<{_SITU(kifcr&opy;RAD~d;4h@<@ zXPMOh;pymVD0^Rz#%J5!;*d;@-zj?|VSAW~sPDj$zK@O%ImL1YZ&H1UPX~P^`Z&Dc zc|OR!n$dArq4N)HfY`ZUiUh6D66dE)I=_Qy$?@16w0|5Q7Oe@n7&5M_Yg#p;9i&7> zfTn)2rfl9B2cMnsFWwI~V`UmJ48d11SXQ}fDL$=|7?quxeR(oT#a zV?|%=!@02h?gJK)v4n$*)?qils=Vz#DgIz%W05?$Zt`4^TkVi}GmGbUKxOv*CR3K! zku?WS5#MtcO_+_bSd=EV%JXX77onzk<{OBA^R>1!JJ@pMwK&Q|aU?5U!u6GaT zM+Vio`;WR0%oC+Yy1PS~$+g!Ul%Gy_XXf&*N)6#%7DyZKJ2SMJokr`ep`mh?imHmc z+`Fh7X3Y_Tgm(bYJtpT;Se1%Sj~xCH+gB*wx(WiH>D3Dx-9?Ai=XE(=AtL_OTj?8WviikwTG6? zkM9x8=p4sfnAosBbB?bm2~+w9)f#&YS*P5QmY>`j!uQ^Cu~B(g-{FOG4Hg@D_YX z$@nX$z&m-ssb68O?0BAud-bU;enxA^+_t;1naQaF>pc9!bkWZCz%8AUZ@u*^DIi5) z=9WHi%$rLrKHHX1CSsgzl&G-FT?)zpo2M7fMkjrrQQd-G17x%(GF6DY9RefiwAA>; zW-u=z3NUg5f3|{$fLb5uiw4e5`KfC3`Q~Ny{84k zrO0oLoSQ_D5vb(G5J;!`pg+t2cX$`*?LzWYCkDNt4rDq~qjeJQvNpaunVxABA@_*z#dZrQ)1G z%U#bRthq^heGn~Rs9{--53CSn3o>Z7p zkfZ<-$WEQEkO6bL54=txh$2cUo#mHLMaMgUdVMMv8N(ueBP=wWc=*pOGLxr-ce}N# zUMg)~2iDQue*xMOTktm(-EU%t6*7<0aaEQk#O;`7^h2Zf7)F zcceQ07)@tYFl@rg$aD?E$e-U-{)4O;_9G>hau?$RYMKn|<{kYf1z}<7eL5`W4BpHY z7vbDVm|am>cu?E1dKYB5Q~UO#Rfzj^b&Ve+OCeI;t#df=8vQ>;`~MVA@dQ3v9iX7* zUXCpobf*a4zaTRI0eV5P5ddt4V5fUJut{8c)6wat@a*4o(JFE~hY}lK@rF|Ykcx!8 zvH$jOeY*N#s1$;gyyrHrNAM8=vF9h!HgW`!$Y$VsU&6awPx!E2i%i*G0Iy`iJ%|LM zHi!!g5jX{>lWBznMInV`Zx`%|@KKJFECqSt$%Z^AjoXdkoqxMG5|rFE;QliQHy7w% z!oFBiSU4puvYC9k*t^oKy4f1dyAS>_L)({p)fcVod{nvi5h(Ku4bIGygZASeYcbPH-P|C0Ya0 z#tEDfLbmb415Sw@2B;cNd2^t7tlvo+Y+f=$G2{`_`gVB>QbG6&A0S6%&rtyy?1Nwe zJ0y}g0Y|GIgp7uT5O2`;?y|ED=zjg{_}^6m8)pN%d!WuSq=P(S$aUmLEXPkb;GOAY ztc@7rpf*~<%PKW0%750b^L?c)e)*MIQzU^8U0UCkqHONFp4$zJ2zqrgJ$l7mChwoQ zXvQEpI;zSsIZz(cH#O)}$e;cFj#;B_!Rs@c__D%un{SQHeUp0F-sRwS0eKkxM|Y3) zHKwS%yA7aQE(c$~1%MI7uj*LJo5z6saH!5>d5&x-1r8t|(C?<8#o~Ua5XP{@c?6V# zKP9)wvwRtMPi$6e-T+FtzYLK(OE?Vq!AmrV7|}rBQ3zMo06N}8N!{@FX{Wj{PI!<+ z5af!20a*D5688bGxa%F!&C4pt41y+0hEuF4$X`9*fP}`C&S)LGe=Z2e@OS=YlZ7o1 zXLB;3hzZJ~za6>k2NI#;bT?+^BgT1|qW`|Ehc_I=UH_*rXFS*WLD>9|%r0ac%Tb%N8 zi}?r`ac{d&rIj+9Pyx_41wP?2wr?+(yLEOr+13+je>&%Y#YkaUZ~B^c>5(JSAw?S1 zj)r6T8pQpWuV9S}o>|W-o8aqiNP-8J4D^5bgbP}4ht}z=NIq%rgM|u6j94LrWMjJi zfq&Upq@bfa^IcN+>J*fNNyMVSS_(os2h5|+kXxd5{fBf5CRkTp^Tvd-yHQmuXRDwZcHN2pfsrLB z-CiS=m*9?mkr=r?*JxYa?QC0{qiZu7M#`;xlug!YNpF~e>7Z`BLZP>7Wu8!2b9d1` zECVJWL3L@sRVe+(+s)sAf$tEV^Xflcpds@5hgj~DTtTLCsQX!12Nm!K9r|e=loI`3 zowdb^E`iSyB&!TM%Ne`o6qX1;Vvs?8S6XyxTGXk6HV+P`EXFXILOzrQ|7$$ z{4Nm@0`5tjl`A^w=%vV=M11M*$$S17FIU&7p0&kp?9}R98bK*x9Jr`p#odn+Eu@L05@OKEO)F*|8{K<%acxb331PC z{gfj1)}Z={!yoBdfS3gXcE&A$M??OwtLy{L1!CwI?LFv#R zdCoedpGq>f{PX{VtpDlg)JuBMcjp4s?hFHb+RJ$LHC>ZIJ6tYgyudWsxnj}}(@ z=?uB5w%^~J*NB9%@bcGYfs&@L&|m;8fyvr%o|%Jx+Y3^i9}<3%h{vIiIkx|5SEimB zTyaNRVZ~4nI3{|U30QtzE&$J(x-!f-%9E>YXFWrAAHIkOeYOLelJm%uMLwHkp^g>V%Efp)pEsD6P=zF1FfK=rvb#(BvYjUGLv1qBRkHhB{ zcgaRdw7H3WmpOKQw7aEs_C5E<< zwPE7C{m0p_5708rAfgr6a8rseeB=(6U;{DVIFJKRZ|GYVl%!`4*T*#9$A~r=d)mcw zLYhp2_(l(<93y$haA=gf%ETP2hPA2-=M0&b zqpc!<%cKDVD4mAMd8@MAVh*36J-37$3CU5f_7h>CbaLCziF03oL=435Blt~6f&~*7 z-7^dBqi5abTZBg-@N~7ZG;^?d&B%2cRz(|ru?6V|l%zX41s1K&+gn-*%#~_11@>Bd zxkFgCl!FEFhC=5&s(j78y)`t<0+c=OGt9)jRubic9zelZKk z#&EK|bU=luAPYVDU#9E8ndSCPg2Ca;bvH!}?Vc{!@kPvn6N;7iyCdKy;u0=K&^<^_ z`gpqUaZBGvqK(=_ppLM8?QMxO40RjkO#}88KX11YuzF%*tA>J%JH}V1Z;q+TJ`wg5 z9uZYhYJE1^d{aAQW;N!BV2#<}9>Jb#G)jo%_coR@u4cGgzGO1LSNDu5V5tZUqh4zG6pIcR; z4dU*k{yPxqm>(T+qYqkGn~o=UbgJCU_|YfA(Ir`_ixZyVwkr>`3i|Th7H@E@P0E(% z-{DL*EDAL8Lf=f+PAPGnYQZC2_jI#<^LTIm2g>fn6;SEok)3g%EzE)Eux_<~OFo(b zP5M;+cL>Nff-IvhAmD(C@p@gi0rj8!iH-%dZzs81%^OcvQL(NKOLP7_{D&(7xs&W>PTEG~YUIR|xujdq`w}w65z^2l76~e9(whQUedI?rQVbeBhXZEFP@F4p9bPqc? zTwdEE47Onp9_skNdIQO0@=~_baRkwkhxIz8R<6a9fW`?L#z{Fp3O#cEdmN|ai3V|B zLScKwV+7s!bux=da=P>i-6S4{2sEj1}t zFwh-K)fZcqeEmZ@i5M zj1`W=R`U4L@6YBM68q^XaGJYhigoC0NsZaAy-fGh>6`#dpd~Uu3LNMdkws^K-NBIt zZHPFAkN}hJK{5ao#W1-yF7DA`L>1bCCQ<8S*W;3hgxCB%EET7K< z@G@E>C8xNvPfXnEOM)-YYH`xRy|^i_;-hmz@SZ%q* zJoz9G?v#wfkZe5nJlx?ux3In&xckUT0LB-t>IYEDuNJOXYORQw)KtVcD`~_=TLP z-_|m5DSbxo&H%8-v^061t9;LNEY^U}DfcOKbo|V8sbpxisLw9H;|kTev*^mJP7Da> zvLZeMx-3VIn;E1pwtCW>_f4CI1j$`f@4{KuFYW(dh33}>)2Kn~*$6TpJNAEx%wNAi zvjE5OXclFW$hH%UQ8Q81Piau1zsHc!jCM`K-BLAVi$&ZxL)8xn_gs(|vkK@{3w~Xv zU6{U}fID@*Luohl2s6kIhXh;ES2oI=Ka>L!;XYKUhgEaF;}ApU%rxz$BuQQ)@&7V~ z*sdvk7&MjlM=h@0m_d$yH%`a20G-a#uH;v8R<2w4Gm5%#$y|?#~ zD@GJ4ltU&W%C<MKOKCFXG^7j434#Pc;l*{Lu1aR zlbSEHHtZPPtw@P}`5+ydn(HwQ?}d}h6yg8Vx9RvgsA~5@Y9$2!)q|aC(Y~^%W3j8m zM>SnN_o>;Bmz`bNMz)B35Mb~={_OAqXvw#6`I^sniQ0YdL%ewzI>UK7S#no(2^GJ} z?LmiY+67Cwx|OA*_xoDO-3&cj@-M7_+~OUzD8X2yF=*Af4C`Hb3P~;Isg*wff^ zvZH5!2row=DmQ;W37CsP>?vFh-bhlyV=kwS9Z-k6?SuGE_; z6nH1oj0g-naost_ERr?`KKdM~iOd6CZ>0Cf_GnOIc;h^3FbA<1@>}cNggwSfY1cem zh_4V*88OBnI&dV9!7<`DbVm*wx(6;;>&-Yr7@-YPeWVDi9z@qpolzi?T>28o606j; zZeL~3%7G69^N_Sl*eDK+lv;^6$oymR(!|}%aiXg)1ljs#&#g#0{4{#-;8$jQ|-XY7y&w8BKJV-8jpc}WzwWgMEt8&Shz(l33EQJCBwVWht8l+k0;X8c$*Mxl zGWE8g`cZ#I`{|$gJi2>~HXag$g^9X+EAJ`c}1B&YE1yL%2G zxbhGF^XoU@w1uFvDf7wUQGAvI2@A%40$UTEpRx&DgU*^nKxL=MfY2nKHGw^k>3#r7 z7b2;!KZvnVennuYrLKCbhz<87z^<2TiIzl*}!%A&y0;= z=35i&rmXliY*%rA0hAM>a5)RM$0_MSCo_fcAG@bYp-ro#Z z?6?m(>P~j$p`VT-{1IV}M*1mrW4Y%EGVWv*>%L!Z{3dxh<@TUr){w4(npx6}WRKe2 zRa?{xU)zc?c`ghxM*h0r@h_zSyxd33^P{0`UhzcID@YpGO8NlwU$$(LiLK(m)VM-Wq8E-cL6^DSY zzueSl-mtxk3`IHE`UXDe!u>xnU8Kr#5~MxqWGdPsb&0X(p%n1)Q_jyJF4_PwUv9&~ zlT`q-wsDT_=7R^mo~v^9sC|k+SgedSA>k^5s4U2bd)=G_LM^vCPWM~&FZaq5{xV3J zSns)D)i}w^dp^M$Usprf%;%-h2KZAJoguQusL=Tj3cryt8mHU^_}iE60hd>Mrg|zn zql;}!RilfU>!;^FFV*3nwEoT{Hx+N6;*Y*$9asmOx4t8FC&0@dI9P;hVH<#=C-yG_ zU}ZX1P;4MFU5n4=XFw_PdPeWS321D42ywur1e!@J052d!w?u{i7P%62(jj-)dNdaX zMhaZ_W`4iQH`*GUN2ITff;Q;s9_dC9rcL1A+TaW7VS(D=UzwXb0c6FH8A~64U{j;4 zr^E2rl;B1{+gFC|bo&jHvfT;{z&w)1)bLJ4d2}`8ExA_2Vm@V{RzVyqSkQ%WC%8sg zcjr0{-{`JP0yyB{%?Xhb(z|eE!}x1T2R{Xup7mEBjySWSm1Ar@xP{>KBaaS0aM{lq z0sJ(uh7>bW&tAuumA4^w=h?nWD>-SjtS{8CtkB0b_ZK5{g7RrCW@#3n?HN9GHbXBz z#mG(1gX#LWS4;iXxu?vZg`GO_8gz^%Go;O*@F|(+<36FG;~|!9GH9Xnjr^KXKr?#a zK{W?7Bnob316r2SFgm2PGs*gM!Z5>I(BYXu@pp)*1|F0_`zr4}3KRWcE?^ntU3*XV z%Wc6UfoyF&N7)e}9wr(D2-fggI8x*Dp;7}>?%oI{yCc6xM?T*RYP6Ei1v}7AntVGX zJO~)JJ$Wc`;Hjrp0ikUbgxK8o)Z8UsOzP&7>8B1lnZyLE(H?yjSmPr7wHe86LbQ{> zB*m(}Id`mOqj&z!JYp$*p2<}YCx1xy^f5D>_S9;}czdg0bF@IY*Y|a*Z(pU%#*;kn z)_`n7kxQG$%$^(DT4aS`2lUUZ(WV%4koubjngK6ZfZ*$2n^mcC$UKI(|M>3^bLbRE zT=S4$^v-?J8Soy$Vl&63+FLY6>tAn7k*$TK*b)d|)nt6^)uh(1e>B;@`Lmt~K_c!W zz7YgBfouN9&iX<%fIp?T%K*3Ts`A~qC3A_J&w5`a9KeVe1fUI0IH|@zZNrT2m+el| z9b7cwy5#mA0nWHEIUg3M`x-}@-m-eAU@NIctP8a;t#w7SlGX(|Grn*Z-vUKrhTIoE z$LM=o`toHVkcJ}o_oKqsQC5Gl5Mg~sLhhfim(GWaw_mGSV_B<*kkj1W>cHb2T?SLxY=t+xz?dM>v2pU&aGe}}eO zqyGvI|6jfN|L3FsKNRF|-K775*7)D@HeT1HSpcRo141GrA*mH1RS@Pu)MMth1aJfo zGVK9hM`X~5zzs1nf#={J*r^Jjy#N7m{*m3&9`7Fh&?aIi8F(u*(Hn zv4lrWj9}>v& z^PyPw0QPVXB;2{k!K4qHvLVoN*EnVCr}#M_KJw8xuOCKB#BK!0|TBvd;K@8!PFUK{<6Lx?B2-R5+R_jXY9kEAog9yJyynd{SbE&!`Kj*tYN(I z?>nD)QJT1UBBK0i`GQV5;ci93uvWKxsIcid)V8k=FuBOznFlh8!9H`==L`fU^{4<_b61SXFPNUiVlF6?a|`V%$+burs`wE_ey7`nL!7#QoO+ zMj_KscHHOjL7H$~_CJKGR_eQxA>oCvS^aa-giACpCSirD5i&$n%fx&i&pWYjDln=s z`}&xG?D4w6dx4a}KMwe7$M%smWM9_0Ki;^*%^EkzS;IhS2)X&z6wY?}RrX+MzaQXO zDTl@2*Pn%hW-bQ;?#`RO>mKcX3**p6fwVo$i;g~L{Q^g?3C#Giq4wAT=ZdOfMKYIe zESxL^P{XAgvWK3{p7(j{Yh#1B^h+w>x!w5q@2>a1@r~2};~Ss6-2DMJf}~-34EXB( z6RdPXC+cL-PfmnvE`}X7d(2k;QsoeHla>kb$&UKwpj|gL#NzEiKk}Ji2hiPrcn~!DIOPortnp(ikMb%Mr}1e{?Z@KR+u?1+OD8oeF>C^&y! zh8G|QWOR6FYZEhsAUTE%(w{QD6AF6nFKCr?+A=s$iXeGQ z7Q3ZdLtV$cL~;#OsAtC=lGXJZUs>DV8zu%XAI{63uFNkN` zodRIUt_s9NB%ATqr|hTfCpQX=&mer=`!Mz^jWr>!{jb2kKfGkyd&rj11j2_e9v~Jg zxGvdXD1m;15%pZbqUs{xAqYZ3CqpM|2zr0nJxpfz#V!K&amG7KJs|+`Z0Teh+X7AG zC1<}6b#zy&qb8zDK*C=^LDpv$Wpp1jf<b8b3(<2TtSiCcGm{(8oruLW6JDYzg)=A*z))TmKn=jdY7$bI#8eIlgl@sp zhtQEoY#x$4hD3u+IllX{0~I*oH?ZRej;no_hEZLI?nOA35eaS|5WhAcVx3|~`KgEQ zSuC(;V2+1>M9cy%`vDS)C6U#3{`FJO0nCyS5+*C%!FQh8!U z=*D@bV_M7ouw?)VkU|g-q`D!xZ}Wx6aNZ*cYOJjwfG!=Irh_(-ZglAB@KBf91Th2i z1AsC_Wq{=UB5XO7$wq-CN)ZH$b&;aLer;YwCCBuEr{5ugtRa4L4s0M3Q|$^Ri5g<~ z9aM;oBSR!{FQC<#FmSp-!yYPpd@E8)dA_939dJd09zF5$797V zZ@A`JM@IptcBoTB1O?F5qW?4}3~ZV;oCXM#gh2h>(R8wHij2&t z;JdeO&6En$HzM)aKl7OTVZ-~hrv$N4Yv%VtvL#@KNUpCnEb3vz78;tAA)ph47rr(F zgCAgIla=0AeK_Im`VDv@8BsmMF+{r*3Yoi5$w^aM&EG+#^@)^^NEKKlXZ$h^OWR_% zQ9-I;xO^5dBOp{5u%ujLxk}5sQBPD2!%Pnx0nY;=s18Dna*$9&QTu^O@>O`r7$vhH zZ^1vXLM0|qZy1g+w5+E= zyW&p`M}kV|TdY1L@QopWumhr-7xWW^#gATV0H3gWgpHgnnJOdp+e)crCZSMIrVcf5 zQ{E93h5>i&_qmba2>pcN-mq0k1hzoSBf(exbyn|FCkf@-6+2y>Z3um@!NgV*#VX&Npy=KJScfM zH^i3*S5*(x7KRb|8Tg#TUFi9_OIy7X@=k6PAL!$UKWsbvx%9H_U3dBJ=3Bivd#sf5L3W~ z0i-;Bq+`c)amNoOs*tN})-D?!AvPJy_?v;N)bPQoR818k*TwtKBwpa2jv-Oyg+J9> zGN|ovM+nXp2(FE!3Iyvgg>Og6QeF?4vKM07l++e)cY%Lu0Wj*!zR_)LUErSV1WDC2 z`7)T$?K3*trEh2HX2+Fx!G;Bs!)vz|eY+~vC_;sEsOJSR%3hzLTOAVOYo%{xCnxje zK-$19I2(<;m^kB`p~*nfS0D}UDdW>YuW6}9*i*(4nPGK)WB9uK+jMmL2;R|R=Nn{~ zHk)fz;PcY7XeL zx8P13X!0@nmHvTeerZLz1t|f66zaO#ouaVf4mxp(cGZIh-84_{VCp-%X)pP5H=1DIguRM(@~58XMxg^>3n0c-@&3}kWVo-c1xn_?@4>p^?F=|R#iIuMjjC_bL}gLpOXBJ{q*fQB^_>*PmZSF zyo|%Ik#sb85F(}Kzh4x*PBys<9w$>!gSiziN)`^b6v|Mt6B?s4^-LnQ>ofTBzxBn_ z`JiLZD70MpI3^;pO*PKrSMaAC%f<6rUMxYoJU`;SmV0AqP{HyuJ|{#WiO22pvrZ1J z&bvUvxXDL|tSY`b2ST{hGq=#u@E$*;f?T?k;;laP@LsG1+G6stjJ5A2X!@4kAYNeA zk@u)iSamc!%>~xJDX&tiJKLl8+@MrK$MJIF0_ytvwLd;qOCW%)b`Jq((tO>E4&bAR0>uw}t30+2vXJV4n6t5A@x%&Y%DT z(!)e+lLQJPa#Q>{b}vnFFctT`nQ;cxf~9o!s(4`#eSMnuo>mQF7SA1D^0&Hp5+Aif zeJ`V)4a7#d&(9z>Lx;>)1!Y|`_tDp^rSNPkqFixB@+5KL=P&G4vO$*jI<`#U1YP5-2;Zw=@rAO>sUzWj|u)aU4ajA2U>Cj z(XMA8*uUVBxV1$OH1+%La2z;6%!O8Ir;6|)!(*8jrx;Sf3O^l5Y<%}9dF5CO?yJu9 z4o$SP?4(kOQKqV{6O zDF0g5*gGKfN~yQ*>9g%vM(y!&@lKR`3z0}=A&S-8!audv-=tE1KsIsU==hNHjQF}{IWK)TtP)YXZG^vulQW?3qMk6pd2nmRrzw}SXVpQ+ys`6| z-j2G7-V8c(%r4DoGp5eA!c_N^jhf>)r!IBVjjJmAnfYpUPUbUzW0mEX?fRpVXVZJZ zom0|9|BgS`hok!PiDzcr zvKkZsMs;`IT=sk^+49pXI#{}|?iKYyc){#<(N`HcAwIw55j zBZk5(B_u9hqKk6OQnEk%f)4fb3%A78xTLS?M&(6}@vg_U>TWYQVnhna7opW*DR7R4 zVDe6cVYC*UbmM>0^i|U0uhF>tA+?>4 zS=Qn38BE@v2--}XL@@5fhv?t3uS$ut3l7_7OrC#2DLtV*6N zLriHaO+}^TnLnLZoE&n}hs8s(%_h_iO74BheQ=UHy}K)ljhb=i&I|t1g<(~bc$eqX zeNM}_^*?tn(5rOPf>mwzwich#N{8`~cYM7aU*!Xu7x?!qy@FN0g0T-hcTL3{St~Z8 zzJr;4{M0ja2rqr}gmqf*+X8HhkBUHS2fQb9I9aXb44j~m%UvxJR`|+G<}nN^^C5K9xLC{4&_84?f1D;0=i>^z2I9CtqGFv>5G&2Z;_ zgX%E6(G$9gDM%Y5IcQ%_L1Tu_O<- zDi_+K5^L3)#Po;?I0$#>-$fJ%i<&x&yIXU^?I0cMl^Y~+lozRs@5TB2V0bCQ5*GI% z@fhiIfqk2g0)?X@OmSTyg&Gq{EEV6Cjxq?6tft(QnNUn98XR?~*)MyVI-Skycj79w zjzM()MHK4rO(lgZ+6td5j{D}e-D3l|z9c49?Q^zpdEO~SO$SNXFt%ifc`^Gr3d#P? zLK|gFB zX7M-O(&Kez@XljeSo|7gQ*uUd}-J;Jr6AE;~o}RbI$Qy~bkhw(vh4PC?!5$0n98@l9%+ z%%is;zfy|wip;bNV$VK?7QxE55+ zP<9vY>)5G9D(^TbPko23&Ul3@Q>&9{CCPU@xxnM$Swg2)FC^y=ulpHA{8;_<{!Y#M}@lSfM#H7|{K9k|cm))>n(#ry9 z>9$yI<#S7qH4is5wds}$6p^|i_?JogJ&+7x`KlqL69?!crwSo$SpBwGWoY9ki*NIcwK}e*t ziT&0=j!Rr-}?%Tk+R?dgM0bWME4Q?DY}x6fT}o=I zPqTjWnW!XjwZJJz4HE~$cj{_YTwV;i(b zL1G^89iJ|bv{ndJe8hLV%Yq{`-~mLT^*e@yIlDYdA|Hz5#w5Lnt0X)cG+Egz;`8fD z(3iwTi?=d%bFZrS#dMyKmeioLrmz$~(Hy*wzLoyu8jn(EoK@|Uy@Ib(zHIk%02KM? zF0IZ;i?K-NMD#GSU#*O?b}oNN=)$UIvQPB7a!lo+6$UdjeX8qMNNgMRG{w~wJPJLN z+Ld!YXnXW`rB3T#RTe7~7Y1p!L8I3)=?sy%b+}u2S95Tq+t{m#vmj*E@TrWSu1K`LlSRifGH;@l@S=^*(4;(FXn$*Z37IxC+l@Q_ngy=$fWgnU-cZBOv1Y$ippJ=J^ z*miu-`yy_oCk7J4;^|h$H$_TU@+|f(e_L1;r@=9z44;CI>V!(?4{=m?b(x^JM|YGU zeVzEwV3BPMhu{creM=M%T7Z(!SQ#K|^7VisxR!`n`l=N%P&kJ9qA$WSj!K&}Y|5zv z9c6!yuzD*g5?cU;Jso$8mxp?#l&?g)FjZE@mN|p#mS9sHf!VxeaH(hHU<)<+v=QY4FrDR?r~2k&C(`8ZF5ZBp#U&gONxzS@ z`Y7_)9Jon5Vdbz`V4Woxz+zG<@lDT7K~f z?Y~%t(46rOvTW$IGd4PYOsJBDaeTjOs&$9XnQmmhB@YY=v$qP{A%I@9O#y+aJ79LZ_H-#e4^C=4Vu;n8P!E3k8X zVQo)!(i5J|An)bN+cniZX`))){zrA^ z77yjV#_?BEWQ1y>Qj)Z(v6Vy)E9Vl*F{1{B!e~&ILupFJA+jh|Dk5h&%rJyOWEdSS zsf3|Xp|l9&Y#hdDKd;Ze*mwJ4Uu`#T4Db7n`TzgV^ZPyDpNuDV^CsD$mXyUSTaDs3~%?dfqVe_~M&H#WCOdXlrPe_`VzNEsEWLu4%#psnJ{@ zYeN}YGfB|DT9HwbDuL#=Pv%vIiY^n$)ByPWTGt?u7@JW)Tz7tWIeSj}T|g(v-y zu{i~nRdi-`<8;rk4LWSVIU~TOVe4gFMnWt{I-5J+^sGT*H-!D9?zX;vPR<`eJkj^MHb&sS_Rv{Y|tUl!7;>pR`S#WqJT-|$^!!piwkC4vOdPLMu z1HjGIt?y91WrcNDD;6}F77D0C`z69KKN)dELZqOhG*EpwVCF42N3PD$DJZ@0W;)dxUMJC4z;d%M{eFtp3EC8;u1%2RwoL@ zC6d*gq4lm6D0uEh{y77xU0p0T5-90YX%}mootR{q{u{H``Fpq`iT6OJ_rLG87?7X+ z7Mm41o0_>*4)ziIZh9m|em-Nw2z(dy95}J2=?3Q0y*L9;^lTk(t~6Cm(`7{WI7ns8 zUji$&9lqvzDQS%e$m@*Xy1UHf{A?!92PZNRuJmnAK>Cowwgf1->Mq-2?{^+^s?7>Y;kN+=BaAh zA5%FZ&Fjq6Igf5C}7Rt<}1g zbg-4e7d-Y17d)*Eg_(*-)IsRYIszH}3a2;h$wvGOC~+hog*MU=PA@0Xldv8Jz;|=q z_-Kk^e+Acj&w6N+Btwo08c3IhbAx9Ky%3C-q$wnD2xLScUtkJj ztqBj;D14?5FM@zv4wEP~v#Lh6agx1y_(SVRqSWJYT-6`VNw04gL+-+r&o8KO6Zt+3 z`B)NRyUin=wwtf8IWNUt$w}shc7vAY2ODqrQ}WE|+m9xvv4wNxl*2x*Id%uxK{=h{ zksT4H2?U8E&}on9SbtijI6&^*)2%Rr(3^#a(B!(2?7r+gE(KTC`NJ{~!Y1G<{1s;)$ zO$N-@wp&J@5B4>lc7H;<+j&PRWQ+01V*!d>wNOeiGhg)80$nSh>fhHD#kJ< zbiWv`@4PtWWu`eHj6`suZ`Oi4TA5TyPV9^9d!u<31{N?FeU)J6q-Hb~0?bn1BhC;K zR@5nzx9uYJ_?nLKU#LNN4(dYpD#I;ZtCLp!wwtHZZeP`rCi()+?73Qajw-u+LO7=!rw!vGl zvISNX+O}VH%|Yrwj~}bz9=+;Rz&C{rq3m>cfI6>wQ-IG{8XV2P@}%|Cmpb;yuW73c zT1EX8jhbq9iU883`?e7zJA@TO`4xWISMBbMj-(w5U`4GZjM6Eb&EbA!`;Q_EKzeeC zll+pQ?rsv*2T)T~kzme8^7^uf;gwR~phY_Tz^5n3&6VMzJ9pTXJ-Xn>`tiVdm7hVx z_*};zv1dpn<&QgGwZYS~`z#8J>HnEX!r1yT%tU2Ko$^!7s zq|w*;e5qwh1YVmR_Jmxt0I`0SYsGkTvF%e>((EYsUhs*a7=ip97O_uGB8079g zrn`MhA-qdZsk&3Q6nJ%--^_5si@0pIB9s3o_w`-}KH=kyr+*uL*pr3EMnW_mG_@{N z1rA!Tq+xFc(nY&iLdgy+4JalLCQ?v}Gm1ppveRM4|d_8=t?~wB7rzDT18))?>ba05?rm!AO9sVk?m< zZ9e8zqh?2euA#Z_~~BY(}AW<+v*mCgzO#==k%JrelC@qtDyh z9G`u*=_Z?eRT1`|1b<{)^?}SGKopijPIKQFIzfj9<1qemVpps}Lt5(a2WFc)2COLh zk|!U4JF#}8Pq9U>J7K#lf~-FU(rJ2KhW@pxx;vj!Epq)jYAuz1zX1~Mp7*8Tu>;Qc zTSzC~#=DMz*6~W>NMd79VzKNN`2Vj4$oL`u2I>2uS))B?=Of9$aoJo+(WN_ar}ZC( zBD_eFf~@LkjxkH>Yv(VGn&Cx2ojw6;%3G_Cd9#I=D!2j9(4tCv23SODU^U)1Bz3kR zOkw83YUI)WbcarhPw@A<;p#pc{A2z_zFEI_?`G!~6)1ZY(P6{27+Qdo)NQTUyH38h zN~fm!8PfM4mB$0f%D*RR0S8B^i~-_u?b+niD2t?RqA^XZ%zWvh(EAU=;!Or%imnH^ zIlgEcn5REe(R6{}h?g9H?uzrq=11#E-E9;TFk=mhM zbDn&WN<2FvlC&S-7>tdu965!|N6vhH8d%r7R_2_~RMQ)rt_2wOAX-{I9t}Udc!t7? z)`0O74)dEWAqWUHqcFi%+c>@hp;pLgdiODe=4o&IbimHNBXt+7s!`zp4m^!6%N{p? zb9*I893}`q%Tra4EW^zBmjJ^}MAHPgJU?{u*HD$}XjEeGmUfPDvfz$H1)9d7lOk=9 zcnMqy_h=e6`dMH;8eOy-nFPCN@EmIFvNAsnOHSMb#D3o0#S@(48bTA@$^rdwM~N^S z(FkZDfPN&%3!3jSz{}WWc^-O6;Sd@-v4BQqf`PpUvX(X!dh>y z*{X)dn?cQbjb@h{y^`yGY6ec9JbX540o}>Co9<2n0fbtXwfr9Syo0!2^R-0-#`s6R z)K&jzP3uBavB9SllwD3afHw0`Hs?t9lUzvKao)grywjP`j!k<18oYjI2wJGIbqQ23 z_~!l%wsF*>-$f#p=xjph(t$7(>kDm?u>)`j2~I|dqi0(I=u-j6?ibxkCz+yb04&2x z8($tlg9X-`F<6V_9w2;_z&tDivI`r9Xp8{r_FU7RV&nm_?(#P5gW$H+kZv=MYr#=L2RlqTRvYjtB zS_H2mA8b_0Lscu+@pQ$ByWY(E@ygRW0N7u6Xz}yn7()MwGdWrBKXJ7G9cYVQ373TRp71bAM3p?&3(*2e!Gu+D}G-!HFxLbO_{I1jLUF~bNKD3;miW9e&#_x@f+cJZL|z)p;&wt0$so^0SR zCjyM26$yqQA$()DWcXIK*KQ}7lJJWY!V5KpHH1f#`G-9UhO!PQmn66se>ZIkb?R1; za-+l~oFx>;n7y5=+MeSvtnrx|)->=8k8}I|rf_#{AZ=}SAWhVBE7|GW?Tg3HUP&Ix zf)!L~NH<+ch!G!Uj&mv0i z&$aIk>UY#ma#SpQ!aZXOkrvxoxDi&u%FN*JFY}39Q}@Md9h*{~)o#_auguYG?1k;u zamitdvei3FO_zGtgZYP~%V&#qqP+JuRURv-g_w+|=SZsy2SP?R6y*$^vfOV%Z1wM1fH?Al5H`}~Z>nL>CyfuvJeia3^wV`iO9P-l& z{g79CLWoVQBr@jW(6VdAh1Xh)bZ%oWaNG0MN=c`U-Rq|X)_*um%$ru!TjrLUmd#gf zQ|qOr{%{_!-sFupd40R{uQ2(Sy-X}mjo&Az6rD^uJsLDu*rm8B)KTg%;jzCc64drd z|G{iHz5C?b7tGjzc}Lfo{07OW=JcNqo4*1%I}cFmJ|B2w*86L3w4A?z)*(5kWJs|3 zWbkGEXeF^-o_+6?4yn);t)fxFkU2|%OzeSpn3$d00jq)QY~OglJ;0?zC>~<)ct>;v zQ>aUhNvQiU^S+5OxY0XQcu6{}sdBkhsAoI`)}Lg=*(-OYO<-;_IePmHN^AyMI4c&e zCzWu%F5UPQ$Xe;XYOy4`w-iyW=CwUrB00~*A+}IM#5xPp=q&Qft+(Ga+Inf6HTQ$j zc)w*{j=b>V=inimsf@JTioLDrU{U8#dXc3oQuJR9_0DAb+wd0t;fn7AEqnu3n*{^yx8&Hv z^9|q!>I4-3?%+PDu9}c<-);x5f(bm!E(UZl3pIu&G@ArcyP*-OP1kPo8&t z&}BHMnWsx$VQ6nMDb!|d_k`HqhVkAWZwb9a3$NyQQmANVQ1JflymxpAoe@8j`JYN`r7GfN%LwB9?53RMOzyLB$rKbI6LUk4Y_97mBW$)maGo3>@Pz9oig znAiu?euSvTbA7O+><298-pMP`R_|r1M4d<0VOT4ExD0!0qLpEr|xXZ4PK z@&M&J!yV-(=h6NfhdrLYGTocEG?kU7@eO|6)D=oyyN{Q89rEiavUFh|hPR1svkf;4 z*jDS6j=hv)K|6YUsfm-}r*mEFRx@{7Ol>s;*0=WiyX%@&o4=zQC*r;J1$I}uvfhq# z%5e_Pgm{pe&c1Kr5O}`1w>`hSwLfd?ot3Ne#3X+w#-a7ijZoKM*rr*HDG~bGZ{SBR zUuohfw@L~nyY_Kyz&8GIBZ;c>R4~3hjzDcEKlyu#OA<+=yKPj@J zYyvmk7TTj+QB^lxT2F1M22sr4qkMhnL7qLtcXbq7@}gmLN*=iZ<1pi)x4kyIJl*?> z&bjYACqMj1P2b}*e4m_Ppfq{S^leJVmmW~Blh3civ>JT3Z(`4yrktOicVl4m(7hMy zv^25z^<}-p|Qy5 z>c&`@5(_Lf7n^&d_b~|Su`@MI*$M@*jy?D>!_g#uY;|a;7$~Z_D83$R8 zyTn*#`6UWodOctqDk51{Rh&7VnU!l%SZ4e4DP?P~)B(*Kq*p0Q3}9GUAsfkswUytA zRMir>&QopZVTMN_1~;HP1WP(enurY5SS;=ESO`qC96FD&7|(B@9xK z&?7!XI?1EqufDCz#FcTJJxaTn@AI=$i6k+JkFcLw^2tByk;+P(>UrfmFC}ccNXuAw zR@7rdx}+4B8lBlR1B-}BmojhrCYi#kyzTYs{lQ6N$q}-jrfdZhaSuZ5^qyFe{-|>z z73E#qI8WH3m1^0W%XGlOcu6!!*292F-iF00JLfOUsq&)VQI>qSc_~q1x7g@aOSwb* zuuFkoPk#>5{n@06QxESgZsb+2^-~QxJJuF=s8rm$f67pvb*3wMk(aGiE&JYR$d3WF>5`QBQ``oC}X2rRcSo`Y=h(vsi0xXwPEjl;UQyH>M90G z1CRTwO9ichSzGQs0lj4#Z1q~+wSFh_^kL+wEwf*=@vpwA8VwP;wZFH!y!p#P^TS%d zWfz3v*al;vi5LfJ&z)1o%XL?shdq`R%J%oR8@_M1D|`G7Z2LxZQkbiAb+?HeLKr*p0)4~4G!>6gq!sI^xMOXLvq{8NM{CHHEO+3u{~F9ug&pg!MZ1uN}`*0<0eh4dVIr*eojq7M7y|To~wD&Q*z z#2T`vH#fAl;~I!*gDWQ%e+)xz_Yh^SGL!X~3v{xctUW0vAMddwBU;_alEt%47m5+A zfQd@ty`yxVx$yF%?%Co24zb@GOAvlj^_;(75HqEXe7641l`fEvMSuP`^X}Q@4r z-&^0|7pwx%O-h*Gk+J9Bo}{c3OIENY8Ln=YDQNB(l)04dy7Qamaa4I|WSxWjYZ_tK z1?k?pZPN3~6<$fV)ZX!uJBQ>#n!kAHHJ0f9lZuib_HwzWcSQp1QX=2F$)DlQ-}`ye z+swI?vH38mp~Q%))`Qt12a_Hc{Kzbfr5OKWDQ6~GV8Xj^%c04oNa179cc9WFOI|##R1#O~2gKiVC zo18<#ByO5ocb?K_Q-0i-CsqG)F~;tC82Oi^Z5S#IeYf$Qv%zb8ELuqc4lg7qzJ|X? zh$?PT=mZ5uTrP$16h_;Bl7dR_!v0!GbxH&M12#V%m7ioi0@W}|^FuNrTp3se^)_C$ zav}!xD^G{XO(wa)_{H`2n|rs0t2`Z^lV3C2VTU;3 zw&^>V9DPJp>H=Zguib$5JAkZYsx=qmf?Kx9+wMEs=sfVeAN}bwnNkd2zgd#U9$IB2 zdQbAo=gMd;q$K(v5eRhdhP2$P%4pl3tT@@h&iko)a;XC1>TjE>$_ML;5FtEQMZ`8n z9@BWXjSA@usDBVp&=bq89#s&4LSy+kk%S#hcp$))Yjef458WI?L)*6c0@hbIMnl_% zvq&hCqL*}RE5oam7Ttn+v1O%{QQLd+B>Y3J#V%Kz^6sY8% zeXA7IMfFqK(IjY{aj$;W@|2!o(iPU5tHbp)wz@f6JC*a_`?4)bXL@{ff-@?s^54Bn zp9`D|m!MqgpQ~Ox8+?D3nniROJ2y1xH3axA1bZpw2DQL4dq%=OVHrc`CcYo14 z=-Syk>wV7_sc-yf0N@$L2fJL?gzAvsuVvD<<#n%I%lM{AGmg7Kxx0txT@Ck+eU5z7 zalivGd)rTgCt0K+k|~Cjk7Y4tK~U#wXY}WPvnVI8SM(yj^;X~5w666*Z?4|*W_`T_ zYv8X^3C(Z)b2&vuKZ;0>))!62xy_YM{5Bl@remuB1p##uc8GfFd%-=?;E}1vCq=1} z1}Hw>QnTevbN*3X|0#V^@wTqM{K3=lKhra93RMNL!mYL_>Yhij2`^Jt7y zjwVScUl9fBMA0p#n54}p(RcEo5Z`%8HJ%(OLpFCVr^VdLDWi#9XB`?&e8f`!+0shB zo}?U+AyZ2V>K^3kUW=vvb1hg7-Dmj}(bUFV*J~$z&xX=)Q-xa47E|TfTInbG_P7Ke za9yOHc~^U)<>0M4wx-2hE4G?IpUjOD{LDksFn|2<)d2sq@>Eof^y(cchFycBj)obp z;S!HJG{@+&mM!?Iw7k-(cwMJ_;J5_;Z2$KR8t$@Cb=oujw!`Gy#VFAzIJrlbICfr57oU3&=<9>XJ`m*9@M?h%@e_lZP8(mA$X7wBK-eM<(=L zg%|Db+kx0`i53?!Uql_HssDCbj!mU;YIUEETXoa4 zVSYEyQRLkx)*ub~>=3lTSWgsGbG}}lCZ5r*49Y0`bZ4k+Q&s`RWE196g|V}A|0>3o z&z$$8eOb;^Ka;QWol@QU{%vfA%$>JSDH_Snoq4F%{CW$4r~wH(d#LDd>?w3SBnvysYUk;Wlk3G^!TfkAL| zD*hg6m3W!7u-;h0j)K5!r*5=keC9omDW&s;wNbFW;+;h-*(T$_bKq~rdc*Oy^L*GB# zMM0h>;wkI_o{UlH_4ZAR6o)!=%3S2#tYCg~_hlp0){*g2I}B}EZ#z75lL}_WBMkJ- zA&u&r0`jt5D(Hym)qjw4Wx%R{ciXFOx}&}Q?G0847H$P4f!m%Fu`ZIOCjc{kBPof# z<9n6`?SV_SF7UiY>KoH26|W&7$4O;eZ{Dz0^~q3?H7yhPa_gl%ps)6KZktaP9kH$6 zb@tL%7?k?q1x#Y=gr=TTkL3I)K&;G_sUFM1s?qJ40wojqGt*+3y^BzcpT!m5@I96R zRlRWVQ0m$$9gmr^iT{PyW=nezIQJTXWzgIe>yNJ2Z`g?AU8Vl!zVXv9TDKVJm~cds ze4Te++%%ix6eR=j)I?R;is5>jjv@BZU@Y5}aQZ5|+#Gf&z8WApR1bq{bU&){?`I?) znJVhlZzAe$_;3f?WvM135#GcY$o8^4RC2XxgIbC$^Kqh(x!T%IUL|=R(jINBvzD5B z4&Vs4puE(2XUpS>pI9=BG;W)O?`$aoljO!2SPlG2DK&d0@x@o^%z=$LuAxcP?E1r4 zI&Us!9)FG&mG2bXQIeER8*D9HwZ}^PSxg0OvZpPZpHqj&bhPl7oo}FWx?jGL&0e|E zi5StwuqVwz^A$4HK);4YWJ_E^anEaA`cU8`i{$LB3c39BCs zDpGXnu?IdlYs_kQTo9~R_dv*fg1t(0aWG(lc2GHKz*6mqqz>tDEjAx0iiz!z##Yzi zz;*8agvbTv{#WVU1CTZ)%dCYX(xR;8=gH1MnE!(z`2;!xq`XQg!>3YdNtNI`z7~bp%7r0(b(#iON7-}UJ&<+xw`peb> zg+9Z2TFh0XKkd8p6)l5rMaMQrC zL&B}OF7i%^?QoSFGmgrbtNzw8pi{lWxVs23LAfDGm49I{T&==DqHN|}MjJ;E$f1f~ zn3{PcMED%GN($zat@SReN#bK{Y)h`YaXi0pPmGmdVfCdqYK3Y zHqp5seg&9Dpy)t<4+%V&#>5tN3@VH>?zP1Q>-Pe!c(_Uwy}~70REn|lqjUb!L7bYAc(t&u91{TCx+y3i?E}+rhAi~@|j~aa<$ibQe14)`y zs>}+=YWWC#cXMs1NvGc{qL;a<3bWK$!n+T~IQN^oy&hKA)Z=XtoX$3182N~Uts^Hz z=cMMog-h!~JjEPrufk4Q*5IWo;N^$LoD4h`C8qlFje$g;zUqbaD$FY&J3Yc2GOj&7 z7+lg|#!X|6clpr*~yc2^24*`{|*)X?F;2PW%lCMpYJ%{tvt{3b;JA>4omO z=IRBE{AE2Txi;$}?Hez5K`MNjIrJCr!iMa}UDGg>k zXDAD=sgdbMgSv#lnCT?;_CjEOtwc~?0{T?S2|B^CuGi9N<%?gw*|!dLbn-)@L#YsN zZ*LruR|eKx4)llHQHUB9KEr>=>)#$1zL3!>JN!t3GL7bI+{^)8H!I0S9{xpChA4ml;Lj4 zor8^oNjRhDT?-_3Seetb#=JUVe}ZDH;P~T{SPH`@-W?nYz+$og62IA$egod@Gu*Me z4%stDW1pZ~HDbT_ghct7S=GO%<>5(PvQ+lq~r3ALPSt`9rMe z1+vC|Ki%Fb{TU&}zPmq$C+J)t2n${Ye$3Lm#)n|@J*3>fa$uDkJ9!th@ab%=bSbIM z4^=L2e)`ugcKmJH~a@Pefe-&l{P zfdY-qhdXhxj^XMWGTg0KjV++KiGsX4RcTS<%7}&q-iN9g2q@7v$sz>#r76MSAa(jdba~? zcWlF}!0~$^Id*5Lrr~wY9t|CKJrF8_MI~n)7+{*|BW$1;Qw)>8A`=C4Zr8O#bgz7m zoqZWG3v=J^y}#Q9TE>gKQ0&RkBB&}2qCHL^cD@o{^XX{~szeRxt#1gh@p?bQrV zrQEWqOkFyD*hXo%w0KeA%S?M}N^WH&BkqD7G`#t?&dQ1wDcI&)~(&)Z#R!KDL4E6ZRVb(2h6p-9_&mq#I_W^xyHL z>ueR8kI@Het?=_{x#VEJ{!8p-lZgV^{^G3lJlU)9`#Tru9GbBi2Gw5X9}iGAf7E$$ z75f$BRI8V3pan5AOoQGiixX5AF4OhZz8qaS0dVjD_d57h{V1EwUU@#R1&8ptq<|#TOJ} zfOKW`evNbj$;Y)2$i}hE^Q9AUJP^7j_V+g$#D3e&jXXZtx<1&B>SlwsXq&Rjha&S8 zu>uMe)ujD(yCE5_>KjBJ<;^w`~YJ&Q&b8 z>fWaae@L25KH63ZqNF*fo2@;jq@N_dXnvSX478WH4KjiC8QjoiJsF(8H4Bk(& zw0lPYvcla@~Mr!kq#MlwT7^p2&J%ZH7G0cDC$H-Yg=|YwLU&M?`?hheUO$ z4C4n1f?be7Xu293e4aeCE}0*yn?4ftr=$l(!K_gF9((-@;Joy<3%ubC0FQqL!8dK?1K>wTkR<&nwf z3FAnJjXwoygA^{MUsWnYW29RR5r~=XOlU{=m&FAxFI-wT7AhV9HTUZSS ztcLVv=No3_ubv=Xljs>`FZt0qWBjcuq;A92m|MtJHP2FBw6kw3SXOvAUBhZh)EnwK z$>FT@FKn4N8zvs8pClBA8!~(qFqL)nk|DtXrbJ8biQa9~>A^Xv9_(d5*E(FVLUGD7BZWD+l)8f-Ph;rHg=j zq}Ry`$`vnRef6F#5~Fb19i!_PP>#3;NAO0}$&)Lm9sJUG>J=7&f1xB}*ujEW;4Nz1{b7MF=4bhL8dd{?o)gNblHwE!g=uy+PuGB~XoDD*{e01S`_S;;8%D%$xW& zWUb1gWU>C>%wJ34es6mGu)2VyV4^GFX@g&1tr$lE~xiI`}LzK zDX+Cz+WKk9AFCWVYs^!>+L2Oz0r4vh5f&q*k0?g#NKIk$bTMTi&xPRRFdVn(9^{)t79&#mR ztGh{0C+H{*DdT}VEWc%|nx`bDiQ0=uhS!T?00+;`KFU)UbJF#kulp;rF?0AImH`nl za;|cH=trzzT$?Ow&MV|t6231Zw@v%`LqJw;nMH1LP_GDKO;U&;%OG$u3a#&G9$FEb zck~D{+xWLBJ7xv1o5N&$xn?FG(0N)S{cL!o%6JLD)89SiNTHh4NSzPg8$F@30Fvc1qkq@06>#Pb1#U&%~mcqz$Mw0r}l-z{TY2k}?R6g*b|1|eu ze#jr2W2CZ|s8_wlx{;FeWJCu0HuxAiK|zT_Nxp4gKU8$fOdA|PT>-4Qk!?405v7qu1iJR6d&WN=`fA~5!=$#aMinRtIhx` zYT$T0WPgC2V9|+=Dq8gz-+SAg>VV>wQ48VI1HK()BL^Sn#`M0V)7F`#L8;(9p^r5iB=&*1Xs?d}Yhvkj8z|W_^qnEy zM}r` zQXg#U$Tev%?;Hh$q$CJHdwDZ`%(4+CT11=N)7T=I_ZtkFzP5RHr?wL~Jib;2 zN^KhQKR@zj??Am3xXf^X&YIQTR2DTYM`|9HMlXOL?j<0?@)f`{V@;6Ink;w?jT2wm zL26MNm2MTDA{3}H#h-oS^j)Sq1}%0&<(3*f$EQ0UDd# zg=`NJ-jY2;#bN1^3|`<(Ln1WS!PM2L_r?!oG(;l`f0b6dd;)ljye4ofw$Hd<(@Df{ ztNzHL$_HscFuEo;i2>&J=m+#qnz?`QBz?@igCPkyZxgd-pE!dEH)c1_u)iU7f?220 z{hD0z-=B^YC@F11UfwB_-L2K8H1NYxK?O3B3)^bAh$B|esWA;9cB;YL(p743FG6t| z?bYjk4ag?^Y8OB`F9h8p(+8bx#v=S52{1L%uy%K@7$55?R+RxcK4C%?ulkuhptVJ5S(Nq1I&g5M` zgH$DO_O+M02r!Wg-$cg*Ifc5v(p|K=@{!)M@7*1=51S_*K(1$wGD|$X)|aDD?mWAc zaS5ZX7|nD3{a?oTX!{>s>81q01N!iN>ho^r%fM%t|1=A_Q}#MowlOtNI$gRHv_fuxc#@Arf{0nHpB+h%cB7SR8O*N=@t~x>4>^8C}SDf%4clh?tJZ`?WS|;QZYL7$U_@QP{EC|KfK# zgScBGBXHo%)dnow1;S6)*lr~-spg0~2bo4M%CS?Xi|{Dhb47viS1SPjgVzMF$ua%m zxJ*G_SAbOS5M}lLYbSz_reG*^1UF_fs9mCi;DAi>6uaSfLcuaSU$P$(Nf(M*ufhLQ zIOIM(-qZ#}cHzROqiHB@B@cLl|Ir)-Cu#WcN>zF>p$Qy>H2~fj>0mJ?t!_xs-oLrE zJ-i=h3WOJ)gayb|>OKfLC0+we?~b5=q(J4#o7amdt^Lr6vmfu=K#nLp@s1?9+U5@m zPyF&lqpdo(n4dp%FbhZxW$r zzj{ZFl_InpXn{1K?Nb0I3Tf^-O}3vmSO*S14T%Ckj=2~wu-s^R`*&1K-^)6_|B2ub z?ep5gz1{6Jl$u_xlRo?oKO?Ly9^{0c1x|UuEI3GQ&2M#q0&tMf`w1RzX$U(;x!Hks zxls9cJiS`R#6J!{;mgN6PrRqI3l!av^_*m%(NK6QJ!xqZK(OUO+X( z9^G&%V_H7blyHs4G*rH?zW*g)3m`+FhIX*`m^RVh>G};=ZlfzJl;TB`DngAOg{IE2 z-Xzg1x7vUiTgCx0w)kV7RP=zm1Jr3Rkx^Wo?xA_WoOYh7sl9PIt>l|#R zHb4Kxgyk?)lfG_=-xnB3rS%;D0InueQr?7jRCuCbKgMWa^_-y$&KbOG)6|xvE+kxOe=xs zxJ)N2!BW}@{L|G>SFpYYkuaBCXvg)h>t!1+QuC~ghKgnGCG}hZwp9rP0_)@n!i0ilk&BOf0x|3!NT*pXa)WW=86LT`4fAHF&l~m=i z68+#8L_bPr9{97kwt1*sd-8X$6Puyj1YeuC6zL8|p-g>B?sbm$Ul`pILlE7Hm|CV5 z4i5+^JllPiIYk;}0D15PGV0)Q1Y%nwFYvH%i15;vAHk?6MG3b1m~;t}?mHo)#Kgf| zPnqP0s2tb;+?8_UgaFtku2#TK!hIMGp&7R%WdECste=6~Hb(*gAS#a&m5d|qqrv>Q z0KIeqj)`h$qoG5qH17DuTR(bPs5lLBR{}AMz+->CrtMf;2_W`Xnqn-pw=`C!j|47m zGEN3^d_+N6pO5*d8!$(m$l`xj^T&1JZ@`O2mJ>^z5&+l$`TfxNw2|+K>rR-GsDM}l z44T;L%y;fXzNE5fPD#RjSLZ#3cl7vruD0wCV3q5wr<1z`j`&ddeC z!ro(Cwm{R<;G6+Mefjtk-Wge2N^;B=to;@A9pwBR40MRIJ~2_|JR5aFbYsM^?~b2( zo)F-@PE{q5@M5THcZPP8&UVR?6QzW-zQTdlXkf`;bnp1ocnF!~F^&4PNt-PoG=49ujFbN) z6R$i*Q_E+5{jLoDxBf4-sI_OpPR@0&x|PxGq=#YQomIBpR#PGrhb^OB1j1bM+U&mV zx-$r^tNL^a!8AMj{cp`853N2Lba06PMSVo;y)DZqFH4mF5`JQBm8+dza z_Lf`*W=3nC9Ny_OoCO{Um~MQ9%( zI0qle(O2ttgyPH-mbBZ1j;3Kk9FpU5W{0A{2+a`ETGwEBZJFYQP#{R<#{EU?X8<(Cbv3{dX}ae5S!F-~~+*{wSM*u#G3r zkt}BlrgGFh_U(ZadeN}L!hv7}8J=JfJ$3}+e=3JwS@%CLsuGMK&tfYA2(Zf2024t2 z9%7*|qk7SYF^7OH+1p#MPO}*-OM!B?^6wYPZZ>pF&VUM@0rEF$IFNxGa;nrs+A=T? ze54q}-ryb`<&XwB!(1CEImfb(lzOW*f_j<6@%Fk)w^grap8$zz7jE) zJDS%15{BwnN4enxAH4VX%pra%A;%N_{z-K4{+FGa7)0P?KwiNC7?K10G0)uh)70| z1bI|&P$J}XMIq9&=7Z)IhHD3sJ}$q3-g0RR!rfo{tCJnZk8&x|Z3Nk-Fc2*?FnG&{ z%cgWxgF45|=>Ylq_dXL|dGxt**%P00FxGGIWXr!<7MPg{%o`z(#O*ir1KNI`FQ2eJ+X#d;y|H5zmKWyH=nG)7yzlzFNQ(veZM-iiF58~_Sd zK%z|n&tL|=0T}|QJOofBk=1TgNRE%>eFLQ?9=r{eD*!)Jq59(l1h}6yAowTXd^0V> za29pnVtVR*2M9C;h%FwX5JcB|@3f$0Ww>Jw9wG-p=bh^7UjQ+D7|KE)#Ehy%pTrzG zM)f>Ia4LPr2)cCzN7s7rwi-c8Zet8ZV)Q(O^)QI6`mw}+U9UmxS}ZiItm=!P&r_xK z>tZ&b2`~+HHauxUa^4D(+?{ZpCWUiJ57y@t^w%L=rJ@|*wY#-D2R5IPb)2A^5a;=B ze@{lA4l)Ug*@4hu#WF}mKLGM94 zZVvtv1w5?>7w&@ZjX>SrrnUoVO7SWhhxqtL?JW5=w&$lRkF=YH=ZKh3Mn4R9&m$}} z@23*qzT)gf|6{=K(A3@cFK3yG7}|QgH?j5GOofC9du`P+>^&C+PGALEY5tW>V@NT3 zVzELlt!XewEgv4B=5S0xE3wbz6lgW(K;D_n18;!#ncj^w>>oxRIo*~*@1|nt|NPgvjvRjGjXo$^yYS-0)CLUw5*xm) zG|t8i5TYDd4D>2B5#qtrXF#fL1+@+b#g3*sco{p9gbr;h;XGD-?}VoZk@m~iXZW}f zaD2+2WrMgDv}{N)0F|~JfqVM)fEPVy_{fs%Z~8~)0^%Ovlz}WcOdk+wsM9{wz7((s z)@i1X+5IxY2;y#pHQ>+Mmvp&}KA!MI&j+~Ti9h^2{)!1b7Wo7t$oUlqf*DY}+~1G{ z0`KjKG9;R5!8Xt`?RW$JPX)|h*kJxV(lB~x|iY)Q((Y{ST2|$rAD+i)x(NfdKG-V5qLR^ zSO`o)5wQ=xm46sq#+}gmCHBb#eN>$F1jZ6L6O&a}E%bjg2kjZQ8uTx5)*Q?+WG)0F zjg9-l$3v$$jPL-y~Z^00y*Rm6(gcShHgf? zvW(aV(Mc!PJ|QjvP-OqudqQ#anwbf~2(T#{p?>%%#FxfFFakP+>VNzHAGsGp=KD7^ z%I47Xf#$O6(6#r>1ZW_AG_D@Ca|k&5h&4Rm&wCoL99>xWS=$9mHVXAVplz9o2pV*v z0-47y`r-sO?J>3*|R`@Q&_#s!(;bMLCsxlwD^XcbMvWqHU>&~FF8q(J@(F@{P zJw`+|&?%k@pWqPzpuDP&A91rHZ7U2FqQ?=p@1BSEbrx!~^I<%|Al?a8~3#> z+;e)AOM~9>N~XbGcoMX_m|XDMtbqQ;jZajJ6?$ig5%QP->U(6QKN3MjSowbNY}EGC>Ibjx_Fs zpVm2c1$$VUZ5bv)vE2#nexEBDFzl#V%nosVmiBKx%t6>vL=6YTdi>}jm%?CNkRH1= z%CH}a)-`j)8)Rg>ew6(ck|_@V+i-8M;KJa`LR2^?o6Zyvp|8N?{Ds*8mI}f}lGmgb zk|fFQ#-bMzf?z>Q_71~9+qZ3%8H~SXqVC;tT zX~64FnF5Kmggy%i>6jGdh$A)c5ZMc(&71?k$^89S=BBp>J9u{m9FL-a*iDdxbEb;i z(I=FTIAvhi*kB#vqa<=bBM)_z6#zE6oU@Pcp^JhZL&3NZ4d;GLS5o6YB`7NAj^e5(S}u1dd8U#%823Kg#>FYOm=L));tgUO-k+901LheNC2d<_K~O z3Jl_`M={9ZE*C2us#lG3kY=qVo!YE1oK z=Rh&Ud+0^z>CvxbxVo=PAr%-a*L;jxBZnPWx+3w}f| z7k;_z1JcoSA!y(!$V?;x8K2Cm5?vs{N_*_DbLhk&wBK_N3WItz4XqjQr`@9H3x-P` zG+?|6!~etX2AG%8E7~~}G#FnrI}il^24#2%We9EgNH+>gjk3u6zvUZy0OJfTn(8cyBv}DCg;ij|ISVm3$pwk1y zK6sc^i=Vt`)d(z7rJ;ib-w^X}2Wwf95saYv<;F>oSIFs$cO@g;RRNH#x$jw1X{62o z^;AAp830$nD96c&GZ&)f?_D^%sC9yA1=$>KO|-^#A@4ZKq&p?hrjUXTGD2kNP}WiO zIj9iRF;;*j`|WCvcfqv8A#ThOuth3doCEI$0ZvK**_=@D+=6;h=igfOqrB;8Lz)%*4e8ZgeAWvSkG0;Qr zPA`B46ECP&aB#^sPRSrL{LV4yWHu$RX@tg)sA6b3>#p|RF9a`2YAh_eLYrRj5irBC zP?Er%zTR4$j%d6)QrOk=fIOjvMdv}Ma_`rcsuQcp!32ez#`nG?7i=$x#~!=nPjoh#i_iZvjx$kQ=1Usu=~8JQla#O;^-yW6a_~DWLZ~Dxpj7Y z7(}{T|Gpg1TiP}LPd!c!ZvOu;I_ZC#llcB?=(78VBTy0ccl{`@Y@JGI_wWFPG-|UfgG#Ln_{A{P zJmQ{r8htFt1-%2h9;trr!{LVVYlq=UaT?|WcqIQH&VB4}rt%rR1!{J5#T-bg@(6Pv zv}JO|t{?Xg^_4z7HjDm~YHo~~ZR$IF1veuqtbFl?Y;h=$$Hu~R$k=tw-D?WP4l$s>he@X)IENnpV<(y^Gl3gem%0F{YwYd=h0vIWP8vZD z;H-#YL>5A)W_Hs*1Sw$+@Ku+=lH(BEI;kU+H4p7K7Z>o!zGkBfCI#7q0PO!W()E}> zB-a1$M`gm#|F78g|Dgi%i3rheKOy0q=Cqcr5z#YoH+{xvOcoto=4Dl0q$ zfqi#BZvG-Q&ZrYEWh}FO-7+EAys{8m=(_eon|<~uyTX_>s9_zhoY1QuufpW`TZ5WHII8%yv74G74%!&+Y7^V9 z_-Ya`9bkeoK*BKqv>y;{*9BZ+)mIr&eP~bg02U?iU-Z;shNlir^c`|OEu-ch>0V?| zE+w=BMMDuC|lN-b1{|&BpIwGl0Ae_zdBTytyw37 zip}^@53wV-5;XP?Nh*LHA?RG$DDLHmRz7_qu=xmOwlbkr==4i z891gbkBF*gO%_?YfIEA=?+sPbvbiqBAu7gWLCReHuZ_wNZio?zn>~eIhf+Mhir64n(S_77XOJU35(w}&V1n;| z0@*GN^}f$1!~+#%_#_7!|*KYj(_`;*_*5SpXn5mnUstuGuC zGU%2V7Xpc1-sd{67(ErsHR#xt@{~gR(s4RL9<GV|>Mf~QFK6D$3bCK0W6Vi(>P?J)@cs5m~jyA`nQ zjm5hA_zs*^snnc#)le*RteHV*5G&b>BJ!tJmk#C@P!{n8Hx&NvTM*zX(rcuxWHEk4c6GBMz4|8vy6gr{hb69^qrgtyEo`SulWLWB5AR*Z4=5$Oy)iCY*aN-{jt6V54pt|#(;(h@D5q~^JVy0gu}t=HW26v#GSTj7Rl{*O-xUScqt?9K?Iq@cPM zR6<^Px)rw23Yc)moSFLr%v=(8{W;uFO3{W8Z|~HK4)$Xf)bBX|53WPr=QH=wNt>klA3=fUx9w#E5g(oL z(WLkIEqm7is;TrZ!%2ypWkW!0n7dsp2U8Ocy5PTtdn0q<-U}hrvEr%=jE_x~g4)^> z)VHEJ^Vw4bj`-fZ@{^&kB^O)Nu)^3U1U6v%50pZ~=wJj<#v!|k_C7n*lkFh%9#}}= zA#y{H&!YomP`5nZq<{|M8^YLbTooTUJ#YzGPLTyzt#Sr7f-I~aw<@;!9|817JBBI{ z$%%~SsBco#X8~5^*aD~H{E!IBt!@+}MnN0{oXTEF1;N^-nl^rDj|tR)<6{ZFrfmq48}@DwNC zlcdaf)ItgVb0dy>p>tg500n(h)JZNh1}XjliiY%EjGx&w>&$6;5K&{_W9V4$1`bc1 zUSWH97`v?ZcFjZWv=VegZhFipHz>NQosl9}oO#zk;EQN91}zXO1Tw?<>P`{WeSG>0 z8z5Uvb#ohez>z^e&c_V6u|?(^{tb&X>YDpzI=_6x9cN?Vl+a}p)KGi^lhkO72>v^p zBpB4X|9>@i=8sVC@B5#qP>3uuHI|}~eNvXlzO*397D6ME3{51mhZ+AJ=-uHE1*W>=u-BM}(PtSr@g7Bb~SHw|9g*`{8)hD^UGL1~=n3{6HjcU9qLro*{W`8-(Bd zame4EH1y=O?G^SyB8bZ<-(UVDT5SA!YG5sH-DY)<#g%d-#43b@ zQS~*>tfCMi@B3%=iErS~27h?%revWR-QN<9u`|QnAkvGs(FlWva7E!~r z*_A#|G~SzWWI?}-e;~*25BceY~^g5)#b3g(6dX49$XY<2`JVKJTf@$q|w_CueL#rQ{ zo7xbHL*;vtdM%8mqKhVze->9f88SBcqRys{3rUnGpN%z}sj%3Q#J|=ABmJ!MtQ<8l zRFkgLB!(FlNo$t2dWX=w5XttNEEQvCbhANsr=!Endl1hrL8aA!eNRtwkjhd{wF5Vw zyfe=&I7mv33eMQ-`P3Os88P|Ydn__CGe^|f5A#csB83nUL2}16u%eRVv<1RDzR_Dr(dZooAHUMSd|<61Mt-rS&m8WuhWV$#D7}WhJlIz|~NWvsn!2CE|aUDKofrreZ7{MW=EBW19{OLQG>xXXaZy7e% zmxw!0nRgBtOUVCm7Vs{!s>5_lo-!qO-f^jmti8XfLMfV5woiF|Xuwpp9Qrr5VntRCTe#Ud?9g^_ccSoO#N@g?JoMeJ1WdGGVoSXgdk$lIPU zS_+*9p@V6ECd3?1k}YR_Py6eJRcG!z_dc4?4qgmh)AVzq0~S417>z?MafxuV>jyaq zF|WOxMt^!#+yVs#>D&xFos}0gd2!< z*g6e!oJ!8Qc0Af_vw6f7s0~N<6#H$*2Ydt~y~2|;j>Td>zVM?6-)=mkhUJxdqAvgQ zr3zu%Cz*78(aqw+NepjGoyS*NM6DdE*Xb@t;Mv|I(q^?dguit($0wh_`?r`_sJTuh z#!aPwfFF7_?vGAy#*w0lS8(P}ZA;8(Z)Ermn$hw+H-{7N$eC7CS3F7_5wM1bFVvT9 z-m*;LNTQ~kurdWi*Gs%1AcZ8sxWMvvUt^;aY7M{C@cC8TC!X+M7~J+pG>DL_arwe z5xSe&*8q}r@P(VB6>gPt1S_(SAtb|Ep-ggp2xXfJlzx}zLlJw+2t>Qv7qKx21HlK3 zEQJWTe0c+?AFmI3HN?*RJh92!+d6to1F%vxTpVZv&gxJ>wIUu5jSZNpsX&={eLb}p zBoFz3HUqF2E+eu;5r_@t4jT0UVV@m=-AJ%p=O1*sW)z_MOg z1MO(n1pcTnd`GQNlDx=@oB4(ue+W}P9gLiE8~tCjPn^Nm`$0kb!WvrnE^Z=-$zz{j z)=rxR5RhXA469GQAT&Yil&t}vvT^=J^okmB4O~Hon??}JecdWFf5l}Qw2m{JsWlkZ zrz8FQ?+y6Bvx3Ap2kEo974Jc)VgArF>*YsXnuE0A%hh2KYzwknVxCWJ9kB+q!l~xo z3sQ{=LOZgo`ieAdHAJa6uR&-5K9sBR)GtlIG|UHVfJKjCfSnFMT=6Wt!zouhA#i=t zP))l9DP7aVe8oRZ)g%UH9b=m*fJX9$k`a2L-gls-#I>wCH3)I@X z&8zU&uJ2q);{97xgt1uf6p-8!N8%SwUYLMRqMdsx04qVBN4Wa4*NB>0HyPkyGASc&Omu-BYPb1TB80OP`_;E8AOT3xK(WOQ5X44#C`wgI z<`H7$XTe;?_%R<>W{NuYNW#ckbdsKCA$6E6G@yG>Iz474{wk6_;;qU5Hz}q67Iw&j zZv9x&s+3AqEVzj0iv;NF^Di*u&{GWv`}~5ntN3L>(vMo|PNhR%#?u=1q~7USKgOeG zM%yH9)tgrp2oBE>tjY6?|R@6_>fX`(P6CR2%)6YGCaPnB8{Cl31k)79mOJ=RhFe@GJIHM zhmxqUD*`u52a}?nr)P;gqU+TK!tH|jZis|o+(i$;?|;;10pcKEq9yRvPg#Tg-f2f? zeTV}&+9(S5KK|S=%0kt-+I9xbaT48p3!16CgP??2KDqZp3KJt2ki2#efsp-UnlR*L z0t`UQsu~u9w3bJ`ea5Uac~8L-aPy5{iEP=~8UF*>jgz-1jrhMK>p|kOnSTxCba7Rh zB-dRkj6}#_sgp(3vy!{cmiD7g|bIAL>Ks*NDOTl;_5 z7~qbd7yZNIRA+2yLoJ0tfh8m zGYZq8a0oN32V~pW^fVH9Q(R5jr47B$iotppVib)GZ*RcHPoc%OOK2@K7LHLwSK%-l zOFDv&?7%mrz=^1jz&^q$h4`a*N+D}&(hMr^C5fBP5>{rD5fS)FcUcndnm`RqOvnf z6dAw2B2P`IF|A+Rdp!%z-|&dRJ7&VWzSns_lAqFNaFZW~A=HGLfXU&OI(wYSdCIhN zwT2|VeHaW)YURft5z5X6Mo?|Y$Hnr#l5i~V)dtuh!OA8tC^Jmd6*iR2SNIQ|LLECm z--kgOdKrH0Xg}msT5T3LQcYy(rb>Tjp;A&fnKoKY_4v`}6^5Y=!?zE9VD4}ZsOT0p zz{YbJY44$#esr;#EL(|LI^tVRM*Rd8vo+%-+{(*_$V!|V+@E8?YX(}sQ;2D)u45_L z^&uDEJ(bxa@5{;pkf>!vhY0o) zed|vc&COk+HG8(pELn0*VEhm4H_Hq1#2kPsb<%J5UfYxnti;vXW1 z`VCFZ2T4r{OMO=PXng+xh4myK?-&(fJR{mcN|%&$R1Qv#G~FeUl?!mN^EBL=kl>u$ z%YMl&QxsNesXHr`KsrvY130t6-LtB)d}eVp@3tqWlVoebaHaESFu~M_4x=_b&HSn! zos$Y>@Q)p?t0elZp!Q>EBV~HpMcNsu4Ej9njg(P{1SPAA#{ zNexDpsQUhwEbl-&FKtAk$!|b^g&4oS^(G7-3tz=Z8j)}(H8t1ijKKDG>Hhcm2?vy2 zx0U940^}&V#Wp;^*y^pMepP9sD-_fJ4I|@NjoVieb@S~uC(*r`TLV@RRArijkQm`? zfH^bu2GNe{cO6L{)`-ejO?rYm-5=%RE{pw2!^w6T6TRd<%Mj9xUuO#K$h@LIlIwS7 zw^L(F`+cMw_>=i9KQFfw0nzCHkW!?6xO3|{$S;Ir6+>I{ZYnm_=aZA_b(LPhQTvTI zjb`StIhK|s*1yb>B;V1zA!W`H=R;HF@|_^+x^Rl%CO($;KDoCcFDhAP$9XWJ=0*0= ztsmJWiZ)c(M~J+tj8C^Xl?Mu%5<<#EY<=Mo`_t)F)cY2v%=}0ZM>=b3gehS52){Cv@EnjN8?7&-(9w)Q<6T5aD@Wr zh8ix>Y{b)cakNUeNHt8)Z3KlItL)UDu-8Ky*_Gml^-N3oM-Xv7@|Uvqjp6)jQhI55 z{fM3Z+A^y$Rkl#X$Uq4C2qX6y-d#o~azo>5kq+1Di1t-^bIQmz-x{( zw@wKp`)--9ys}{faqep3dwDbm=^ug<=6U&q3j;H|$!+p2X(^wUvYB^h5yyPc7Y4&G zA&3^YTmStaWXxN;B0GIsghM;e?LYKoPufon2kr@)*4H-|JN(f@{IhaW0hskfewjS!?er>%XQG=CTEQi#1V1<$ z**K!Qm9yXc+`-7w4YQN7QxZ?y3~PiOr#eNrmPFL6Zm0#^xPM{426jMhp#<5RbVs91 zpL%ByDYCk*+kdeWG!o%jOwP4%shlgn3dyFo`4w#j6ary$=CUKYOep9`C`lehkCF4&H!zq-BW_{WuTZTDGi;uX+%x52bd}I#u+Z zS0O@k&sjxmzZa7P)Lz{|k;fIP4HqWV4 zd+Rzev8(3#T46s2L<8%=6c>W=H=8r}&AM4FQDZfcYnQ!>5gA@@KBlu_MO9td)#xw%a5{FACL=paM1OZIghQgM8)sZ@RIkS*%$DpRAmNsXtUT&_#Y{fq$Lj zf*xKniK?F0RxBV=@zVUjv-S^d5>Wi6c~C~a?Ajk8JYANzNEd!hfuzPJ4DQ6EFSITY!ek^EF7myptY_dp@(qyPWDRj!%DNs?G}DL ztVO+rV!2<*glr5$siiT_uN?0_&(cm>%l5lXRZCh>yJbLbGi&E@C&?vcN~u-tOY^_F z$7@v#by^3v-4`cm&i#fJWh8>~nD^HpNLg1sI?S4kfa zvP3xIqU)duBa?}1svYp0;w*=W!#NkY6;1Ri@p}<11pi_U_HBp{(QZU>KSY%r#u7^G zW$K}^pK6_u=6H4XwcKF1#IZ4%nnNp61hZhr7TlR2smbJbQF5!u=ESR-=wYHI-P76} zGw30{wKRwN`mobLdW>=%xad#5zC3%&F73ND$>G`j}eXQsK3 zo_|YjEc4E*fLA22A$MU0f7>Ya0zKz9JV5 zy7vpKnma#cGq)&wL9L`XE?I<0*78uR11C{dXm)0#JUfLweJNntoFyqySK6hyR~RME zwILmEBzp*oq8M>ZmP`^aPj2XgY0Qy5${BgH|K_(pKiA_@oO!^Y>X)S0jYHzi|CnQT ziET}>71>0=($|kgdgl4T&P^WA zc2g?2otI%^PwaUmSCKEC`br(1ziV{ulelm8o3Fz&&OI5hVxE3p5O)RUETd~w-Pm2t z`Q|%bD)o`*R=wz!!7z&mtfJwhH<*?L!u53OO^p-d z*g!$%t%%iEmO2eC<*uDw>gzB)Yz62ENn4(squU@d6l11A<$m&R%G058< zNJQl>&`bpn?Bv=?Rgn6G6XCB#{5@psL*WfW6o@kUvN(K}+}rKYer+7!-xKF~_e57@2 zg?Y6{CAgOg-uP@HSak|pJd&Vhk#|kt8T~+5bmsi+r$ycsjaZ=~ANgJP=M*81FAf3@ zN=JtK9qe6N_k%#aN{5)p(^*hjySU+|Zjx`@Q~ej-fi7I|8dfBKnf4}C==HJGJ854ZuA!!Paue~ynvgVLZEaVry!`3zD3eyPd^i8 zvuDG>0#xwkN_WvP5~|5C0!`Y=rGF^0{e(=<2BiJ+%k+(-=!?M)Z!R(yOxg?GN(2!3 zH#@hQb-fZ`G6fvWASi|(delIUO$UpblSXltofkF3~g;^PY2*rm&7;TwPgzm3a)-u;Ou1tyDcUPF#@q;06`7I6AXTLBK0&Upx0QuRCZ zPv~Qdw~`tXbdJ=|!JP%xbml3nE$C`X<`#n2Nl`X7 zUq}i|o`0RiJ8zV4hk>=(!oAdUuz*rR8PT}sOqjY#Vc?k@A&~vNkzVBofoeyEY*sq5 zU7>3zX^7a2yp6@v07z5!^=k+H`?2gCLUr6gPq>v35IVdM=+wpSFnb9%I?&BU0QA46 z`!}xFY8$uhLtUWIq5bniAP0t~;zlwv@uKPK_9kena(^8M=ruZOtXDs_q|O{^Hh&~h9svYU%Zi#X3f!EAp0bYba`Sf^j_1VBjv%Mz zx{GRdN`_dlh{mcVypw9(JH`+EEb zwhh^%lp^z$oM%s5gQDbab1hr;HM_NWNSnAD%bhWk`WpQA5#_BS)QOl96=)iZ18b}} zJI{HM0}sxixgMh1z?-q*aGqC<1s+THaA?ee>F(T3@A6sO=^p~6q*UvMVeWKteu2Jl zLJ)iUm#UvCie#tRRL2r7c)6se%q#Cw__a=;^8?64yy=k3)|D-!5MHnY--;fQiTc%< zvNe@nb+~(u2U9*2(e3aP-QTg4ZPjVgd21E+vgW&A>vELea=Y`>fueelORTD%fZ$Pf z74{G~-rfc+szyN%;Z@_-WeCqx4cRkozzNf{kddo;5V$AzD3ycd`#8xbmQ-(1(;`=;k)nH`X$%^`Wn( zFfRJO&AhJJtZdT51KzPKm{*3-DhCHyOG)-mWSZ!Rb2M=8oZCZpZJ_Uke)f@` zvxNhIq#$PhlNV~d!BR*PhzW$*&6iuqE4e(2G{}F}WPxaDg9t|1L<0pOa@$Ze!tF2^ zkG2kxd=rhvKj?>PSAOr9-;glI2;cut=ALJvM)GwVxVTf+HDdSX5|G344`U0#mS;y* z!w#>?uSpAv3uXwm_S*`Aa=XeASy4y0>bqghT=v67#F%zZhEeBCS}UkHXHCT{qJeps zskzU$FJa&wo6Id#1K3aApyb&6F|cEq*8Q?T`H2@R@NpaV0tH)Fb-~<}II{*sm~7 zQV*Fq=K(%WWn@B@eP5A*O${t1hC9r63+H=sQl?mVKrWd3Brzc6q)L1$2=g10hihxDZgP4M`7tW7ao!ja(L z2m)2E-Abhvhw?OHsvN(*;NpTp0z{K*U!u>;j}MV zTZ)?r9ttRj?WXwCqTa&1c~qWPDJrR6f7G%6kin`AGDa{wjrhi&A%R?|YYa88R*yYE zIgNY-{?*zWE4rqVRxC4Xs~GU%jOMZ7L!u4+C3=csh1v4k&*0GgOJ&=3dS_|=tE9YX zVu(sAh1pr`I>RC+7);5hlIT$_qG7Q_AB53KIOXvBP(WU+fMS^%b=xy<<_5<)g?oP} z^4C5RHzE6Igtlwx&Z80tRigxYl2LLE#O)o6C!m&L1HvSk0{ih_{6ce#_+z&?1kNpR^xDOI(MUe3UB$_CLXV?VxG#zxksn z+T&n#?5KSXlfk;kC{=8FJ;dOTBi>jf3i=804{&bg|9&a4;%I^0S^Em#%PJN=DCpAoDv&p1p9_K-+@y{L>Xi-%k*t zN8f4V9~ZK5>iA{dgA!u55BUIPJmBi90IP;gIRvsm7+&%#=A>?!Bb7(-#%}R9uHEFB za=e?I!mLukxc`fJd|6wCp>h~`!Ym-7WrU{o-{;MK^uoH7B;~ry1G*2A-rcmk$l0$P zNg3u-J2}Y_Kbnhdb1{$)Blw(i5JziGAiwf1&S$4@h%wkxwynEQUEK}vG>@WIGoy4E zn#BYx`C+TKJN~E`EcLs@8@@_L>dd@PI427!8*<*UnZ*qS^1I^ypjbi7&vVNd`4>*a zeFmv_5-aN2d3MBK{t!KFv+7PJs*_hzBc9XI}l0sUM>6KzeTO9yiPjzzBym|wGoAJHWXcqrf}E%2lMRmAFM!5uqy>DMvR zk9umW!VOjP{Wg@vUBh9A-Fq?{Ee5I#q_*cs$=R#S_J)@KYp1d*|PKt*;8Vd(AKKdM&eB5*!{Rq1GiMh@& zyKTE-R>)}&CjMqDL!9Ge+~zo$yb7uFqUa~d+A6{a#qVvgC||e3eq%0=v)xLxEIy*| zVX{HVU0xt=bUGqlK3jyZI<-^kUi>7y@|EY3j_;5UQ5BX%mIGIwB$k-4cl_UP>aO(g z_m<8A{Nu;}YPSx<1f+&_tPb39bGk8!OE|Lp6wO}7>XoGXl}3YzAnqDH?L1E3lbhy4 zj9S&?LA^Y+`oI$^{O|=fKQETdO<4MxHiEB~H?76Q=nC(a)UO0bueB8&i|Rbkbj;f%eRwyG&x>G^ygv05xGMYY7(Zr7zCRY z7L*cmn-Va=O91#i1}DfxF#3L_^qm32qSO*e^J62AS0wVhi2jC6K}JVtjRplN-K?c^ zV5Yyi-!o)aG_b2@N5JQKe7%u;a2X(3qp}rL{VRJ@1L<4CL-e+yTIGdzGHipbr53=3 z1#7Au#^|X*)@7m~lDt+>kWSWAQ&=Vsn&1_W@RW$$yDgF12i_?IQ)A2nSO4b*_Gc&E z(anFgq}~V-O(j4d7p{O{O63!-(#)jI?zV)UAc-6l*Po_)GfY0PJ{%C|Qyt#pS2zYY zJm=elNN!GG?Z+w$opsFB0cdkNHhDG=k^Ca4cwL2k%EJ3T-%99YSYRbqoGsG5h*5&& z;#gJ+(k6Y->aL2gSZS96-twYG-T(~e+b8gOQ*WFldf^z8 z$-KAiadR7`*Nuv`M~PAmPiUtl-)({o0$;E+-_TnWc9m+_oQ> zRFd5H^pWa?XD&(y&g<10%|{&xZqK^po7x!cS$yQ{XrI=YhwqWa@V@?zw17|6C#OGu zc8EGkxFWzSwzI6D>WhKZs8>qI=eE-0fp4|YDp!yGcR|}Z@I$cDLz~CLxnH2!EI6~b zM{9?S!FYV*v9L@lUqZ^(*u2?VH~L;p_uhr0uA&C6`|d7M9A-+xsl$BY&kqF#qu%I6 z*5JcBsTPj=Gb5gTxt)TxK}Blt7#ED}pBQ=5ZJ5@%>o9kCax&ohkI&De&sw#1pL(oo z-&_fxk|gGe)YEc#~cU*)| zUS*hMpt0ABJt5ce-o$79Yk=^Fj=zWs*QdTb|B+R)zq}_PCe^?hE7R`z{Fv?2zP}G& zE;x7n*HO8vPwp$%Noij=WIP(rvpVU3I&I={>+(er0^9X;Z-yuSQk>qE0pqNMG4nO} zD)Y&&)|qEt^a_QsInDv^C?eEUxALXK7a?46vo?g{_gA&{Q zuj{KzTFX=Y_5Gd^=G6inDd9~`6peSQg`EY>u*|7@lW!4W_Q*p{MsvG++Xd|bPI*vu zx{C9{_K%i9lXJlW8(z_EYc|?5qd9|c_^&a3Dmb=;Pyc>#cN5C!QBY3?E(oxLMP;@= zE(969P5m|D!SDY)#+lYdGf@rs?Io>tD+!^n)Zw{pT*;E0s#H}tw%zemap`%IxwF&BYx(%(LEx$&IxC46L_ zZD!KTEu%)24-7j_-aVsn^JHJ}PE^^6^VqsL&n(oPKEDCQRbJHka|sovkhTYEFMM|Q4m|Randm7l z#*Qb1P50Lq7neI#m3;6q?1#j|k0)X%h1HX2Q{~0e7MUo`x!+f{`bw}fA?hU~dxl@H zdIM&qbX>lpqrK>Dd%f!B-r9)rjfRHDvweqqR<<`3zZEnFaD84;^ep&)ewp7boRmZ$?k4xqpVE3)|9SNA;qx)ob5$iWL47sP&qkd&bEcFMP_)a6f)Z2kZS z_zKTrRc~)^;HQ6W*2ds7_ZXh zI-IM6Zq^#S9+02={U7_qVKi!B!y6$giIoEV`S;RDTgLCqA9V>aceVQyVgxq7;r2YR zLB&p=+Oh{e=4m>7{1~!<20kU-^Pi?2s94!Q{|SW>%-jaw8488b+_VSzSMdKB$6Kxc z$4=(`ztH5S{{{FPV*i&$_G;K13m=Wr^x`pwfBrxF->U|HOAG#WvN+f5)ivb1J8ofX KUS#TZ^Zx-q(?OE} diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/charts/locomo_comparison.png b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/charts/locomo_comparison.png deleted file mode 100644 index 727eba31be6f7517eb5f4b4eb3d8d1cef495f8a5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 43814 zcmc$`1ys~)*e^O@fB}k%AfN~+Dvi<-DkUHwQc8zKcc;n*B~>IO6&PY@q+5wWQgUbn zq@}yh^RmD5-E(8z^R2VaUDvht8r%*u|9Ic$`PK7(_fSEa>^RMF3jG0Xc;Y%mz=Nc10(jNO}07!2;CjKp0P$Ed#p zN5Vs%ZHXuQXBh3H5|N3;a zvZ4qtuW`rqCb9h;JGbjx;!|DentOviZtDXxTMsdw&JzEZgEe_XAc2E_aIr;XJxcBv z#$Wo#Wpp+NXBW;85)y(fnJZ>BE1pU;Y8IS$&=kQdXVb_nJv~_FQao2Y(-zKc$Xpzt zU*(NG!BKdbUt#jsSMdm^@z3r3HZ9R=x^Ansy+%pcz1?laRX1tF;FoQu0~CJjt`Eg_ zX+V>)LTo&ux>P@@37aYcRg)xbWMy&#V9o2P4W3N44qpb{Ee(4M*iX z^6nb5qg$$bQ$O4!qc>!Hbp+$Y&(Cok3x)>rcgpv+W^Ah|q!XTnK4GT!&{|fA7Q&8& zYI4k$%$KKF43(c14hmKfy5SK}yNMU9&Ymrv$(ULt=*H@)8cYefuXd^%RllW}T4`5Q zl;bS3HSWZzx>t-8rk3ksDpgdDVji|cS=ZnAu=A;jf5^3ygZ}2z`#taxk%W|~d^nvj zxBcC9-nr497qRS-hJk|2g#mV5PhQ~WZ#UoQdS@dHSH!r#R&@JsFu?<^c|E!+#ss6j ze6t>p{p}%|mc=$XPWDKn#Lve4(thXJ>*p%=_i}EumN?Akz^zwYaQ>U$XFggs2WP{o zY2Wkg+D0)MU-uVNSVo<51-Z>|D!Tcbi(ce#mWoj6hB& zz*SLxj)l*tE#Yscx^8sZ`qXz4dN#44?!jka+G=Y*uf=}a-I~dKcIfEIIy^n$WyS7V z>PoRvyi|zYRNK{x46D)FXFl}qj+?6fT8r^I%+ zEJnw|bU8HbI#rjKNbkhDF8$mQ#k(vmRGkXj-53qpbSq!@abnrTu&jH+YwcSl@rgg% zZ^(tNR}d<8mgIU&GRxGu;-!O@?2LPBLp9Bx*f(0xT0P;ijz<$OL1lZZjqqTiIl7^$tlNQ3Tx*_mjrlw#@Z#-!(?O?+rUoI~$%po{ zd4$l_`600i^+mhY5kJoH`8S19s&0Fm6Jxp-Ru^4Xr$dH@1cLT=Cd8hxrsqD|9Pg0CC&U6YJ#^F&Rx-}VZoOuYtwRzYRQ+bjSBf3y| zXiWbHg*2Vg>0K2}AQrYG%Na|Gdu z{y&uU%$;v3n5^F`_u{>9?5R5VJkuP40wm4WkBHFW5=!aFvMM)lfRa+606mmFB}bQnBg($O6xVQ^4%6(?NCk?>GD-|qR>;orDhc4c4kD%MX$l` zSZS43wAAE1;z^+qnose4Hlf=%HN8Gl>F}Uxjh(jBSje(aQzSqA{7jw!hWf67-H;8% z18GU)w6pp*FkE)fCp|B)VAxdaBsX}h|I``l(lgY5l;ikJ@Y7GE6xrJt4e_>9@VznM zP{VcYbs4u|tx8FN$Id8yhY|P=i;CS)(Vc~=9f1sG zt<34%m#vJt6TdQbD`*M|7HeqEL}w9*>wL3bk$|EsGSz0H+=>m(k=h5;y&Y;8*D{?KAzR$ z7a9nCU1CwbnfT~h_rq!5+al}QO13)bGG4mDIfHc{nD8?z8pjLb8cgKl-k98Q^_@!# zjc{_&kfw433Z;cQy2}!E3yRP}p?F zD5`EHb2`>PuTG^G4Ay_Ol-GXabMnHYQPJ~Smfv}bMof00<~9Za^62be<~6p=eelN=<0*3$pxblmDsM7YkN!mY$YU9rR!%j9 zkk<+C%G@>nmc&Ki_L*OlJ^G?%7n>)DKWJP6CO4V1@x2EJSUkg%J%&2ZimR6y$wk|T zG$PKn8qm$!EB011SD?8i(tGT1>sAS5`VNl+VlwW>CminT+{BCReL#y|E1l}-N$U6J zaG?>V7@ICjZr+A+nzXSS@r%&*$5rGIMmgiBO7~GrrL7XTXEIH$iYhw9nqNIXkCi6Y zmM4X^iC(jhq9lCo7&Rp&WG5Pa{Y%a-a{JG~vT^=(u7uvdN- z#JX?YpjSMZ=?=T!W??;cf9IOm`dXhwm^!VRTZMtZYR8@iF(y%j5|uutdr9G9?4QOz zr2ZtKcYQx^7?|e-4Zl=y2%7&l85&~&-Zhc3>R*Fxf_u=jc{8ZQd8PfSm8N*}REq=i zqUAEKWJntgR`lxNd!9$83tOhKsXyUDtrrI#Z{to{;J&n&(!*ylQd`6TWs(l9`Boc?nv##N7XUkZwy>ipObl zvSb(}8@??fAQZOr4C_tFePy}%_Iy;vc8pG!hDo|BJ)Rn`5iIJq<-Z=$%c6~+FW=56 zWw>u9@wON?_XK`@z(F@)rpDD3Ysjg+Uc3U$Z6fpYG{7Owsm+PlmLS4o!vNU^S>0>A zO`BBaCA0Y|gg6ETqnIj1(anUNO<1C3uHEKSGsQv{rHfLVuCIc(@(yDjxE!E|9N1+O z+p!*dg3*p6?>aMVdY*-asIwwG7ZPd*fK=F12mpdjh?YFA8PA7>qvq=~{nn9fh+ncv z%Q$P*xNi?g{12WrE$g0!`{vj+BYF{+HMaSCuD@TAj0?^(uL|w0SM19JfO(deDG24a z{rC4bGb=)BD}cMhbTe^rB^D`d)`0laa?^M|tu5l|Gx>Pjx2BTX<@pCbCFYSDU*qu? z#OcdvzkdZre_N^%=*v8aak(+kZa#0yxAmqRSXA z@poK|5tLphI7_)4xC5GTtvC3*FJH2%rVxC83=q~uPGMi!@|UisZ9Yp@o+G1R?0)x) zW8A>+%b$1GV($CUS>Nw(mlUK!9^>;0=?2@Ax^Z-BASm z?FicK*f5=lFsN4JMsUvZcI@K|x5qvyFBMh0T*Pldbx#DOUS9f|6E~@*VT6^<^cx#) zR}^dd{ufa6C)T6e>+`V+V!JD@?tajQc*ORu#k$_Ko9h?!$y^`u@YsUuv%F0|udJex zQ84H{WvEI&>=J{x_a{VNoT4Mlp+dX%!1w5mipBu4%mH-n_PX!!i)g#V zt@6<*gJ=PKM89pj{8(Sa`*&tD{|Ui`o6oard>ZiB-+iV$0;rkAq$g+Wa3&s>FyG+J zQF6M+`G0d76c+5D47>LLkxL!0Fp4fz3*NRrHr!zjj;+4wM4T}jE^K5I04$%nHo!Zb zqT+OBkL|u8Yf2ScqkzqUI7U_)0|^_imnsw??g)VM^sKK~ckBC%)n_Chbshiw!g^JA zv^Jm}E@2p7zh#wHI<>04GGj)o&`eq|0&e}1Yj80e#qUi-*)(hMQc;>?Sqgo}V_fF6 z?oz3GmAR}p2XJ|lUyPu2m*CYu$rL~cq4Dm>OUC&x`%M=(4#me=^cPwgE!I#e!v1wZ z46%NqKJ|Kr!%(@~^6I<#hHzcZa=sB0wg^j!Rl^#Fg7$j*KC>{N<=Xe7lPk0#8Lh_z z0xjS!{#1yx<#-&G|qr)a3Y!1e_e2V$FIIFIq?W)I!WpSao6fy zP$=9~j$by6DJR{+V4h2i|JUS)Ean5fHgCL1lHGaVJZ_ zB@rRJ$zR{ckX~~AJ!&YtX1zQC)Z7LUv*H+mLvI%>{&r{ExlyApc<`*9X_*4kjFAoT zI7C98HY1MY6+M>e{vw-*u12E&B^BjyaIq-*aJ{V?tgLc|_`s(1%8piO1U&5zZjEhk z-9OMfj!@|jAu#2AiarMF;i+Nz2Zt#3@=SYf=9v!&!K%#A`R5h!9r&EAe6N zUKTQ1J~pxoH?MfK)U4~ko>q3djP8x+>r=aAh#gGBuF|(96Us##1xPWmhGmTok=f96x-WiYJzr<@kr!fy@&oD_CLpt%>ebUU3*;|Cgk z^QZ~@3BYc01YHj|)IF9r3t({$g#yyH{c-GC0zh-OxW0_*<9&V;hTFh^@0a||k&Cct z7HkKo7b>j*ZVSl1tC~metLc{3t9psnw%9o*5v({oF~6_q&@kj zp`8l2BjKV$KR1aF5@g8R%6HfLasd-00ll{L%~*$W7s>&Un(n%Je$VqMKhE;|En@qw zS+Cntk z_L!CF)XYTGBR&SM4mb?S`&_htY)r_|)_{hwB&zE?g|!7uXKdy)ArFT{yo)Y>vyD($ zYn7yu*EBDBMk8qb6t%SWf2efALqU3@+mplyxo#}jBK0B58kDW*Mpg}^j(}cz*H1PV zi0>auW6+LT7&PGCE$6^xxC!7!1hK)X+;)YA@14~rX?U)qvM#t&0*YTdBGu)8r;=iu zzYcsV1SHlDJvin-iI{L+pNl?(X9^O&xjSp&(eTv}aIKwcF+ke(S0tAN26JAB%(CJ5&;p{3+T~5Z(NvRZwM+roDeqfS+I`BTCRcQQU| zndNoX6N28a0upUj^(g4Kmg|Ln$YM4hA0*0>Ip@d4`?`@7(^p`jtJ74sagqor-!!?& z{(cMz(L#3h!x}esPQX>u%Mdydrt9vswcH|ht+7-iLiiFR}_DpyGBoUA3wr>(58IM8u9h2E~6h;Ud!8-`id0H~#cis>jTA zSjqskK!sNO(GzyT@_avKY(RmtzF>BxPw6#M^gBGTn=|&%E|(axDJ8Sci$7-S@MFl% z8_?-|pK)b8Kk^K3S^0`<=jMj2@hu5H@7u8sgHBUl-jK_}0+DW3V&w7v3QKdx=F)lL z8*#na#qdUkL5CqX^%U!y-Y2C=FHnAZAVT(>)H0z{$zF)J#&P?4dI0C2yedqj21LF>k`mucS=Q^)^_4egF-t5`XHQfQO zp@WUcQte07c}>V_}b2A3C2Hb5xm#dJKycpLW=B zA4Fm8Ej?=gxe9C`0qGA#UaPIr^i5~YYQ4g_7oUG>l&aBhq=f^)aEU;BRNZ#<*GiLV zOrjdrZN#sG@|b{V9Oz@ZPMTJJ56$oOWyo|g)N}-(FVd}3UOuq^Fyg1~iV`_3dl_qB zKv4D`xAYwqB6pAesaNSKJELIg@`1z^WZF)S%&E;@U&E<3 ztXx1}z~&Bf4G2GNhCw2s-#!+`g6fLbyi{eAl=mp_H&lxEx4BY@qnxX=f*H;%ByXJn z20&-AiEvIAkdvLlV*4)mwmZA6>6!&g3dNngn{S6^4L2-6o3K{)dj|#AZpqF6UYde! zNA#J30sFqg1)4&!y}nuOLg(>W{Ww@}fmOj8oBV>~a7uKhfwcS_NPo<8mT#1&j@vxO zdlsYM4SF3LE_VLTLZQ+QMAYEEu&;crR@0+k{Ea`23_O& z{EnW7!&1mBU8;Ay=oEh!k0_iZp7zxrC{Nl`MK1GY{6P}_iAh#b!Hmu{)urwL;a%KoNz-`xcS?%N8f}VW z!hYxH7covO=_lauR(rZ5$NJw-Xm7H)bHcnX3SVZTi2?r*8j$)6^Zh;y18_MwlV^;5{`9)|-9=YYzL zI^G}m!ysU4=-7i0l?WBx{$oDXE*Yvv2Vy>Gy1YnMb29$+vYtZw)u^zJj}(4Vx&3CJY#>ZQF|`b`#|*Bx^0>b)ZgQ`Qv^mP~!379r^tO}Y)jTY0ML%Hb z_tcntTMJlZd!$AAXT-TzKzEp6DIq<|q~eQV!)+M^milh{&(In;nsyq#TVV0dl{)n* zInM9BIkbnS;a=eX%9k(-&4P5czwR1^sH(}7Ij_nh3O49*ShsZtcnJSRi%9Hn2{zMb zn$C6Md(rWm#MQ}J+rX>aL468krC#r|sG#Es4J;VpGw!uNIU4;-RF1&Wm0G+5RapU4 zWBYP7its@oE=3sDoL~X`W3@%zY#kM@+U@gqe^0i?FE3+d8?w)K6LL>)znwX=9e}If z*o%BGkYO~yhE?iWp{~|GH49C03K*melJgZpuGPT({EXU*aLNn$^gO1iAt+tT{0_E< zV`zwe1AF9>!GlM5r|LYB#K{1EjKQlT=j%Ej@osL*jGih;o8OAbme-~y_P<74M@>O9 zoaozKPaWX_&ONbxeUy!i zBx4#bS9&f7+)GTy5u)Eu-*PWtk6kr%_niI|Gv5n4u~ZEG8<*r^&tQgLIebC{b`z88 zn<3D8HL0TvH89MyFUFEM1K?J_xsZmQy4~Mw{P<#jtMiuF{-)yQvd7M%^xVmFXGP2b zux_rR-*~7_(`fn`8kxNi(D8=-i?an)(7YorIsN&tG!fslu4U5{LXVe7Y5mSGl&?Z5 z?t=7jThL_^<@tK9q=QPC0>Vy_h@I_hfc_7xefxiW>H~~`?c}c&pggL5ZUqJ;g9wK? zM@rLKNoYL({LK3F1{fC;8I>U7SO@`+UP|v^M}$Jp_-*T3tim_4PH+sOkf)LhcHtQQ zf!3R2m=e1gzMn&MpyeBv%hv%Y46``+I(puJZClR^x;$E77pf|QR|VGSb86x{=nc=Q zDRz$SGY4=~qQC7ggG87BIE^R0XgneX^a~@@QJ0(ebq@e2te_`Gxa{ztd4O4RndMu? zy|CQd|9)$U*(Q>R|Ml{W#uC!i5c09ew43QNf;!UDBx?e|B{Ly_e^8(acDmCprMeb3 zs;lx09YXMPQK>%??{0 zbyJyBq8oZxJHx?>!Zk!?^R@#;$pF}C59*qxB0+A~1)5l@wpjkzKEGLVYOc%3PAg(C zrL&2$h{7kV_$LSb3x*Ru9Y9*BsaWLOB9`T&N-+)AmV@9g7r=zdEiodj+Nsj?ZagI3 z6t);jU<)rUPlXZP%P*=YPXkxG9-rSINt!h^2uiJ`fMM6Q}SU?Sblfj zP`zpeUo*O@f^<4Ahy6$DN74$$N!)atABQaHN+!La!8XKhBN-WW9<66o3|(gxg+GXA zoz_=VDTXWOHdVUZ-0(_O=I~SCUFtg5bUeUK{>16`p6y5R4a&ZNg?ZpW53WU7-orj) zk`8=;6ie&``$9-oJpC?ox&~@mPFc<%;fb}bZC4YwY_Z)h`R;o&l5pIvJap<$eIx0$7x$laxY)FaIFy4zP?>;SfqK;EUH$MK ze*y1b_X~fTQ8dm%%V*jx!P$w#9Kw8SJXl~yPhE-%tC>xX52Uf!gn&Uv7<*PFaY&5V za^tP#W$?U=^R|VNQ(JJmh_B~q@Xi)cr4qFnC~Ip@yC7O-^^O6zch)KYah&y&m;<Ph%gNaQ)0YSl1yjL(raAE>RqMA3nK5>*D79i z-KQTUmjF9?=uT~hD{`r%zh>&oQbM2-E!Ig$uZ9aH;%slORsH zeK~jpuCpv?0Xi`ioa49spVLxw*JO?1sys;GHu#+I0T^}7$57QGfRQEvxBwX(i+TQJ(DV?@edr0%dx6 z$CKM8HLTfgkAJ z2M!lYn^0X$@vwFmvFJlTNY5qZ)vo?6i-m4b?0cJ;YDF!130LWVKD!AW~zgD-wnrs7_`T6#B^;M`yB;}tRJd@unH1lU0YGr)y7a&G<_F+Q0qtuQD;;yx9nR|7|hX@smy&O-olz4rE_FPTU2wm1P@Zz+15eabX;f z=%2E9d)9~ox;`}H9)7=BaK9NR{TRNtzGW=ll(KmT>I)}o#L9Nm=BhyIWs#C}iJ4Zv z3;B1#pLo~kqfLNxE!aX0c+e_}0w=<0d^FFh^W(qQ00oXJpci)|0IH|+4wbM8ByErT zX5JsijYlT&vnn*Y_3jsvYxsS_%IKet5J8B{uJ0oO#yce zlIMVvYL^E=yyX^&f2-$Jp}E?uQBniBi?0k{TGQdH*+3C<+WRqSSqiu4j|pYp4%DP& zVIPRJO@Rg?PivZW*ZAdOPdHu7e(Fs>mcrzZC2)#1R|mn24vAE$Ug(bzaZy;F$*m&4 zod959;vw4_y-lwq$w2a6AM}hQ0gNyrAfSLmR23E;2wAqNX*sgJKu6nv=LifgbSJ;{ zA1+Y}p#~NthzCVe86SAD9r|+=x#-4kVXwj~uYZs(GFEN_g15fh4eB2qPU`K235xH= zGG6vT9BpH0xw--vx~-ZFH>}c1Ak);fwPUc$pF&yd#H1~d{9?$VooSt$f`>iHf)7+yy`vpKQWFYT^Yjnu z_5bGJ-xtquIQkAys#oB~W4GeK2ign=;8&!gI{<5YlNxTsy2?4?JLqg^yZ~IX__$C6 z@?DXi2E8Btyrf2{i4}xU0R|LgoWkDy_l&r-N%y9`87kpDfFP>wjh;vGs$9rsB_hO! zl>7ATTAC@4Q&Sgd0xsNQ-ks6iuV3zi|@MhSEw#1N-17I*Gw*`|H>z z2t1@AXcU$PF842#sGlv0=AK;z7DG!mNTn6+ifB$5`JZdF?TmhGA2XHv{|h4Ra-J4c z{sU|)6+ojpx(R2kX#|=@1EI`md3@7t5+oC$>E!q}*tW279$y+MYRk*m7^Dh0xRJj- z6u?UBfajw6) zH+X=B*V4<@dYvLcwNdNU_qm93YdS)u^bk(!1*6u>{v{Md5t_dtxHK6c0oe{p#k;z!b(Hh?)^JV6;h zq!8yI@M{MCYDm87fC=++_^Bb&2$p*fh`KsPJpfIkV=W+%(gPViU85dUeceSeKoK%7 z*oC)*qHFvN{G9Za9nit*?FZ}#JH3D>ElUX@`MW@p)STU64X_SP9J?0vaLT!65v^-Z zm(8&-*0MQZ0F5pD7K5c}4)cSfv-8mF8+G@uqV(GgJ}>1b^0^7`3?nZ6_mW{}usrHq zvb0)PHDRht@PTXekTVKE3KBF@Rv=z=F^r>ZrELXJ+&DuSph-pdJ-8wY4g+@T(;X?w z#lTLNfA#v3Lzh_j6RbG?H6QI7n^o<%#>I}>Y%@LIUbxSJ>nw{vHtimh4vb$r}S76x_TDYBH23@nFH zsCF@*sx$BnRLVvY(|nbo)!3JFGXF^3-Xhw5+J- zf%k|Eo*trpz70DK3)x2|Sd}JDd*1=9ilO+Q(}pEVT#?2-a;n0yg zn_{(_@~^E}`Sd#3k-*A12r`5x5TwGXY{n!Dsq~+Cy~BEPdececdna{hqiuCTDx^!3 zRPX=E4HB~Pp?cIBhSd4PhP?c#xfxHH1ZHu}L8a(g488>OqTQ{Np0bwGtk!4q@( zzKHpDh12z97dLySgp5-F;*zo9+w)#$749R=t{8xp^743-iQ8=4cd5fKDV9RUSN%5k zWf@QUAFl*X#!q5wlAfmqLE_G*W&E%MH4&++b=iYb$R3+y`ADLh&;D;W-Hs~R{tvW( ze};Gj8^TR!{5}pBBFwyWu@-5d4_0^~OWv^Fs)uDzQb$>Qt_z4P3!l=lJvs07b4 zWnz5&Uk&2{5WM0WB>D$?_*PKzeZO%UB_gUO-VoarO%V4>^l>r40RiJ3X+L zR?t9N#69I^6fyu)vCTr-V+xM4xk(zhqhG(ifW+7c$R)RB`cM|S4dS>_taGTQ*}|qB z=lI$BiK&=)9;mug_Y&-_=)}B={oVN~i1x`|i*YoV)1YH8E+s*w13?~gj0NQJmpFcK ze{u%=T}Sk?ae!JpT@$RMFx1LrB~hRa#H_lMSJPGJgv@$#WfX-MC8i;}acPUDNf_mf z`}6pU+vmWF6SPFks@BEK%ei&7V5Kk)x-4W|L1g*ahZtwlb&M8VyY{PLS|5E_r3EgC z`@*H0?00->P3OKbEI26)kie#H4aAr61@<46fOIQro2%M?K&~h@vlIE5Z}3p%#VRh2w3#^&XCzYaC>a+lKtsf+w0riRaFQ$q2jczta^jf#Pog% zCm-21321)4Os@lyUmpjaZt3LHHBE}q-|`re&+Nzli}C);5r(t>BEQQhuWdorgY?79 z?*Ni?2tGHfP9OZ^omadmGswyYq`9>{v>$UYTmUtJDn0bDC;I=5T#OrM_vC5BJz*1J z_9lYF_U?m>W29lz#H(ClKih`PW$@F>b0I#N1Wx(49U+ha*;~XB-NLHty73;2Rlw3w zM0W@fw}LYyNg4>dEgpNp#SZf^1%zU|nOx}ANl2!sHQD?BPBuh9tOKW-S{jQc43JSD z4dHI1mC4qSE)U@NZIBF%MJ{CouSqD#vZ1*UTTe90?_m}J(Ic3L56!6I)gY{$Sm(9BPo^|PKm-!x^8dp6x(p^tlBjeX8N}S7b)r~W zFuBOeFX5+lFhOx2QCsKflqBo>)D7psT9NIWX-`t@g)ZI>Or;$HQPBmo_`Lsg-G`Du z{k$AYfXLHVY4YU~m%20bd5!p7o(0mT^K9Qz5GUOZ3q4MZLYn;?h zUT2aB!0@i|;^}PuEZn+R2~7+Od8jL8fC8r=gWE{-!VyFdo8K3{>VnTN-d`h>u zmj`%i&xMA)&a$x6LPEe9@B)hn)z=qZu1XA4csfGzmDhR6X@0oxq`iz+PQt~uAm3-~ z@S84!{-;*lOpz#6sd0u#)62uE$pKVA+3lb&i-4&zI*V$uFFF2xm6DdmR&mX$|0=4& z?I5&<9I6DaBtnv_ipaBWaJ#~7cAq$gW~Lu~B$pg7t3Q_#5M=sSz)ET12;!McT>y_M z2_%MFAU1M2?-+8CU7LqWfu|Pnw;UjoYwxFJ}UfKV(}&sh$7{f-k)o z%YP`oWelcLgSZa}Z(6fUlINdXfzZqDdPq#r=3((pmcox9LYes@$kZN)h^2o)p=u#8uuzRgJeUlRTe{hN`$sG3 zP6W^Am598y)l?(%O2B`tB%VsIs8vI;EEOLGntg|Lf1Z(J`!F9LVIze-hB95he%Dxb zown(h_eC_dcS4TAz^)-)>3(wWW5)5}6RlD4LcAhkH{uwrP^4vfy~fq2)s%_2ggRf# zZo}emIJcGnR8nCRRB-}2!qpNTB4u2bA}8z8L6AVNmY4(W{830i-KR_-eTUFnlh zV`RXqY0@8sxNk_%jq%g|SnjFd- zAra&*RuX0GbneuR@ln6p%ZTrhm*ae!^V{z=7Elps{^fsN2?&gz#FJZEho`|2qQPwt za|lj7lq7LDFgpO~m+-vdJ;nsT<3Ah~4Gsi_vt$@1VSfzQ>#BbTIe<~|Q*t}}X3Z@9STe0Fn&p+){ju@!>Sz%3N56l z3jTLVvHQO&DaKpRA%!r`O$%sbQ8Cy5gPZhBAZH5Aa!vi; zBt^-B5$cz8BnOLAA5T$T+y9DNSkRjp zZ}p%NChCGGIQ_}-C>mF;mfeIpPm9fCHMLL$pyIjTkJ2W1OC2oQg$kB$$*=ub4q_VW zBPb42BqMpUWwz(r1W$zmxLV>U(n(%pu+N>qD>Ny!It1P1b`8;QNS%aSf(B1lTVq!D zo+sW|@1QA;KjgjFC=f)dCdZiT_qBzaUoY*pLG@eL|GB0}(`WjYCPZ>artJLC z24&wXs_zgB$(RxU@_JluaNN1$?6|Li)R;}#-oj^{iVZE2X^oRbXv%bXl+-wa&rE~h zbNC;A=Q9=mhoSR72!w(MVa(@uTCXCgm%h3MLnx-^zIUts9Wf3kA_G@(0J?sjj#z{Z zDTbDr=rXLO%9F{U0?JsZ8&~}=0owoB$g*-L5V$Y|dI46B*LNKvz60O-KOFagUiRN7 znt2?#b+!8uhl@c%QwRJG4}hD4@S+=dD^3tqcv3v#%i4xyDvdnjkX|UmXdn>+T4J4! zWKf&ib0O@8{pY;Kbx8gQqrB{akR{gn&S`n%vB9OX{|XB3%`*uLa)5QFh_rpjz8w%e zmiSRv}!9idwk&Rc;5>Vt*6|5Fe&a}0ds&|)xr13_v;Nu8jAJ&-+O z1BI9Ba;wkCC7^+2lfpJx*0FW23~_OW0|vGcI~sVK0}Yi5Uhf5i-{z$)wEQHLcLF2c z!U&?6b^7f{Zb0$$1IP&m$W;;xG==F%$X1LM75swctgS8ce=)KxZ*|6+;ydbE=6;ofo!DRX#mPYC>#Y{ zUH@98WmiC`myzVX)$B+f+1>xhbbD@%K$%m7DUIjkvp-;j%%-plhTrJYc7a(IJwnq` z4GkV}qPZYJU=c_Zl;{JErM_Sl?Dq*Uu;&>&P$0M=OfpYH<<|gYGm1tmz^A!5|CrI% zFUtfFE{De){r-k@G0=1y%ME~ISXAZJG2Sq5Gj7SbW(O0dV>T%)YX@~D5V>NTGl2pl zZ*Ad=CyepHLnd&LkSd+c@3WqKqU7}tty!)Txdh78(cS5i^0KpHk2lVaLjZMkvXx0) zPU;~|=ew&&Y{|IikEJ*kNF zJCBzpd!^9;UKKqBU_wiB{r8(wiM_tZe?UxdYLgVh^d(iJ$pLZ=pLE;de?5lbyk)MT zk0>`|aV>Bjj_3NR{d*^q0`(v*pXhYm^5hJM(7#g$`)5O5sIuqOG44~v@fdwfB{I>t zwA>T|#yQ6>u3^rEUDYZ2l5?=v12#6abpcO)?0D?Y7i zNkObexqQT%bdlxgbMOb6f+iqj_)cf&amDoM-swQBt#G!xU4tC{Oo&hCXVCDM(}h|e zGo1wWv2B_AlYloV{?~BCr{+995PNgAe)50aTxT)Q?C`vr;n7I>m=+Bqmj?-+yEYD$ z?)-r!!E=t%$5){~2Ny?dd5_XTRW-^cGk-hs)97dThiWOpK)4vE_c46wYi>EN-C%!L%S=%#N$7w1LrK+$ z)Y42cY3MYiCrl4mr01*SZ2isaxU~xTR6UjKF7ye35xVqn##`YmV*H*dY%JqwI9B$n zx*Nqsw^Z2a73gsc%cGOBT1)Vk@onD#MOlM7n@(?rv$wv^LNigE0^y8qeiL#6xA>pG z! zlEi8??ox%=0L_=*%z5-;KwuB3p6c;^q^%4RF*=!h+t9BZ6QkO;P*%i zbkZqvoP+cYO7C}HncW9s+w#31bcs+j_gb0`PdvD30iytea$q68$jkKGt<4%Hy)?>8 zab+&6ihMIqjy+GM9O?CAGkC6X6mNthBMX*p;CI5(P{cm7E}#U<3JMqSH@#ipikTr~4qcfQyH`yL z`K86`6%bDoA#HW#)MCKpn-Oo20LfdFy4QI?)g)G3PaTHr_6wcl(nVpXK>DICJtaP}5zP9JL%;5)^TSeG`6YfzK6n(& zYZy6h{o;+m3oJth3JVs{pPXMXRh$G>npx3y6Nwh63l&Ue1fcT&s4e{I&HQgDHkH$T z-!VRo$K2Cr zW(Q!h$>-(IvVhDvYGOj4hX){7&vGgC2rC&EC%g)FE$+D4~|^m4<{M~BKaMI(U`FzwEA9RlGK zetG<|K0J3MulKnrs}Ko9{THX8_}Aan`-uXb;BS=i0GuamQQvl$(;Ce|rY(#ySe43F zU3lWt6)?SZI{8b_gNs^0&T;pLCV1FH2{Wf+r&j%1?7n0KT)fSPoc1Qbc$?))@ep)4 zKOw;_0t{YI^9iu*b`&ano}1Z_lasTvyAGP?#C*lE3*a~z_dE8Ix0#(5_e}bSbS$oZ z9tg1_>gQYVLp#i2+NkQ;9u=LO)|z5{P=QuZUc#c83{)!q}ghi z*&yFMK(ZM?p>2nfbck35AXY5eX+ECl5gf?>v7r{=h5@~(mv(xc=2f?HV!gv)Qt;v3 zz16?11!OYSY_!m@xDXUT1b2vXBMD#b`>fZJuw=1XGe*~C6-5F zWW%a-Y_S#%SZ^*O#NLA^gLC_JT5_QP;F@Aq?4dluW#UTsYSE!nmxK#znduQ&EJqn= ziGp=dKR$_Nq!?s%>{OLd zalWD-tcjQssFurG=9D4h9xx&};Mjnssg~SCA!{5GiYEO?^x&}=ieSq`{Y_~=dCuI) zIKT1P>a{o`{zMWjGP=+lDJ$gkqNMO8B44xLUw5_TF8`7U47X;rq@R}Aw*JlWv z>X=%pM&?Mz5fS@}>%lF$>06e+;+-cCZJU&FCiDafS+&VPTDvXGoG^7cC}o&?RA(e0$eXj2M^ zr@^k|Z;q15qRA1I53{~sfbzni1Tnriht&JIXAe7}8NCnHah^h{s|3Vi(!ruDTK`U^ z>-r&6M$3=-k=9DuZ7>B;aswSi z5G%}uZ8Y{J&3H`9>j^MUk7BqPHm4t-i8f^td;6Y~IO*et13c-(`i?p;hHdw@&TT=2 z?_3`s7*-6S9~L{0Ku=cU0^BQoua^`mIzTK?ikvkzjKew@SPO^9V)fZ6q?w|qxD&VP z)2rCu=^Y+luMoE+r?D2H**s{bHXDvGOsS&5|D(B3Dh~ZUpw?)9)aUT~sXC_6s%z_bjeS?M~i+ZG8 zDI`d#f1><%6q@jeGLD6Tw-B9MPwU=yRCw%1*{iP~X@{qJ2{SzVRJ(`9|Au}qGGqbY z`BRM^I@PQTkBY-o`XA_Wc|c$tQMT;x52SV`UH6;*My)_s zBym_%nH9rk#>%n}GzeRMgj*#)4iOd-H*U5yU@XZUpA3+x`}Jih2kwt7Wgyl~VobMl zfm0$+w={p6Ta|z5CT;`e?=_ zubstySFLD;Q}o;d%(|)jT{)`-s+H!_>~;=^qru@>#1Dc4v@M ztKIQdx=i^>qfziNB!@j0foZo;x&||iA3X_WSTnCduF7LnApUp`4zel}n`EiJBN>lI z35>>E;C{!M@|x#S3{_gC%0LXn#*vyIj#{r)-O4v}dVD+RecDLBy=n~?!8fjrEy7FU z$Zr^K4fpeA{OkAnE9t(JKlKNK--N+A{d3oAcAUA}vJ9>cCcORdF1rE(tV}4(B$%v; zxTTdu4*il99yJLgff8ZR|$p$aylsxJj1rbw8z#t$OiSy8QI3 ztYVZtQ&awUtVg52c^y+kGW1&k^+ju#Gqm5ATtczatBg(mNdH+|t@! zuB~iL$NgmW)l!HpWqtO&Ny+2I?luNdb-HmU>Gz}QPS-w&_}L7-5tw^8{g7^0FI6p_ zxp3CE%Gt2n^xP9LGQ|b^{raE+MO!3AS<8|OPe3HC)LdntW)n|Zk0Py3PoMW243>w} zb8X#{xH_EM<#9v3tF95Y-_B*I zh_0f{!{Z^wH1}vI9eCd<6e$U=5AE;q55QrKE$@eX`caN%W=X6l>T0<9Qf_$D;ddM; zFy2sv3tXS@$dVx?_1ydm=v^&$4LaRal4OFJ1y}s^Tl1cX*@CS;q)v8TPIlO|@yT_} z&i-Lf_(sfGCF3)Tz#Eg0ywa8fV;>y+hoT>DoL`=kq|`ZS6Hv6sVqX@Z$I z{NLL95^$>be&1zCWr{>9QJD&bGS8Y==DCs(DxxTqnG&H;BqAxpVi6kbL^6czQdZ`a zh(*RSXIi-5@7nMCo_o%@_uPA*JDqc%`#$gU?7eHP|61!m{C?l>XZrSj1?;pLl!Zw` zI-=HcDG|AUta;&3{uk4YgqK*29^BG;1=?8~&L`j+jU1$HIdC(B z;jmjV5M4vCpVUKv%hdq_v(kV+&kq9ru2JaBKM6vDeW@VK^G!{vkZwag1U>X-a{TNlyi&G`c(2eC@_0=D`HD-j_dweFK{s z8&J3`wN5$pDB!u z0$P)l$CYtGS8jw~{;b6?>_~#Y1*+_ruZcG5ioB^>Gw$&DqL*$Ai-c@|@{i>kPo%Hb z#=TT{`t7Ln`H3%|FI|hepoX3y!Syy<8c!)TjG!-u_a3s+8fk+LWw87A8-28B%XwV- zTdqYi%RR~2?!<-^p(CCya?N&AqVAp@Bas@fXm(_^56=I2^?Re|9s;UC0$W|dT>EeH z0%F_nuAFHB0xoWgXGb9Q^{* z?x*zb(L3|MqxAnGcT`JT{yc3Z_@@x`<*9;favK~a0}3r^f{*oQs5{~KA>=)8u8SEi z7p@F|B+TYy5+^-e;)ori)(O~Q_;J^Gr%>IMw7J}Q*Z=}~n}Pb=xSb@BPsirHZ{hHdc48;-JTkHUSq z0JN2-cN4T%xAPK0$db8BXnD_bL&ch(`O-$_5P0z?t#HP=-ES)RS*4h-kr&k!K#b9v z_o;9P!#4O^xozAgcQ%I-I1Vg1AHoKbV8Fwuh5!6_9L@^o{y%QA?EnArbe!TJk-(d6 z2%lkTMuz|@SW!6DlBJ(W=c2#|WBgkI+W<6ZXChcB@7g8>K{?y8`@t;MwA?yQyj|mS zwO55;>?TG|03Y)D4l->;*a|@-E4CGqc3nUuXowwV=1*i=ef^QFL2Xl$nf`9Nxu^&P zK-oy2oWR7nQ5!t)Lxto_mR{&bbkU5BW_u9a1`NY<9kdK}@Vg$+o-Z{K#Z&JfC1dUG z3HpS0ykXn~TOgjuAgJueM9W0{FPDpSqp`_x4ie)y03i*tNM`y%tVPi)IxYk~+zzma zWLkaohaICCQu&hematRS>#=i&7!iy#${wbJLzS6lxfs5~mYxU!s*q9u>T>FCJJ8ED zL(3fxy2AvNr&ynxg8Eom$O=vF2hx}Oi0`uIqEf7F@ zuvig7ekbby$~j^UXr<@WEs$EMU0%V&lPQy4#Ajav=Z6u-_XZXI8H6N@bUt~fO3`_X z_Ul@!KAs-_4g_?!-7`PB0J8fhcYDU&zwJ4RJC2fi;ZG=;a7)*=WAOKUO;A1cPzhb`de?k13qUE z7;={$1(M#kt0L-jvI~jM+9EZID*QiG-a=(;RHP#FLq;kpEqrB#E1|SKVAR+d@_@7zsjOa1(Nv;0EE_Q#l- zLXVB;d4-o44CV(3{3m?%J!e~6`lx{3e3*seCUx+)L(UlH1PHCbZP8j_?q@4*k8c<@ z3ta;x!cRTP{5wKe-comufzz>oaPv_as`K(+bb#q2z^<9A8#>#)-IzCF?$L*Dl{tl< zmA=nCf+Wy*uUI6P5z=5PX#g0kGlJOX-FE0`>Fg*5T%3At3~^fs^{xLF1fdRJAg2^g zl^0<4cM1{9C3t65F^AAORBC!h9VDwJE5Pkt1g$j*0&|pJ11dsyVxzDttDPNUfl#F$ z2$d}LTp>JbWbi1Ep)hj6yqpuk84@z#T$tTjVm=N}sKrRse*BaqK7eNV{UFBnz$*mS zP*+&uCmp_8$MQSWmGRa^&a%X>$Ki=>Mx8bA;KkE`B4#7fk958Av_C*`&1ggY*nlbS zgtHmMv-YUi49JUyRztm8%n1{JKrT0Bp`t?n%+k#kbheQoC&#FTfAZhG!J>MWU&YJ0 z6z=(EG~8f4(E@>LULf!sI0enE3&75WO@KG0LY_7#Z!waFuW4}wzqb0eFp`u($dmcuF)fTl z_4Y!1fgt0U_nfs7oiz{#m8C|@BSIo^Gtllb-(oOVgb?=PJ4Z-5`8nYU+*w0NH4geAxd$a|wU@FOzIA7^3L60=!^zmHPs5$x^0EO9kOM2I zkRgpc#f3-9wB;D#B_eTR+pv<|3`?93o&PQ~@?wec#{=23uo94YMz<5951H5(Bl$l+ zj}G&5Piy&Hs#a|@07Tty7%VAxW=4i(@bLEGR%aOOUqV;VjBXbpO?dR@mw`Fek5!x0 zMS6~&x9$bEQhWb){85$c`NR{y0w7gM4~Zs=f9msS5cpC zA$S0*YVh-Gu`$4PK^Q=V-2BL}34C_)9CftBro%72e?qebQt7m+w=f0(p_a|S#M09G zi@xq^=R7BPfRVrekk{P2Z+KL#P?IwZ-{HHZsV>I|s+SgU;wuTPg>WShFh-2_C;SBT zS9*JcfE-pCXYV%VK9Kk8Ax}@2!h$*~nU3zGoDLEaZx)|t=hYa>j34e(qMw<=ZJB!{ z3HqJ{PC+%LnP1=h%e);x{qk4gOE_Ib7>91UjhPvbA7%4bklltNy7-Y`@}0GkE=t2d zG$G`bUBJ4YJ92=2TWqpTJwoWh(0S}@A9YLQJ&tovRlRJ4XmKe3iz5r;cvQGJ*$m$? zXZ<%ksdN|8`3hW;ihVi(BP&xokFTm^=Og?cY8F4Tf^OC!K-3HX=sQlw=6?coNY`|5 zGG(xiL*tR3SB+mdWRrF3st~|E&z%J~mLCAh%@9;@58W^2Za`HMF^9tz>+gPVJQ*RI zb`k;rb!$S`J(#tvsVeA{vmHS8v!IS1E^^2et_rxe%s_}HR%-j0X9RyNVV?;>2~-R} zBq}2#y+PTfbRvfSC`OUsa#cXES4lJGXdI`iz^jaGvtjXVp81ULe7G~Yr8NeC?$0fd z98V8*13-|j$3}m@ZwF#~XyH_wwFC1?GdRr6|LHkrBW#3*wFT-ma672}nr2Zp4pCy24obWQdeP!dxR*VZ7;9J`l8=2nvFN#^_tAw?3>G#`&AW@a2RV z83H@^IL5fn(|`Ad%)eiA0w8pn21t9Nh48G6cydUM3Z|udsbd5WVP_OP@P)vu3H8mq1^C>vRDcKnW`iMIaM|fuh#yHB^m}Rq z*fFVI7O$;VBd#!@HX>p5smMfr2XgCF!~@FcHJ}XBst~*#P)el&7}XRNaFv}%ZFTov zP}^AVkP0%PX9b+}k2&pZIDEDat%9ZC)1s^VP#7JMmjkkHd6y=U6IfirZD06LH=-pW z9GuA{NUjydrHbwYn7(lwOodILBD!Y_IYmW)9kl#o{b`UO=wo8Q!tb9cSg>OLLs|-~ zL9VUa>5qHP0E0b|IS5Bq1fhH{8)d4nzy_44nDC=InGm1d@O~7vL^exMJXGjWuyQXM zahF^E48F&v!&}bX*%FE`#QPV+_DEQ3<8X(5%4@idhG4_j0~(0pNkBFLuZZi{j+KjzL%0(n_H~Xo6-q4XbPc7OaT9y_4{RWDev#C&+9L z^7_?T(&PzKEcPeE_=hG`z^|yovq8dm1WdO7&$cFl2U<0L{N4iKh#y7*!scpA9@3Fx z!Q_eI!JYJ#>@BYGaKW2U{IWa-DAFT)<`7;Eh*1M^-V4qcB>f4e1K{4f1ehAuDXD<_ zl}wJ^F*kZQwQ{JvJJ)lS)m9055M;{~VH~Am*k53R2=*6#AQ*0fbv@DQ=*h52Zw-)b zprE1%T7|aOlmFN@eu9Z@qY>aB7XewjF)0uXTHcFj;+4psLck8|S0KEK{&6$GHww=J zl^@3B$nVjFpw&qv0|`l0ebAiS7e>Gz8-|AZ@UI;-0hV@ne z(A_hn_Y?FhcE{O-;X5XsVaWyUrgRqY49e%AQ5^y`2N6`E2G+1#HiJe_-2W@?IbWbq zx;RHKk7o`+9+MFGIa(lx5mlrZ^^^s+bv4+^0*A1|sF!+fFbq#0Xcgkc{1-6>dI6XR zOwrM_jHM^?kq*L9fEFDM2!^aG5;Wa+H9?xN=!N3=x+69h4sG7+^rwjXE9Bk4G^mSN zJq1$qEp8!Qn>hrRakQvRQelaSDGtCm3S3qJ@3MGM=Wd4a-X+8Cu1Vd3RYaZucys-1zV#ou7gC)$*Mk$)I`16j z4q^I-%cC$pi8MP8JzNalp2D#KsJB@FEWq5<%Q^q}#lR*v5+}VE8{3^*Xhu3uJNA(B z&E?~O-%`)M*Fh5{w!w1-s~?u5dlj_=ZG+hPT7QO<-wg2dUDdV4ZXM4G*3mE4=JSNl zK+Xqff{J=``zDP5pYk*uJyZG`b*btwjKFuf;{3N#!vseGC6Dh1^OzhttJCLPe2&pu zRiVZB5mVz2D$jm5bIJVtL+aDOuc9CSP}LM_29TIc>THxoa|LX`!FB5|M#( z&)$jgeCq;p`K|Zw-{-zYov3dr@LP$RM^bY4;{l-gf_^auGH(Ndp%NgW9gjAj!wO9! z(-al(dnH^xn@qZ~=fCcYZ;=r`3zCg$T(r@dhMK7?$b*r;?gv*|4+Ag+3tpjqc3*>1 zMw4qe`=+>s*)jJuC{!TqdyJ8i*z95QDP~Mf!|j<3$cxmvbuqb>C=>#{b9%u#y4-`5 z3ok$V3=04LtwZu^F_1;x15(PZ?pWVg^zlOusS5q+^A4(4nBK+i3DPK5h2a&T*S> zp#cg#i&*#OgF1r@M&Kb>d+1xI-(1i2W{hn^2&eJqO^}Jf3@Lfdm~)R|y;5d|h1T8= zj%bFMEyAma^2woH`1(HVfOEbQLPHy*gPmrD3(USzR=sfT$-bjVq_VqZ- z6QbL)po~rw*wF=Z5QbpA`jBu15ZEJr054g5IeXeDF^!;)OUz{B&+%7P0j=M=I|tRM!*6ckClso^&h0D$cLfh)Kh=9vFY1K5NK z*+R3U9iCXY9}06oaF^l#~%1zS7DddvcF;SNH#Rra;s;zz)lGSBrL^K8ntzro|lsLMy} zsr*+Pt;(TA6%fu{6#xFY3tP{Tb{OLDw{7;>X z@Y}Ri3!SNL|HK5&T&H;1M``NN9@`df81f7R2!wor>T)~-)h^kIP)^CGf+m07@D;T6 z7=L3179#j&T1@x?nA{XUhGFwm739xmA>*mu>1PJ`KU@Ga_z(+#%|hpPA2X1Td!)Bs zIh1TXp51c_c72OafW#K)sLPK0xho|+gq(NkiUt@=(njF>OfS^Zi-4_3X2EmEUfb?! z8t|jK*w96He_^u0fUy9CZEaEG>So06O!a z^B#~Sb8!xK)J4NqD)!MKyIQv~R(b$hIj}hRVG$6yL`@+}ATxaIgP=ECjZyMX7 z2nRq^cJb3?ctn3xo`*IJYc@S(NmP0xgv}u3|Lh;B7?+R0Rhj=7lDpJW95Y7mSVB~q zA2L>QdH$}=S3vhvK%RukDu5;oLo?`4H#x8BZ^IEd(4-0f`*$>Q60X+&>K=tT1b^ed z(<?_zQ6U7a8NbdK_w>z;J8@X3xz?&}_~rgp-FKQEllC5b>h4D} z1YOFVP6#oWF#nr#DgO0A{x=U%<&gNdH+5Hkbp7F6PHeI;W7pdL%BrUA*9GEW?an3# zjp9$2QAU+K?AqBVpTqzL)QK7)pp3JFnlRuJrNlT4kNBkvyFN@03D5*)B@2l_`SYDh zIB}4U?E(KVFQ-ZeAan9zxy2cSWN_NiYXnGG)(2X{rxZl7O|To^rRQ`2PUQvcZL6O@ zEMS3Oa9RZ{qD>gbj9?Oiq=*rdu+za@CNdDMkfz|4{DA&31_sbZ{a1Tw(-K9HURs|U zgFfF7wS?AiPH@7+@nsqY>7srO^+9Np%mkZ)Lv9+K|1gMx6c_BWIjQ?C`Yr#!yg5t& zMGzj4TSw5O40fWpj7**D4N7OcC%y7}XDHf-=xa_0@``+aG;QrfHL zWXjA!^%=jV-oM&d|G!l4^xZW}H*Eygb;nCrfP8pdBaOgx&e{+hv!W2+c_QF+8qEw*)n`;%*46IXl~;udp)o*T zzHV;cwOcY@bvta9+Mqtb5i46! zL9F5q5Rf6dMZN&(Ic3rXOo!%wVN(a#)}idD_i)QD%vUVYuZLSYrMLt$Fb{1!AqTa? zIWh#!m8i&+<FWbppf+_Jgf?N03IehB+Zk$YY{HK}}65ml3A~w4~Q&{$T~#!#aq2 zPReuxqPh17`{7mJwO4Pw11!tyNnb{+gF$W}At@=D*DQ>;CxLVi8g@GT{6D;>)8ij= z>A~bso@(Gyf&s&1a8~Zw0Eae!vIJ7Ps@>+I$g963{Wez1kq?T$(!7Gw_))20! zpq?@dhta^9&ZCv^S{>bvM4>VbXKqe;Vy#l00R^fa;OYYomFzG%x`m91JJ`R_9StCL zln4$2hk|Zi{tN2`Ab{u{3J^cT(@G3UYNoH%!Jny0JE=>wHt+3Mm!G z3tD-_H$cpK*>@S__LIN0a0=3JCm1Q>fHru6QxviRU3)>BhgVCkyy35A zw}om|4sa_Kb)%5*0=do+z(A#f_b`?X?=OSQ`z*M-)OJJRPY?X{EL{fl6GOv^KpFdg zfrCy?bB5nyXi1%;#YmK+gmT3ni%poN)VgdczU<)%>gOzI^>%%P4ak=@ZSH>TTbSQG z7TkGqMw+c|iQ+GH)QF;Lb4C(t562NLR^oVrywNCyd z$SmOwSrP@;xuHI|)8$K%%W`LoW@RvZj&G`uKrzM&;?w5>w1v^%L+G0E0ixTK0MN^Z zX%_uaW5723d3^26Edp24>E6QYS@UUUbH{hC3Z*;6fU_yu=ra7p(9dgrFqO<;-$b<*b1?F9g4SFKf*8NQ04+&<`!36= z+A@;~rbDzHbI_jOk#>i6>w%pU9I&A}M3R%6#eRX_mLTtqOAPJCNm?(zWv48&)Q_;J zlNH+~w0jdPlg8Y^HzO4sNYmWDoaBiEt}x{42-{n=LPAS4Oi%tn8Jc7I2zPIC%6Kh> zilW{xAV;pxE2R#MgJD=BxZpB80JIs?pRaL3%Fmkn-(-U?lZk6gD&`uPNs()Q_g=rA zWGoWC#sbGvWCM~Z4BfPmvIx8Ehs&Vnd{@nib|sX?O$rHO15XPj(oDFgZeXI1@1g_) z1F6uti@UnEcjKG6&<2#WU1!qCy2L&bt=V=hhrn(+v$1!bCT7Dd-;6C71E#5As^YD5 zGCX4Uin;AKOGN)k{&i*m>I-4a)97D8(@!%Hjd7Y4s3@Hr&@YVEt3+kUx|wLr-!(P` z7zXhLvL~A{OjENxlE#IxZ1qN5h~Wny|wp^6~B3z z19kDb2@0ds(DI|=;Wul?L+AQs#3Eva)bxT+z%7@~D)VBaCo>K%@C_u8ehv7scjOmb z6xJBa@ROEM-%;rbsm6kWmq0Y#N14mjn__$`eSPA_sSm4pS7I~=Yot?vWUfyLHSOo9 zELuADx|HnGJEd{AR_Hr*pZfAALCrID)2gPmN4^~H?44=6FTK3NrI}Zk4+cBm&+c9; z(WJnMeStuM*B1-%_GD?YGR%a+(;_laz2C(MP5m>$G-oZQuHMy752i|or|1K6_eY;L*qTXQ_!L2d<5M9;QPec$3>2|O^$XbRF%(Y$i zw9QHF2@A&34=z$}Oq0q}wNll?2S{^jG0G1dKD*Zr3Fz_efQ-9$)m08J$AGD(kA|b& zaOlL~BfUi-IcJmQ;0PHm`{76mwTUu3G39ylNhGCX?wG2?G_I#*Zs_z$?SLhsi?(kL z@U;7{=HI$XT5&W0Cp5oqfqTO;taOytZp<#u>Zufk8#w5}3vtpMx!@=vXA6u+6eA85 zMoj42(RT1B1jw$lARlm#u;X`hm_@HGW$pSNJKEKa3BvH8+xP*;H0g&Z+5=&U7n>^7 z8emB!gCL<~hm0+zpC-6_0*FS@^r{KMFh$z^?UgX0*CKxw7V$pKhJ*b&7Q?kdr_!~} zX}w2TS*TO$8rd0Fr8T~0RO!&-jx4C{S#<@(Z6B-b9`jV#9A0(#!xl?<+6~I0GuQfH zc1HWj6)ihU+Grq%b^2T`rSBgDg-Jv3%}C1cH8eeQ`8yij!u&V2q?bLWK)GVMz+@Ux zQtG(DC?ck^@e}#%oFbIFYTpjr>>Ps9WwU}?Qt^giZia0pl37B89RW9bN+$H_ZnXnt zb3%c;S|pdkWW+dP>}bfI2(Vy`rf)+9AZpU9{bkFx{b2V6Fh+9e`}*n-$2l!G=^&3? zkwUt?*Ap>-ZspXo=>u$pKX`g9!=H|r z(oDrj-Vt`|Gnz6oH!#0FLxb(-!Advx31Ll;R_GPRo;Ge&azf1m6ez~gsl5zIbCdzz zywTq08b3aCK?~FbYjm<^eTKO|?U5*{Bl5QL1);QA-r%;F5I7$zKqy%*l$49Scn*kC z7K9o0r`5(nUI)sd`qJKIczVztiIc7=fqua=G$&`rW2NicOxE3DFsOk*lgTB}CfU9K z-)Evc$lM|GML`nEossu=k)OdnL7#HpKav2W-AhLyKSjzFK;I55(q91bn(!L1J>O%{ z=1g5u_-r3aD{XSsi4i#V0(zkzr*MhSf(b8!8K+P!*n!hAy2=ST^HZ;Kp*T&aO`!5d z(=!nkoM#{gE*b@5X{X&K41vS3tJ1htzfW5M>)m@an_|ZD0A+rztgw^isrF`>KDCFf>&zNFRPzm zVAFbj$h6}!lo11zA341-@5^#}Z)`s)n>Yd}wI!KBm;EPBZfQoDb6#N~Tt~J0#YG>n z=Ec^NBI*q_`;U@i8E>BSDmbJ8R44($!A)_JhtkbNb4wjrBcUqE|1zIJNP0Q=UNFH& zR@%-7K5dHf&9rKx`N*AQ-x|s%SdnaP1)E1}&frkzdpZL)(1G`BHbcy?Ol)st$xFvO z#$K+4Ml{VHKr$&?yBr%nCl&ew+D=Ig$xvQnqUplO$Z>R+i6qAselXw!*_?Q?QB$tQ zq>#UjS9IyAQCWF!UnOs?w4dx_NpP}Na4*+NhjS~ubsCe+wiuOMUPA9b;mP!D1b;GD zg}~eKD^V4?ci&XxR@gJjl+U@v_%1n^%??iTGuVBY*!aj!&PpQi5;iaW--4;nRl_gV z!VaYGK)XWf14iOqb#-sz`zD$1FKbt}^d?e3VY_8wC7j=iZ@Z(xTJB((FF+eibv~V_ zKNV1}b9pu{m*Zf_N_*G05P$XVSfQ2A(Vs<1&a^k~VKc4&c+KR-J;Q6HQ(xp2iCa94 zAMxw$kOcD~$#Orb?W}&1=*EkRilQYO<0D|E-0vkjO8eN#^1^-hZci27k-4C>*b$X> z^4NTrfQ@Y$WpO6xwVgCsZGqJE>YefX5>l^ioo%*Z z`aww(+A`Tm%|5t&%ibdQIS?jpI-spfFk_av{HxPPNa37wYc+v8X5s2VhoA-b1FUQ! z`I`zJW=XsayLUrQah2Tgi87xHBE#Tc*=o6UKJumkXU0?MrGqaAr%D${eyHZZw0dYZ zk?)2SyLoTS7S8nnYjp85<8Wt&R5lI(8OT5-%&-Eb8k^-UpJsA8D=RQK@N(P zQ=eC_J@jjK-t)^EhtLOBY8#k6?j$+B6M`CkFn+{OEfqY;9mYYE@oo?w6}`3At0;t3t;A2jc8i3f`JS40dg>!N`y1(gZ5?cqS^rEM;w)=H@~uN<9B|$!t7>`eIhPu2R*u#pn%S)5P=H2Taav@7X?{;5Q`Ff? z>4h}oxLodcO20aYgA)U#U#L$at33xxKC-NHCG=`OP4S<+GfmwolP>_vPlW@o9n2%> zeJS0}lFQT_9+N4pC`rg-JrE}y8$Ke^lf!&cQCj`5rC^{IY%UuE4;&RS%c`1?VsU>G zk+ScM(dh3h8TL}sgA5m;HmY;%6v>QRx#n0S!dbOH?$GwgN>I-_0V)yH8G@1d7X-HN zHt`J{8rw~5jX0{iKRTD~R*1f3r=LLUq^=UFd*!pcZq3meA=FKl#LDxlM%C+~&;sJ_tqRWi$Gs_{6^NCI7oO7hf)qFK%iV9eVyU z$@oG3`IOb`eNb~MGNnYUHtlYSm0nq{@n$7pfWznU)mv292OMR1)Qic3V&&$oCyvx_ z`~1~7mtBzIt_1tXprMhvbj)^-U@eEXM_d*7sPydBN~BeKOG7{FJYvyun`_iW**Tj% zT>;R`Q>{#6AxnRe$w7ug#yb}xJf=f(wKlJbw-C|(lliII`GaQbs%TC!u~&#qXCnTn zvEg$oahgdAxD$ARf}l%DdmJ|U*GoYB*|cQS{cSXFT@njPwy+uiC1O8*-v zggp!5sa|IJ15YZ2=Zk>}t~sJ|_HffJLDPT>j+N)_hYUZUVi}c>DaxKdE~b4fs)KxG z(Mvqd2knSv-`W$`-H*%n&hsjieB!;0mZLg7fAy-&TKCO)W> z(_PSL6x?xg+n>OHZVg~}4YuJ&|jlytU$Eg`&<&l-lMnZ`ya-Aa$%ZMpQL% zdSda1VnxZD?&vM-v9o$jQI4t;aZ$Grc#?e_ntMxsPl<@a5$A5Liq}8^KYUs8Orkw} zdUN<8--*ZaAZeWE02`yn<}^NmBXQ|1-8ztbgR zd6YY2o8x1(w@PlI&X+;_(kTon>?~|f7O&{B`!#D=z3A4y%~G6lQ8)U-z|$>t4!6sN z3}_A$5l!z-2&}FKu7%G$aEjwwqSUeVH;g5i$cJ4(4*k?LprkuGEh}xnKUOzRBi?~} zV4eB~j?c=Y%VxF;@i;py%8wA|x+y7ui#J^fQIaA0mDu{=PNq}IESjc6_NNq88v z*#%S1?eE!{womAE?;@Hl$-XY|_h1p*3PU7hO#>nK;zpGZkGRpzaf_Kd) z7Mk42JbOTLRot2W^{%@#5`SxatUD(tPTHrPD#7I^-#EML+=n|iCTa#ivsjl#<}*lp zDj}=9WIN4o@J>_mkO*19!BvCQ?(J~B#tz_hhtO2&D}jNf#@L_@?<%JKSVCkxVvoME zuexe|L9w>)($6ooY2a;20jZInyu~^GN*r6F zn|z@risN2W&GGWY^}K>2bA#2>fH3db=zfgQWOYR_UtsCu5so(Al%bzBGTCSjnOMwp zt$P2q@8a46{(tD1ovX3xm7V=5Fm`GuPr_HENYx%VjG@{RIN9^uX`6a$E_BfY89oG6%aP4Ikrc2noN9Y^sy?Up7l_EFO;4qdZ8~$Gdnqjr00s?fb(w_JHg+2} zs|}9Dper7Qz?le2&!3wx04Jx#Xp5?Mt(Tzjgj%3cDd({zabg-#_Cn8_utrL(V31qX z)R~28u#ITs(v-HxdvqrHGz2QN-)y=r>=~yuxMM-vwCisJ>S7+`~e54em<1RKm%YY z6)nZ4kw<>b{d#ZTqj&Xo$~w}`lZV)h^7e~Zs(#tQwGVu(0{PrIqxzQqz9?M6QCmuF z^_3!iO0rdnyG8*i^t}YMN$c7ATQNL5BU`NVF94QQH;d&+T%68UZwmnq4Jb<-wC}|Z zNjXdrHG7pgejIZ$`=|sg^xVA*wr27>h*o<{?@CMp30ao2K(+V)eo~>7DOzeq888#k zn&O?=JnduxdUsvVJsv;fAd?^X^3sz>njYUPN5SS2@tG1h4{>6*mtdob%%}i}RZ`W5 z+Ye3t!l(3|j>oxKU!;f5BrA21wR+B0A@5OOyJdCYK+K<{jA8+vKZRCN*Hi-3 zz9}KNzsnV{k7>z8m%o=M3+B~$88w)k`&Q;VWj$8MY-cpc)l6pX zgt|nSb00Z%(bIa4D;OJi>q6@5dwd3#%4s^9S_R@Xqnb(dEH3t}3qI;NmiOKKH{qfT2spzP<6)uI;{c%--RbLZ~O@<>hPBRwffG zZaP*=u_&tq0H@)Eys6E)A0O`V1TOigt7k(*j%mQw3L$gxc&}^G-S>vx2u!?&Vp@1e zC!MS4Eq%P`qe_Qbc~o6Z)2N z#GJJCGJF)S!rEs2ruP>>{d;Wp={W`rwb(1f1 zZPTZN4l&HufImC(To>sCz0&jfsR5mvM*JjgcPohs^7tY!B0?8Zll1$$A%T2L;lM$| z+_@iTP&wR2HqVae@1zJpLHLwxXZ>+?!KUf;eR*EO+0z=SS4`c0c$h_$hr0D;@*IdO zQi0v#R3_VHjmWnb7|wGZwKr|8Gh{rqB*0_;m&$dy93y7u_g#Olr4XSjV0K~;G-GSA9hlg zmKr_hRgtF9+u>0GpgmP1jX_XkDQVou?W3I{fA4LHxttfI^jMBe^jxeD;pp%wbT?0n zezeP+wBf6kZqop%;a5mpdZZs@7PH~VRk^SIu1FV54>f07RsV*Dm`~x0bB6N|RmZnC ztvLcP>(S>Mup`v5A#b^DB%+$(6_-KG;d)C~=$r)mZKE3mw}M!KIswAm<=8qu4J6jeHG;*9;c_hz&xo|?ktmb zNo4*6L_F_Y*`lAj*+MJP%45Lvo#FG}VQd>ugC0EM5}O4rL9j&hG>;CsDTDF89#^kI z;lNHl^#!FJZzsM1P$kta_u#0J_x7}^fh7o3lm2~I&r;lC)%uaG1@QZo`}XfaqrW6KXz4{af+`^@VJmbHQf1(!txZE4RoGbIn12|E0iSt*DUDBvzGq?p zV_D3Jnxc`zT~ZG`)i=_zX9b{{o{$IM zsy2>(may?^JQN1R7jLqhiy6&E@Ho`K8<)p)<2J`P$C_Iz@_ri{ z7*ZZugu9jvGczn)DXFT(Z4{qi2l=t5ASKheY5UUSWDn4^k-6jGx;s2G2ZhD)mtd2A z%z8U>WxGRQ?jTFHYeaiz;((*CUqf&04K}QA)UrWL#a$dZ6bvKj*sy*AzDMIiS~<+k z;X6%vaN_qq{jmvA^3$h`CK8UI--$i^Q_jISX#B_XH zc>9jy#B5>(P4pFvi`X$y_a-)lG2G|Vc;cZQb#HVTFS3V3#u;1_(XNwaJimXq^q8(r zRnm0rMrT&anZ}zB4vj+gNh0PPtjucuY%k{{YVzHc+gVU}vP{I1&3gMZxOg4J+2qS1 z=qA;9uwf%n+j<9=`=;Kiak2FwNVBThp1-KFThN(L6Dru3oV2+*&6rFFYm4<$_n0YH zZ&OY&MO^z1!Z+^tkaOSn5MBA6rruqLA*LAddusJoFB7!~X)P%&EiE-ms!ZxHNo5ck zD*Gvtj|=h^j~!THj+>r`_Ax=bpXKPErfcV3buaB%P;1PN&DA1BHHu%e-8MZY%%JPQ z2?%wd)EZUq;SH)0Qu|BZJrvrYmpylJ>b9SW;e91pzFQb$BxPWprp=c)pTa9kl9vN5s7p{vMFi#0?)!Xj@=~~f_;=2C|GwKgJIav< zX4KlmHZAX+WZdaA0<)tQUZ*ojJp z7{Bd-wi;F_w>kz}K*rT;QCtqeCo1&ymM1iZbf3P{BwFu_pw`d;gR6Nh7plSkMMo;z za9#C%=l<%(u9jSJqO2MR>y-n^B7q&p7fH3LJ z_!-T`V;3eJZI@@|hs24|OuRuwqdg0KOBKlL2?$$jfX=2vY?*fz~h)Qja7 z8Da@#)$S^so)=pdu?)TQsa|{x{J5LfswtR7)LD37xN_s>?X`!kML*>wL~mM9y`sR5 z;Ta;cT5rZca~W%!>mtLG&?Ybec4S>CmdKIrg|_0ey9v!{LrXOj7%?M|CQMwoXrCcy zBc4NGZfCTneVS6G_@5^xGyvMQ6^zd_P!k-c+JCBxO+5guIswZ6eYuQ7Y=?ODl*w1! zPhDR3?c5y;!=paj?!>uAkmS_?%7ygeE+E^trbrc8e9z$vz1?&nAFW_4I~!kI>B_4y z;`2qRuYqgk5lP;w0<$}ACklON1RD(hr1q;-3Kz|I9pcu<8Js-xNOw|NQ{F7D{-mT`C#0o0pAH!1kv@tGb$rju>vLTU9+t==2Hb@@>3B}DMgNv~b z^4aG;gSIN2VppDf>yY3Ug`#5*^-?N4)snidl!d!gGM5AsiMK)DVa4Nd-dakCp6>n$=WTlRLK|}{&Svqlk6@O zs*01X2CLsY>6pwE!_Ig3`DKSoL1xl*I0TWe?sd|f?&F5yqYo(~a}!jrbZqjV51mlQgaXgBE?3GsY96+n<^~2c@YKRE{wZuIS&DilmkUO%&Z_ z4A=Om!NO~(XN0Sc9&t+)A;w80@P@>RkQ?p!S0{DqyO+~ix-cKa6$;xIUPDc~%cw7| zJt-(Mj)S6bD3WYKVAa)C6a0}?YKwp9!FxBudP9uc9E~H-M<7kYV0J6iHtddiy34X_ zt}O>U17+MX{2Dv7pu2IwB5R1Ws-S%QF^9znqC~Tdix3c0gCZ<7RoK|56bR`*iKePr zZo8PNOG8bfU!jgo^gmZ#<*%Y)aP?D78hr8G4(qU{A1ZJdSFTn+fuFS)`hf$hNQxJW zW+Cd}9ggZn1ik|_t$o4baQes$qx7%Jn>%9!Mlm-KN*)6n*fY2ebz19k7jY1{(hezn zCE~#%#OMZ0J3a@um+S^7Xpn0ewnbK}!KY`2@1q|n3t0*jUq8pFraUtF9nP`JVU*gs zupJK*#K~|Q4%37!#9}5fJ}UMeEgs%)^?-ibO|474T-X^pz`Xt(S0uv~{6S5LcMoWZ zz=^4u?grM!30Y_Z!uc;nL&?ht_~Fu`lJpZd0~?&*=n;5}wc%f<@@c<{I=e6&kH zJZkV__`gJwc?kXyyp#1bqY3^CoP2wH$N%$x_+FxYg=vb_-O%Rx>nEtVEpHPLd=PU?3?$MI$+M<^6IeB8%+a6kNU4Q(dCA7Q&YkL?t#4DB3so*AHIb?mIot?bNAbXgq?p4pmM zS@Lo6aC2U}!fI@1XKgFY#bxm?dvIDkGvfNLZr=oFp|HO9#1@63k3jyB+_!)A5rrzh zd;j(grRR~e{S=Y1t2?!e_|PoBtRM@jCp5n~AFN&Jq0Xi~?>l?`^Y$bC-*S&Wo!L5g z+~9J_z5GKxIsU)O`SLd;$_L%>+lCEghB^%mVVLE`!a&=JNZV%OfNjmxT68m>uu1{j zakv3wlafAxrQG|7qOv`RBH8y-$_j|F_%mo}~Et(7uiJ z^@~PLky-sGSlh}sr!Ny+%$TkBXIw&|j?s~@?mhg_Q!npNtrXW*ER~s8}m~3IwEVjFkCaiVR$cx5#c%TSjr}&Lyud3(2cIuz* zOr6`Ttdzudxo#{aYgf1xyAhpVSFlwO*CMpsiS~q3?OMy?8?e3XM7O08F}rzHk(XJe(0jp(jSLGcG+;_=4i3w`O7~%51+bb z9b5P5YU&XN-isz}U+TgIQcnq*e>JH8zUfrijn&as^DjFYHiX7a#JFqqWExN81?UJlV_7q63kPTH@aJxoILu}5a5B5h@?gDMcC^lhll_5{pOSDZ&~_sIe|_w8JZ=|Vt3!4xpR|Kv-r;qe3*5=BX<1MiH4m6_bu`j0Q@SYHYlF z*O{v7t^e!Ya5QFZ6{CHu$bLp86tg?!MieQ?|;G(h>0h3MEk+PrXE0u8OT}${x9BQ4<#SmCKsN zx&6_iPD}5go8Nw3-mt;L7GqKcL-ORKBsnHW#&Y7CdP&8`><$DOa}OJ6t#{A?N?r-k zoOaR;Fd>!Ckstoyc?+KH`6iml4A~#rlAE~`S3@vQInBD#*zO0h zQzmN^J^S^B_VVn6g!PaI(Q!x^Zmfc(E%wGf7cq&AKSA$^1Uz=olf`|D=bU!AD-O>y zQ(6{yJ>>p}b6IQ{=n)p`vLy{pM-LwSM)>j|q{$6e(m!witY2G(5#Bh%()ws1)?Itt zu~T%sA&e&3Y;(Y6zNnyVV?2VbVtdo<*WzgH+=O67;CLP~Mt9gpM#PSwJPLgvgnN+v z(IOez+F0)2(Zi;z3k7Df2%3kw#PI0VJSWB&El&727J zsn^n2Jr*q!>X^AK@U+sy8vE+it6R^;>c%XM)@FMwSVu1v+AtYGa2?KGILGDjMsA6ScMY!{G6r9mVKO{j{3+@*l+j$J3okKGn?z-<**kg~OtxS( z^p?3_{OOlj|Fena!w|IQ6~8`Y3+yx*#}C;19c2=JEjl^#r??{2o8k%#^7+-r_~JzN zNWPgE_w9{~d0d=*RVb8CDuNd7N#bZx#VL(4LJIOXyi-N=zrQ3e+b@=}<0N<^I4QP@ zTUjJy=euA(?=8N#*H5f&(kVub`(Dqu?xs&FX+Mwc=}JPj(`cJ$C$>C{Xw?8!~E+xqZ<%v$5_Z*l##N|XIW z5p)ISk{#-r?RbL=^Mn?rqfcF<(_P^>9Zm;@`l&2K3OQglcEhj@RdwW>)&3Iv&m#eY zg*JKwSBN9R?%Soc#1z#`PHk=NKx_l+X559`gN>bDM$t|+a*{*PE)0DVynrCaR&7Il zDg1a1>Mq((cQE7YA@*wy6xy8ayM0W8xDrIj(JFWCnW>W5KN9XSg7cL+V_OaZYR*k7 z{w`%8;fr)|o!q0RrzPC_^h6qMvh{zD%w3Ll(a8o)=UzwK|hx+)O22ndu(FoTD1&aal7j(bi|@^L+zDspr0c293e(>fSuF zIbPKa4TncZth8Jg@09RQRLYzj@!df41^v8@KjpoRE+w4uHe`=2!tgZwWc3O=JrO9e z?el&{e{Ffqii@}5%axiCZMWp9_9U#k`{EdqKQ+eab|Gt4bAz!HuJ1abkM^;Xo?*K^ z@fiO7S*h78366ZZefZOvnT6phg-xO3n~v)`TB9uuX+H*D zrj|&&HAxNoy?#Uc0$)xxMM8Z2mP@#VdwCgGo_WusIcz~e4poFyFo*K2-;iT7^J&$+ zuA+osOm)gX)C%`bIJ-<@JRLGoXZV*ThT_7WHr{)Grdmv!u66Be?pa|Af6IIJ#LZQL z=`H=W5<=wNr1a?CjkX6|JqgLWcAjr)07>LR!X|7%U_Xi%R-^@Jb&HWBY&usQy=e^S zK=3)w3;zu!-eZae#(l;14~~7H@X|QL=rKSf5V?0`Z%>@^J{jyT5{KPng*5ANMOdaf zVIzvS!gl=eIGT=2%N)D1(k_pkgW&5Mgy%+~MmRWN>`R93usL{I54;{=?vy-#hCPz| zMetO>sUE?fUM*~OFrGV;6)#jPXXJ-Vb6IJ=I~#xF)&8}|<9(1WFsHbmdT3fjmkdm7 zOvD_+3)7sL>&~hYkKr>4Fzrg4T3kP_xjcDD)7S6^vz6v8F&m;=?RbUn*Eg2xdIVvo z>62pi(@L_D*IFI!3u&eTbh0eTji{4pk0LaL@l6kuxfC{2$7Ij}z&}h2u)fmkTm}!` z0oUi$%r|m~LWZvN^z~<=Ejjqf_`tiSrwb2=$!REs@$73)l4r?yYohVz)J%(#LzvCM z_o*+*3B8Z6F2S4D_Jo<#EUsJPFAY+HZ{hdZm zZaB}R^Xl}NP0VnWH`eC2Zba(LaBI9&XTpP!gQ+t&10(RLZZw-fURi*J^amp`@Yev+Io*5L*8J3x-%JAG!jS#e$wN*@? zJX9%Kn68ZukWw#knYI&w`E34jYF3P}7+UQv9J!BiIvIM;hI8oGY#R{4CJVinhq6=#7m zB3&Hnnom(rI+eJ>V;hxzQ!FP zv^cs~OPg;%4e+J8khxWVIL&$KFDI|66{Zeo1S=Uv^^E&Y$xRN6UD)jQpe^ zY(fU<#s?aYPY`r!F7sFLC2r1C(`o14_{5;tCgR*qmvujxx$nup=G?>IGu zSLd)oO_~?z_}*IOnRfD9_FW^)4=Qbh&FH6aH;PTZh)=h2nYB0Jl9g3Fxx0;jFaR+{HHJ2r)@byzRKDHe{9!{Yl6WRsQ z?p|{TgQASFKIuW4#8AoKqNO|&SMx(#nw9JYEc?2>gxL%2JVT=UW_rWxve<5Uy`K6! zmrlskxp|)ET{t&Ji7p?oMS}y)>44MM+x83a2;|3`yxNEoeDR)P1_woYPT;TSP(+#g z7?Pg1MQdtS+~OO(;fC*b%o5*@(W$BBs|d6zUGx$;a=4K!_%I!3?y#Xoi5w^U7k(PI z(AqM$Bw0y%x_JZ;+(Dwe2lueY6ua?FAsXh;ca>Z6*%a4gR)Z)Gx)E1Z4IlyCEiunf z%d7LEOq8xAGf&h^9BHx}e8EGW^!^AK(J zaOm}z#8cdYVbjtyeW7N?8R7yFNc#PLUjxt~nFE8kcPwWl(0t8WDtd8`n7p5IFhP^=3m%H&87j z$N88eDE2k7Zo6+SN|_5C@Ub`Cl4m> zp;0rlKd`6i>IEvmhm1{3Twq~gS)Cu;DGwmD%S$A~fUkSU_6kTN5*6a_inZtk(DU4ccu;Ud7vd`rQS9lOB`QEuWQ{jPr6L8;Q%G3I z=5v)Ik@VZcxSP&|xdH+7rrlg`-UDEp+R#i^*nA75VEr>Qlgi?@ej`;ivrA~uH6CgR zgdENvS@0Jcc>Z+mA9rx}yArA#7zQ0TxsQ|E+08HZ9pE^3?sjJB(i@}_)QS;xdL|pr ze?L%cEi|39ugs-j?&OiZyZGrvrbx2R+7EP=3X!2PLg-il5r{`+K$)}40B^)r*7gQC zij&Nv^_%0omS&lAr#P3w=hwApjTu>0$Wh3~nootgTlM|bfjM5dl5`kvLM=KQTGmke z@9%==LU~XuK*ezO7)VNbV9-D=>V!aL;Mc*EV^BYCk3}e#BUQj=_&-zMyC#)mKM-T( z6Dd9i5eSZK9Dm61t;Rgew^@zUaAD^jbTfAFagqh02#Y|)$B3T55ukHcKu7)VFRmUaHS3~4qLVfK?P z--*lc4riAD3SuBX&TC0n4TwUbuNId;Z{`B^6Av|tw@CCFrTPtR zFpu6{q{Gz)vJ~{chX3w76lX-8Q?eh7qQ%Nu=jxMJ1kABt>zxhR#uJeX``o8sN0mNb z{48kMH)*rh-@>=B|G)vM&Cvd0dqo%y7F6pqnH9ZHUfBNm73VmJ#)xa=kpJV3t1p~` zt!sek{RasCB}bT)NKs!=ekMw&nkz_khg$gdEC%`)WI3u34dVJBo770qpR&MO2O79E z73D71fjs*)dJ17RycsVbX|pOXQkX`jSKjl0tmyx#g?33AIQuoOa^(&Z z8wwIu*ctV6^xX?f7nFxj3pqXV|8P9_?ubGC{JO59Z~*1e{ty{9$o5}$v;W~JBJ)75 z@t=JB{~yozX^6BEs_4{53DVLLg6X8Bq)MPxQ8Mt}7P1=fTgn2>=`+O86p$i7;0va{ zAeRO)GgU51Snb=>8-BSgdynmNNa8#zh&P>I72^L)OeAqAr(C@A=7d_7t|ua5dLE`T z1x0ZQj}Pp~LYNugS?@x~PZE=ZuY-X>pBt>u1{&oR+78#(Q~vj?RO-|+3z6!KEIh-1du95xB47&C~j&O1c>S#;?!iLDkdcc|Slz-!Dc8a9R}0CUp4(EQpj zUMjE)Bp>Sdm_2lkDhwf`LAh-$);jd0s&auC*-$2a%j?J~AhZqhFU(YG#ItONwl zJSd*;G4Vag)_>pIOy;+DjHLN}Dw3gm3+!y0(lng0epKA>OkfM)?ibLO9j-rLcp4?< zBG@2Ea|Rl|P43{MOsz74x!B&pRA>ra>UXaW9P2fJdbt#*$AP*v&v?9j6_3$&&(qrx z+nkICVKmZ(nu_me?1mE%FVa!MtNS)j{JnU`c^}jQNTv=9R8&LuEPU5}Y!*kV?=iKX zu5UQq2U0COUc>$uoInPv0&#GHo}PY6(6Ihf<4pc&ZQy07Lx(viM|1oxx$2fs1bq4_ zq-QSly|szpl5s8|HA7@_z(xpnp69L0{f0MIcI$xDoUhJKs)`ApD^9snxLSAJn=IOIm$?8ze696tN^=^3eSlRvHd zcxi44swyo!&v4Xx=w|nra+Fh59CY6Uz3>$f32qhOK?h`et6PmVdd#Oz5Vj# z%QI<5OI`AZ*JuBXz3aAo8GpV6&6^)FWNeag$MUO3;hCaWI3~wYE$W=Yke_Mm4qQBi zF|5UfG010$VXc>wBYQtZhh}W&%*T@-;k!F`;+%_-u&i$=7ajMW`-k!W_8UhC|3jeN!x^XTWB>&^g^S|Do#Z3%w?AHu( zzh=@WzW2~SB`D6I_%)4XF&cU`ehkC;nd${iDk$&*C_$eLSa=?H0pDF{6JLvV+ZdxO zX$ZdTM)~V+H#dT=7OP`8Be0+H)E9u|#h8ecN|2PgC!jvIQ^hd=zo7Vnlw1j3HFs^@ z`?KC64`%`Q!7e8Z3ArZi{AOKO2&I5tdDfHnZpXvr0acI(J>@C>hZ9Xvh@>BUNwLPu znS_Kyt=zS!DMrG|XoUz=?vK)O7@xg4Mu8xtG2@;GU64lx5Zp7T*!KG9S||gr{x^yF zzM^(qd$z#?7%L-j=(~FtVg0_}qXy!Lc?%wtJX7E~ik+4dK~zcCDsxU&FR+vbhsT#d zWf01MslIjgu3rlXm2-ZT-@kuHVi@oSajM#AWdP8=t<8T^+U;{U$UVF$#?CM{&FQI3s`4N6pn zr%z*D2+?9L9pJJcPDt!1fKGrT5JA~Y0dC`WbF(~D+6v$>%hjzu1kN5sgt(qt1Z=dj zXxUe&^XdT&M>s}|xY7>EE=Vr&S{$6*bY-%2vtox(L9i6si%nGXq?LfUra+x3U{eE8 zTRBVjPTBg%Tb(r?+83Y*70^218Bq|E2ff_cSJca#^S}vWh=~FD!s`Y2nA*V#k{Nf) z`|f5ac#-mkV<;|NkO_?iOIcw75C&3^jAzh!40~7eDbVzg*k0Ge<~c%sJ0;?f1}%Hq zlBGRrEjcN=4OFsi$${p{&eRZ*Ej}4zqheOZL?${;4WEw1Ph8p=U~x(Tj%IT_!s^Fs z_q(5O`zj^Mg#GKYaP6xMzO352hzA$j^JXHvphQ3%gBTY`RWcwcWjD))jFtp*X=?#k zRkgPM26D@ZWW4|ezMB5@YhcR3cmB3#_n*Bu5b;MQ^(?@_Ku~lTdz>N3>Vfln}qhoaNtg z=iDz8H<$qOdAZIYKAR|IyxVEf=PjI2(+}ITD3Z`S5Bl`lZ7dlOwt%Kk@IK1aCs6|L zt;*}Aq3eykc4Yld?qPVk9GoPRknpu$6~O~`FYYwxv*Abo6`85#4FHrqD>?vv#=d53 zu1P$!Nbm1khfZ9KMc6CH`GNe)HqeE^R--Zo$n%F@ct+DeC})eI!!Y#ex=Xx?3%Hr?J&YoWgS zjT+Per}r)7VYG6BJVJ5#iHGO<8)i!lW59-4;`cx%LW6YQzC63BM_AA9*FJ_CRbXlC z{(VD6c?kG^-MsX@$Gz9Bn@;)-y1m2rAiTNDV2eW#s^$(*GO3Wx)$&Y(DO=QXjpJZ| z_F-HSWWw4w=LJ5+rXIEeyV)7EXgrX84t*6Zu4r^;hNcL#|9$XXD1Pas2ijc6K9-@X z<&vD%wzkik16)0GL6KebAIH^x-Xi~b{`ddKhy97LTgR$D10glp?>Ly(ta+mcyn`!5 z{NN0jRuBeprBhYAA5Zwx5Z2Dwc#dUY7w}C!7Z=}g&_T~;)r#Uq)!Zxi=>LL^#ca|R zj2CEf4ONi2_T5%Did8U-EwpxB+$C@W+skjEm7v%apoZ>m)FkBYJS>58z zaj>J7IL_y528-FG8?is#Ys8<4k5f^utDUN=O(OH%6DIA6%3#E_pfc_Xvms$oTA!G{ zsh+KW-x2IKUFqsZmr=5+17Hl1nhs4QWDlpmod?HRhe4tU z3ccg}U+VN+A&MJ}I5?XXW-maS-P}zwYYN~GL-bm?+aE5-Gg}On_siW}0#4M^sS(P- zTp%@>u35A8E%NIBihYt}cnJz6=TU#5*|ds8jutyML0Uh<#38`(5^c1ed+p#6Pg)7~ zS}cP*gy@Zy0b6hz3Ff+NP9^1m$5y?}sQ3UNT&U-M4Gv1w#SFv2s;MgJcH3J{Xo%** z&;IyO5|Dr+ZAG`+#3cR&k$~lIzvQfgyNZp#EoSGPU-PSqn2cM>=tfSiA9itY%lC}u! zj}7ywvTH1h=&`J6JB!-o%0RGsf?9Xq!Mh)`N3>UDV`Y2O`4937bu{W9l!nKv%cyw= z5Ru8r$$KWiBv|+1`~!0#ioJt;@VaHB)==tkJdE|ABua;HTChM(-43d81|)%6hvmKf zJf2@w*;fgb=QP+#G2qLXGp~T2ly_{|6^w5VB~E+0p?r);hQK~-v8o-;Z^Cak`4Gu$ zN%ArIfy)!kX5HU(1ds45d%(-~O*aJJ46-e%rICApA+2a@Yik9rn*uo&3eP+jbr`}O zVC^eu#9%PEzwOP+Ozr{1ydu!7%4v^zG6~?v?ZD;0F!$~i+&f|ACEcs2(r06Dcnurq zhusWW{=qVPJU@Xl1Wi+gWpLrW{*hZ)o8ez#-kA1ibyR zJd3?uJQUZfyBP7)oev=FKY}#K4{UH5yhtb~L;BaChBkqO>o8jZUZiB;&$BvZ_6}s# zdYgyh25viC%q1U*RZ#Dnz@XL1_kR7lyT=1s#}bbyi?C4zl1h#iO+>dt3$KE}kC=Xh z9cDu{S?KA%0)vtasrG&N#lL+(^7IiDCCx$^7&ZQ(4^*-RdF-AExGZZ_q3whmdVSa% zNnjXs!#P&N7*ItdI`aQwU%BIEBtwNZqvnMnY`6Aj5<)E?DYUOIjP3GgfO&5&c45~!si0X5R{J}FDm@B zvRW&Gviz0_LOjDZ#GiE%6`oN68TyU2M2@@c_6Xo%JMn^wS1>2yIbZEe~#W8Z#0>`A8t z5krLM@2*H^ZVKa*I&k##Logro&j?x#q}hztAd%4FNF~f6xT;;Q!DrOu_ismy-|<{7q9C38fVogBJnA07*OlKsdjcHNVpAiQP59Y(_!@IXO1x16AH-xvPK z%?hG)^y@?4@KN6pvg%hTR{H(%f~`l_oOn{yjgaf!aI^w+T$L21`_L+_09ce#Rjz_- z1{1JjWJoM?NvkyP$*xx6BSCrWdws|J8!M-Sfexawl#)vsFxNcZ? zHfeKaeWTm*<`y;eXKaDx#h%s6OY+XLV~6k2W%hrD*zR$Sql>f>;(-}VBbjbb3Pitt z{rW4IusF0z3teKMmrsMiQIHj4g3r%jajJj+b}97hSyaTQg4&U>1@kxoIm_$x=N1SJN(;yXE-wpP5BJ+R~!}jlF4G-fd zfDog|^MRjMa7%a6R5CJvF_VODgu>;7YOcA4{6=;XriOCngescQcm!J$-9O+rmk6hN zd=s&SpnfXekP7U!&RlAjuXa#Gp(IcLLu?pHWjk9t)ti@EcPjL1egs6NAzBhvID|@; z%pJ0^cdi8h&5@bw#W1VpV{@AqB_q;*67C1x zz4YkIL#QtJVaCpZXZaAWk|4-lhF5_|xlG>Ht)K5O;{~t{-7gFqegGSm0^T*}Kkua= zxSm0xhC>XzER!z3Wapf@`V_6WFK_Hm+xJ8%Ps7Hao>x-n<*PEcawhk!M8{(Lhr!u*lrPGw1Yh~6ZXbib`-*_? z%!-Z2;kmJtIZdV7lK|j6c8!^+mc&7a$)b~1)3Zf~>yC%sSv6DY@_Z1I=UioI(Y;kN z&ZQOS`9PHe^xOBs>PX!TS=8M6F|cs7oEndTdF?dIEK3BmFo0oT5p&MH>qoD%uUgoC zdb0AxJ~gO5(V-Sm!uCtOiJ(xYO1>aXQArPPiWW=#azCioqGYEp8SmntSiHjUaVAUD z`Lp$L%~ESEk^G&EMLj9wMf~7Y1o?wtL5t#xdG?qXk&c~L0}u=Rr9W||9;RzDnMu@M z?Fk+3EpRNTk=+vRGh+{6iI3YS(E&FEC+tg6EBo{^QybI)caydWycELo&f#jI(i5=g9wwt7~RL`I7S?zQ$R}RajH8+TT%P^Q}HD4g$-gL$P zl<-5ABkSj+Euwg*Z1d(jG@}Nm&x3uv#Bv^|Y5J(b%T2BL9yZ_nQR~S0egcr_k8kaF zjfKWFS@4lpF0p*YU5l-7EC{>VBW~3;dp`K8cI$o*llFuh0_6GUH|d^2Pav~w!b?nB zx7ZjoZuxlOXNHgN_|-BuCbI2d`o&p*q(|7?;YFaI5eGuDOc<}u4~cjP{HB| zU<3GK6mX9cL?M85SJlZU?}Ou7)XV+u_>~ueGKZPA8Me27cg~N-A$j`)%gNhC>7Iw^ z6fn$X{lhWuGb=|^2ES#Y%iZRtxe0;H;@zAjW$hO<){XxBR+V&G8V`IX0)zO+uAVAM zTHs7HEHQsnk@$g~ah6Pa}wq2inejQH_?`&d>M>j9KCY_ znfQD+h{ZU3g*tHbXGGV!8Y@hfGZ|O#S*)5B4zd6$9MB^_@7t|4m}p*e*Gg{x;%ldV zSvggwmZel3SGqB~6yqLHf{7B|2y~e1v%K}IMppgqCL+_E*^N5yDw#xV+Bj_uj;$2- zlRY+!pg?q@lf#Q3Lu`v_WoR%LxTv!{InIPx-FB>pc!tr7zK)h=(zZ&!8sSzn;S%xt z=4WaYs924onP=w4Hdnt6s5#5PcgX;J8gAgJ@bIC6(2uHA&`@@9T!smEo8jVO z#k~v7ezlk)7kS!?kMAWvO0seE=F~2Ym5bi0oyjV@|M8;hDyVCfRWc4foUK(4J;S-WIJUPFOLIORx4YQ!mc{} z`1phaFJq9p^vL=@W+npI|xw~%3JLm>bZv?C@XM&I zYj^5CN(T$u7tS3nk763DtC-~0beA0)hVjjB^l8G@QAqWk3A+RUj|DHp1I_xvOev}O z8JS`0;03v_fm4VrsnnO&WB-8>|G4-z&QBFnzIza91w#^3QV`-OqGbI3;w= zJY9HtO^(HX>yMtjDgF&}eUK_SGuM})8I0Zf!i~AX5~){S{tD5h8~wB%%Bh$R7}a0h z8sBheW&3>~I;K?xO89Qe)I+a(@)hX}9@65<_tZZ%KJxD3>XkI#{y?i%U=tDTe5OL2 zsYB{;VgNFQ_Y67_=7X4~5)nD4%KO+%$$WV`nAFWzV<>PqZwge!#yF$;T*1@tq=*Ux zP0#|0QzsNE#!NY@W>&_1KQoonZeL>BGOJM8&bbo5B4HX5h8IAKJ*vER2!ij!g z_)vM_{_LZNmc+RoT@~^D-}=oKo}Rxv>12LRlwL8w(pNQ!7hEyVI6(2Nr@(*sQ%+(U z8Sj~$(3IgJ^E>z#hA=2j^Njqd{)XYDGfBO4c7pj4R8~N_s;f3fJ#6{?Hhg>8Mw6m^ z?ib;{MvY@o;j4M}T~&2NYhn!O8$n@!6kle zbQd#)uMT!_sc-vON0!f*6XX!myV!1Owix8v9yE-=1gPLr5mz9T-{1;_EWr_cQQ;

pU+nd zHGw(ff(IKuU1d}SgnjNgnFl=nhr8vPvvJYhkwO_Yzn2Hr66k_iENIMK+p&M)3ym#eyjQQJm8u6r+HtU31QJqJY} z6a(o+zJs@{WU*Z)&_RW(2|(CF-hJLUj6xzj*2 zB*UCSt3h$Govr2GBHJVknp3Nkek}FfS*fFeGmT_ty$XKcfk&o3#ZCfkF8ReNLKD+4 zJL8{xlz&(o5N57ny5i-Hv zAq@JF^>B58*!z+<)eEiT049I$0Tk||V~4oXJ=C!wS`SxiGE-frbPqLb`8eRN$Ig|k^-x&> zgHZ<*Bb>$S2t!Afo)o+xX&6k(;E**Jhz(nH1X0I%z6{?{;qIOR_bqwkGbutXT2>KO zTe@GA*-y0@pbPYaj~-xB^{*StUBcdX;0>JQR(1nU*nhEsNWG*$A!=I5EC^ z!o*BLvPgcD55PsQq0|s=_qJS1>si6R)U6f8NM2~+;23I;|Iw_(0CZLdbmESL2Xr9# z;Yf0k0k-)2-TP56ZYvv5Hk{nO^1W`7AxQSKysWI1GvbENs78VNc`rN;HP#Afh#WxB z;TFl~rSi=OD)^Zt+%zDBwt>Jjc7j2XTm6}g^hgBH-N2ZZC_G-{{%KWB$Ck2D%f1J? zpTLRq+m$#r5%gGm>3t0KWrfqzTh<~9)YU?41(u#08{jP4$v8&QnqHEBN5xLdNQLO2 z-yWCf;JJu%JseHRYa``x)@P=X^I(QfQOip?$)z=G zqn&dVG*P7J2Kgch6q$>#SOwT-d_i>8#TU75eAwngrwkY`XLE#HPJUUGgOzIJ7+d`v zyInkK@VPLNt3xp(ZECfC^P3*3ynj`;#H}ffLD>1vH}KotI{#KQW9xd(*OB&W0lNj1 z$aby}c4uAI%6=;LP{Lad_lz{>l&6-RDQkY>+ut%@z6>ZHtTGo0AZN;(r`v>R@-CJG z1+<$<$iMZTA7^fMRnUNl$h5;H{7TD(dQacmcsX<80+;tsm?U-;^gXZ|OFgi?xr z`M(H!W3OHAKO>)1`6cUl@pge7pPVvkbD$`$P`}+uWJTkxko{U8`PjjR&zk}qGd~jQ z)yi~o1kHOB$=|wl{7ATLVLU!EL$JCY78Y97%r%RtL;R4SNC{rMqV)CMAv(@9a1f?| z)ZPn@f0v39Z8`>67ogqe_}N+dCibVIfUTjX*JFXbRA9LOn4J$d|AV!v=!6`IOm%<| z-ME&)?fy%^qPrZO+=X{pDWK)>nMC|48ur z0UiUJ4-T{4ac(;pbx`IMezzIe5C6Ds(Z7-^&Lp7W;xOM;=&S8baHXjYY;AeL(rfCW z5jdvOIh}XHl~{a~W%~h5O`tLquASfR+tOZDxC{}M361*Uv%2Obj=SLB_;AiB}5Tog)^D)NlJeQ)ZyKXB1>7jhv!l%C&MU-Y3 zq;{pkoJW|9Ecv>YZ_pd!n_g7~lrO1Z_`+F0j4O!iC(>Vk_66LQiO6J23{0G_-yZ;S zh$=YCEG8!}UE6}W(VajHT4S>fnlm!Dm(%M$rc;eSDEyz$C}T%u@VJ2l_EodVq=dJ* z7WM43SGC&8c9sm@3g}lJkPhb$+aI{Dls6-I(6*H5;<$qyRCyF+$dzoq@kS6O6TunP zkJQvbh08_@^h&vt6`E%OT94X)2n67aHOSJfK7-a+KGhz4Zbc6f+prLw`)7bl*_ITC z)GsL$x=aidK5eJ*giqr0FwwcNN@6riD#zs4j(tIMtsetT^1J|~=xjai5SR%g1A=dW zSqf@i3b>~#fO#;V^DA(Wy;&r(w6O^9y9oF$EQhR@dk$AoNg!NMtqAn)<@kPH;)>5R z^~1EBpEJaqOR6N!YQLyeuF!!c5s1B=I%_OquHQ5rT$1Dq7n*+sWqK_AX=E9507&xs zDx`tL!xq0k>}(s@@7fR}vYyJ2KZRJN4AAtR|8>3nC2K3z{$_gReZ^gO5>t=@JU>lq zXP)I|L`P=wm-|)na!amrDbf+_!IY~qT;4uukdZj&k4|ZT1;Ac=ZgtHW;|zsSI(P++ zGrP=;Yp+%pDs0BeA6uOsP=Jh-e9SrGA-``^^!4<^6U|b|>A@+T){ZxhUUeLBb=YW@ zsO_JTi`tSZaj-04(;rQ|8oz=VLZl}(J^X=sp8=N)<}c6hm=KeE$c}ivP_H%sC`9|v z45U8E06T+)BP|?rLD*qraIxpvKgb(w(K6lcLQR1dk(Wazp0p#Rh)=+bh548ZQDLoN zN^yGK+03G6rCw|hD|7OLux$9f0w~g(bI6kJEzZ+-8k$RlR0I;SI4NGkM|Jo79#BF` zegYFl_;bn-<;#JBr(Z$FQw!$vP_D^1_QN#msrW$gL(Add4C>}Avm1sw$45fu*b#YT z$(`~<`M?apExyr3d)$B``Jt1KQaVif`s3>9xV{Em*jaQox1v6ONSg>fYvCLCZYw!~ zvS+I$5KF(*zl;xh~qNaOR8^p5O`F(X{Cl{-3S5h zh?2_-32*82wusQZf4q++m5hh=XdHBpD)u|BPAh{OKvT?RP3gnA;o1_D{^C61Wh8W z`>dAt5%A95}o_qma_|Ef9h)nT7$r(dQI&AnyfX2>`MPj?0^94ha6T~{5 zu11jbz%7xNvl{Mj_YNf7kHYq+fXlwIJkfM0XRaVUg5UgMw~nsvquLCy*=IlBk*F14 z+)Aq=;Bj5sXqP^Tqz5rKe1p!lPAOin-|j2&?vjJI{^BO}h znCct0M9h9%aL%ZPRj~~6PjYD|wN*!VBwSX_U7C>1=UC{GHs~y{8obBE1X74eYvFo=W?eEsw;+pm*$-8cSo9UPS~9|ndsq?R z_eG{=3`W(;VJCUa_G_*HL(W8i*D#&}Vu<&w*&RF%wR2u}}*;k3%7 z`V8j@^WOwn+<7@_uz)N_4(!h9i2Wc>+xE#Irz39ltsqrIg}!H(MPFi4^sh|$1FNks zljFsXhKu;NDET7rV)=cS`4knivHfpB{uWGjh z*N3c*ewfcQXurF$ab5+x7NwNf$EUNlV?AUoVA0dt)McKsf1)AM5UGPW#o3PONz(Mu zVI;e5+!#189$+(#|+CH8r=h8*I|x<0J+p-)KC-!}JQ zen$`OyGupX8_qjeqnjBhzbk`R?)*_Aylkp*pczRQakQM7E}}Uk?4;1{E}n7udjXS^ zgm3-2CAH&1|2ab&F5f484Pt2!8_A4YFEaiTj@Prk_C#QHOuy4&D>) z$NE!ztsDqjOe+%u6(9Q~!s8;Xwr0^>$irdMd$TeNkkyFB1rQ(4-GI}{z$0Rx zT0x>#%Q5svmxD_aOdGO**;Sy*Seeafo(Ap*0u@pJ*3F!&^3kF?0Yb1k4^)pbFxFJ! zZhFqz1Tu@IV6$!;mOtDgJNw!uMZeov99ixuoi*>5JOkm+0m^_o<(hqC*B>*Ctfr7g;ORqN9yoXnv z#5nZnJm@VQuws}1OdkPe_&c!}YKyPHY&)KNXKCd zNd3D5>*Nvmew-fO(Wo?pr`!8TSm6@6cAd^HC&LB5CY&JoeHMLJ)bk}UXgQ-K&e&)n zE3J|(u%1?>35`;m0=i*gT?<~|$Uhx!cWrWzJ3}%$QbCyU`m;X^g#$7J!xklB7k?f+ zdh}?H_{EFAIBjA1uEo%SE=U!73l5;DEa3GQXU@U(eVkm+!Vexm|7RKL;J0sOIS4|2 zpId5SYqnU(XNFz&Rt-RPepTv<_IxsUwJSIkqdVrOl=irl6G~{DO-YyqGN(*IYF9?? zOjo#n=P+#xA|_O02D`#?CaNNXXE3kwo-dBjiz!0C%+fnx31$;01C9#a_yyp<2-Y@5 zwGPom(Lu{pK52N2wq-x2I%mj~t6$w*r<|Qw7tE2eVxlqxK@kf#v>AZ2t^cAxbvT-zk}~H6I%yd1I(|vyp@kL@Gq40c3t3hJhlC9pFAV^Q;E7cBfF!1Np2|JLUogttuuXaH%^)IBfx7 z)yZeN{_JN)Gf+~w8C}5jD3!Z9fsIodjBmLXnz_abuSZubdJAvNX_;iS-C4Ti9&p6L zRdjS|>_NDIUdn(uAv;C&V)B`dY^+q2>6n14wL9PtwjX-lhPxJm#X#okX$2*vaoN?( z7@DG-uck#Kr+D-|EXfYI5M4#K*JbDJW@0%E-ljZ^PUCoTv0z%9;I|(?pQ?95s()T5 zpV2VNsiFnxh8koZ&sa>mW`=kI^$u*4E4 zDJo+888R}g%EuQ4puSd!FYZJH`SWs-k3sBuwD`dQmY9{5-E>V8%-9hsy{eVs z2Q{@>4A^mECm01Nm!WBz?jZU)s?a}bizQwCnsUNhT8GRTp%p$f|CTM-(;fgv=o*@{ zTEl2%3|>9K#}PoSQQ|d_bOaH63W@Nr+zxQZW~u97oJiT(nvf9eFdHZ>>c_mcMcn*g z)X@QJiLfJXOOjNwJpicv$v^7Px z(4r^X2&1~Qp<+;{^6>NR9YRF$UcqcZFJw9?iRK;(0@Fp^m*&oLMMrzL)DSW4nt}e3 zjxTPM#9X`g`4&B2K36{|#%q_GXL%i2mjtBWK6E@L#i4({PC$YCyyfrO+Fj29qduo* z?!&*H1CQ4)cEd*@dcnb)dpM>wRgp14F8C&xM=9GAFl6@Zzv>|p!h*EN*|+$R*%RPt zHXAHQtCc>_aw)s~ldxFJ(m#W6x4_NF0^)lFlNl|(8W*2r1OXqJdMYC7m$JD!<4v)1 zgho0%MRJ)r@n%a9?-$kQLebXhXs7iaVX>Bc=atD-jkh*iQ_{XD%eN(099|HIr{21NCD z?ZQKelo%jL`v*lpKqaI@1wrW!2}Qa@=^Dg91;GFmBqWpu5g3pzK{})d7!akq1{h$@ z+ITw{_wIM##os=;b_UfNEB2vrgHLCd^~<3UDdJgo zQI4kfs(iYiUab2es6zu8-%8tg8F4Bgwd65&+FWCh>KNG(deU-K)#2Qwb9ez{Cq2ha`Rty}9(7aj&_@c^pr zu|FgM9udT;JGBbHPXG`D@oEO$r7{T#(er9}Ca7kICUyW8EN$cqqdtKdsSGgX)A>xx zw14Wqt9+BFG^bi)IWqc7_!-PjWFx2e;lU~H(t%J?v0O{zX=5pplADz*Bzj_0JyC8% zc499Vf zUAT_>6mT{E$FjHcy04Lx=fdrG+)rscyOqXm*++qqw7Gl*mGYpPb9^awSax@}WL@`46jt#8K5qs>E^gfVY@-sfU$?$ha#*pk+O*mtCT zwhUT2_e8d0wK`fi9WyDf8WO+Neg_sWR;W5RASDY%J(dtiCDD1h39zKBy%;!JM2RpSv5$0|ZJ)hl-C zq6svu+I$M6IFs);BD-sy|lNC$~8^rGjPdEQ**>P*a z4}G;y+1)*!LqsBMh|fo;sqeSO zWD{Hl7B6{!dBeCa(|3#`jPH7JeU@%aIo?f@5u`;2I6V#}*4}kxP{-C#;q&hszvuQ` z6ZKHat(RLWYbGE%GY5)o@Id@c4k`UGC8+{DU$2Tt+83aujglZe-~k+MsGoVDpQ$YJso6|c@D|j=U^K? z#vxq@PkoCZLRTRJNE$V( zvF?a9zG<+VeWRl)6U5v;vbty89Xmfh1Q;k4><=@+%x!gJqX>&AW8k@IULs zqSbM=oz-ySw-52oESq5b#<$E!;Xme=-G>cld(o+gha=>{8lbzP#g~FQpw$4?2Zjv8 z7(n*iEiq1m z$?k}5gZE#(Z)XeN(`s9`aFOCiaPzC`ju>tfd974My-JjL75QD`j zV77*@>I1jzY6$vi#@TErm%z9!2M+YUHGI7T27!W|@a5dOmB)ondWKLz{{$)^sJ8B5 z4@eLv>`5ip{xc4Qknz`Rkl}`|UWE6H1AG>a_FhXAcjv_5k8RwBqM~9#@l@-~#`#5T zu_ok&<=~b&m>>pGWx4>#tnsEZFiasI8ELnuWRX$$) zWdTIcqjd)`$Jpj}P<|JKOP-c=kd(qD>UiDxfGXLpZ(oXFJv z7H1GB-J~&jjQw-7E7WEDCWYz95`x5;4>;Vhk`F%1?M!pGLoY#rHsv76E8W1mE*5kG zA9(ZFDbRSnxCcIGoE1B>S)0eAVxCm66Bcvpy$GNvn@37W;@@=$KmC6>-s1~76AHN2 zrht#@K$6LL^yo*(q!2|kcq`aMh8H+v#p&22v=BN80$Iaw2)|Z99i{>1dbzp+M-Pvn z_pd&ZRzn$*2^szl6sSs}9M(+0Rq|I-adNW}$p8ewWdG^9kIk_7Ngtq)lK$KOM*G;iCHHPaxvc?$l)%&o!*Xx8fVH;E zQegLm(5H~9$)wCPCx^E6&6!ZTLvYt}q$C2d3H9v7$CT?%C5GEGX%cQgtKx$8%r3aw zFMI{#F^q4=ng7usrq*hEdmB-KISiNNBOdTT@;#yc?v52S0o*ANCJz=!4Tau+4#{r! z5Gy0#7OF2KKZfG?4?#1CD>+=sW^zA3c|bu)=?MHA{9Yg|yrP>wKMm*|hCe_l_fDQcM5n1OMN^q8NJK*XqUj^J=aClmo7^BAi2(#RdnIl)CU1 z{O`A#Po=|Wlx|~U8bQ`~%I3qV;=$|#7(xyQ+5bFefbORopTL!G2;)nrbDehE5qBQ` z=$R*KIP=eTr2O?m(sBuI+pA~VVlV2uVgihN1tRrc`wuRpXcVoo$dyzH`OfJ7a;8-f z8BH*D4M(s4xLq;whb~HRvn1h1ypnzkWB;?$WRRi^%uQie8eM(UOeyvGO!c0u-qmlq z@=YH}R;RF>4oDH;;i(^?C;_(N zc`MaU!81DqWge01SA@JbioDiFdZnydqD>ZHSzaDe(*~H z?qNd&r>mdx<9)KMQ$lCJ2Vf}QDg+EZPfZ#j4)BdrN$6Z~?}?QE2~oTk48O71I)8Zq zqc0x~^2~FPsWk96D?6p%ap<&nu{}S#xI^?X{z_3H<5H$}S#AJ~ya-Bg z?GbH-vm)^>@i=Pjd(Yv+hnGtuV%X|S0MrbqFsusPn^&uz^oC&*FV^QkgZ0;55NeOJ z@v^KN!mr)*8yMscC>g-FLA-SSN1Zv6@m;b&Z*TKNNBI9vtG~ezvcKtr@NJZ=csqJx zIc?JQ^8Z=%;0(BAuov>Oz>2Rwhu|9Ydv#|n&k+Ga)Ce$r#i?ZK&RI)8FnHX6-yEPa zMiQ2}EWM07jW@*Keg)*cHCJRjE+%r}Id4KI>`(_XRNyx)TE!&31cE8{MP2#2<3GG6wk)l<5j>tym&Yf}CBd&gCu6J^<>=Z@;04amJ*=5APA?Dwss zKLIe3FB*R_5?5^MoijJ~m&(tZWuGRK)9Sw4BP)oV1rqDlTmmk!djiJJ- zJxLIl7Lh$~kbBGi(fNMIlv+KWQQ0Hzz*JZV_iZfx!igFm9gIIk1n4@>IDTx_PiJ0i zDMhdTo|V6s9xv*$f~QFy?yJ>2*=4%4?iqqs-Y1Qi+g5(Lu;$@t7&3_d-C5(g?b#tW zUMtV-T1ndc_A)vWi{wmaAf(5-nTH7&=6a7zK_v0CZ+P-N9C$lOZ=|RLY%DBo4~M^i zVM2$q$l^s4#+q7AA1qT(X{UGmGDh~fBRDM*dei%qBhrfv|LVBiA}M9OuX~!hOK>Di zmocc)%jKLAWn$jHc2c4J_s)k0umSw9-q zHPUHv8QsV2ydnmw`(O+zF3iaemTnt5?|VGjmphp{*PlDMcAG~d?iv&-eKR6cQNS{| z*I4Os{dRajV`P_Vrcu#Tmk}xp#diW;a$X%r`{&g9$K<^bGDO{3L2jq4W4@fO1RTQg zQJQ>jBfkc&>CqY%R9#VHN3V}+{<>88p`IO<6SNEOoW#EwIPtR=4IvO%plN2KvWGqF za-E@muJv~lMeB>7`GBasr4N%s+;oK=%Swrybu&6d2eiSZej~7ctluGYl?XpcXgU!` zfw+s+t<^iU2()Toz7+QcP15HByALz+5VWs${V}NJzy_j7xjN~T$nF2+n}X zbv_UOiXx2D&Yy%O=d(0a>?ma(U{%!BrUI*iv+&W?c=QP=mWaEpsgYf~+#nrSVLpBh zWp^l19Fhqu-tFb}Go8CpgBQ#+lJ5Xb4rzZNnLJWx90>vn#;c?Iq|Wlb+yL#NCqKs}Me z%z4YO&|CH2m>BasY|7ieC>9=bjfO9%CtnMD3k*z-atQsgLWPJo84#F6tr}>-0hhPc zQ)H|#19%|4h_^*|BRJ0`vCB8O>s}nU0AkIZj~fuy+;pJ`Tj(bD9Bt@YYk2H4Nq9^3 zU%{wJX=5<*b}qJDqrCM~sh`}@hBS2g0vff+fZ$N)YMTaRzWHvvFv#2Odin|BYh?Ch z)H%z0@K2VN+t9U6E>io?{8GvUYC({pn=Mxyb065im5GY1K^om#G3uY;-gcoMxW=E3 zY*4rWCjfC70Q{L{(Hm^cbRY+F+PWOL>s>1k=X30RJ~0gF`HyE<*iPA8qCfF5=2b`Q zAt${&^AVWH^l;KIdVA5P!A(f_a~LV<{=!o>eVgFpt|v-P{iSWF4cPdiQ&nOvf!gc* zmZxW=05Q4FeNQYZ#Z*~YjYPSFPQ!XPw(Uv z8TyhAFnfbV)F}+8XiK@MY?F%5(Ys$1AP3Teu1jPq4(CGHdaHeL)Eo*=Y4C~!!DN9e zoEJ=K2f5GZojh=H;HZ<=t(n?cWr>a98_14m<(4X-MiN0LXrlE%N;5CfayC^OEDCZ; zOrd1}-yUHX4kWg5nk}9DKLXln!l#_}53{l7SomF?I2)J)aeijZf>5-YcBb?gTXyAM zh~1B8Zocn_yk>Ka*O;<2_L$uNtXl7t{PuGmEYuDReBO%*;Em$LtG(ZoXOp<4Zi>N7 z4d~Ie$4It8-5K;?`>{+I$&Gs!x5#uqI9yFj|9=-`dpq{mQ5BBl@#6!$81|(9+gi4s z6=C2*><;>IpsabX{t3DJ?&(2m&S0a!{`~CF+D%t+zNBhHscW(&1x`S^t`2vPSff$(D zw^yn!y}4lkm0w-?Bv9--LEWnlRVV8_A{zplRepydl$t;p$$**^Fs!hIfK6lY0dfVS znnWiRJIo)*$68lxIudd~;5h?~bA*W{y9jcqjUmT8d&uJGx5MBtB*;j(;2(DoT8Rbb zl+h704gF1Y8;^8y&;0z>(F#IJlZrX_5)m;luSR@mKoRJk=LdF0qW)X=5I_Pk9sw4C zjR2U8%p=xdhkI!i%u2!q|LCRY82g>H2`Dx) z%ry^<648N$R~82$qjA9oa~oc8js5TIij~hW3hF5!q(UYOw5e@-0{sJnTnqX>fH|$! z)sRaN&R%l6?$ApPS*c<2(;)cz%g1K(N|ENm0Dnb1qd4CMiWBjvyKrIN5H4d``~ z0TG1maeq(#Ux-amN#Xv6_ZlOiYa%AldVs%}8Sa*bVIU(m@Lc{DxG}xxpX+xH1~hKrn#W zaKIFGkX>sP0U2Oy;~v28Jb)|$H|j>!?rJDPb3g{RLfk^si$$RJ#aDu?LmSFu@I^5B zr1JU-v>W?D@lnUV2Gc+{-%CE21pB38(Yrs6%C6L(0Q=?Q#H5=9Lk~-$dBnnY0QMx^ zspW?Vz+fFJo6F15lARWMQmf1`c5bRDD|r`dWxh5}kk!vqR*Leyl!st5j5ki|yxVkM zZ*(2Uatwvb2mX{EkxIWYn9Lu^n|0%4a}MD6DK&e9S)dn^CXxe*g-{*BxFTX|r*bE1v zNDf(FM8X0FicbMK7PD?rSUOzHoudW7unuAHN`rzCcxHH4cdVCRAg`0D; zfPs35HPgX#T~RwrvL*z=QTE8R$40A|yIpvkHTQSal->AVn>zX~gtiK9p1@2~Eu|p={iOLy3e&Y&U4LuN=aV@?V z-7|H*C1a+P@WT~d>A6tqAnur#h#s+dR0Qdw0|?7}*SoGSY$AOEh)sP652>ya5m!VY z;5_SUOmi75%}i)HEjojBlL4fg8!(Px=tUJ697^6Y1=Yc4X!s(U)kP22ij8QbvHGYE zp{hZJj3MnT2_Rsga}Hcr8KV+}UZNnjOv0iOwBL z=TUuIYOyvu=Q{iLbt0eD=hoZd*0r_b!eV`|NCNazJk40Cm6bd2I~jU0_ix0XRe!%B zFSK1_@J}UfPEbGlmRZy8-Uv2Y*wiWOD3qX|Yq;g~7fR`B%RNDo|0V(y`MSKst`B*| z;7`D`UK6C*Q)unTD!~FVw`i%rrR_$*{)~#p{1CSeatc@eDj4q(K3@0p+Opk(OrY3P z_#0UTglf=F)VU`Dk-Z`SM4hV_Z0}bGkx(BwVnE#A19au7WCK`l%s+UbrU?;_0T?Vm zR=0ZRw>h7`G)GOO)qObtg;s$zFbx#)WErC z_m3fBtRe0!1={y^gJT*5hvbDmsDI-QhBu)~dNWPrh6jJV6=iG<{Q^NvKwLUB~I5tqywz)G^!d$wUdG8>ofmTXJbWImw z-fz~uIkoQ=3#A=%EmRYQEAJBZ@!QI`F&{F&>9@SI$0dxE-WxA>*(;u#AHRwo{8Lj^ z?j^ZdAJ$gv@I7M3kthejMZ1_N&4H8CZL$sbvaI6Q&sfZz#1=g$qg!FQOvn1nx4;*i zH|*u!@_sg1>ADqR4|Rf+`*P5IAerAl=DDoU9K_yX04J!t#C6Dbv5dW81ludsWid^X zv?qRjdmpq2;dj#qN^5SJIvcdeQwuxE`@5_%PaQR1FnFRhv1ZN}qe2qkA~wj$G;QK0 z;w3o5Wke@7=;rYWx#ktWiuT?Ko9WzrdFtY|iALJ0sV%>E!o($SG$+m177%)TwW3gN z0H}B2>o!e?QpwVwh%FLJ?n#g;Kr|_NvdGU8fUYRTy41@0;`M>8772<9=mB8!>1;I;?-XChur73erxCN+vXp+bO5mjH`)Mt3 z4muAvqFZqst^*$<=1ctXSiydeeMdRN4$OvmcdELd#S-L6zvK=+U*?{TlLCMm2y+Um zw<7sQim8soVtP&epvG{HqL$9LKIiwxrCA$_oJI_sv0g{;v2~Qh1yVNSP^Q2sj+Q+8 z(B+!+teb9jXYaXtP*l3F+)2$u72KsOH1kvcK*Pkx@<|+WxvAR+-i9*wKHtgp-9R^f zApl9?SI;SMmd&fnDRY~6dQvdXHt}6xX`J8QleMXZOBYPHTv|aQfPGP!UUF|g&di36 z7F{lqra1#4sW#`b8JdRS{N1dbNrcIQ_xZe9^)}`F{ywf6nxQ;Srxfa146eUzg_Y42i8z1bWWP>q;7?7?|NP!#=Bnf666A z>cU!VYpdvY*0nm(jYFZ?K%4s3$S9hW5}GmZmxle|IivdDgd28>B@6#mxKXvKd>yDo zLmCd3X4W+8$_=O8NUI3qE>8+zSGM^(qT*sQxVq&a5ruDz3VU|DiA%WyY6^UsT!j5}xM z!)mKsZV40b?mMjCp^4BB?C8t1Dx-_|Uz>bLe^~k#m!%91Mm{@EzeGKp`b{1Yh>3{P zQu#IwI3%kfl<*_-*evc4`b?FsU*P1^L?%@R-#+)i@9Toq=YaiIV816!rN1ms%{#B{ z-w7jmqs|-aKwrNPhzIEaEE*#udZY;n1WM3dX#=y**KXnB3Fr4hP0)qQH+$F*oGaen z>Cna~E$_YQv&wk5peiHz1EKEzOAc`rsPTGY^;=-7gh`a&=QmOw0oQ?ccKj2*BVj_j z&7%AUxBMx~tM#Dan0bBuQl>`5)7hSiG#mfDPyPKEpu%Xr%leq{me=lQ!v;cyp+2mV z)M`xjw?=2$^eVX<^VOtq*LDx>hbLjzFWPhSC*#*Ysu$~SuNlk5MVic@?H#PA^ab^g z_%%Jf>f8ZuBZb$whhMBJXzBTGZXxK2!9aS?AX4e~5y+ z*^H}pUD^1OoF6F546(frdAcjC5A2;pZiDob!(u@Mvnc z7;}X{vLe|=juK=)geqXjyff1a-l{o(YZ^e)lqiR>o-0mko_lc!7`g>}(&B?Vf@KR8 z#;ne`Br9#ngjDNR_3k5+XZb?&d8{u+U-KFcVL@&_cqlM%ACL**$7>f!gOI0?CsUBQ zf2d@0AnAv7_rsS+`G*WCw&C&Y4mk>{0(bkjdaHmCnxkJynr+s(Zj1cN;67(UOqpuyaC<_toL0k&1^SDQA6skjH>U?$W5^a#5H zoPmgycObAxLuN;I$)=iHh;?g@_3 zC&K0Zr(eBfUM($h8mf-S(aQYTo4ZMm)~K3F_!=+mSuYW|k;&LcsPW5DT*!oZYgato zzTNS(&f)hY0oXH*=#vZb`I_H)EF%f-0jAp3p{gT2S#Q6vVTAM zloMFjz`-_BxT5)ixaUMKhp9}u8oVE1p-HGM#dS3uVV9boa`0~}0xrA7zUzBw$ER(( zo~5!lwB<1RgZyQq(6bw|AB}T-)#))pam0!WX@|#x#)Z{#@7H~{aBx+vRiXR_7_UZ6%&l+j;co4T(dWNj zc_Mt6xVyQ0S$4~dcr{=?cagq9Vsq>vS)l(Fr#;8ejrqV3N}_wN^ilxV?$S?5DK9&# z6Acx%`kDI0tV6i_wQCzjS(wX1BEscPw$^R4c<0>*kIc{`z5BZviRzoxXW;%6I=HhL z$iH#hXr22MB~dP~zCXfQvKB4z2#xd@B$f=h@<1KV!3BfVcd1CKk-?z|5p)i)CS39T zKQ|EV*a$dvW6;DvbURcRIbbl@bR5<5?OF`_b&b5X=LZbQb>D@eekY*ndFnO__3cNu zU*u@{a5-_qZ$sVDC@t2uY2<;0ruI!WRXL8NI`|1S_HD-o4JGl~!{!gt zBQARu-VHV#yU{2v5EMN-*eDM>1}ro^a%mnQKpiv>ZR;=6OXbkdpS=sI$~sHXY9lAQ zUcwyvxag%J_A%ppDDASDN3Z@km&Lur*+UMYxfps!R@Xb^F;Bs%Xf?YYY(x{*w~*$R z>j!biQ&SVvv5dmT&5lzU;fZ?bMuWx6nVyNHk4(i~rK}Ate7Kj_P7uB>)U5uFofgtE<&%P zZ2T#!6BOJl90foFWR}m;eir-q9N9e(^(CB1w@myx&9DoeWr>_=1Q6p`LYax*@y#SM zIn~LUffBQTSP9j22hs<9SQ;u+`orQn7zvj@SLLzhxk2b7%KZ5T&ipoKZbo4MQ#V=S zCHNN5i*f5+mH;XaXLEqf`! zy(NX1y#k+>dqK;3!6>t?# zRl3G`5V#Jv0B0%={c?{$@4YAQoTuN1GxHj;MtYYJ4*Y_gLCyp1Z%LXbS9oO%AIeeu zUhjp3hJF=cMwS0DR96b7txk0zGEyAD1>nta_p|5a#2CB!fu*u{U@>9e&4V5}?tZRv zGO=xj-*Ix#2*VrgfJJbtrNpEiZ(__XpWqY8gcQS9}4` z+##|Ugei{RE5`NiJst{3yFa?UUD0Q*f}P|& ze(gETGWeA=j);4%n{otY>l0Sr?#o%P=iI7n?aMJ^pPI;|PoJ+EGlKo_HWd=p(+aH` zg{lOBt`!#{Fd-xdl(eD;9kTr}u|Z z;rn(;>F$l^ZX zVpfN|p`}10pIaRl?gPg4Z_W@j2Ll{i=$NUG^%9W*1CL)o=xA~~vf{gQbc1ibF zKnve`Je6x&)*zR#wY1#vz$U;J$?{;F*YOF7J=Us(%_nGd4TVx2z6#&2V*2e@{rRRU zjMJubwN8o3@W%E(X1M*6dHwhn&6`fkb9wax_xI*HeSxUrx!kgLIj|b)m@We!G7y6LC#a^iS7jE1%C_S>!JlFSp*JDQg+rWB0Hbqc zttrj@SH4%uN=S(F^=uoOh5`@^T24YZVt6LRMY|?$__-Zep_2WBODkzwq#N) zEj4sAAo0twtLDX*{xqBJ?^4LvmE0NBF)g!+($DaHZq`5Ri*Dzv+H>fT-%F>Q;MdFk zRkVMELuO2|gl?{Xa1i|O@Cg;QRaV0TU7PDx^ro-xe9v(jL>uK6Jn*^tdV6hhXU(Da z+c1my22QPB*I6>qXH07@(AL(k2={;^vON8YH5RiVOt`)1gfkf4+1VTyY=SF#O{+|u3#2K~6A_2$6Q#g-?1YxlgJzWFE!_3dG?e8R{mXnD{>zH4=P!Yx7xCvp z1{XYv%A`yetj^qf2lhqG9U0nL81)$OmQi-GS`os3%g~qNu;u!uWu>YG%PVv&X3mh- zVBn|+AlYzUsombkpEKGbo%8Jo@cfxmY+Q4N>2X9Aas2uM*4|>^mlBQBO7iypt$0(# z?*{!iyMFWjp<;60YkSIu*i=Z?mM>HKp8eIs_g2s^oXw!ZMzAdD>h&dl=v5r$(><4K z;H;us$S(*;W--(PUc1=GuiOWPSn3ilQdATy);n9<8;8rW3YuJuPR+JszIp#{lf84O z$CrP6UYN9#N}hiZU8uX4T0g!QJ$jcNdH^X ze~1e%L3cKw&?dkTJ>l;U?VoJm7Vz!FBLBg9@KTimFw}EgDqqtGjxhAPbMV~7V@M*e z5Q~7hDYoqJW4G4-zFvU$`i~-l$NZT*2MT4+j^tQJi^89T@8ra)JPtqc38i!8ocr(< zqMskTYy*K76;${P3CPb(Lc%#Q0oU<^H7S$}KlX`-$VhndQbIPdHFejWmy)bJx2E0p z%TcIQMcdR(#OX>!^0aB`lwr#Uu{`UM54{Dbj@H1Z)^gkh#-Drk$~aX+VssiK*z)Z> zcDM4OD4Pi(=COXAXbd0ds-s+3t_i^S5l<7G>&!jDV*i!$!57tD=r@zc`WUZaS32EV zJbpX1lnQ@{f4v>n6HXj#*4k-_Y%a9@rX*nfWqh4Y)^nKX;9R=iWyL;}mv>++qP_@)-!D|TgSf3 z0e9K~1!|Y+sH9R<1gHFHZu#mB$iVVIu=^8S>F+=-P{95Cyj=yDdOBh;IgIDDs-b*> z3?RDDZ>gdmnAvaU^I$C?50x<56R7#Fln^rzsex<7c?hVV%^tLgVW2bO2cFv5ijNL? zj#bZCld6Fz9)jPWVgP*)HbfZepkT!T|+G+01*k!S^({#=JNLb?wV%VMxW^dFh9k?5Dzf;1jNM+q!bR9 zN8H~!kCqP(6}!47X`NFdn*IHZ>WsV3P-9TzEEOXjlW<&5mgHgW&$nI1tuIA=D|R%j zbyz8fIn+op3r*G6VCpL7kA~)%)ACqNG@v4c@es~=aQ#u)L?g=)9`~e@ z;v_Yv{Qtg8;f7RQa@?g&L{ziFk>XtT6|zI}#HG7aGVyB^(5$)N?c6$h0ldC$hn_6k zWWTZS?r85NYsvRR!lyIMym!+-@y3ikOUUOI2jJqn(VMT`SfhW!cvaSk(yi(`dcLfh z%r{?66TO&k>M&TT&T5MoZ$zo6YICB(4uL_#D)ATtdDuyxj`jq2zveSUoEO-21)8R7 zCEb<3{K_co4ByLq@>7%(syD1;6dS6I-gGnHJQ@$}w>wZZ+Pv5gPp3yg%YHDh32Vxw-8M@gT zK62I@Q9@=?4Qp@i%l&;Z*BR^f!=_~5ggcG{AL-4pu+d zKeSJ1;!WJbBQ3*US^xT^H)m(bg1XZ*IjaIzzB4(4J>fVZiqRXKQ3!bG=Y(o^Ai@W5 zru2d7F5^^Fb8t)j0hI|9>H@5C{@z>r(9YT3MnouzjR0z-Emt88a(GC5qdkUl0$mzR z&K@a%*qmt%4gP#_=b`0_O@Vs7Rc`rsg4MfVq$KK(YBwH6?JX%U4ol z@XBC8qPg@6V$cZVewMKQ&|R82^{5$D4xA7t7^^WZ_3-b}DmXMVP(Qhnu>*2xTD};Uxtcqq<|N_IFxbLI1OhAY0u@ zKnuVk)|v=h50a^^0Hg|mN;hruCfTbaOvix5;)JkIZVFfQfWpQV7&6qnQulHmg@vQI zUe$!bwgl}u8(?P^5pTh4CPJh9{3tFAd0HqK0mYY*Pp2N)kc&q= zKfp-ex`^A!bYpib_cl#gFWr3x7r6;6B<3enK*qFyToizL9?NqCkt1a|V&5WnSrXoB zYab{$LuB+amhcpPjeK?Mr6&Vxl-DYtA6s*Xmf4asVEwgg79X%*#Oyk&tH4D5KmoV# z?l4#U7L4Bt_$No{F$4e;5Cl2{;RXY~nike*=%mko{@?=fj!OeEhJ$+(3wtsGBYF1f z8xBSpn%hHaey~?V;Ayfrq7aM#bfsQPE&vu41CPNNm_ylcmx|x1KTH5em)#XWO!|i~ z&MjNPv=OulTrVh2N$jB5z>G65cw=z{3kmNuHn2l#F8f3{MbK8m3H4;4p#x$9gt48+ zgyc}6uYfBNs0k1-mW+d-+x!I@VXGnJ_k!2jr8^L#N832Jrz7A<)Ccz>L_OCq!5TP! z6Ia+3g8gB{563QCy%@@!cDP}xh_Q~j`L>AxePFw!+Hs^O_$maS?W}*U0*ZR{doVc; z8sb|E0PW;}q7{*{0aC(4PWCLK;jnXl*EK>gjnJ2V3-rh%^c&dJg@`GO<`XB(X64mJa7x^(=EcQ6=9KY%#l`jN zA;n0w<|L37H_j_CPW6jdSpJjarmR?vRF>$3OV&5MuN*<;3l9K=>wcc%E(E}>XiJ1` zewnOGsnmfHGR6-s)e47PkE{bRYI`HJUISddf5Y{51}N3MQ3eF)fmM^dihLv^X8gX&IkxwAmyy zRFs^1(Hg>^xeix}{QZ1IC5%4yW?KkUwTAmwNuMb){bAwGE%`;=G30|#C7gnUep`5t zQ?P=R-;2y3ry9KC>zVjrhS4#ra+0F~6SW{Yo!=)c6~j){S+Jugphpv4YH_EZpMH{^ z-PVxNDJ$tCu(@Ytrx%3U$q}aOC&|v_kuD^cI&c3&pCuPWJ?x$$xV-iOD;1fMVNxv zLnm?niEN34>%f;JGnlGmF1!q#qmP(#K@xmqUw~3)UVM{j(d3!lsfQ4P5yaz9TZ>-p zHTH(yRy7(XRPs+SQQ9-PwszEmqJNS6%-MNhd%&MDj%pwMMkk?iKOF?GVM?uI`#&Zk zJ+i-&!RcOEB~?&XLLF5}VJP`YX|Sxwlx=gyhDL2Uj<8tPQM@xlrh{XGY_(k$e+vm^ zX@ga;Xe#ND!EAR)NVU%~Yew1^_-;7R?;;LOquvvA(%^56;74*@;Y&Us4p^IVz>dNf zVYYfAYsZu7E6p$PF?0i%Dm*P$4h0n|hCZe4c|N-IoT{0^>ggm0*img}%R6`d+z#G; z(Lg77)~+~MEkQ<7(c=b5swR>dAc3H4ga4S~5?b&$1B?jpVn7TsSL6&IoM^i(D%_QS zJnT%(?k8wS8UUm5CwQ`v}@>D|f#ElY%EX3HmMpW}W> z$uQe$TvsLJ0y_AvU%c@93G!m_ z(8|F%+`iJ*jJlQwb-T&ZXhmJW9FPW==Z;&GWH~+0eR9W@?43Iw#wVZR7vo1umpHUwr&cU`5A3F{EPL~KZ0XBdX)~=r=lc7=AvPXuyQ#P2bo82DHKflD!FDT( zObZPJEY+ea011D97avAf^NH=>^(fVNCwDiBBAq3OKP(HE*DH$ZQh$rHe3RW;iD0U% zWMBj)I69Rc7BuT=Uv}>c^D*+RV5d7`C*L4ZdyMl{JH?V~n|$QOY~|Y^*H~Hd$j|xq ze3G^~TQ%&*$J@WfEw@2*<$p|XP_~)+S1|jtz}Hq+pmNF9%h17iuiPQN*ekJnh>UQQ zQ?~0;uvTXvXLxzDS+8&R+S*AE>Rix##!Zo2A|(w{U`eHw(M~dfozbgYIb06Kc(_MA zj*I((WNet-{XDK}sG?+n1S=k_=Zos4Fgtmm_2TySM z_UB{=xf(wOQ&djMp{uj&-=KQtd{nr_7yDLbYwVHgzAQ>7S5gbgIawbrX3gN8kE%%z z?B~DzdLiJSq(~yuCdKzm!{@dY zlwa{(6Ovsn?hZ*7>f@Go*He*)b9pfw6ihh(qB$sPi(#r>lS}2cO(LxTbNUqTX0U~6 zh;8fp@s57V6uEP~U9M1R8ho)-B!VwYh2K~Idt@qdLwz8>XXRunq*KP4yb&oAI?R)K zsuS`T=e&x|*a(W1u98nNUu)_JfLf#l8^$cocnU-4ig}lLhyB3&kufGc^<|ch6coq9C zyhzo-gny+`GIfAT_!s9eP;fR(7#TKsDGiCYbYz7U7+R(Ol%;nm|}ci z2B~KNUh#sk50JvMp$Z99%$ejI@_iTS#6qzPD6bL$7H!3;n9@LSCEqZ=%AhdBK>k~d zvWwYm)Mv_P`>6H$5#|fk1>U_pDm68~KT=$$*aTAf^n}hWbvo|G)4d`Fm{q6=16uOF z5}|A&-#6H?1T)11^aY0xaY7|gVkrc;eVyZfhM4}Q_(uAOzd3K+!|XRDo_Dj(grXcN zW0|IRwDeYpKl;z5ekZdJIoo4$PD`*8nZ@B!v0)$pf_zF`kG)~cr6HELJ!)*)&cb5k z-QO!4UH||xu+)F&F~`r@z|Lb5_v5r0!V{RdOes|edU!)qC;b-P!NANSeT!}Org;uK6eQ>_|Em+BljnSij>QTJjgDoBCwU^rQ|58M zO%(Jxg}z^D_;7YXY-~Lgny9R>*YH$`cAC)v1g8FXD8kW2YhS-uSLzcGbR1 z;SaT_xt%{u&)m1K@Z?HK{=u|-Ij>OQV_JJmX?l2Sjt~UpFG#{T3QLt9b!a3=pQwBa z&Hpt^&+}70@AYb->CH4%aWP_0Pt-{kGKwjk8fOZW$HP1vt2<}ko?GuQmAUQR2d7c? ziNS&VG|99HL6>$#Ewt6;%sS_6%`CG{x-Fac->HUqB{C}M@l>;0@$a`)ZqdYsomQB= zxiG2l`oRkrJwh-_Ijcxb4g&2#&i*C){@i#Ak@$Hw)IR5_plTe~k7$TgFE!nZJT%uI zevUdHkUk@@wX8>70iig{Ol#Ugz$w-#Yw?NBtd%RBOjHSGoIhJVp;?fqoCVih9vwfDQJ$CkUBEX;;L--@p;a!dlExA}dA5fPg_x868BC6$nc9N_1xxD9SmdjnnpGpDTF=(8^B(uIHO1I)st=9Ual4aY@O*{0pNovpcq8mr-MpH`tS!@fpjt;(`K-9Io3Es89S(`T9 z?^#*P%Bx(hPL#UGZzZvcuU{wVln<2IHG-OE@1m?Q6o5%k`V%^!uBYcrnXsyw1ST1<3QZV%p%S|a_M(TO z7XN_`4+un|g)4{vXcOR#@ouZmrZC0cj;2h%j=WY1t$$fbZ}) za|9T*H*oqvO=Ts6fm^l{c;s^t6c*4Irh;bTd6~1{#X`X8$#^>6Ym{o_ev9BZ&JRvo z2-MQi>HYLvKG1Iy?g6s}Edaf*fHDV<2cL$3tMqbPmg$gr?I*B{+P&gr4O0aGfX?iG z(;2ytZ3qf)7mA#4m3xl~@86=HUq>C4cs}rU2A1Iyj=-(k#4%7z-8-VWQ0OO8NIZV4}6RZv=45)q~t3mgu;O$QKVzm)NAXGrPUF_ca}6mEOL#)^CJ>L(y;nR{z@O3WdWa!$BpEVvWA9d|2% z9<{Xq<>T1W6nGmYx+nd^Ct9QNd<_=4FK5Gsc^tUAL>z;pm2Lr7CEtbaJI1ZNRgOr7 zk|2H(0+3cE4FOaWPeC>{Hu3T#2h7c7oA}1^7OCM_F1y5ay#AHK^#>7vmV(Ra6`*v{ zQqtx@m7ADW&p1H-N-Fa4M|@hYYVhNS)rIyyck=;adI}SPIVUbwRBa4cMaZ56(B5Lp zgkSRH(U`ZWrblPFs#VW9n-y&p_8M%Q)pJKf*Km|k)58rAJ@gP(ztCFKbhmIs$)o5_ z|B7yGMfYHouRKiPVBXm8(Y|RxFS8u<2T+T4X9d80Hv3t77=b&*xwjAb6&76W{&+tcL9O=+e@gGyD?T5-J` znh`!eBmtbFF-(+HO@Kc1q@vdb?fweK}Q^#tk&D(jAfkwlA9 zU_}nnV<<^fScCETm{GpEvma?$th>uPtl;f;>y=Noprmg$aC|Pdv-=Uu z{Uf#OG3HzgSdjNlLzcX`;LU$RD^=xK2TPy7;sb^#YOU>8KN(w3f7VRo7#&;&jA=Un z3RG%3<_lbr(52M^zQPrRW5OxoyFM+^__E}yKWx13kjcq=KK!Q1^B;j~D%pjmM4{>1 zxPw>5fp0@A3s_(h@bw>0!JgIS3~UsCf9h%9t1-2uN4ms`L8E|4;?3*;m)d`DrSYie zX;f1Lit8SArrR;6_+BL+9r^cqB580(jxUIU3YZRIDHP8n0M52Q0JS9JEU-op`zb29 zJj}QqO_GBvG~s2Fpue?@JE?M;h3t*vT&%;g{iU2x|CjV2>{(m6j`e>;vxHPH5LLI5 z1Hc_vuEwB3!FeLI!g}iN2`5303+AW{Zll)FxnK!qQ@uG|;GuLO2m^h#eM=su{m_IG z(DJT8`bU?z<_;swMmkcSw1e}P?}#wPDY5-|0IP*pJ^Z9Z-}x+ln|f2k!5sAVX1Jv4 zg{S9lm0W;fYgaj0VS}FZ2#z98qEHn)5{KU3HgD#61A}fo&ru~oy%1=Y0xsEmM4)$T zY{};5ePZzMSVtC#U+iR-Y#K1E)LNr1x8l?Pm?v_~vK)qKBO-gO|11;Ek^ z;{6uCsqXt_T2vd=iWlD>p6H=C@F}9); zQiOvUgpSpYLaRFP|x>(JIkqJ+$=Pp<8-1akE|(?Qwxp6*!YsgRNqHkjcU{|2%4m08nC> zODlvY2dF8|wu^TN35n~%mZ9OuAt#Rr^DA#P1A^I!LXXd1Va++v(dbnUAJwXckW~z! z(*ONe$?;Q4h8>7qWIqIwkg*!ja$Xt47Ce0nqvA7lJxd60q^8In`X$xl1}Gd^1MuVw zgwgqM{p81V zJM46uLHkH99Je4Z{M0v)&cnke|M5!%Y=QG3C6geV3a#P7{aa`$;6He30mRIR8?4RQ zFkk+f|F~-+>S&sLDV=wNTf$%CKZ~)`$>px$?*03E(WJ(SN)w zNtW};hBV$!4I1`TY4^TbMn7*$*_JsC+B1^#Rfleds@UYOC`1!#%e@HFKP{?c%#bPU z#%o5f-YgL4Eo@(e%<0%kCB>cJDDy{PW&YQk-$+*RAf6Zoer;IC-525mQMi|5l5ibj zHmP5+j8DMz=-LrzXEQGX{760OFqqg=H0nMiD$TOg!HJ}tgl9~GWfx&t+i_+x5F6uf zSw2h`V_C!J_bU)t1owj({d#~#6fUsVg}}yNDuXuXi*Ca9OC=NSr>y6`k2GwFU%U$> zJsDJ>oT?viPQWWX;#TMVv*#|qpskR7gbSw;a|QIxGS*XfY-;GUJ#HW5x4a~X?grZ> zYsbue`GfN9!>s|cW8LTB5N3lRR~Iy{#-OZ@ZY#v4d`e=#3NuHO7`7~0H6 zRd!TKjjVZ^gpnJuPmn$Pi>8WjqF!bU?KPtbz+e(2*d zo9|&qw^`~s0)>hr{awA3a~8HBex(^`V;@aq4!c9fRR)8p-=pXw+NOA3x>ql{Rg>t6 zXU#{^HSg~{f<_5lnYXhyTd*k+?W11`3QNk-TyQZe(Q+W>S-YZVZ44$DQn_w9d*?H# zh1gP!GDmm7TUz#xeG>ysN}CMY71i`EpS_KJP%;cmZ_e9~KqA;&vf)?&k+_sCA)Z2+ z??Y$3p)RX4uqG!keT)z(zTn`0;JY|kcKS@w>lMh3I61#x4=l`oOiqX4t-;3UH4rbA zBnBc1h?R%+;ptS}zSb9zUe?!O;tkFfxIM1AKnwopfzp`_$>Jj`psLB^Xm%Wr@@uvNVzpjz@8`tY{3FbRA1Sy}>)z>bLJG=5@^Z@r52Az_noJmC^$A zmB${nkX3~^)+w}f?S@|O4)0gvL~jT6xjM7@;i4|AIrY28h-m7yNoXf_L3U5tU|lT= z+3lB)hXBf)aRa|1Iz-$b0y%dhP^XXB3PM;BDFp=f3`NSdg{_NEEnJ6)`}+#C98kl1+Sop(C0sDXU`+e&1N z`$o>y>E2VCG=)$|aFhLyPu%MaszcQ3BXu<3oiAKU3i}QkcvJ| zB8mpRrBTjg^JI_cF>-u_#_qte`fZq$sGAt^J8L5S_$JkmmHU%R>@HhQrSVoECd|h% zJad=JUMW9bWHY7CrA{dX#oFnJM<46se!Pf zYqqe%9%YzRlz5+sJUeJU7A}g)2q%Q#~YVMXbpGy_!M%LwV5UY zk0d2bk_70DXUoj$miXX7e8t5gY#B28rBJ*9ebKyudH*H1q38_qHjXXQr!$NtJ<*G| zbPDfgdksui+vW_SS)B~6_>)(;<%CSUGVqsYoH`Z^TyDCVYi>0J2Lo-cF_JfVZ|%@w z1<`)lzD#Y8>z(heD=h+F^)@YHurci~dEvJGp@D_2-9XCyW(W~!u)p{znl*(*HsWm^ zZ=#57T=>MHiq?EH6OYa61VXERT16+@>r^!?mn6(vusv(arPU!~%CTKcgUsgKP;l0B zL1_cZMeQHVCS=U@H>)zsD1v%o`rl{sJU_KAP?@iEj^z8)`abUA)g4N*sCyzC?r-CE zqjK!8l&vO%GbqU?#N$zcrJOYmsxGp<~c_ zv(#DZ?m1gF4LZktA0vRZGxI8t>br0%iR{784}N%k#ryU}7$tg^bK2!YA#lZ`s`5LV zW3wtJz;Zf*Pn%c(qp&drGvwse4D9YUd%*yV&-u8&g*2d0hjyKc0MY3mKm$3$C~e&d z?YalSa-KyeHfC2M-kg(tnqbYdoUVOE&Zj;7QnUV`(3nY2qYoN82mOU@+}7^!cQdiQ zQA0oHN~QKIhTjoQA(x45GT&Ku^$Qrwy#6fPfIwMfdNoTKH6SF3m+mwy5Ii_;AlBCI zW76%P)|BV!(yHMGKKM~arIckzo5nG<6hE8dz0n5e09=b&MrAl=H6qucJupMe`RUEV z)bIwkJjMvV7m9Ccz4{FBfxEm28pcCG1&1Co*VI|X_RVfs#Gb@8F%Xs+N)IvL?mO1G zk?b*-zdyt|iNSn|! zFg#X9Ogzmd-}PQOeqg=(bPwS0DCNAx;Pl39<~2-tU`(pKW3}$qCWg*IH79E+B&~S1 zUdAwlS5iL1ZXnnP5Pd%kB`!UnZQH|*ek4P-A{h8 zpZ6r#SIlmI@}aX>8xxdWlJE^xC++JsOERz>u>M1*an0!o>u$dwqtdE~#u*!_pFBpq z{(3kPCDw%5{^L27yGxum9CT-oP~MT92~gRhr?e=s3spu@@L5`flxMbp#7OoPI$q5P z73*GPI(hW!fEQuxvcXQz%50GpdTn-=E3mPOpas-RGIi7q^Lc(nqW9BL$6CmEO64>h zG%HpfAXW?>1-ZX&RTShNQjndi!9`(ZYUH%t=O}r{&QsGEFqa#38V0JiKou_fSDE(qjBxFQmK&8Q-$9lgd!p*giltHCKcbXV1c-nXA;7&Tm!etQ3s^HE& zq^nsS)vV$W$Oo$O$^4>6qbS_~$)e8pj$I8!{M3gC2UZ zjDKnhmyafq*iDb2y6=6l|0n;TX;DIC`!{a6ix6v8H4pGPK(2}Mb60n}fp6M|&>Lf6W@TDp?8^KHQc&%s diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/charts/membench_comparison.png b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/charts/membench_comparison.png deleted file mode 100644 index 8bec145c818b33f8026ce642a92fe0022d439ebe..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 33854 zcmc$`2UJt(+b_;L4%-}l0UUvvu%9%S8b7`W>=Tf2MRbG1Tg+;e~E;Oy>TYjMKU%GJ%* z*-2bjOjP*Ng%c0l-5UhkFGR#5!Kz*~6wK?$1*+ z;%*i?4UTr`Xoa>WE8~;IvGny{{e{k%pX&u$;n@ehpRF=U+nb9ERR^`GTW2+5*`){F zbsUPvqjDC$?o%GC4YmBT=V&8^(Sq9+KdTvUko;YyHsm;)O(MoR^V3(B)-ufUuZ}bw zhr&V6#Xfr?T9z`En$+Ac=QYo*5i9ZBD$4L|v+PXfF=;20(8Lxmvj5R@Nq(*uN@qW} zW*X)WbTvu!U*Prm^^P+ksjr+m-czS zYo*<-zg<#5bNx<)%H=jH7S8C|jiaU#xT9@uTGR+EW6EoFn?}*^k8)3%Do#q%oPav4*&*JkdI`s{ayuDUv zr9@x2*8A<%>4q*d-#LNsG<~93FI<0vIo|_3vU!>CS9GGWPgcd|v=w8Ef=iX9em|FV zLCN~Z+bdRTBIWLplhU?r9~_1&A8o`hwQKsdK!+gy^)(eAHVki z=NnVcUMqBka|yZ4#kqV{Z<{;4TJi#n0a(&f$4r4Ruy@Gcj|E4pK3zo%+kG z>9Qp3)~1y|Z7pkw)45_}!aYf=Axfx$*MFr|poO^ZuSp6{uV^7m%6Tsg+bS-f4wrW= zaoiZBV5K`TQyTxuOq%kW5{+{TL= z(Rx`%0B(oCIH&2gH}0AEmeQ>=ZZ8rBaT$q&EcR{=Yv?gb)c$58KrSm2BL*!5Q zsyP4xdih=5yzmmkcyKm*B3=n%uUm z`tAq(sySBOhZQZey*?(mr)X<)RgmlA{aY2auuk+n=kmLw9P?UlT1gK)ycS7*-kcye zNvZak$-?hWacA&Xm;ZdNR(~y8vfCur!Fmq%+4$T{d#YA1d=hghzx4&WA>r;k1;_C1 zwadtoLLZBK{C-7UmE6yscYsSOb2fumAy5iMpq>;w6rqi&$S>4;k)hQ8*;d%Z>-M3oq zRUMFSboAbwN^MZw{FO}iQ(PZ@^4ZwAJl`P_t;c_ml)lM}rYm|)wVmO^XGdb)ro1Au zW3boS_o1M&EUJ!dz|wk6QX}IS{@aww0H)pbtg^q}(iIZg_8c|2%|EO)nL^R_A$3Hp zkVD#Va%~g`Rks=1@2F)irW2^!mIRzhYWGB(vze63XmwgxuUm^kLsjOADu=g@>WEQV zTW0!Yfxx@AWT;3cYa5eE`n^%?*>AQ_COwXyejLKy`~h`emH#q7PedQ4=UxzcL(x7f zUoZdOl^UuFRR?`WH~nuD?8bP5AYS{7@eTg{{Af=F`a%`D;9keaJL8WYq-Vxz*VLyN zI~7}{Z}FcH+VhHEH%6O?eS6N!BDYB@&JOo_*Fc-tRBqB8 zetNorWK?>!U0+3_f^Sri8UOWIZ5AW7FGiac8W;-4lVWxOf2hVfrl--u}rHde&be73&t5aTk`rR0-*daFnRh#l0s`qB_FGKVLui z@%D>uzgNFwVVU~aGQp)6shDYDQ$yvxE3>`gMAAx|TI5)H``*0dBJl-e#af#lx^-3-AU$JVLsBjnHyo(D`+3iu$A$e`ve+_QvGo>fzHjt9Kuizde`VZV&?R%|G|K zL3rHJ=>Dr4qX%*8Rl0Q_C!5&}141H(bov729t77SSHe6w? zM_Q5;r}4Z$sf}W9L?-YL`Ykt&TV6_8{Ik6Wr(@*5+BsIa`(Wb?-~?wnmBdJx zAuFSz`2Z*d9FN+&q{Vh$*D8$=C16f=Z`a5uuWeHkG36~$ms!WLFh#I`+*%(C@2qI6 z-rG)7rXyQme2u;51MOCqfn_N7XN`;m+g+iy8j81BywdHj#s2+Q;HZi^nvRC>f&tP0cMo5_xqjg&tws!|JEZ*rlJhiJ{n zDV-)0Bo5zFsiA1!*u8iEH)+V+!NKl zIjF1avu68eyoBv-k2Xfm%MX8`)8p8!GtgwR{{Ivl!(snUHSIicHwHd zRVfeW0o!G)Y&51(+VU!t+iAjtnbc3RO?e7d#_hQpBO_4L{%qHANtsW7^K~)imGxlx z?B|B13nR8SM5Juhl3l}%s)PD!DID0&sGW1u zqU5G|H)j6*Hh;rZHPQiTduB>gA0n1~So*aI4>%`9{kz*TN*265)g0xh18!ptBBYQ) zGFEPTqXjo1Jdoxiww2vp6pR2(-AUat(zNb?l8B!fl(IuWm6qSUFg>UQXd~LZjImAY zoLA$xjlfwm;-$kkjI`L?J0{9gPy~sbnmFf*xtC!B5)zj7CQNcbRdA2T1m1S%qMPh#Vg<$qEWf{krjJ4;OY_p zr6Z{u+nHBD@fZu07A9WhmbBAtRMWm3bJkYHiL|x0B*tccDpW?_wcFGS?Q0SnBNG^) zlZc$(Jk+11)fZ&Ki$U9rnW4DHKP23o9974}5&?9f!!`_k)}AmQY#w9QO9kwRiFrJd z-(_SUR&W&c;wrGrMmJMMo&4Lb-_PKyBeWD{n#OcGb!?Xc`rbT_k(f6`ziR*_9N(>@ z-{=SwB;BR|HQvmJ-zzvis$+~GViDu>RZCVmB3K}>_PCgV;Qi_WBdx z+1#9Q)NPZU1sQQQOoGjh1trycKVluA>(LfPnnPli)ShFu7Eg4)zbOzY`SpIB<4V45 zPR)_vtimB#APJ=L%NI4u763-cbpec#FLiJQ%BU_$n+lVgKL%*m#!QRt^aq!I?K49F z1YPx8-dzi-g`zpuDBjZ8@`3$Wv!?CSt~L*yRm1djRSvhb18Jv3_sH%L(UPf@Vy?-7 z0_bSt`fY{Iron9*IXc9MvN3e{kU?0v=r|NTI9)C5`Bg2o|X*YLl`39IC4j_u<4wW^QwR zl5q(I$n9C2G!c$4)%;0LjvMG{aX*EzMP}5rYbhmAPWo%6^--X?gjTW2W%{KIt#CkdE@*RoHIZ$ZS`M1Ut z(c5z+GubkiZ;9ihtVOa^fpFVzolN;;4Nd0z6(jUX(>~81SD4WlCl>3z0>`-BAK4uU z`W_Ry0TI>@G_(U^(lb-y-@dzVG}h3hB2D&l+m1(Yl=n)j@^Z#m?Mi$JYii5cJ+Zc) z1N?*lQFPq=rz%!4k)q{hIk$y`bg|SowIS+`LLU9u=L6sJ|E2zpoV2}?N>4w(m6%>S zKO0rMCIA~OCMqfKgN5tGnN!#FJe<;9CARrc9q}wkMSGL}RvX_3 zYx+cx<`f@IBTlM`!zfMr{2m_Lf_Bh56ulfX9n<`62R^f}c+7MOF>~>Es4>+(Ij9)D zJ9PSkuWF|T(t5J()H`+ff7AG)8z<%HD&2LBRpu{!h32%JT{@rM^sR%=F#gsRph}o^ zQ|x$*CEn0FX23%x@F799tXN2)Mc045noaBHi|@KFF$ILlHv*>-T(KV@91H zY)~TY5Dc`eGlJohOgHD>oK1*{a4K7@9ng^WKP#A(r#+uyyeXXO)IgZ7MrcR9$~A2( z^G(y#mSX$v?2YaA$#^U?zb^e(a=1V!?st^BdHH9F6ChgMws!(`qX+2KatQ;&ifVNF zpM)LM|3b5Byif^GgQoTW@b~{iBJ288z*V^-={zJ4X!{V4touR{avYYQP}-m04CSnW z)-~DV`#WlKH$Od2!dBZ+dN({D(CgrIUvXFE#^tmn=blgZK(s<$W}Pf-T2dWvz5g=u zxBu6=9A}s3&cXY1^z@E^G*GX0ccLRhUxZWQ!Atz|DW(%}&<-*FSsLgXV|9GE5KwMZ z)|qp!urOjJ?SU2NJu7hRZ`Qf@c1H-DehDnTZegU#ZJoJs=&t{4PtG_I?w#`1=5p;X z-(8pCa38Nh11McuAi_Fq{cP1>aQD%ki8Q-2(R@{;#j(1&XfaERm0ztJ(otA@Ak*QP zNB!32chlQCqDt~=PI<0g#xgIAywQE28%I&~=RZnZ~DD!X( zmUsk%Hh6ueE6Wl%cUVSti2(dvqN-&!@Tl1{kd5T~@@(0V+knvQ2RIti)(Fb?pt4;FxP>&%#$GN@nX*rgA-9kc4_Nle2imb zb>YOdZJ=HGJ)kpMK0g!nE$IfJ@%M;E`1Ox-)kn?!?rihg;$9a-)}k||*~SEqgzjAN z0RA%IJ2}19aW(Ktd`2H4wj2X3g|#3Nx4pq#v;~Ol1t7-mU%jgovJ%4AxVf{7MqkL> z;*xcL!0rCy$|4ZV=C)-DsH-M^i#0O02S8I8g?>UBqFr(ynhRTdyeAxpqiK9H6#T_c zY+6&blDwf%jv`(1OXrFoXh1H*kC-PGD9$*VF}K%%%Q$T>D|hs@QbQL66$up%{E2k` zjRFwP#w!Nybz9qjj0PzcHy?+p31v#LeWRwmO|OTL{7{`oMAefx}>aPj?YS+(tL zDygo>Wh{lZl3l*mH$adIG~uWs{T28e@W!tT{=3pnWm~M{XQ5jc)8=7Qy9F3V9*`dR zRKFlH)kaWrGOk$=vF*wZXq)*k$$HCt_?S ztXPav_cLCPhJ3vn>sK5kq0fc|Za!S}q-{uEUL|5c~BFxk77^It1@ zEm2^a9uj9bMV~H6wJYMU8xu@ZIojJXC=<$d>HfEuCR_S?dQpX`*)50|yDWaI<$a2F zz9o8WG4g8JNL7%2I-JE*o!?^w&c|SOU557yU{O7(YkT!MKLI5;r#7NrQ)y=u@mzhz zr_+&`Hf374vW@mw&T0l=!6r9fWVs*eaV%vE(0Ai)l)>PIFSbjsKUmzbV&RE^I&ev= z@@u2`K2bRCmc2 z2H=J#UkhR#w>+7$mF7Diua|y=Tg>8m;-i?DbHE*-%D)9;Wn6qpPx5;mNM){6a+Hwq zgo;p#4G2Xo4T$Vkzh!%K(9mVMNm^^Ugg{G6V=o{a57sFhkaFx(&sS=?B%d6K6Hf+3 z6fBhRXnBmQiPxZqr0ls+jpz^GwbuEVPQV*o-UA!fEnATW%X$o!I+6V>!LAfh?=f2+ zlfzy>xm2h268^Ygl=tAU2!ME9@uieMWV50082W7UzbpfXLBJ7F8<&1NVOucN(R>_f zG4k1D(Bo5Bpa?A1TpE9)W8uW#w95~~@Nrs98?5r96cPiwLEa~ncbk<-CvX(%mvncZ z3^7AIBEWBbB&fK>CpO_+c^m#Tq1SeLN@;sBT-<+9LSYrr_q{j9@os<2f({kM{9Rq? z>B<=3(twTDuGE#_jqh8N6xsglNb7A4ratR9$0LmOIwe=>9%TC=7Lb{R zs?w$m5xoakMHp-z!7!PL=pxvg7Bw9Djcm_89QQ;7o+4;fHYMd#v7j*fHlG87<_KcI zlz}paA%lUz4#jb4zbWy`lfB^j*i^gJ@f-zFSqsqeYzNF9IdGSYQEAW|)@Z#$U_POgNg6 z`B~MEHdmNSn1efBIgk&cc`Z_tNwh(V>xroJS#5*vT$`47G5Lkc-Q$BA_7=eX8alE3 zm(;)Xf_{MCU8B2&e_U$#YF8yJ7jb!7M|Asm=>(fTIK8L&Wx3_wPA2*;{dR_bR&KGH%@UDB-05h2Qw}Iq4-&Iv_B`q~ z^L4~_TCLVk+)SOrTvo-jAC=m=?YlZ1d96gZRyLO-z`1Pk=KN&R-fDHF$4LKnW&jj$ zZIgdETxs%0bj5?*bO{3+k9Gu>UDi;K7Cou2aRza8Ha@VT(;7GeL<_MtZi3V_anpeI zmCjtZzA41tIp|#sx&b~2ry?Mzk>)B8s^QCDko##mHKy@5p8dhJ2YT>F+{ws7)^g*$XmDGIGW{tVdrH}WySXn&%UvaSBaH&Hk%Ve+`I-p?E1T#-4_n58$^OqdA%-- zXAG8K+(-f7v&-KgPRZJaV26`E_*I>6qShbOxuXn~{zJK7Dt8zPP;1b_Ta`lvdGO(R!*ef`qp>xu~5^&-I98q4c4rdxQjWfW3aa8t6do&MV1xBedRudivI&Tor8pZ*99U8a1)wWr*9}jpJ=$@ zhg^)iQNN>Xt|InrgHU1P4B~?7(nzpt5Ycvd6MPjV`=lMuym1%PumplVTFZT@JvK@H zDSci5Atsl4z*hDL(lSsSB>@+w}AxRpC^ad-R8b^;D*^uVRr9QzAG5ioYd*dYhm zF@hX>R_6!hCV$gXSk*0q_bq<-05Ou6f8Vj2K?RNCi*x-OMDCw6`43u8A*P@=l;jM@ zs(m75hhS4-c(sIiNc934)nK$pH&8uyKyB*Pl=}#xdaeO$Y*UfhVo9r^PW$5USgs-XYvyG*KRw8HS{Dp-4m<}Z-7}g8)h>lvf@{* zH%}s--#88jz6=J_iPo<{H0=t1@K+Ecd~#JnbQ*BUF#w&h(>`hT6m!OA_a!i;EQ5s# zz9~(x^2q+=TLBl#ReA)pT%z|WxF(^Oz$3B@mF#+^A=UfW5a{9F;5EBt zzg!anGcPi`n}lQZKxJ_YC;&_~F_YrS^PgCBi$Lvzb}$BTyq+x~`uA|n7dA+`Nc>QXZu?XKXqE^RGfPB^1QY*htoY?$Z0OA89bsF7V zVeGPZ2>4O=dMpTXCfg%~6Qch7?coGq5ia%osXz5?3+^HdaQh-==_j`lwEjIFg#)f? z2-Y8`)rE55cbD@MLWWb~oL)e$$nfijN=q}@@~mWH{<*Nws?68K5un1dw&TD9Lvvg0 z@6sE+}>jXiy`A7sd-W%Wl> z7aK0!14>?4`Sw$Ds)BDLAXXNn-EbP14s`T@r4Hc@+y4AypW zMIcG8sOa>+VlcHkpe6dMZ5gAZ#;bn8*(jZ550n&qDXV@V$+*x71Etekug{G0i8Da| z%%yT|>fpwWX)}4DG-a!BUWJst#FAa-(p+Ev3;)2OEJBVa42bVBMDqteL(hpEI4L#$ z=|16PL)%Yobea>b>}*ra%i7!VF_T?T$By%U4o&+fcw!v^eo;%?_QoAfva?ZJNJYC& z_2fm7MRz1hlb&{qHvHtE-CVEx{D6#z67Wv}s(zurr#<**W+{umr=o?vm#OV=O3^?h z$;IOGbEiRLIO0?5q} zdXJtrcg*ixGVp`_XEp%4Cvpw=8!o*-jB*%-lvVZP(H%rS@L8T&;Z3EO{nOMIU}47E z?9q(GAOsWM$S+K>5V7a8A>{xr-xY4RMzI{Ai+B>_DVNhRph-Or3Gcj!UY3|v)@Pf=?0b$B+8<-KKo_+ z>Su??>$rVqb6h|WzX@8wDC9Ms0mPAz^_-2yGegkD!Is%gb0=OUcu7gS@^J*fFZ??<_33H;Ao33=>p`ephmM*&yb>C& z72)Mh85&F`?*1(Rd2Y`F?A!6@vUYZGH#npie4o`0IgyotEte>o=-! z3~|WvoWH$1iJ>*_eE>`HJSi0+D+PL9S{!v#zQDmP!6FU|JYsZyx|^6(SEq)+a43r& zvmolWW*HaK{}*6|wqiy;ch1Rq(0)<6Qh9xRN(Ed_Qcd^y&F@aVXK=BOVb7rHcsn zA&5vobOs+lX21t7wPi+DLH)_TH`<2^7HhfczA*E}U;D^D>j)mcOi>AJmt=4<;2GyP zd`Y`LL%Xbo9nD7eTh<`hD?lGbXY12_lS8FqM_RIBFUx#XV=^iUM;LMVOe_{57ZDoihL%ylvb>WbYAgj#Jm&YU?+-9BDz7@Tz7V)Ha0DE?c9TW}Mr~R8? z_trXNm*cqQvq9I-8SS=-uwD@5ky-? zkycnoa6}lenbtikmi>YiEQAn@*<^FV@;_*l(gli%i8!LAj?k^33NF@P*0UM516bB0hv}%V?7D}NS143xd`!x z%GMx((!dMB5hQYj?o;%f?fDFH&bcICh{>sYm`(M;j=xWsgJS8t;hP9CtH?xsJM-m< zrsWFglND_kI4Sd?AwbHXLC4Wp#X=2o+TO z;EG(9Y=)qL0T|$dj&i_S--F1y2(2AG-G)fGb#sMe-rx(LZw)|3q4+cLy=Az9l>T8X zKpP~9h<6r2F9Kn-9PdYN(K?6#mZ=GzFA&OF9PBV{L}vsSgAtMVZBUoCG9_M>WSai0 zAA&4`j{N`@-OzV^!gWs?Ao7f8*rm~D0#px3(-fZZ*q8m$FxPr997jLRDdi9?_`8j6 z3DvDx<{ao6w<{bGwrct~BRxc&BTgm#&;YcAl#(HULwo>}OgbdbG9j(+#%XbLuuA?! zLTx*Ym;MF7t6T-RR0IA+5O))&_oBPYB^|ZKcr;~r%~e=Vr=AVrD@8L(1*zl6es9RY*n{Tj* z@|7m8jEz#CE)+^Utwcqt?dS3DAavCi5^@$_10t0orJM=8$3Z?FDikfS8<1X=0k8o# zRP@kkG+#>2W-~PRGNC;Ixfi$1wPV-cm&_qi5myEZao9fLq7%$^_L(_1 zp*i@a^kG0@(l6eE?jpdK`tat5PmXOLREmw~-;9_;bt`OqI8JUvqO`11I^I4IKkfV# z52a|*%Jy;O`+p3A#=Q!@YmH;WgqTO|Ati(tB`>tSy`UvbsK^Vbeld()lnQ-?e_W&+ z*bP)}X^UW2sckiq>CWI0y?c28)Rf2qIIJmV*UOIabfS&VCqT|dq@~N`^N*|h#t(_2&tst$+|eyTAzn@6Swvr9{mP47ACZqyPQRrV zVgG@pi2e5pU=bKSGW%5xnJ3OHt8EUXG@nlvtLFSBAY$)vJ38E!BR&&t8TK4Hj>3+A z{9PGIILcQ&bOknF1HmKtYu)yMw2<(x3!Hcj)DkwlITn+6MnShlW(<4DY8 zA`fbbVzUBu0Pz5#ZG<#W+5rw8{kIG>B`(l(D!xS98EHi*4NK{i?5?I*$p#t{#5&Ky zbOJt%Xk>!(kX?n3z!nvIVq%YwaX|Y(iHcuwuD!Py$p&HY$k$n~iXm&31+3{b$BLr4cez979@mu31C~Iy% zFwpWRUe3W4Xc{qQ*?mxM_LH3rZz7wf!fFRSb8~Gj?nxa~zH>NX+mKrKa9i^m`lcRe zR(+iZagQtiU^~k$jLOuPl9s(EuBQ6LSx@=^mHuExXS36j4K*D?Ya!2F^{=(5D`X;=F1sC&(f<_W!d%l|ExZbD~uy0bwf`pC`|yUbGn?QQbVA|NQ@T#9JW zKal;uMC=3X=qv47J9fbTok}nUeO6E@8zkCdt~msxApyIc@cHoLeK$eiv4l1m4w^&( z9h^}b8Sxi_kkkw1|7AbD=BW8k65hb#`I6Wk zHLCVv5d=<%^)nx`t88)8swRVL&=+fv&ho1_*QO$jJM=`Wf8pI zo}#h5+ne-1?m4&pgH00STsh@Z?GPDMX&))gdC3}FBT zfrMqCHWQT(0iD%<8A6lz7A~7^1GCm4=TnJ)_1pRGGMj~ueK3g-sA0RBE!pW}t3NsHo5}~^=XzoKqqDk{WEslXGg$Oh<@q$xONg7!}$&&f6RG$p+ z;V|e(ewl!rHlkr)$?OHE4uJt!DV@C`9q~Kh_%2JtIj!b*XcmU{Z7--#)mQfk=~m!? z-pTxkYeVFZB0$O(KzjtpYcw)LQJ=&EA$l2kKgb@q?YaBgjn>gBE!6Vhu}RMFo2s4_rY6SuxefuBK_*STBLc(Fkj zqt3Fe?J*2Sni21RCGsX7o#@xcPtmFZDAt95B1bl~H;zWfH$AmuP_!|`pW!IE)6xL8A5U)%i^P{MNSsj7os zlX#SKc$ziq{03=giooWZ4FRh zq*n8t$mxO>ds+8iFYEq;*$X5c5Wo2kACK!Gn&YC@TEf9>YkzIMl8|^x5s!JEieTB> z8aMiY)7`djNpYKO#%%nD0h4PCh;;d4sPd&OJCzC3U6*&x#G#9uN(5Ea*Lph;-*BdH|>||HjJ! zaXew9m&j}mu!4Lp?>UTYY;f)p>>j^FB(;JvExfSl(h}^0ELR59*(flSf!B`$mMClxtndpHTtvf{j!swON(mSLp<$X?V@^jf4&7ne` zN~f6pO^?p~X;17DKjP3Y`xLiIUT^W`yBVh^G}eyC9bjs}ZitDT3#I>cHuQnqYR%cL zrJ2zE6LBSaf;X^OqqXoXaAwfyU*vD$O zWXI-t<(yvauKXGOg!;nM(Ws70EEXE_qgHX#w+iZ(7Q1sbpAql|E8VD<7`+|Iq0i|z zNx>Q+4TkA&#z}2N=u@|CBn&HDr4OJcLb!q|CR#6=>!Mh<7#sm=1@?zRbx{|SV)25g za}ZKV@>{aE8iR_+1Kc)!b)fE80Ib4)%MzlNJN%UWa_pP2uMNbE#l>^m^8vqx9G}bd zot-$a>oWXuz^?Ua61i5xYsh?TjM{6(#dMeFIq80kr`B4~x@ng2k)z%$A`-Yav~b_ejti zII$*A|9R}9!}TGsdynAFy_D&m$)@i+we7Fharo3R)g|(h{b;87Ei1`dq-RxH^5>!q z-7+3jRNJ*W^?JOgt}*|lrm1pDi{zgPO{O9%B8#ecQ8x1vHgBpx&*4+mqs3V@BA$;% zIhPb1>Lv46pGcvT6D6gTmy^hV*^%sxjUOW$v1(Zb81?u|V%x48t2wR|p7%AcGbVj| z=6m?@h#$z2!7zCr9b(veaK}8b^)K95KXH^>`*Fk1g7m#g9<5WhbNCg}=3k zmV4#a=!e!|OI2pEzn(AF$5GUm2mV8qfYyEHJv8Z7sE5D6f{lOp^$$p-o2z@!p>`gJ zM(K9}1ZHHMY-gOUWIEjihE~QWZ|m;547V)6F5btCd}*ObJlQIEBB%GMy>f1f#(T0q zBW$&>U#Ws`_1dgUkBUnh!<0KgR+v3*#Y~V>;v=8*AO`19(0^~SL4*xxpea>}vs`NE zOm@Zi{DXmycN{84WEcB+8|t{6NqjV3*!%7SF`d{W2Nkv*SXoG$oBcfx1|=<~YL155 znEMHlx|fM%$MsEc$_}~IJ7P`)>pkA(lF4qVtYQnjc-MgS@53iM&Ag~pnlHqK%jA#D z7u|L!o!Z-4f(|a4*{*Y~q<+nC*6^V7wU90dS4>YFQ`*M=%3pHLnX~4)jt`WpeG;w| z(ceamHMaRhTPclb3C(aWmjj)M&pvB2uE#Zb^}R>KkyggGvgM%TTCgv#Us?S!h#sVB zsH8`btfhq0z@-9^$PH`~JS@Kc{f7$z#v$5jIJJ%hBK1sMY7ht45)eQRBJqG_$AkE{ z+#Vfwa$sg|!Y%mYzeKdVW*&1{)M~YZWKIP=11ZR~wTxNh4jv4jG{c@|wpZGc7asDr zzI>s}>Qxz|{}s7Twttq5fuST9%{)@J4Cg7hKdzLmRhrEpY^9NrEFE#QRQ9#0e^ zGY_?o6Cn>HT!(qlw>;d z8ddn8j^%5jdNO$`$xhI)deLR=wU)+}8Oj?OGk@Q-7PRf9+%M>SrK0{<0wCWx* zvoiJ^uuE12@;TkgJU$+jgyjvI&vk=iN<8>+LQ`?N(!BrTM^|G3)3Kf>^;M)c3{G*Y z=vUdpAnrq&hL7-Hi} z)FLg|%~n($iChb(JL$?xEcAL$KGCx1_{C@OmBFV(8Ee7L+0k25lj=g&mol7}Ru$KO zE`!j|XF2-8oJbu~PKq>l4py9*SS40$NFjWYdz8m}n8-A_BELRN91}$FIeMNlQ0(Ro z!B>4&MC{6GyvW#hJ^1p!M3U@;qltFL!4n$Gt|P2-Zogk5LK1fVy zVzov;O%|1To=3^*Xy1*D2Wn{wQ|sw{v9-eL+T-`oeR)pD9gVb1{qYxa9!!>TOO4K` z_7kCj4BYUZ)l$;$-{z|g35}Jd5%p^~{E6QqpGX!Dy&NHFRw9CpGUWU0S9n2NwSQSO zUqX8NtbGX*Ec*YPp1$s^fb41oBUDUUA>tf^XgsA@s&48cP1wUJoj5{`d$rd1?1h4b zIjzMPqh8UWu}*Ifd!8j_sKgv@>-t7wVyLrlAd-r|(NxmP>iys@D@teFlI*4xcA(2l z=*U#j?RmR|Y_Glz|FISv$tkLITr*K#M#Kxewg*G>?%R2?X=)E!Kkjtu=&HU#dp32H|~P=gCpl%$oNGD6Bm~n(KqxIte<{ zyG(Sp7^@G=65ZNfxh$B;Y2=gAdxO;?!%p7}c?R93`ynVBkS6S@>Tn&z@91 z+ykh_=aJ@FG}VlY^is<@uaPd6e6lZ7miihuK^0P&lie6lTk63D3%Bum_E;n@`45%~ zy~0bAF}RQo7YD)u<(p1(@Rw^(TYmV8P#Hw8)hw^cA366|4q93;EH%ShmFTigqi5qJ}x;SYx9b%ST!izW0vqxO6BuhS$8$75dL&HNh-Y*rJCOI#2b-7;1S zPIv(vNyHIo| z9l`H$nJ=EK<%9T=e6~5{xZTjhK!3RbO{%b>kd(gx$mmOp$iO_P;3A$tP1mdVwhzJ7 z>%C0Kcw$by>Il6Q7>G2xOfUg+0GZrq-O)P&(n3hs+>Pn93!fMyOYEoiuDEIelHtj1 z?A)B#Ld?;z4gH}Im56i7b!&4LX=jdvo?ashc@9kDIK149%6$%dnRwizkH6p=?JDIB zu`%~iV8+Qg6yKpw{at0bVHK6`?#C&(fHbFE;Z0@v&6RzN2qxRpUl11BZ7!~VNs+q5 zCBCw)=03+wRP<41y~bR=a#MbNUA)!y-5$%rIjL}QEY83LW4UVT2}z8Oo`+QjlEGeC z1P7-YiZL4~U5#T~3Erd6G3GZwN;E5jOMt+q8pz}yL&R#Jm5M+l4?*U+>5Dnb@4+9< zdNVbXzjEWXr|&}mJP_0OKLPpei9V-YmlGb2{phwU{X^80jjJ&1UHLZqYM)o|nSRM< zHxHyIN$Q~t9Fh+5Bnqy#_4p=L&>uEBb6yxYJ%NTU@(um$`~L#rGo65)2q;|TGV(Yb zU2}Af`(KdVlNj5`Oh&S8dx~U97mRFLg3A+(=&R;7S@~Trz%QzX zFGl$DpyK*nEDv=2jV(YrC%Fuhv%h(`sSf<&rW5q4}s30>nBd6THmK z68)CGw?cAgX%30GgT8c%XbuK|7+f1pQxP=lS&(nx=0~nMg4vhzWiQvyRpgQt3?eG9 zsMm-2&r^{D59`Wv*1BldORc)U_bc+rLY~Z`2k$as)m?W}j*nX{^RuCZv0tabCz@=| zPyww*$+<%CtX87QRBZB1#-Y&Cl^1IbS#q5f+YFx%bOk=nbq9j~SRLa}A-u`ySdknf zI(@n5cu9E2xU}hV%u%xjoO53LmvKsjneOb4n`dMf>Nj9r%5y}dY_n_s;8F1u&EEp5f)0l3n{DKW)Beezfmwg03JJei zWCnWB-U%i*7XfkBCV>n63+CYo0+sBi6hW83Tzv_`jJ4FzA8>0MdG^gRoWo^LIaSO& z@}8*RvIr$-%XcpKe4xf6tfTtHAU-<}z73!P_@nH_z48*G^Vs2>9@`m>dg`j7*Q)nw zlb74KLZg<*`odJwQTSL;OHL+6pF(l3&jycAHh-KSjCwXBEOSnO&3}vh%4PHwO_`Ff zmypo{1v29F=+vZnXklWJT$G_*P8s5sxb*xnS-t8tz=hjv;8{J%$S+JzGry& z%ai5L_TAGubGl3hi#<~+ra0g~R31Vb;YNC&TQK6GU z+&p4wNB4&*IV1@HpzS1Q0(`|^XiK{mFp-ZVWZuV-tXb*0WoX#YM3+qfu=I`+_sE1}4y)$?^`TO{Yc^kylFHQbDy}T(dQgH+dTlCmm zW-xN9Fj%tm8M8dm$s2K&Sp^iYAWl` z zyqT}yn(WW@V;B9dLvmAeqD~bj^6B!JfqlN2*uvDLCBTfDvs{@FY+k;~E9hLc+SSN( z_jLgQwUBm2WIW)(R=$4R*vZy-KfUi?dHR);Z$?&;Uh4@N{X{(+C^Db8Ju_V(dl(h9 zt0;$|=S6W0uPJ&iifBykda1CMIJKock@`cJxEnV%5*Zw2AZ6 zTH9wh)y1yt*0ImIy!Gt6PLMXt@4C&FkzW;O%wtQPxPl-;ccI+7`$6YtO!oTbuXYiO z4UfXm^Ksk8v+W$4kpUh(ohI{cz9| zjQ;u=xOVI6g-C=lT3TVbZAOMZli%j*(o|zJzKGJ1?$Y*Z31`OLnJXr^ppv}TYP`qh z4qE)ACRwtKM{JZ zC-+@BwpDtnGo#y`C=H&&((1IpUbP4uyX}A?y#25~8K)#Ga&&GWtzOx+fAAoh_R%N1 zLjlC%v)jwtbuP4kpydJyrdukupTlRI_zv;mr2aZ$`ay^AT-^E$B7kd0(&R+wGexaQ z2aX>Bs<3*U8ziFH3*}1r8PJE)_Bl?+#vwTfojE>8A!MUWkhBn_N~7mj1vKXu;cFvO zCxkJJHRd+Uh8BP9W`eeRxz(LgaNkT(?AJKx!dilJrJGTuqs_Q2=g&X*3G?r43&;U~rIYd@>ls6Sh?zdM$M(F4suCQZ#1}j3&n4I0_kNHk@j8{GN@pq@snT8NoT< z=?(o+t)r%54UTWuI-a9MRdQF)A_1>5NGgo?>!61ZD|5f<4wqm0wp3q!AM&wCZg{7Q z*n5DvA%9zbBHD5kaRu`jFx>JSyM^6)HJ?+*>$Z>$7c`*nW%@c$buWaaWPbkcl(R7( zRF@c(#4VUz=+EuaR_wCW%B;Ce*ZwFCBAJ1GC;4Y@fZ7pinPdACCIfk9-}Jn%o4#ru5O3`kZW{Ev)ifyDEjGjYP6H z8;<9UepaiCXesAL$P9yGK_}&~0GrYp@X{f3-@>l+9$j3hywdvVPF2wy-_hzM&Gmfm z7aKa|pWo-fZ3AmJ_`ovA^04)3k5wEI_Qlv zFj3}#%&s9BoXM9D>FbNFFjXlZHVj{cbO;g^4Gy+$wnv6-o7m+^y_;{$1^;4-UXHzh z%dOildUW8@ieyU+W~jP0%klwH=|=h5pv8C31Mlzb&P9pi>^?mVkJi_KIRXS{lqIsX zq)6{u82IKtZGyV=Kid29a4hq_{fpMoR8x_PQd3hzAxTQ8h8Afflr<)mCD}5TxLZtR zofg??if|(}D2bbCLb6Skh!8@EvP8=AJ3p76nfX1>`#i_*)uzl=dR4Nc_zWLsam~M^VnA_gk!~g zMQ)yy84Xfu7}ZTKUJM<8Deg1P8he=AVD;=ts>|puV37dT`K1<*(TS!^}7blFUt?e-mQHi z%WJ;0=PJKykbk*;+Ll7g_KJ96@E^RvEmTL3-xA!O^+;M2*WCV{?#tGm_ynWjpcl9I zU#YO%T~l;=o~27+O>cP_7ceY2x-(bCW``QB zd8TlT?fK87tpCXlegCj1{(px)|BKuS|N8ExEBYJ`KDTLx%H1Bvc1064_wP za1a62{9_1_BdtIzdK#w{a|DwSv{{Q_6Oy@4y1_h40)qL}(1h{ngtzQ4n$oGgbf|`_$E0+aMkMdP>?YK zEAuJPfS!{ec=%z2=Ir-^g#^mA!{Znwh!c8xAQxT2+k^Dt%=-h_?5b{RKm znosDlbP3&0wV2xnJj{v$DHQ3`vhbNJT@+{TzGeK`(3<99WPho00AD+axBOqXQq39)V6Umngbj$4I=+q{gATA_tcO%nfy1 zcW{}6@86`jfkoZjJ z!}iQH_Advu%eby0d@`SWC3mZnT1F}GQJD6!Tqdj*Y2isFd1#`}+S;`#Y||s+WD^9X zq_~wUkO6v>MJk()%B>x*^WQ#HaMf86=oK#4vB8>pj-%jDnM9Fj>hI3M4Pf+OzIVxkB-J$5|g3^Dxg?7k!m`K+aqv)IMCYZE8d5|+(zx;Iw z={}5fDsu)ZH&WI}Pag#;^XyKEN~?5wf(PnGEr7#0lj_{L3q6?~Ei z!WHlR;I&7fUqt5s%%r#}Z{baIp47iXYw68mSA4Dr?dAk7{RAGCyeZh4{^2L8CH`ea z*jE;Js}g(ZHK4ZDM@yGX1CdXu!-gxc!|*($R|85~@2~@nnqV1(pP*UoJD4Eau8Juv zn-%1mk_y*2*W+9PH(-U=fes=BK(MhE{PaZD#Q?5(ws@l)$eQe44?JG%|LFPQ$PK@T z0HgiO_nAdp-SmW@*gb;(6p3XV)swZhe7naXc*^}<;arBb$4wvO1N~z%mWM$N(Rppb z`+BwC8O*y;^+@HD?D+B%xfXsvuwoHRY{>ExT5Oo(X~pxs1dnYVvE0eS;NsH7SSLn= zV7QGENl|RdhUX1mRc%{veu7OzTHq4RqT~yFKa}{Qiwz3HoW%@ zrJ?wpAQ72DC1amxNbk3MN$PCo<`*hg;UUwuKmneJAkn=sI68~|QBX5+0tdPZKLoU0 z18HzWk^&Siqgze4k5SXIdVzpk*8`a0cZ?&kQ-?=7kv4T;0eeF!o zl7&ERUCA{;tCO^Uy|PEmU#75c)N+G4PW*O~RDiDLsb>)vZF*UAO@2)zvPBiN*QVf_ z`yf;+f<5YA1Aig=7mo-7B6kD{E z3bSmt#Dms6XHu#`H|?u3)G)mp!M^1S6`--A#AEMkE6aN&#=d#hL=N;qswp=0$^>1| z{voAFY6WZgBA#%dgWrvqfdJnuJ3842uroH5xedsikkM)vHYLQ)!Yv?ub+1nraIo0K zrS#^i4HXBN4>2mk61o0jVnh$~nJahmZV0^#()355o_2oI-Fi1qhhHSD#VGFWk9W)3!Bi;_MnukgoMv-H@*7I_mGqajc4z- zmjOFOX9OKsLLGdeg*aMBCrPH*M9!Juy^-d#^2B9&XA_=ohY$xNA8V+aHk@;ZQp?4i z^aSMGfGKH*#RF&d3XJ0`n0H`#`jJ}^Z4#v#8{Dy8$wUtoXR5Gj4TdXW zzs#yj(1H0wx3MM*oOU4(2*M%v<}z>5XjWUWx2mubH!uTQ^XnKbkhfG0%sa%eELYZ7 zLbA6O@{b-K=!>t+=o2`D;*1-#U#3KzHOh3^2ah8t=wxZoWnfK^3L~h7Wev%JFJw47 z2GSgMteZ{$JIJ3uduRB%=H=OK+wvUVRN3B2VYi&u&1d}Cpk zOv&$W_-nOG6WO4FF)fUxeBjGDG7FJvBZ<7z!=sHuE=fxL#sJkDqb~2{CYpa388mrX7Sq)@@CRN!t{s>y%JZ;%MO zf7-Me(9_zbQE<2L-}Z?-tkZC5f8N%F*P4u#=I&nnzyBFA$`BKGlp%xHgG5EP9?0c+S-JjE-5(nomssImi}6q{X1-z|S{5aFq3obVW{W7b1(VW0MHzwY?=ayy zPfoy?5QGxZdD?JkJTlfDM4w#XJz|7@Sg)Bu_qfF}RfkWn6jP-+&>rl-BwyW6H25Cb zBguy>V%z$lym8r=1V5~REZBJ%NzxA{BXXo{GMF#^B|Sk-u$JYa6J&9CYeWRdSaj!> z1Ktmv^$;(fFLVM*vIbW2JTO`y~= zgAxbOhg13AZh?*^-%HYdzd-T!+$k6b8d}S<=>Xq~ z%@yemZF`T_$dxF~N<5J2g|EMIv=;Th9wz#<3F4x4*tI>Bh_wx}zV^Ht8}#^;lycA- z>^oZOzgqS&zOI%=BKt)L^|2pG4n@P)W{;R)4=wQ^{%kLr4NxhwB862-ws_#<`=P2= zMsCKPoMP7uT1`Z7-l#_M$WBF_msbou>2`3Ol^Al0x}oMh+ZOx@HGHW?hyc8N*Hnws zbnvQJFSJ-W2-~DAkNtfs9<9>H>F4h@!?EmbhEq>MWR{kpHYhAX?rb;|$kw)12sTx67|!HHUQgkGDS zalLpv=l|+0|JCTZr0>1|y7t%);D0xvJ_t=Ddqak^f6rn5Y(S&Z(kQ=Py@pC=u%Dn) zvQ4d&Mmqq9t-ddfU4bFnVg!|N_F$GWiY^B@tQLc$V$sas5k@{%P^O%2Z6shom))@4}LWjLqiMdfe2H0Cfegq+prl=aC_Dz$uX zbJLV!FBsx!Kd?ciNj^RwFnYZ%zAi;Ze^vI2(zK0}ri+-YoKum~UZ>6!x;F`2xky2> zy)rUc-UZW%uvw&NC8b~Qj)$m#<2zgE<5g>?GWu{*3q<_|3LsnWp)k40ugRHM? zQFFI;kGc$WZP2f#w9}?D>89+L-i@JR1t4Rf1wfBiP7ZqT5c*yV4)v7k-r*s6!$9zkdQf!qlB)tk!e#Xady;MGxoG^<)>u&Ehkm^1|-b zOJ^NzKJaed~=FQ+`6p$up3;3M+%vcmLrvQv(?Zz9wAmiG|g zT62d{kvrB4G#^c2_Qi1y#po2804tPx3gc=ygF3)Pc93~KwRs6pjdi2_$s-S{2a{sB*<{rb zvc?)(aWCH=|Md7-E17+xs)lNTeNfj z47I0+R=!ep?Fh-Qir@YNxBK679L!&t@xSB2-Cifm^ureb zp>IAp5w#SB0;b*2h%*d)P=FQw=Ez#Hx{eTFZyzhTsIupkpas0I2B3&`_OPP}lGavzC#?MvqH+E>NYA&Dq%R zIV4*`H1i^ubb<4$i$}_C78#_;(Fepf39(sq+UQ=B0aRMe$Geah{Lrh^1X@w0?D zM2sxJLeX(8#N!DHM5P}RH}D!|7#$VlJdAW-EH}R7Jf67by>+*@HhtmVYqsB7f$v)J zkyUM>?8k$zv0r58WKM{s&NZK?5z$*C1#f3KWDjt_s;1np!|0s8r(-%`=o)m6dTle#dg zdToDrckOV@1hOnnXY+`T7{ZMEp$c6?%Qi>TnLmCR zAo)mZnElRZdx#w8qlh}_gQ0Y3KaP-DO7>hIu*~fo8_-xr8t=Q0>A`ihsP=tz^Z^Uo zT|VQeugL}7-b4Xb9fX=S0?YFI+yxul=pga2ERMYDdF~3oyIc$Z;j+k)B@Er*YtPCh z@e`kUFJ|f9fP)fNQeKI>+H`9)nOHfDwC(9h?o6|NZ8?V7KaX2{T|B+e^F>Ut&lq^* zc###H{*|L}Gyj9nTl*`)(uVncqWM2l8Qoqd^e^*s`Fqy28OiAeIvc`lD3vslZz{rp z`Q3YLw_&c1L`O7h3SN!(TO+(N4q5?k&*V8c5-QB0bQ|%mbAevKKZ2Z8O@VGaF2Ec~ zigazU##!#F`r8nNiwyQ}_qw22qhL7n24KD{h6QP8VHznRV+@)D*i8^k zgf1NkhjEqbxBK@$7tNHW2M`p2jwJ$&9A{_Pe{Dh0@Z=s&Gf*ZE>5Pcw1Bii!H#!V~ z6L!NgI!9f^TJW5x-PXZ=lBJVrE8##aHcFJGezwEs%-!>WIgT?l6n!xLs=Gp7RDaIa z6|Y0!A{E>368Gdd9Z%B%SFsYQy9?-g%rzUv(TP=yrO91xSz9cO787t(!qK^~tQ-8` z9^Gr+>_WdNvrI^t5moyBxDndv)M~T66HIcq!>Y5gfgcb}PLTBxKpT+65kaE(l46e2) zKjO#Rf-gq4Sds5aMaEasEplZD3P?umZR!l~H zbo^;bKGlVlG|!LgpFVH3xbB-3fEv-^(rgpe-O1ESjam=N$Qe8wCQ)eG!Qvm!1eKW` zWCoH!quxe14M^8VaQXc)ug-9X3AfI?5}QA$cO}}D>oFL*LEc6f@aCaZFyx&i=+_0b zoF_juEe*9=b?z-&U)@YJ+K-=dKJpRo>#R{I;a=L5iGn!~Mn(2|_h2HI2g8clnxP{$ zS09mOT0f?9;8DX+MY-ZV!Spo8nl`M1dQz#b9%KHQWX6SgfvoQAtLXycnBDedloXVS zGRuTcX;8;B&$LLs(udxH&UzeDSqUB?YHA$m6@P)9dSNE|qDaBhJ| zCJRQ%?245W;?V8iEnE>Sph|y>^&ZcmKmVU#MVU&-0-USy7=w|zIf@HD)?&(f2aYbe zD5&&#QyI?v&Ct4Ki?Z?K7lb=GYC+r z33PiOsv_x3B(j5pP0_kLIMRu&a$wry14D2ZlrHvEG;p3b!pXvmOxVc&2_C~jri87- z!B;*Dob*nKaZLG4L--g3t)SBl=!h62V?ed+j4!|=`khVBt9)6En|F!eN@g+qj7*Ic zbWYBL^z_rn+bQOqbS7(Rr8#FZL>lB~-kg9y6QRUp8!SYrWOBD1miE*YIhXp=_9Y?X*52Y&tLLlWKc`h^xK>l}XEJKU zXvYl&wEd6xj@SG&8VvR#%sscur0EY$(KBdFLnmCAVGvwYaSG%NHf$2ogXrdsdnO4U z?7inbJebUB(v#3W^)lgyQ)B{-%`LtXptl~*5ZU!PQ z{Jx{Ir1mr@>++#*Co>!`TAz-30{vuctA{~E6O3d0G3JzEjpnyYCJNH5hz72A{if>) z%zWU`cIiwO;YFCc94<)ZV`2~dYH=64X;RekjMY=fNYLKt1$MZ0#A2L z;zZvWuK^Qg*MQuQ58#r`R`qUHHFrC$c#;gmlJx`)1nT!T8%nsvug%Yc&onz}8^Tn- z*h2v-sFiRW79Aqr;p9G^(1YQt&oC?ONQ21R-l@rN{`y>Kj^+Y|49!`On|y%2+Ju1I zN{wIibDlg%No@3a5clyItzfdjAy*}X326wO&F_-R4Hd#w}SY1Q+F7$f#}%IL!ksTYYq7*q#aGb`hxu8|zraPYTarqjUqE!&5A9 z#KFZhb0{#hKm5&*q^B<^yT%z<-1k>V?6yL4PGpzClUKpGdhQSdn@^3*pUWqWXE+bZ zK(z^6-SY^(8vvf~K3Wgb#*fQB>0wtC+#~N!DiB(aWsAAJ+Xy<>PkWsj=I&W1?fV|5 zF+R-2d45}fMRJvJMmXz9C&w*lf) z$l*cFQ^wq4jS6>EEt=#K?(D-mGi}`w?#f@~N`DZ1LoO(~!a5o1Aw!EZD-*PZ`4LV> zbcGq!U&w3vzKD%LH5ri-64_<e;wbZS3fcYivc)_*XN+^V~g1#_`aLI*Oj z0Pe4{L3GbwbTFUJ03h?Bi0mdhNmmLk{&FwsN+-*u_wzG_B4h8jcXvOLjR7Oz+`?pO z$E@)EB38FILR6mqiLekh*pkMa6>v}i59YiSYX{T5rz~ga@D_BiG&(l!;~VZ* z889cJ!$S(>_+l=o?ENYY9JN;**`mJn@Ni|THuAGRWhgK~tGS+xA>Brhs^`(!1Q{p> z2g6qXlt&iA?!%>3r-V*|T1TL7Ygf+EQP8Gkw5%r~-0VgxC=*DW4yrs`R^BB|>0tlr zFSZC&&>WT$W8KxOlxias7523i?GQ2)=m7O7O=-p!Eb%6A2=W_@FL9YNy;J!FC|IW3qHEs&+{ z8Mu68JDlULt(E`oKvL`o~))@_n4s1C2(IG;( zs+X5l$}d`=hDd)E6O-B%kG<>TQ&Z)2`KF1ky?txKVry$_XLS>bMepj(*$n%20%c&X z_VU_IS(D%NoJ)v2>ycxIxLc5?*R@Bg@yL-Qk6RBH23<7Dau2Vrsj>fYAilv^oImr! z@QJP$!PCT)!}RT2i$Vi`%|mg~%Gg1~T3hYup{70b#VEs!h_T^h(Zh_`7v5h6VUE&iT&!nwpvwbKTYC>q8Z6?B)pX^B^?wSj*I>ArC-BD1Y%H%uh@uLf^sLi)Xyg z)ljPS_1vmq!+@_DrnzC37;1yBAA^&l|8Zp4HR6tN9q(jlNzUE+!QR#wP$-*^(`ii= zmJcD(v-@PFJ!+WczTup`CuaLC-^`o7Kxymz`SS-2HZ&NQba=f>sv0kP)`MY&&rrdz z^UnJFMdE0783nUsSJo8Lbi6qWbPhfaIW}C!pyl zMf;Yh>0-*Sramy{xy5}NmN?Bj&>eDGh2e~-QFe(rQiA7p%)>ZqU4gTyEuIl>Tg~4)VfDg-1#+HGL!TqJ7 zAW*stCsc4dGgW4_dgqNl$>2ly7?(Cw|AT4h-cn3+yizbdTa6mZBHCDcRBa#Bb9E1# zl3Z$`aOnBkH@4JUkyFn*+3{?ms0gQN5CrL6%XUTVz+f_ep#FiA%`Jfh>$kqyNwNLj zz1>1dqsRshErr40-z|PC$Ed2@#Gvb%@UvTn`r&tk?T0tr$yD&0F>$3aPpM?k`*^wS?!zI6C;Evfbj2hgW`BCI-3?Y~ zFZ>Kr1WQ8XB~$O-y}3R55>U~XC-Gr1ZZpR-ug|P`8Cz&+Y1uKGnZ3^$H+&~#w!6sa z*E(|sqwZ*O)^5*)IeaE(&)r##RJW+2zYY#Z%}JXKW+i6Y8m91VdF=$w%24_YLG^%o z(5B14L$Yf&-n{wD0N6e&vKb~*D0%217H;pIyVWZnX6wDVWAe-CA^Vh1?ju9@=k??y zWPR27i|(K>Ys|;rzq$)GUS{ppB59l` zla!?I)a19OvyPviT^}nTC3w)}tVefPON+UOJ;S(*jGE%6dQacyF?139=8K2lt4%Pu z)2tvEzyVE92o2)98ZtZkxHWojeqPn%8vD6h7x8amm=ED8uqo~PskpefysGN8jR2GF z@dGyMOR=CybM#$ZkAXzb@u01XpY4TTs?wk0vV05a&nur{vhAPggWu3WqzuEENxc@s ys9qTJSG)FTJO}^D>^9|Br9b~OUmlk+GXA*WT%OmtA2!leZd2R&?X68Fr~eP)-u7$& diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/comparison_table.csv b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/comparison_table.csv deleted file mode 100644 index 619c3c8..0000000 --- a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/comparison_table.csv +++ /dev/null @@ -1,65 +0,0 @@ -benchmark,metric,series_name,system_name,mode,reference_kind,status,value,example_count,dataset_path,source_path -longmemeval,ndcg_at_10,old_brainctl,brainctl,brain,historical,full_same_machine,0.9253,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -longmemeval,ndcg_at_5,old_brainctl,brainctl,brain,historical,full_same_machine,0.9204,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -longmemeval,r_at_10,old_brainctl,brainctl,brain,historical,full_same_machine,0.9894,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -longmemeval,r_at_5,old_brainctl,brainctl,brain,historical,full_same_machine,0.9681,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -longmemeval,ndcg_at_10,old_brainctl,brainctl,cmd,historical,full_same_machine,0.9247,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -longmemeval,ndcg_at_5,old_brainctl,brainctl,cmd,historical,full_same_machine,0.9206,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -longmemeval,r_at_10,old_brainctl,brainctl,cmd,historical,full_same_machine,0.9894,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -longmemeval,r_at_5,old_brainctl,brainctl,cmd,historical,full_same_machine,0.9702,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -longmemeval,ndcg_at_10,mempalace,mempalace,raw_session,historical,full_same_machine,0.8948,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -longmemeval,ndcg_at_5,mempalace,mempalace,raw_session,historical,full_same_machine,0.893,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -longmemeval,r_at_10,mempalace,mempalace,raw_session,historical,full_same_machine,0.983,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -longmemeval,r_at_5,mempalace,mempalace,raw_session,historical,full_same_machine,0.966,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -locomo,avg_recall,old_brainctl,brainctl,cmd_session,historical,full_same_machine,0.9217,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -locomo,cat_1_recall,old_brainctl,brainctl,cmd_session,historical,full_same_machine,0.7614,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -locomo,cat_2_recall,old_brainctl,brainctl,cmd_session,historical,full_same_machine,0.92,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -locomo,cat_3_recall,old_brainctl,brainctl,cmd_session,historical,full_same_machine,0.6616,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -locomo,cat_4_recall,old_brainctl,brainctl,cmd_session,historical,full_same_machine,0.9738,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -locomo,cat_5_recall,old_brainctl,brainctl,cmd_session,historical,full_same_machine,0.9821,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -locomo,perfect_rate,old_brainctl,brainctl,cmd_session,historical,full_same_machine,0.8817,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -locomo,top_k,old_brainctl,brainctl,cmd_session,historical,full_same_machine,10,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -locomo,zero_rate,old_brainctl,brainctl,cmd_session,historical,full_same_machine,0.0438,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -locomo,avg_recall,mempalace,mempalace,raw_session,historical,full_same_machine,0.6028,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -locomo,cat_1_recall,mempalace,mempalace,raw_session,historical,full_same_machine,0.5899,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -locomo,cat_2_recall,mempalace,mempalace,raw_session,historical,full_same_machine,0.6921,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -locomo,cat_3_recall,mempalace,mempalace,raw_session,historical,full_same_machine,0.4604,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -locomo,cat_4_recall,mempalace,mempalace,raw_session,historical,full_same_machine,0.5809,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -locomo,cat_5_recall,mempalace,mempalace,raw_session,historical,full_same_machine,0.6188,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -locomo,perfect_rate,mempalace,mempalace,raw_session,historical,full_same_machine,0.5534,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -locomo,top_k,mempalace,mempalace,raw_session,historical,full_same_machine,10,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -locomo,zero_rate,mempalace,mempalace,raw_session,historical,full_same_machine,0.3499,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -membench,hit_at_5,old_brainctl,brainctl,cmd_turn,historical,partial,0.93,200,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -membench,simple_hit_at_5,old_brainctl,brainctl,cmd_turn,historical,partial,0.93,200,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -membench,top_k,old_brainctl,brainctl,cmd_turn,historical,partial,5,200,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -membench,hit_at_5,mempalace,mempalace,raw_turn,historical,partial,0.885,200,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -membench,simple_hit_at_5,mempalace,mempalace,raw_turn,historical,partial,0.885,200,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -membench,top_k,mempalace,mempalace,raw_turn,historical,partial,5,200,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json -longmemeval,ndcg_at_10,new_brainctl,brainctl,brain,measured,full_same_machine,0.872,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json, -longmemeval,ndcg_at_5,new_brainctl,brainctl,brain,measured,full_same_machine,0.8678,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json, -longmemeval,r_at_10,new_brainctl,brainctl,brain,measured,full_same_machine,0.9894,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json, -longmemeval,r_at_5,new_brainctl,brainctl,brain,measured,full_same_machine,0.9574,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json, -longmemeval,ndcg_at_10,new_brainctl,brainctl,cmd,measured,full_same_machine,0.8753,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json, -longmemeval,ndcg_at_5,new_brainctl,brainctl,cmd,measured,full_same_machine,0.8715,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json, -longmemeval,r_at_10,new_brainctl,brainctl,cmd,measured,full_same_machine,0.9894,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json, -longmemeval,r_at_5,new_brainctl,brainctl,cmd,measured,full_same_machine,0.9596,470,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json, -locomo,avg_recall,new_brainctl,brainctl,cmd_session,measured,full_same_machine,0.9145,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json, -locomo,cat_1_recall,new_brainctl,brainctl,cmd_session,measured,full_same_machine,0.7533,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json, -locomo,cat_2_recall,new_brainctl,brainctl,cmd_session,measured,full_same_machine,0.9107,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json, -locomo,cat_3_recall,new_brainctl,brainctl,cmd_session,measured,full_same_machine,0.6607,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json, -locomo,cat_4_recall,new_brainctl,brainctl,cmd_session,measured,full_same_machine,0.9643,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json, -locomo,cat_5_recall,new_brainctl,brainctl,cmd_session,measured,full_same_machine,0.9798,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json, -locomo,perfect_rate,new_brainctl,brainctl,cmd_session,measured,full_same_machine,0.8751,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json, -locomo,top_k,new_brainctl,brainctl,cmd_session,measured,full_same_machine,10,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json, -locomo,zero_rate,new_brainctl,brainctl,cmd_session,measured,full_same_machine,0.0519,1986,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json, -membench,hit_at_5,new_brainctl,brainctl,cmd_turn,measured,partial,0.895,200,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent, -membench,simple_hit_at_5,new_brainctl,brainctl,cmd_turn,measured,partial,0.895,200,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent, -membench,top_k,new_brainctl,brainctl,cmd_turn,measured,partial,5,200,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent, -convomem,abstention_evidence_recall,new_brainctl,brainctl,cmd,measured,partial,0.0,5,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache, -convomem,assistant_facts_evidence_recall,new_brainctl,brainctl,cmd,measured,partial,1.0,5,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache, -convomem,avg_recall,new_brainctl,brainctl,cmd,measured,partial,0.6,5,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache, -convomem,implicit_connection_evidence_recall,new_brainctl,brainctl,cmd,measured,partial,0.0,5,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache, -convomem,perfect_rate,new_brainctl,brainctl,cmd,measured,partial,0.6,5,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache, -convomem,preference_evidence_recall,new_brainctl,brainctl,cmd,measured,partial,1.0,5,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache, -convomem,top_k,new_brainctl,brainctl,cmd,measured,partial,10,5,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache, -convomem,user_evidence_recall,new_brainctl,brainctl,cmd,measured,partial,1.0,5,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache, diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/comparison_table.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/comparison_table.json deleted file mode 100644 index 31ff305..0000000 --- a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/comparison_table.json +++ /dev/null @@ -1,834 +0,0 @@ -[ - { - "benchmark": "longmemeval", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "metric": "ndcg_at_10", - "mode": "brain", - "reference_kind": "historical", - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl", - "value": 0.9253 - }, - { - "benchmark": "longmemeval", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "metric": "ndcg_at_5", - "mode": "brain", - "reference_kind": "historical", - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl", - "value": 0.9204 - }, - { - "benchmark": "longmemeval", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "metric": "r_at_10", - "mode": "brain", - "reference_kind": "historical", - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl", - "value": 0.9894 - }, - { - "benchmark": "longmemeval", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "metric": "r_at_5", - "mode": "brain", - "reference_kind": "historical", - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl", - "value": 0.9681 - }, - { - "benchmark": "longmemeval", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "metric": "ndcg_at_10", - "mode": "cmd", - "reference_kind": "historical", - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl", - "value": 0.9247 - }, - { - "benchmark": "longmemeval", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "metric": "ndcg_at_5", - "mode": "cmd", - "reference_kind": "historical", - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl", - "value": 0.9206 - }, - { - "benchmark": "longmemeval", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "metric": "r_at_10", - "mode": "cmd", - "reference_kind": "historical", - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl", - "value": 0.9894 - }, - { - "benchmark": "longmemeval", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "metric": "r_at_5", - "mode": "cmd", - "reference_kind": "historical", - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl", - "value": 0.9702 - }, - { - "benchmark": "longmemeval", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "metric": "ndcg_at_10", - "mode": "raw_session", - "reference_kind": "historical", - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "mempalace", - "value": 0.8948 - }, - { - "benchmark": "longmemeval", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "metric": "ndcg_at_5", - "mode": "raw_session", - "reference_kind": "historical", - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "mempalace", - "value": 0.893 - }, - { - "benchmark": "longmemeval", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "metric": "r_at_10", - "mode": "raw_session", - "reference_kind": "historical", - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "mempalace", - "value": 0.983 - }, - { - "benchmark": "longmemeval", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "metric": "r_at_5", - "mode": "raw_session", - "reference_kind": "historical", - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "mempalace", - "value": 0.966 - }, - { - "benchmark": "locomo", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "metric": "avg_recall", - "mode": "cmd_session", - "reference_kind": "historical", - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl", - "value": 0.9217 - }, - { - "benchmark": "locomo", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "metric": "cat_1_recall", - "mode": "cmd_session", - "reference_kind": "historical", - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl", - "value": 0.7614 - }, - { - "benchmark": "locomo", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "metric": "cat_2_recall", - "mode": "cmd_session", - "reference_kind": "historical", - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl", - "value": 0.92 - }, - { - "benchmark": "locomo", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "metric": "cat_3_recall", - "mode": "cmd_session", - "reference_kind": "historical", - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl", - "value": 0.6616 - }, - { - "benchmark": "locomo", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "metric": "cat_4_recall", - "mode": "cmd_session", - "reference_kind": "historical", - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl", - "value": 0.9738 - }, - { - "benchmark": "locomo", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "metric": "cat_5_recall", - "mode": "cmd_session", - "reference_kind": "historical", - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl", - "value": 0.9821 - }, - { - "benchmark": "locomo", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "metric": "perfect_rate", - "mode": "cmd_session", - "reference_kind": "historical", - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl", - "value": 0.8817 - }, - { - "benchmark": "locomo", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "metric": "top_k", - "mode": "cmd_session", - "reference_kind": "historical", - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl", - "value": 10 - }, - { - "benchmark": "locomo", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "metric": "zero_rate", - "mode": "cmd_session", - "reference_kind": "historical", - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl", - "value": 0.0438 - }, - { - "benchmark": "locomo", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "metric": "avg_recall", - "mode": "raw_session", - "reference_kind": "historical", - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "mempalace", - "value": 0.6028 - }, - { - "benchmark": "locomo", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "metric": "cat_1_recall", - "mode": "raw_session", - "reference_kind": "historical", - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "mempalace", - "value": 0.5899 - }, - { - "benchmark": "locomo", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "metric": "cat_2_recall", - "mode": "raw_session", - "reference_kind": "historical", - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "mempalace", - "value": 0.6921 - }, - { - "benchmark": "locomo", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "metric": "cat_3_recall", - "mode": "raw_session", - "reference_kind": "historical", - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "mempalace", - "value": 0.4604 - }, - { - "benchmark": "locomo", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "metric": "cat_4_recall", - "mode": "raw_session", - "reference_kind": "historical", - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "mempalace", - "value": 0.5809 - }, - { - "benchmark": "locomo", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "metric": "cat_5_recall", - "mode": "raw_session", - "reference_kind": "historical", - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "mempalace", - "value": 0.6188 - }, - { - "benchmark": "locomo", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "metric": "perfect_rate", - "mode": "raw_session", - "reference_kind": "historical", - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "mempalace", - "value": 0.5534 - }, - { - "benchmark": "locomo", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "metric": "top_k", - "mode": "raw_session", - "reference_kind": "historical", - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "mempalace", - "value": 10 - }, - { - "benchmark": "locomo", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "metric": "zero_rate", - "mode": "raw_session", - "reference_kind": "historical", - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "mempalace", - "value": 0.3499 - }, - { - "benchmark": "membench", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 200, - "metric": "hit_at_5", - "mode": "cmd_turn", - "reference_kind": "historical", - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "partial", - "system_name": "brainctl", - "value": 0.93 - }, - { - "benchmark": "membench", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 200, - "metric": "simple_hit_at_5", - "mode": "cmd_turn", - "reference_kind": "historical", - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "partial", - "system_name": "brainctl", - "value": 0.93 - }, - { - "benchmark": "membench", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 200, - "metric": "top_k", - "mode": "cmd_turn", - "reference_kind": "historical", - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "partial", - "system_name": "brainctl", - "value": 5 - }, - { - "benchmark": "membench", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 200, - "metric": "hit_at_5", - "mode": "raw_turn", - "reference_kind": "historical", - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "partial", - "system_name": "mempalace", - "value": 0.885 - }, - { - "benchmark": "membench", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 200, - "metric": "simple_hit_at_5", - "mode": "raw_turn", - "reference_kind": "historical", - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "partial", - "system_name": "mempalace", - "value": 0.885 - }, - { - "benchmark": "membench", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 200, - "metric": "top_k", - "mode": "raw_turn", - "reference_kind": "historical", - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "partial", - "system_name": "mempalace", - "value": 5 - }, - { - "benchmark": "longmemeval", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "metric": "ndcg_at_10", - "mode": "brain", - "reference_kind": "measured", - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl", - "value": 0.872 - }, - { - "benchmark": "longmemeval", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "metric": "ndcg_at_5", - "mode": "brain", - "reference_kind": "measured", - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl", - "value": 0.8678 - }, - { - "benchmark": "longmemeval", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "metric": "r_at_10", - "mode": "brain", - "reference_kind": "measured", - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl", - "value": 0.9894 - }, - { - "benchmark": "longmemeval", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "metric": "r_at_5", - "mode": "brain", - "reference_kind": "measured", - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl", - "value": 0.9574 - }, - { - "benchmark": "longmemeval", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "metric": "ndcg_at_10", - "mode": "cmd", - "reference_kind": "measured", - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl", - "value": 0.8753 - }, - { - "benchmark": "longmemeval", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "metric": "ndcg_at_5", - "mode": "cmd", - "reference_kind": "measured", - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl", - "value": 0.8715 - }, - { - "benchmark": "longmemeval", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "metric": "r_at_10", - "mode": "cmd", - "reference_kind": "measured", - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl", - "value": 0.9894 - }, - { - "benchmark": "longmemeval", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "metric": "r_at_5", - "mode": "cmd", - "reference_kind": "measured", - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl", - "value": 0.9596 - }, - { - "benchmark": "locomo", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "metric": "avg_recall", - "mode": "cmd_session", - "reference_kind": "measured", - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl", - "value": 0.9145 - }, - { - "benchmark": "locomo", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "metric": "cat_1_recall", - "mode": "cmd_session", - "reference_kind": "measured", - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl", - "value": 0.7533 - }, - { - "benchmark": "locomo", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "metric": "cat_2_recall", - "mode": "cmd_session", - "reference_kind": "measured", - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl", - "value": 0.9107 - }, - { - "benchmark": "locomo", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "metric": "cat_3_recall", - "mode": "cmd_session", - "reference_kind": "measured", - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl", - "value": 0.6607 - }, - { - "benchmark": "locomo", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "metric": "cat_4_recall", - "mode": "cmd_session", - "reference_kind": "measured", - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl", - "value": 0.9643 - }, - { - "benchmark": "locomo", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "metric": "cat_5_recall", - "mode": "cmd_session", - "reference_kind": "measured", - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl", - "value": 0.9798 - }, - { - "benchmark": "locomo", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "metric": "perfect_rate", - "mode": "cmd_session", - "reference_kind": "measured", - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl", - "value": 0.8751 - }, - { - "benchmark": "locomo", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "metric": "top_k", - "mode": "cmd_session", - "reference_kind": "measured", - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl", - "value": 10 - }, - { - "benchmark": "locomo", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "metric": "zero_rate", - "mode": "cmd_session", - "reference_kind": "measured", - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl", - "value": 0.0519 - }, - { - "benchmark": "membench", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 200, - "metric": "hit_at_5", - "mode": "cmd_turn", - "reference_kind": "measured", - "series_name": "new_brainctl", - "source_path": null, - "status": "partial", - "system_name": "brainctl", - "value": 0.895 - }, - { - "benchmark": "membench", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 200, - "metric": "simple_hit_at_5", - "mode": "cmd_turn", - "reference_kind": "measured", - "series_name": "new_brainctl", - "source_path": null, - "status": "partial", - "system_name": "brainctl", - "value": 0.895 - }, - { - "benchmark": "membench", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 200, - "metric": "top_k", - "mode": "cmd_turn", - "reference_kind": "measured", - "series_name": "new_brainctl", - "source_path": null, - "status": "partial", - "system_name": "brainctl", - "value": 5 - }, - { - "benchmark": "convomem", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "example_count": 5, - "metric": "abstention_evidence_recall", - "mode": "cmd", - "reference_kind": "measured", - "series_name": "new_brainctl", - "source_path": null, - "status": "partial", - "system_name": "brainctl", - "value": 0.0 - }, - { - "benchmark": "convomem", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "example_count": 5, - "metric": "assistant_facts_evidence_recall", - "mode": "cmd", - "reference_kind": "measured", - "series_name": "new_brainctl", - "source_path": null, - "status": "partial", - "system_name": "brainctl", - "value": 1.0 - }, - { - "benchmark": "convomem", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "example_count": 5, - "metric": "avg_recall", - "mode": "cmd", - "reference_kind": "measured", - "series_name": "new_brainctl", - "source_path": null, - "status": "partial", - "system_name": "brainctl", - "value": 0.6 - }, - { - "benchmark": "convomem", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "example_count": 5, - "metric": "implicit_connection_evidence_recall", - "mode": "cmd", - "reference_kind": "measured", - "series_name": "new_brainctl", - "source_path": null, - "status": "partial", - "system_name": "brainctl", - "value": 0.0 - }, - { - "benchmark": "convomem", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "example_count": 5, - "metric": "perfect_rate", - "mode": "cmd", - "reference_kind": "measured", - "series_name": "new_brainctl", - "source_path": null, - "status": "partial", - "system_name": "brainctl", - "value": 0.6 - }, - { - "benchmark": "convomem", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "example_count": 5, - "metric": "preference_evidence_recall", - "mode": "cmd", - "reference_kind": "measured", - "series_name": "new_brainctl", - "source_path": null, - "status": "partial", - "system_name": "brainctl", - "value": 1.0 - }, - { - "benchmark": "convomem", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "example_count": 5, - "metric": "top_k", - "mode": "cmd", - "reference_kind": "measured", - "series_name": "new_brainctl", - "source_path": null, - "status": "partial", - "system_name": "brainctl", - "value": 10 - }, - { - "benchmark": "convomem", - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "example_count": 5, - "metric": "user_evidence_recall", - "mode": "cmd", - "reference_kind": "measured", - "series_name": "new_brainctl", - "source_path": null, - "status": "partial", - "system_name": "brainctl", - "value": 1.0 - } -] \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/metadata.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/metadata.json deleted file mode 100644 index 45f5356..0000000 --- a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/metadata.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "argv": [ - "benchmarks\\compare_memory_engines.py", - "--label", - "rlm_srlm_probe_refresh", - "--membench-limit", - "200" - ], - "cwd": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl", - "datasets": { - "convomem_cache": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "locomo_data": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "longmemeval_data": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "membench_data": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent" - }, - "generated_at_utc": "2026-04-23T18:05:25.112789+00:00", - "git_commit": "9368f288279ae1d3395e37e89972038b74d66e3a", - "historical_summary_mode": "recovered", - "historical_summary_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "machine": "AMD64", - "platform": "Windows-11-10.0.26200-SP0", - "processor": "Intel64 Family 6 Model 189 Stepping 1, GenuineIntel", - "python_version": "3.14.4 (tags/v3.14.4:23116f9, Apr 7 2026, 14:10:54) [MSC v.1944 64 bit (AMD64)]" -} \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/convomem_mempalace_raw.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/convomem_mempalace_raw.json deleted file mode 100644 index eec82fd..0000000 --- a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/convomem_mempalace_raw.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\convomem_mempalace_raw.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" - }, - "benchmark": "convomem", - "caveats": [ - "Blocked while loading ConvoMem evidence data: " - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "example_count": 0, - "measured": false, - "metadata": {}, - "metrics": {}, - "mode": "raw", - "notes": [ - "limit_per_category=1", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": null, - "reference_kind": "historical", - "runtime_seconds": null, - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "blocked", - "system_name": "mempalace" -} \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/convomem_new_brainctl_cmd.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/convomem_new_brainctl_cmd.json deleted file mode 100644 index 0668f29..0000000 --- a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/convomem_new_brainctl_cmd.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "artifacts": { - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\convomem_new_brainctl_cmd.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" - }, - "benchmark": "convomem", - "caveats": [ - "changing_evidence: discover failed: HTTP Error 404: Not Found", - "ConvoMem comparison is partial because it uses a bounded same-machine sample, not the full benchmark." - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "example_count": 5, - "measured": true, - "metadata": {}, - "metrics": { - "abstention_evidence_recall": 0.0, - "assistant_facts_evidence_recall": 1.0, - "avg_recall": 0.6, - "implicit_connection_evidence_recall": 0.0, - "perfect_rate": 0.6, - "preference_evidence_recall": 1.0, - "top_k": 10, - "user_evidence_recall": 1.0 - }, - "mode": "cmd", - "notes": [ - "categories=6", - "limit_per_category=1", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": 0.6, - "reference_kind": "measured", - "rows": [ - { - "category": "user_evidence", - "evidence_count": 1, - "question": "What color do I use for hot leads in my personal spreadsheet?", - "recall": 1.0, - "retrieved_ids": [ - "msg_2", - "msg_3", - "msg_4", - "msg_10", - "msg_7", - "msg_0", - "msg_1" - ] - }, - { - "category": "assistant_facts_evidence", - "evidence_count": 1, - "question": "I'm finally getting around to that productivity hack you mentioned last week. Can you remind me of the specific automation tool and method you suggested for integrating my color-coded spreadsheet with our company's Salesforce CRM?", - "recall": 1.0, - "retrieved_ids": [ - "msg_15", - "msg_2", - "msg_6", - "msg_4", - "msg_23", - "msg_3", - "msg_27", - "msg_12", - "msg_37", - "msg_32" - ] - }, - { - "category": "abstention_evidence", - "evidence_count": 1, - "question": "What is the direct contact phone number for the 'QuantumCorp' lead that Alex mentioned?", - "recall": 0.0, - "retrieved_ids": [] - }, - { - "category": "preference_evidence", - "evidence_count": 1, - "question": "What CRM functionalities should I look into if I want a more visual way to manage my lead pipeline instead of using a spreadsheet?", - "recall": 1.0, - "retrieved_ids": [ - "msg_10", - "msg_60", - "msg_42", - "msg_11", - "msg_43", - "msg_4", - "msg_57", - "msg_16", - "msg_74", - "msg_56" - ] - }, - { - "category": "implicit_connection_evidence", - "evidence_count": 1, - "question": "I need to get the new team-wide kick-off call on the calendar for next Tuesday. What would be a good time to schedule it for everyone?", - "recall": 0.0, - "retrieved_ids": [] - } - ], - "runtime_seconds": 1.514, - "series_name": "new_brainctl", - "source_path": null, - "status": "partial", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/convomem_old_brainctl_cmd.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/convomem_old_brainctl_cmd.json deleted file mode 100644 index 1f53f87..0000000 --- a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/convomem_old_brainctl_cmd.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\convomem_old_brainctl_cmd.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" - }, - "benchmark": "convomem", - "caveats": [ - "Blocked while loading ConvoMem evidence data: " - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "example_count": 0, - "measured": false, - "metadata": {}, - "metrics": {}, - "mode": "cmd", - "notes": [ - "limit_per_category=1", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": null, - "reference_kind": "historical", - "runtime_seconds": null, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "blocked", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/locomo_mempalace_raw_session.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/locomo_mempalace_raw_session.json deleted file mode 100644 index 388f041..0000000 --- a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/locomo_mempalace_raw_session.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\locomo_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\locomo_mempalace_raw_session.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" - }, - "benchmark": "locomo", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "measured": false, - "metadata": {}, - "metrics": { - "avg_recall": 0.6028, - "cat_1_recall": 0.5899, - "cat_2_recall": 0.6921, - "cat_3_recall": 0.4604, - "cat_4_recall": 0.5809, - "cat_5_recall": 0.6188, - "perfect_rate": 0.5534, - "top_k": 10, - "zero_rate": 0.3499 - }, - "mode": "raw_session", - "notes": [ - "granularity=session", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": 0.6028, - "reference_kind": "historical", - "runtime_seconds": 2106.411, - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "mempalace" -} \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/locomo_new_brainctl_cmd_session.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/locomo_new_brainctl_cmd_session.json deleted file mode 100644 index 3d877b6..0000000 --- a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/locomo_new_brainctl_cmd_session.json +++ /dev/null @@ -1,44071 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\locomo_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\locomo_new_brainctl_cmd_session.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" - }, - "benchmark": "locomo", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "measured": true, - "metadata": {}, - "metrics": { - "avg_recall": 0.9145, - "cat_1_recall": 0.7533, - "cat_2_recall": 0.9107, - "cat_3_recall": 0.6607, - "cat_4_recall": 0.9643, - "cat_5_recall": 0.9798, - "perfect_rate": 0.8751, - "top_k": 10, - "zero_rate": 0.0519 - }, - "mode": "cmd_session", - "notes": [ - "granularity=session", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": 0.9145, - "reference_kind": "measured", - "rows": [ - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_1" - ], - "question": "When did Caroline go to the LGBTQ support group?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_12", - "session_10", - "session_4", - "session_9", - "session_5", - "session_13", - "session_11", - "session_7", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_1" - ], - "question": "When did Melanie paint a sunrise?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_13", - "session_9", - "session_14", - "session_17", - "session_15", - "session_11", - "session_8", - "session_18", - "session_16" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_1" - ], - "question": "What fields would Caroline be likely to pursue in her educaton?", - "recall": 0.0, - "retrieved_ids": [ - "session_7", - "session_13", - "session_18", - "session_10", - "session_4", - "session_14", - "session_17", - "session_15", - "session_9", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_2" - ], - "question": "What did Caroline research?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_1", - "session_11", - "session_2", - "session_5", - "session_13", - "session_18", - "session_8", - "session_15", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1" - ], - "question": "What is Caroline's identity?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_11", - "session_16", - "session_17", - "session_3", - "session_7", - "session_1", - "session_8", - "session_9", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_2" - ], - "question": "When did Melanie run a charity race?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_11", - "session_7", - "session_1", - "session_8", - "session_13", - "session_5", - "session_18", - "session_14", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_2" - ], - "question": "When is Melanie planning on going camping?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_2", - "session_9", - "session_18", - "session_14", - "session_10", - "session_12", - "session_16", - "session_7", - "session_17" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_2", - "session_3" - ], - "question": "What is Caroline's relationship status?", - "recall": 0.0, - "retrieved_ids": [], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_3" - ], - "question": "When did Caroline give a speech at a school?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_2", - "session_13", - "session_5", - "session_18", - "session_7", - "session_19", - "session_10", - "session_14", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_3" - ], - "question": "When did Caroline meet up with her friends, family, and mentors?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_16", - "session_17", - "session_3", - "session_7", - "session_8", - "session_10", - "session_15", - "session_9", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_3" - ], - "question": "How long has Caroline had her current group of friends for?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_4", - "session_16", - "session_3", - "session_10", - "session_19", - "session_13", - "session_12", - "session_1", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_3", - "session_4" - ], - "question": "Where did Caroline move from 4 years ago?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_4", - "session_8", - "session_15", - "session_7", - "session_16", - "session_17", - "session_10", - "session_19", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_4" - ], - "question": "How long ago was Caroline's 18th birthday?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_15", - "session_16", - "session_3", - "session_8", - "session_17", - "session_1", - "session_11", - "session_9", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_4" - ], - "question": "What career path has Caroline decided to persue?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_7", - "session_1", - "session_15", - "session_3", - "session_16", - "session_12", - "session_9", - "session_14", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_3", - "session_4" - ], - "question": "Would Caroline still want to pursue counseling as a career if she hadn't received support growing up?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_7", - "session_5", - "session_17", - "session_1", - "session_10", - "session_3", - "session_6", - "session_2", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_5", - "session_9" - ], - "question": "What activities does Melanie partake in?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_15", - "session_17", - "session_1", - "session_9", - "session_18", - "session_13", - "session_4", - "session_8", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_5" - ], - "question": "When did Melanie sign up for a pottery class?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_16", - "session_14", - "session_12", - "session_10", - "session_17", - "session_8", - "session_7", - "session_11", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_5" - ], - "question": "When is Caroline going to the transgender conference?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_9", - "session_7", - "session_17", - "session_3", - "session_14", - "session_11", - "session_1", - "session_4", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_4", - "session_6", - "session_8" - ], - "question": "Where has Melanie camped?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_13", - "session_18", - "session_9", - "session_1", - "session_6", - "session_2", - "session_4", - "session_16", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_4", - "session_6" - ], - "question": "What do Melanie's kids like?", - "recall": 0.5, - "retrieved_ids": [ - "session_9", - "session_13", - "session_1", - "session_10", - "session_18", - "session_15", - "session_4", - "session_17", - "session_16", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_6" - ], - "question": "When did Melanie go to the museum?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_10", - "session_1", - "session_13", - "session_5", - "session_11", - "session_18", - "session_14", - "session_7", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_6" - ], - "question": "When did Caroline have a picnic?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_10", - "session_1", - "session_8", - "session_5", - "session_13", - "session_18", - "session_11", - "session_14", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_6" - ], - "question": "Would Caroline likely have Dr. Seuss books on her bookshelf?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_6", - "session_17", - "session_13", - "session_18", - "session_10", - "session_14", - "session_5", - "session_9", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_6", - "session_7" - ], - "question": "What books has Melanie read?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_6", - "session_17", - "session_9", - "session_1", - "session_2", - "session_18", - "session_13", - "session_4", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_5", - "session_7" - ], - "question": "What does Melanie do to destress?", - "recall": 0.5, - "retrieved_ids": [ - "session_7", - "session_15", - "session_1", - "session_18", - "session_9", - "session_8", - "session_13", - "session_4", - "session_17", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_7" - ], - "question": "When did Caroline go to the LGBTQ conference?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_7", - "session_3", - "session_1", - "session_9", - "session_11", - "session_13", - "session_4", - "session_14", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_7" - ], - "question": "When did Melanie read the book \"nothing is impossible\"?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_6", - "session_17", - "session_1", - "session_2", - "session_13", - "session_18", - "session_15", - "session_8", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_7" - ], - "question": "Would Caroline pursue writing as a career option?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_1", - "session_4", - "session_18", - "session_13", - "session_5", - "session_9", - "session_12", - "session_8", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_8" - ], - "question": "When did Caroline go to the adoption meeting?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_2", - "session_19", - "session_13", - "session_17", - "session_10", - "session_1", - "session_5", - "session_15", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_8" - ], - "question": "When did Melanie go to the pottery workshop?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_5", - "session_4", - "session_12", - "session_16", - "session_17", - "session_14", - "session_11", - "session_1", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [], - "question": "Would Melanie be considered a member of the LGBTQ community?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_5", - "session_9", - "session_11", - "session_14", - "session_10", - "session_7", - "session_15", - "session_3", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_4" - ], - "question": "When did Melanie go camping in June?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_9", - "session_6", - "session_18", - "session_2", - "session_4", - "session_16", - "session_8", - "session_1", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_3", - "session_5", - "session_8" - ], - "question": "What LGBTQ+ events has Caroline participated in?", - "recall": 0.75, - "retrieved_ids": [ - "session_9", - "session_11", - "session_2", - "session_10", - "session_3", - "session_7", - "session_1", - "session_5", - "session_14", - "session_17" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_5" - ], - "question": "When did Caroline go to a pride parade during the summer?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_11", - "session_10", - "session_5", - "session_12", - "session_9", - "session_18", - "session_2", - "session_15", - "session_16" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_3", - "session_9" - ], - "question": "What events has Caroline participated in to help children?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_9", - "session_11", - "session_2", - "session_7", - "session_3", - "session_17", - "session_13", - "session_10", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_9" - ], - "question": "When did Melanie go camping in July?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_9", - "session_6", - "session_18", - "session_2", - "session_4", - "session_16", - "session_8", - "session_1", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_9" - ], - "question": "When did Caroline join a mentorship program?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_10", - "session_3", - "session_8", - "session_14", - "session_5", - "session_1", - "session_13", - "session_18", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_8" - ], - "question": "What did Melanie paint recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_17", - "session_14", - "session_1", - "session_12", - "session_10", - "session_16", - "session_11", - "session_18", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_3", - "session_6", - "session_8", - "session_9" - ], - "question": "What activities has Melanie done with her family?", - "recall": 0.4, - "retrieved_ids": [ - "session_6", - "session_16", - "session_17", - "session_10", - "session_13", - "session_7", - "session_19", - "session_5", - "session_1", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_5", - "session_9" - ], - "question": "In what ways is Caroline participating in the LGBTQ community?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_9", - "session_11", - "session_15", - "session_10", - "session_7", - "session_3", - "session_16", - "session_14", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_6" - ], - "question": "How many times has Melanie gone to the beach in 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_4", - "session_8", - "session_7", - "session_6", - "session_3", - "session_18", - "session_14", - "session_15", - "session_17" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_10" - ], - "question": "When did Caroline join a new activist group?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_1", - "session_9", - "session_4", - "session_15", - "session_13", - "session_12", - "session_3", - "session_8", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_10" - ], - "question": "Would Melanie be more interested in going to a national park or a theme park?", - "recall": 0.0, - "retrieved_ids": [ - "session_11", - "session_6", - "session_5", - "session_18", - "session_4", - "session_7", - "session_16", - "session_3", - "session_14", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_11", - "session_9" - ], - "question": "What kind of art does Caroline make?", - "recall": 0.5, - "retrieved_ids": [ - "session_5", - "session_4", - "session_14", - "session_2", - "session_16", - "session_6", - "session_11", - "session_3", - "session_10", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_11" - ], - "question": "When is Melanie's daughter's birthday?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_4", - "session_15", - "session_17", - "session_19", - "session_9", - "session_13", - "session_1", - "session_18", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_11" - ], - "question": "When did Caroline attend a pride parade in August?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_11", - "session_10", - "session_8", - "session_13", - "session_9", - "session_12", - "session_17", - "session_14", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [], - "question": "Would Melanie be considered an ally to the transgender community?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_5", - "session_9", - "session_14", - "session_7", - "session_1", - "session_18", - "session_11", - "session_17", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_12", - "session_3" - ], - "question": "Who supports Caroline when she has a negative experience?", - "recall": 0.5, - "retrieved_ids": [ - "session_4", - "session_18", - "session_5", - "session_19", - "session_12", - "session_11", - "session_10", - "session_7", - "session_15", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_12", - "session_5", - "session_8" - ], - "question": "What types of pottery have Melanie and her kids made?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_17", - "session_8", - "session_14", - "session_2", - "session_10", - "session_5", - "session_12", - "session_15", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_12" - ], - "question": "When did Caroline and Melanie go to a pride fesetival together?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_10", - "session_14", - "session_17", - "session_8", - "session_9", - "session_4", - "session_5", - "session_12", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_12" - ], - "question": "What would Caroline's political leaning likely be?", - "recall": 0.0, - "retrieved_ids": [], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_13", - "session_8" - ], - "question": "What has Melanie painted?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_9", - "session_13", - "session_17", - "session_14", - "session_4", - "session_11", - "session_12", - "session_16", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_13", - "session_7" - ], - "question": "What are Melanie's pets' names?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_7", - "session_15", - "session_8", - "session_18", - "session_1", - "session_4", - "session_9", - "session_17", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_13" - ], - "question": "When did Caroline apply to adoption agencies?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_19", - "session_2", - "session_17", - "session_8", - "session_5", - "session_1", - "session_18", - "session_15", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_13" - ], - "question": "When did Caroline draw a self-portrait?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_18", - "session_11", - "session_19", - "session_14", - "session_2", - "session_16", - "session_7", - "session_17", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_14", - "session_8" - ], - "question": "What subject have Caroline and Melanie both painted?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_13", - "session_8", - "session_14", - "session_1", - "session_12", - "session_15", - "session_16", - "session_3", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_14", - "session_4" - ], - "question": "What symbols are important to Caroline?", - "recall": 0.5, - "retrieved_ids": [ - "session_14", - "session_12", - "session_6", - "session_2", - "session_11", - "session_7", - "session_18", - "session_15", - "session_3", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_14" - ], - "question": "When did Caroline encounter people on a hike and have a negative experience?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_4", - "session_7", - "session_5", - "session_11", - "session_18", - "session_15", - "session_8", - "session_16", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_14" - ], - "question": "When did Melanie make a plate in pottery class?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_5", - "session_12", - "session_16", - "session_17", - "session_8", - "session_15", - "session_10", - "session_7", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_12", - "session_14" - ], - "question": "Would Caroline be considered religious?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_18", - "session_13", - "session_1", - "session_9", - "session_5", - "session_4", - "session_7", - "session_8", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_15", - "session_2" - ], - "question": "What instruments does Melanie play?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_3", - "session_9", - "session_6", - "session_1", - "session_2", - "session_18", - "session_13", - "session_4", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_11", - "session_15" - ], - "question": "What musical artists/bands has Melanie seen?", - "recall": 0.0, - "retrieved_ids": [], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_15" - ], - "question": "When did Melanie go to the park?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_16", - "session_7", - "session_1", - "session_5", - "session_13", - "session_11", - "session_18", - "session_14", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_15" - ], - "question": "When is Caroline's youth center putting on a talent show?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_9", - "session_14", - "session_17", - "session_11", - "session_16", - "session_5", - "session_12", - "session_19", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_15" - ], - "question": "Would Melanie likely enjoy the song \"The Four Seasons\" by Vivaldi?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_11", - "session_15", - "session_10", - "session_8", - "session_5", - "session_9", - "session_4", - "session_17", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_11", - "session_16" - ], - "question": "What are some changes Caroline has faced during her transition journey?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_8", - "session_10", - "session_14", - "session_11", - "session_7", - "session_3", - "session_18", - "session_15", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_16" - ], - "question": "What does Melanie do with her family on hikes?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_7", - "session_2", - "session_16", - "session_8", - "session_18", - "session_12", - "session_4", - "session_14", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_16" - ], - "question": "When did Caroline go biking with friends?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_6", - "session_19", - "session_2", - "session_12", - "session_4", - "session_3", - "session_8", - "session_17", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_16" - ], - "question": "How long has Melanie been practicing art?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_4", - "session_8", - "session_17", - "session_9", - "session_1", - "session_6", - "session_15", - "session_3", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_13", - "session_16", - "session_7" - ], - "question": "What personality traits might Melanie say Caroline has?", - "recall": 0.6667, - "retrieved_ids": [ - "session_10", - "session_15", - "session_16", - "session_8", - "session_7", - "session_1", - "session_9", - "session_18", - "session_4", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_15", - "session_17" - ], - "question": "What transgender-specific events has Caroline attended?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_2", - "session_9", - "session_10", - "session_17", - "session_3", - "session_1", - "session_5", - "session_15", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_17", - "session_7" - ], - "question": "What book did Melanie read from Caroline's suggestion?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_6", - "session_17", - "session_1", - "session_2", - "session_13", - "session_18", - "session_15", - "session_8", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_17" - ], - "question": "When did Melanie's friend adopt a child?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_8", - "session_13", - "session_4", - "session_2", - "session_19", - "session_6", - "session_12", - "session_16", - "session_3" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_17" - ], - "question": "When did Melanie get hurt?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_10", - "session_1", - "session_5", - "session_18", - "session_8", - "session_15", - "session_16", - "session_14", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_18" - ], - "question": "When did Melanie's family go on a roadtrip?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_10", - "session_2", - "session_4", - "session_7", - "session_17", - "session_3", - "session_1", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_18" - ], - "question": "How many children does Melanie have?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_3", - "session_10", - "session_17", - "session_1", - "session_9", - "session_18", - "session_13", - "session_4", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_18" - ], - "question": "When did Melanie go on a hike after the roadtrip?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_4", - "session_3", - "session_1", - "session_9", - "session_12", - "session_2", - "session_14", - "session_16", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_18" - ], - "question": "Would Melanie go on another roadtrip soon?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_1", - "session_9", - "session_11", - "session_13", - "session_12", - "session_16", - "session_8", - "session_5", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_19", - "session_7" - ], - "question": "What items has Melanie bought?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_4", - "session_7", - "session_1", - "session_9", - "session_18", - "session_13", - "session_8", - "session_15", - "session_17" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_19" - ], - "question": "When did Caroline pass the adoption interview?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_2", - "session_13", - "session_3", - "session_17", - "session_8", - "session_1", - "session_18", - "session_15", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_19" - ], - "question": "When did Melanie buy the figurines?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_10", - "session_1", - "session_5", - "session_13", - "session_8", - "session_11", - "session_18", - "session_15", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_19" - ], - "question": "Would Caroline want to move back to her home country soon?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_19", - "session_8", - "session_5", - "session_4", - "session_7", - "session_18", - "session_2", - "session_10", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What did the charity race raise awareness for?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_7", - "session_3", - "session_5", - "session_8", - "session_14", - "session_13", - "session_18", - "session_15", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What did Melanie realize after the charity race?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_1", - "session_9", - "session_7", - "session_18", - "session_4", - "session_3", - "session_14", - "session_13", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "How does Melanie prioritize self-care?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_16", - "session_9", - "session_1", - "session_13", - "session_11", - "session_4", - "session_19", - "session_7", - "session_17" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What are Caroline's plans for the summer?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_2", - "session_10", - "session_17", - "session_11", - "session_15", - "session_7", - "session_16", - "session_1", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What type of individuals does the adoption agency Caroline is considering support?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_17", - "session_19", - "session_7", - "session_11", - "session_13", - "session_15", - "session_8", - "session_14", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "Why did Caroline choose the adoption agency?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_2", - "session_13", - "session_17", - "session_8", - "session_1", - "session_18", - "session_16", - "session_15", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What is Caroline excited about in the adoption process?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_17", - "session_19", - "session_13", - "session_8", - "session_5", - "session_16", - "session_4", - "session_1", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What does Melanie think about Caroline's decision to adopt?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_8", - "session_15", - "session_2", - "session_12", - "session_4", - "session_17", - "session_5", - "session_16", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "How long have Mel and her husband been married?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_6", - "session_16", - "session_8", - "session_17", - "session_19", - "session_1", - "session_10", - "session_7", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What does Caroline's necklace symbolize?", - "recall": 0.0, - "retrieved_ids": [], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What country is Caroline's grandma from?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_3", - "session_18", - "session_1", - "session_8", - "session_9", - "session_12", - "session_7", - "session_5", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What was grandma's gift to Caroline?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_19", - "session_18", - "session_1", - "session_9", - "session_5", - "session_12", - "session_7", - "session_8", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What is Melanie's hand-painted bowl a reminder of?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_16", - "session_5", - "session_12", - "session_9", - "session_8", - "session_11", - "session_14", - "session_1", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What did Melanie and her family do while camping?", - "recall": 0.0, - "retrieved_ids": [ - "session_16", - "session_8", - "session_10", - "session_2", - "session_6", - "session_17", - "session_13", - "session_7", - "session_18", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What kind of counseling and mental health services is Caroline interested in pursuing?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_5", - "session_6", - "session_1", - "session_2", - "session_7", - "session_10", - "session_8", - "session_11", - "session_3" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What workshop did Caroline attend recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_4", - "session_12", - "session_10", - "session_17", - "session_8", - "session_14", - "session_15", - "session_11", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What was discussed in the LGBTQ+ counseling workshop?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_1", - "session_5", - "session_7", - "session_10", - "session_9", - "session_11", - "session_14", - "session_6", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What motivated Caroline to pursue counseling?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_5", - "session_7", - "session_1", - "session_6", - "session_3", - "session_9", - "session_18", - "session_8", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What kind of place does Caroline want to create for people?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_10", - "session_3", - "session_5", - "session_17", - "session_14", - "session_15", - "session_6", - "session_11", - "session_2" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_5" - ], - "question": "Did Melanie make the black and white bowl in the photo?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_16", - "session_8", - "session_15", - "session_6", - "session_12", - "session_4", - "session_3", - "session_14", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_6" - ], - "question": "What kind of books does Caroline have in her library?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_11", - "session_2", - "session_7", - "session_10", - "session_17", - "session_3", - "session_15", - "session_5", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_6" - ], - "question": "What was Melanie's favorite book from her childhood?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_7", - "session_17", - "session_8", - "session_14", - "session_13", - "session_10", - "session_4", - "session_18", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What book did Caroline recommend to Melanie?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_17", - "session_6", - "session_5", - "session_1", - "session_13", - "session_18", - "session_15", - "session_8", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What did Caroline take away from the book \"Becoming Nicole\"?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_13", - "session_17", - "session_1", - "session_6", - "session_3", - "session_2", - "session_4", - "session_14", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What are the new shoes that Melanie got used for?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_2", - "session_1", - "session_6", - "session_9", - "session_4", - "session_17", - "session_15", - "session_16", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What is Melanie's reason for getting into running?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_2", - "session_5", - "session_6", - "session_18", - "session_16", - "session_4", - "session_10", - "session_15", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What does Melanie say running has been great for?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_2", - "session_10", - "session_12", - "session_6", - "session_9", - "session_15", - "session_17", - "session_3", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What did Mel and her kids make during the pottery workshop?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_16", - "session_17", - "session_14", - "session_4", - "session_10", - "session_5", - "session_18", - "session_12", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What kind of pot did Mel and her kids make with clay?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_6", - "session_2", - "session_10", - "session_3", - "session_5", - "session_11", - "session_4", - "session_15", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What creative project do Mel and her kids do together besides pottery?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_17", - "session_14", - "session_16", - "session_1", - "session_12", - "session_10", - "session_5", - "session_15", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What did Mel and her kids paint in their latest project in July 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_17", - "session_14", - "session_11", - "session_13", - "session_1", - "session_10", - "session_7", - "session_15", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What did Caroline see at the council meeting for adoption?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_17", - "session_14", - "session_19", - "session_2", - "session_5", - "session_13", - "session_10", - "session_15", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What do sunflowers represent according to Caroline?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_4", - "session_1", - "session_11", - "session_9", - "session_5", - "session_12", - "session_18", - "session_14", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "Why are flowers important to Melanie?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_14", - "session_12", - "session_6", - "session_11", - "session_2", - "session_18", - "session_7", - "session_15", - "session_3" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_9" - ], - "question": "What inspired Caroline's painting for the art show?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_9", - "session_16", - "session_17", - "session_8", - "session_11", - "session_13", - "session_12", - "session_1", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "How often does Melanie go to the beach with her kids?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_6", - "session_11", - "session_14", - "session_4", - "session_2", - "session_1", - "session_9", - "session_18", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "What did Melanie and her family see during their camping trip last year?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_10", - "session_18", - "session_17", - "session_6", - "session_4", - "session_7", - "session_16", - "session_12", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "How did Melanie feel while watching the meteor shower?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_11", - "session_13", - "session_8", - "session_16", - "session_17", - "session_1", - "session_15", - "session_14", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_11" - ], - "question": "Whose birthday did Melanie celebrate recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_17", - "session_14", - "session_4", - "session_13", - "session_12", - "session_10", - "session_8", - "session_15", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_11" - ], - "question": "Who performed at the concert at Melanie's daughter's birthday?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_4", - "session_15", - "session_8", - "session_18", - "session_1", - "session_13", - "session_9", - "session_19", - "session_17" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_12" - ], - "question": "Why did Melanie choose to use colors and patterns in her pottery project?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_8", - "session_14", - "session_16", - "session_7", - "session_15", - "session_4", - "session_19", - "session_2", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "What pet does Caroline have?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_13", - "session_4", - "session_1", - "session_9", - "session_5", - "session_12", - "session_18", - "session_8", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "What pets does Melanie have?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_13", - "session_15", - "session_1", - "session_9", - "session_18", - "session_4", - "session_8", - "session_17", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "Where did Oliver hide his bone once?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_7", - "session_10", - "session_11", - "session_1", - "session_14", - "session_8", - "session_18", - "session_15", - "session_6" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "What activity did Caroline used to do with her dad?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_11", - "session_5", - "session_10", - "session_15", - "session_7", - "session_8", - "session_14", - "session_1", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_14" - ], - "question": "What did Caroline make for a local church?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_12", - "session_5", - "session_15", - "session_8", - "session_7", - "session_10", - "session_16", - "session_17", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_14" - ], - "question": "What did Caroline find in her neighborhood during her walk?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_16", - "session_7", - "session_8", - "session_12", - "session_11", - "session_18", - "session_10", - "session_19", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "Which song motivates Caroline to be courageous?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_7", - "session_5", - "session_11", - "session_3", - "session_4", - "session_1", - "session_18", - "session_14", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "Which classical musicians does Melanie enjoy listening to?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_15", - "session_18", - "session_10", - "session_8", - "session_19", - "session_1", - "session_9", - "session_13", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "Who is Melanie a fan of in terms of modern music?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_5", - "session_11", - "session_19", - "session_1", - "session_9", - "session_18", - "session_13", - "session_4", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "How long has Melanie been creating art?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_4", - "session_12", - "session_6", - "session_15", - "session_8", - "session_17", - "session_1", - "session_3", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "What precautionary sign did Melanie see at the caf\u00e9?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_5", - "session_7", - "session_1", - "session_11", - "session_10", - "session_18", - "session_15", - "session_14", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What advice does Caroline give for getting started with adoption?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_13", - "session_2", - "session_7", - "session_18", - "session_8", - "session_12", - "session_4", - "session_15", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What setback did Melanie face in October 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_9", - "session_15", - "session_1", - "session_10", - "session_13", - "session_18", - "session_3", - "session_8", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What does Melanie do to keep herself busy during her pottery break?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_8", - "session_16", - "session_12", - "session_14", - "session_5", - "session_2", - "session_18", - "session_7", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What painting did Melanie show to Caroline on October 13, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_9", - "session_13", - "session_11", - "session_16", - "session_17", - "session_8", - "session_5", - "session_1", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What kind of painting did Caroline share with Melanie on October 13, 2023?", - "recall": 0.0, - "retrieved_ids": [ - "session_11", - "session_8", - "session_2", - "session_10", - "session_15", - "session_3", - "session_14", - "session_5", - "session_6", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What was the poetry reading that Caroline attended about?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_6", - "session_13", - "session_2", - "session_7", - "session_11", - "session_12", - "session_4", - "session_15", - "session_3" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What did the posters at the poetry reading say?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_6", - "session_10", - "session_2", - "session_7", - "session_15", - "session_8", - "session_14", - "session_13", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What does Caroline's drawing symbolize for her?", - "recall": 0.0, - "retrieved_ids": [ - "session_14", - "session_16", - "session_8", - "session_10", - "session_7", - "session_18", - "session_1", - "session_9", - "session_12", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "How do Melanie and Caroline describe their journey through life together?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_4", - "session_8", - "session_14", - "session_11", - "session_15", - "session_19", - "session_7", - "session_17", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What happened to Melanie's son on their road trip?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_15", - "session_8", - "session_10", - "session_7", - "session_17", - "session_14", - "session_6", - "session_12", - "session_2" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "How did Melanie's son handle the accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_16", - "session_10", - "session_8", - "session_14", - "session_13", - "session_15", - "session_1", - "session_11", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "How did Melanie feel about her family after the accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_3", - "session_8", - "session_14", - "session_2", - "session_10", - "session_7", - "session_4", - "session_1", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "How did Melanie's children handle the accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_16", - "session_10", - "session_14", - "session_13", - "session_15", - "session_1", - "session_11", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "How did Melanie feel after the accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_1", - "session_3", - "session_14", - "session_5", - "session_9", - "session_13", - "session_4", - "session_8", - "session_2" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What was Melanie's reaction to her children enjoying the Grand Canyon?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_13", - "session_8", - "session_3", - "session_10", - "session_7", - "session_1", - "session_4", - "session_9", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What do Melanie's family give her?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_7", - "session_3", - "session_10", - "session_13", - "session_6", - "session_18", - "session_8", - "session_4", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "How did Melanie feel about her family supporting her?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_16", - "session_3", - "session_8", - "session_14", - "session_7", - "session_17", - "session_15", - "session_2", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What did Melanie do after the road trip to relax?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_1", - "session_17", - "session_9", - "session_2", - "session_12", - "session_10", - "session_15", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_2" - ], - "question": "What did Caroline realize after her charity race?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_14", - "session_7", - "session_3", - "session_1", - "session_9", - "session_18", - "session_10", - "session_4", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_2" - ], - "question": "What are Melanie's plans for the summer with respect to adoption?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_19", - "session_16", - "session_12", - "session_10", - "session_17", - "session_11", - "session_13", - "session_15", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_2" - ], - "question": "What type of individuals does the adoption agency Melanie is considering support?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_17", - "session_19", - "session_7", - "session_11", - "session_13", - "session_15", - "session_8", - "session_14", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_2" - ], - "question": "Why did Melanie choose the adoption agency?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_2", - "session_13", - "session_17", - "session_8", - "session_1", - "session_18", - "session_16", - "session_15", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_2" - ], - "question": "What is Melanie excited about in her adoption process?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_17", - "session_13", - "session_8", - "session_19", - "session_16", - "session_5", - "session_10", - "session_7", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What does Melanie's necklace symbolize?", - "recall": 0.0, - "retrieved_ids": [], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What country is Melanie's grandma from?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_3", - "session_15", - "session_8", - "session_18", - "session_1", - "session_13", - "session_9", - "session_17", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What was grandma's gift to Melanie?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_19", - "session_7", - "session_1", - "session_9", - "session_18", - "session_13", - "session_8", - "session_15", - "session_17" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What was grandpa's gift to Caroline?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_4", - "session_18", - "session_1", - "session_9", - "session_5", - "session_12", - "session_7", - "session_8", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What is Caroline's hand-painted bowl a reminder of?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_16", - "session_5", - "session_12", - "session_13", - "session_8", - "session_11", - "session_14", - "session_1", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What did Caroline and her family do while camping?", - "recall": 0.0, - "retrieved_ids": [ - "session_16", - "session_8", - "session_10", - "session_2", - "session_6", - "session_17", - "session_13", - "session_7", - "session_18", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What kind of counseling and mental health services is Melanie interested in pursuing?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_5", - "session_6", - "session_1", - "session_2", - "session_7", - "session_10", - "session_8", - "session_11", - "session_3" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What kind of counseling workshop did Melanie attend recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_17", - "session_5", - "session_6", - "session_13", - "session_10", - "session_1", - "session_11", - "session_7", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What motivated Melanie to pursue counseling?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_5", - "session_7", - "session_1", - "session_6", - "session_3", - "session_13", - "session_18", - "session_8", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What kind of place does Melanie want to create for people?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_10", - "session_3", - "session_5", - "session_15", - "session_17", - "session_14", - "session_6", - "session_11", - "session_2" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_5" - ], - "question": "Did Caroline make the black and white bowl in the photo?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_16", - "session_15", - "session_8", - "session_6", - "session_12", - "session_4", - "session_3", - "session_14", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_7" - ], - "question": "What are the new shoes that Caroline got used for?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_2", - "session_1", - "session_6", - "session_9", - "session_4", - "session_17", - "session_15", - "session_16", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_7" - ], - "question": "What is Caroline's reason for getting into running?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_5", - "session_2", - "session_6", - "session_18", - "session_16", - "session_4", - "session_10", - "session_15", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_7" - ], - "question": "What does Caroline say running has been great for?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_2", - "session_10", - "session_12", - "session_6", - "session_9", - "session_15", - "session_17", - "session_3", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_8" - ], - "question": "What did Melanie see at the council meeting for adoption?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_17", - "session_14", - "session_19", - "session_2", - "session_5", - "session_13", - "session_10", - "session_15", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_9" - ], - "question": "What inspired Melanie's painting for the art show?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_9", - "session_16", - "session_17", - "session_8", - "session_11", - "session_13", - "session_12", - "session_1", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_9" - ], - "question": "What inspired Caroline's sculpture for the art show?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_9", - "session_16", - "session_17", - "session_8", - "session_5", - "session_11", - "session_12", - "session_13", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_10" - ], - "question": "How often does Caroline go to the beach with her kids?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_6", - "session_11", - "session_14", - "session_4", - "session_2", - "session_1", - "session_9", - "session_18", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_10" - ], - "question": "What did Caroline and her family see during their camping trip last year?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_10", - "session_18", - "session_17", - "session_6", - "session_4", - "session_7", - "session_16", - "session_12", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_10" - ], - "question": "How did Caroline feel while watching the meteor shower?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_11", - "session_13", - "session_8", - "session_16", - "session_17", - "session_1", - "session_15", - "session_14", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_12" - ], - "question": "Why did Caroline choose to use colors and patterns in her pottery project?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_8", - "session_14", - "session_16", - "session_7", - "session_15", - "session_4", - "session_19", - "session_2", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_13" - ], - "question": "Is Oscar Melanie's pet?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_7", - "session_15", - "session_8", - "session_18", - "session_1", - "session_4", - "session_9", - "session_19", - "session_17" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_13" - ], - "question": "Where did Oscar hide his bone once?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_5", - "session_10", - "session_11", - "session_14", - "session_8", - "session_18", - "session_15", - "session_1", - "session_6" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_13" - ], - "question": "What activity did Melanie used to do with her dad?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_11", - "session_5", - "session_10", - "session_15", - "session_7", - "session_8", - "session_14", - "session_1", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_14" - ], - "question": "What did Melanie make for a local church?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_12", - "session_5", - "session_15", - "session_8", - "session_7", - "session_10", - "session_16", - "session_17", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_14" - ], - "question": "What did Melanie find in her neighborhood during her walk?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_16", - "session_7", - "session_8", - "session_12", - "session_11", - "session_18", - "session_10", - "session_19", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_15" - ], - "question": "Which song motivates Melanie to be courageous?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_7", - "session_11", - "session_5", - "session_3", - "session_4", - "session_1", - "session_18", - "session_14", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_15" - ], - "question": "What type of instrument does Caroline play?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_14", - "session_3", - "session_7", - "session_6", - "session_1", - "session_2", - "session_9", - "session_12", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_15" - ], - "question": "Which classical musicians does Caroline enjoy listening to?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_5", - "session_15", - "session_18", - "session_10", - "session_8", - "session_7", - "session_1", - "session_9", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_15" - ], - "question": "Who is Caroline a fan of in terms of modern music?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_18", - "session_5", - "session_11", - "session_1", - "session_9", - "session_12", - "session_7", - "session_8", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_16" - ], - "question": "What precautionary sign did Caroline see at the caf\u00e9?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_5", - "session_7", - "session_1", - "session_11", - "session_10", - "session_18", - "session_15", - "session_14", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_17" - ], - "question": "What setback did Caroline face recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_14", - "session_9", - "session_10", - "session_13", - "session_12", - "session_15", - "session_11", - "session_18", - "session_3" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_17" - ], - "question": "What does Caroline do to keep herself busy during her pottery break?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_8", - "session_16", - "session_12", - "session_14", - "session_5", - "session_2", - "session_18", - "session_7", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_17" - ], - "question": "What was the poetry reading that Melanie attended about?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_6", - "session_13", - "session_2", - "session_12", - "session_7", - "session_11", - "session_4", - "session_15", - "session_3" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "What happened to Caroline's son on their road trip?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_15", - "session_8", - "session_10", - "session_7", - "session_17", - "session_14", - "session_6", - "session_12", - "session_2" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "How did Caroline's son handle the accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_16", - "session_10", - "session_8", - "session_14", - "session_13", - "session_15", - "session_1", - "session_11", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "How did Caroline feel about her family after the accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_3", - "session_14", - "session_2", - "session_10", - "session_7", - "session_4", - "session_1", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "How did Caroline's children handle the accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_16", - "session_10", - "session_14", - "session_13", - "session_15", - "session_1", - "session_11", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "How did Caroline feel after the accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_1", - "session_3", - "session_14", - "session_5", - "session_9", - "session_13", - "session_4", - "session_8", - "session_2" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "What was Caroline's reaction to her children enjoying the Grand Canyon?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_14", - "session_8", - "session_3", - "session_10", - "session_7", - "session_5", - "session_1", - "session_9", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "What did Caroline do after the road trip to relax?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_1", - "session_17", - "session_9", - "session_2", - "session_12", - "session_10", - "session_15", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "What does Caroline love most about camping with her family?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_8", - "session_6", - "session_18", - "session_2", - "session_16", - "session_4", - "session_7", - "session_14", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_1" - ], - "question": "When Jon has lost his job as a banker?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_16", - "session_6", - "session_14", - "session_4", - "session_11", - "session_10", - "session_5", - "session_15", - "session_3" - ], - "sample_id": "conv-30" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_1" - ], - "question": "When Gina has lost her job at Door Dash?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_1", - "session_14", - "session_16", - "session_17", - "session_9", - "session_18", - "session_4", - "session_10", - "session_11" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_1" - ], - "question": "How do Jon and Gina both like to destress?", - "recall": 0.0, - "retrieved_ids": [ - "session_2", - "session_6", - "session_11", - "session_5", - "session_9", - "session_10", - "session_16", - "session_8", - "session_18", - "session_3" - ], - "sample_id": "conv-30" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_2" - ], - "question": "What do Jon and Gina both have in common?", - "recall": 0.5, - "retrieved_ids": [ - "session_6", - "session_9", - "session_10", - "session_2", - "session_5", - "session_17", - "session_4", - "session_14", - "session_19", - "session_15" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_1" - ], - "question": "Why did Jon decide to start his dance studio?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_11", - "session_10", - "session_1", - "session_13", - "session_15", - "session_3", - "session_14", - "session_6", - "session_5" - ], - "sample_id": "conv-30" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_2" - ], - "question": "What Jon thinks the ideal dance studio should look like?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_2", - "session_3", - "session_5", - "session_9", - "session_10", - "session_8", - "session_11", - "session_19", - "session_12" - ], - "sample_id": "conv-30" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_1" - ], - "question": "When is Jon's group performing at a festival?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_19", - "session_14", - "session_5", - "session_16", - "session_15", - "session_4", - "session_18", - "session_12", - "session_8" - ], - "sample_id": "conv-30" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_2" - ], - "question": "When did Gina launch an ad campaign for her store?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_6", - "session_7", - "session_14", - "session_3", - "session_15", - "session_1", - "session_5", - "session_8", - "session_11" - ], - "sample_id": "conv-30" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_2" - ], - "question": "When was Jon in Paris?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_16", - "session_15", - "session_4", - "session_19", - "session_14", - "session_1", - "session_18", - "session_12", - "session_8" - ], - "sample_id": "conv-30" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_15", - "session_2" - ], - "question": "Which city have both Jean and John visited?", - "recall": 0.5, - "retrieved_ids": [ - "session_2", - "session_10", - "session_6", - "session_9", - "session_5" - ], - "sample_id": "conv-30" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_5" - ], - "question": "When did Gina team up with a local artist for some cool designs?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_1", - "session_16", - "session_8", - "session_3", - "session_18", - "session_6", - "session_11", - "session_14", - "session_2" - ], - "sample_id": "conv-30" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_5" - ], - "question": "When did Gina get her tattoo?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_4", - "session_16", - "session_19", - "session_14", - "session_1", - "session_11", - "session_2", - "session_18", - "session_12" - ], - "sample_id": "conv-30" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_6" - ], - "question": "When did Jon start to go to the gym?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_17", - "session_19", - "session_14", - "session_1", - "session_11", - "session_10", - "session_2", - "session_7", - "session_5" - ], - "sample_id": "conv-30" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_6" - ], - "question": "When did Gina open her online clothing store?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_7", - "session_6", - "session_18", - "session_10", - "session_8", - "session_2", - "session_5", - "session_1", - "session_11" - ], - "sample_id": "conv-30" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_8" - ], - "question": "When did Jon start expanding his studio's social media presence?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_14", - "session_18", - "session_11", - "session_10", - "session_1", - "session_2", - "session_15", - "session_3", - "session_6" - ], - "sample_id": "conv-30" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_8" - ], - "question": "When did Jon host a dance competition?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_1", - "session_8", - "session_13", - "session_14", - "session_9", - "session_11", - "session_2", - "session_18", - "session_5" - ], - "sample_id": "conv-30" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_10" - ], - "question": "When did Jon go to a fair to get more exposure for his dance studio?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_11", - "session_2", - "session_15", - "session_5", - "session_3", - "session_1", - "session_8", - "session_14", - "session_6" - ], - "sample_id": "conv-30" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_6" - ], - "question": "Why did Gina decide to start her own clothing store?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_2", - "session_3", - "session_6", - "session_10", - "session_7", - "session_1", - "session_18", - "session_11", - "session_5" - ], - "sample_id": "conv-30" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_6" - ], - "question": "Do Jon and Gina start businesses out of what they love?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_14", - "session_10", - "session_6", - "session_5", - "session_11", - "session_8", - "session_9", - "session_3", - "session_7" - ], - "sample_id": "conv-30" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_11" - ], - "question": "When did Gina interview for a design internship?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_16", - "session_3", - "session_5", - "session_19", - "session_12", - "session_8", - "session_14", - "session_1", - "session_2" - ], - "sample_id": "conv-30" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_12" - ], - "question": "When did Gina get accepted for the design internship?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_12", - "session_16", - "session_3", - "session_5", - "session_19", - "session_8", - "session_1", - "session_2", - "session_14" - ], - "sample_id": "conv-30" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_12" - ], - "question": "When did Jon start reading \"The Lean Startup\"?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_19", - "session_14", - "session_11", - "session_1", - "session_6", - "session_10", - "session_2", - "session_13", - "session_5" - ], - "sample_id": "conv-30" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_13" - ], - "question": "When did Gina develop a video presentation to teach how to style her fashion pieces? ", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_3", - "session_16", - "session_14", - "session_18", - "session_8", - "session_11", - "session_1", - "session_5", - "session_2" - ], - "sample_id": "conv-30" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_13", - "session_16", - "session_5", - "session_8" - ], - "question": "How did Gina promote her clothes store?", - "recall": 0.5, - "retrieved_ids": [ - "session_6", - "session_14", - "session_7", - "session_3", - "session_8", - "session_10", - "session_2", - "session_5", - "session_1", - "session_11" - ], - "sample_id": "conv-30" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_16", - "session_8" - ], - "question": "Which events has Jon participated in to promote his business venture?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_16", - "session_18", - "session_10", - "session_14", - "session_6", - "session_11", - "session_5", - "session_15", - "session_3" - ], - "sample_id": "conv-30" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_13", - "session_8" - ], - "question": "What does Jon's dance studio offer?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_7", - "session_18", - "session_13", - "session_6", - "session_5", - "session_15", - "session_9", - "session_11", - "session_2" - ], - "sample_id": "conv-30" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_14" - ], - "question": "When did Jon receive mentorship to promote his venture?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_2", - "session_15", - "session_14", - "session_3", - "session_11", - "session_10", - "session_1", - "session_5", - "session_8" - ], - "sample_id": "conv-30" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_14", - "session_9" - ], - "question": "Did Jon and Gina both participate in dance competitions?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_2", - "session_8", - "session_4", - "session_9", - "session_6", - "session_10", - "session_5", - "session_11", - "session_14" - ], - "sample_id": "conv-30" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_15" - ], - "question": "When was Jon in Rome?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_2", - "session_18", - "session_16", - "session_19", - "session_4", - "session_14", - "session_1", - "session_12", - "session_8" - ], - "sample_id": "conv-30" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_15", - "session_2" - ], - "question": "Which cities has Jon visited?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_16", - "session_12", - "session_8", - "session_4", - "session_19", - "session_14", - "session_15", - "session_1", - "session_18" - ], - "sample_id": "conv-30" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_15" - ], - "question": "When Jon is planning to open his dance studio?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_18", - "session_11", - "session_1", - "session_15", - "session_6", - "session_10", - "session_12", - "session_8", - "session_16" - ], - "sample_id": "conv-30" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_15" - ], - "question": "How long did it take for Jon to open his studio?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_18", - "session_2", - "session_15", - "session_10", - "session_11", - "session_3", - "session_1", - "session_14", - "session_6" - ], - "sample_id": "conv-30" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_16" - ], - "question": "When did Gina design a limited collection of hoodies?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_11", - "session_3", - "session_15", - "session_5", - "session_8", - "session_19", - "session_14", - "session_1", - "session_2" - ], - "sample_id": "conv-30" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_16" - ], - "question": "When did Jon visit networking events for his store?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_16", - "session_2", - "session_10", - "session_8", - "session_3", - "session_6", - "session_1", - "session_15", - "session_11" - ], - "sample_id": "conv-30" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_17" - ], - "question": "When did Gina start being recognized by fashion editors?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_7", - "session_14", - "session_6", - "session_5", - "session_11", - "session_1", - "session_12", - "session_8", - "session_2" - ], - "sample_id": "conv-30" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_17" - ], - "question": "When did Jon start learning marketing and analytics tools?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_14", - "session_10", - "session_11", - "session_1", - "session_18", - "session_7", - "session_12", - "session_2", - "session_5" - ], - "sample_id": "conv-30" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_18" - ], - "question": "When did Jon and Gina decide to collaborate to create dance content?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_3", - "session_2", - "session_11", - "session_17", - "session_5", - "session_9", - "session_7", - "session_14", - "session_1" - ], - "sample_id": "conv-30" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_19" - ], - "question": "When did Gina mention Shia Labeouf?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_16", - "session_15", - "session_14", - "session_1", - "session_13", - "session_11", - "session_17", - "session_2", - "session_5" - ], - "sample_id": "conv-30" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_19" - ], - "question": "When did Gina go to a dance class with a group of friends?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_1", - "session_15", - "session_9", - "session_17", - "session_13", - "session_14", - "session_11", - "session_2", - "session_8" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_1" - ], - "question": "What is Gina's favorite style of dance?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_3", - "session_13", - "session_18", - "session_8", - "session_16", - "session_1", - "session_9", - "session_11", - "session_2" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_1" - ], - "question": "What is Jon's favorite style of dance?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_3", - "session_13", - "session_8", - "session_16", - "session_1", - "session_18", - "session_9", - "session_11", - "session_2" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_1" - ], - "question": "What was Gina's favorite dancing memory?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_1", - "session_9", - "session_14", - "session_4", - "session_15", - "session_11", - "session_2", - "session_8", - "session_18" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_1" - ], - "question": "What kind of dance piece did Gina's team perform to win first place?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_3", - "session_5", - "session_7", - "session_4", - "session_18", - "session_8", - "session_11", - "session_2", - "session_14" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_1" - ], - "question": "What do the dancers in the photo represent?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_8", - "session_13", - "session_1", - "session_5", - "session_19", - "session_9", - "session_17" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_1" - ], - "question": "What does Gina say about the dancers in the photo?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_8", - "session_1", - "session_5", - "session_12", - "session_7", - "session_13", - "session_19", - "session_9", - "session_17" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_1" - ], - "question": "What is Jon's attitude towards being part of the dance festival?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_7", - "session_1", - "session_12", - "session_13", - "session_15", - "session_9", - "session_11", - "session_18", - "session_2" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What kind of flooring is Jon looking for in his dance studio?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_2", - "session_7", - "session_4", - "session_15", - "session_11", - "session_3", - "session_10", - "session_5", - "session_6" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "What did Gina find for her clothing store on 1 February, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_2", - "session_14", - "session_7", - "session_8", - "session_6", - "session_1", - "session_18", - "session_5", - "session_11" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "What did Gina design for her store?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_6", - "session_8", - "session_16", - "session_7", - "session_14", - "session_5", - "session_11", - "session_1", - "session_2" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "What did Gina want her customers to feel in her store?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_7", - "session_14", - "session_18", - "session_5", - "session_8", - "session_11", - "session_2", - "session_12", - "session_1" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "What did Jon say about Gina's progress with her store?", - "recall": 0.0, - "retrieved_ids": [ - "session_4", - "session_8", - "session_14", - "session_5", - "session_12", - "session_16", - "session_1", - "session_11", - "session_2", - "session_13" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "What made Gina choose the furniture and decor for her store?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_7", - "session_8", - "session_16", - "session_14", - "session_6", - "session_5", - "session_10", - "session_4", - "session_2" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "What did Jon say about creating a special experience for customers?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_14", - "session_18", - "session_4", - "session_7", - "session_1", - "session_5", - "session_13", - "session_9", - "session_11" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "What did Gina say about creating an experience for her customers?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_14", - "session_7", - "session_18", - "session_4", - "session_5", - "session_13", - "session_2", - "session_1", - "session_11" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "How is Gina's store doing?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_4", - "session_14", - "session_1", - "session_5", - "session_10", - "session_8", - "session_6", - "session_18", - "session_16" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_5" - ], - "question": "What does Gina's tattoo symbolize?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_11", - "session_16", - "session_13", - "session_14", - "session_1", - "session_12", - "session_19", - "session_15", - "session_17" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_6" - ], - "question": "What did Jon and Gina compare their entrepreneurial journeys to?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_2", - "session_17", - "session_13", - "session_6", - "session_1", - "session_10", - "session_5", - "session_14", - "session_11" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What advice does Gina give to Jon about running a successful business?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_7", - "session_14", - "session_10", - "session_5", - "session_4", - "session_12", - "session_13", - "session_6", - "session_11" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "Why did Jon shut down his bank account?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_14", - "session_6", - "session_18", - "session_5", - "session_10", - "session_15", - "session_11", - "session_3", - "session_1" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "Why did Gina combine her clothing business with dance?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_14", - "session_7", - "session_10", - "session_3", - "session_2", - "session_6", - "session_11", - "session_1", - "session_5" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_9" - ], - "question": "What does Jon's dance make him?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_11", - "session_4", - "session_3", - "session_18", - "session_15", - "session_9", - "session_17", - "session_2", - "session_5" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_9" - ], - "question": "What did Gina receive from a dance contest?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_13", - "session_4", - "session_14", - "session_1", - "session_18", - "session_8", - "session_11", - "session_2", - "session_5" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "How does Gina stay confident in her business?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_7", - "session_14", - "session_13", - "session_5", - "session_12", - "session_8", - "session_11", - "session_16", - "session_18" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_12" - ], - "question": "What kind of professional experience did Gina get accepted for on May 23, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_4", - "session_13", - "session_3", - "session_1", - "session_2", - "session_12", - "session_11", - "session_7", - "session_14" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_12" - ], - "question": "Where is Gina's fashion internship?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_7", - "session_11", - "session_13", - "session_6", - "session_17", - "session_5", - "session_8", - "session_15", - "session_19" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_12" - ], - "question": "What book is Jon currently reading?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_17", - "session_4", - "session_16", - "session_14", - "session_1", - "session_18", - "session_8", - "session_19", - "session_15" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "What is Jon offering to the dancers at his dance studio?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_8", - "session_18", - "session_19", - "session_7", - "session_3", - "session_15", - "session_11", - "session_10", - "session_6" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "How does Jon use the clipboard with a notepad attached to it?", - "recall": 0.0, - "retrieved_ids": [ - "session_14", - "session_4", - "session_17", - "session_18", - "session_12", - "session_8", - "session_19", - "session_15", - "session_16", - "session_1" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_14" - ], - "question": "What does Jon tell Gina he won't do?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_16", - "session_14", - "session_7", - "session_13", - "session_12", - "session_19", - "session_9", - "session_17", - "session_1" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "What did Jon take a trip to Rome for?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_2", - "session_17", - "session_18", - "session_1", - "session_9", - "session_5", - "session_16", - "session_14", - "session_11" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "What is Jon working on opening?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_13", - "session_11", - "session_5", - "session_14", - "session_18", - "session_9", - "session_3", - "session_6", - "session_17" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "How does Gina describe the studio that Jon has opened?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_18", - "session_3", - "session_15", - "session_8", - "session_14", - "session_13", - "session_6", - "session_11", - "session_9" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "How does Jon feel about the opening night of his dance studio?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_11", - "session_18", - "session_8", - "session_13", - "session_17", - "session_3", - "session_10", - "session_14", - "session_6" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "How does Gina describe the feeling that dance brings?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_5", - "session_8", - "session_15", - "session_1", - "session_9", - "session_4", - "session_18", - "session_11", - "session_3" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "What does Jon plan to do at the grand opening of his dance studio?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_1", - "session_18", - "session_12", - "session_8", - "session_11", - "session_3", - "session_10", - "session_16", - "session_6" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "What does Gina say to Jon about the grand opening?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_4", - "session_11", - "session_12", - "session_13", - "session_14", - "session_6", - "session_18", - "session_3", - "session_17" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "What is the general sentiment about the upcoming grand opening?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_15", - "session_11", - "session_13", - "session_14", - "session_6", - "session_18", - "session_12", - "session_3", - "session_17" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "What did Gina make a limited edition line of?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_15", - "session_3", - "session_18", - "session_11", - "session_17", - "session_2", - "session_5", - "session_14", - "session_1" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "According to Gina, what makes Jon a perfect mentor and guide?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_13", - "session_4", - "session_3", - "session_14", - "session_6", - "session_10", - "session_2", - "session_15", - "session_18" - ], - "sample_id": "conv-30" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "What temporary job did Jon take to cover expenses?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_17", - "session_1", - "session_9", - "session_16", - "session_14", - "session_6", - "session_11", - "session_2", - "session_5" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What plans does Jon have after receiving advice at the networking event?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_16", - "session_7", - "session_14", - "session_1", - "session_10", - "session_8", - "session_4", - "session_19", - "session_12" - ], - "sample_id": "conv-30" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What offer does Gina make to Jon regarding social media?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_7", - "session_14", - "session_6", - "session_4", - "session_13", - "session_3", - "session_15", - "session_17" - ], - "sample_id": "conv-30" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_1" - ], - "question": "What is Jon's favorite style of painting?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_16", - "session_3", - "session_13", - "session_12", - "session_4", - "session_14", - "session_1", - "session_18", - "session_19" - ], - "sample_id": "conv-30" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_1" - ], - "question": "What was Jon's favorite dancing memory?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_1", - "session_9", - "session_14", - "session_4", - "session_15", - "session_11", - "session_2", - "session_18", - "session_8" - ], - "sample_id": "conv-30" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_1" - ], - "question": "What kind of dance piece did Jon's team perform to win first place?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_3", - "session_5", - "session_7", - "session_4", - "session_18", - "session_8", - "session_11", - "session_2", - "session_14" - ], - "sample_id": "conv-30" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_1" - ], - "question": "What is Gina's attitude towards participating in the dance festival?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_1", - "session_12", - "session_14", - "session_4", - "session_18", - "session_8", - "session_9", - "session_11", - "session_2" - ], - "sample_id": "conv-30" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_2" - ], - "question": "What kind of flooring is Gina looking for in her dance studio?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_8", - "session_18", - "session_3", - "session_2", - "session_4", - "session_1", - "session_15", - "session_9", - "session_5" - ], - "sample_id": "conv-30" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_3" - ], - "question": "What did Jon find for his clothing store on 1 February, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_2", - "session_3", - "session_14", - "session_1", - "session_10", - "session_6", - "session_11", - "session_5", - "session_15" - ], - "sample_id": "conv-30" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_3" - ], - "question": "What did Jon design for his store?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_16", - "session_5", - "session_11", - "session_2", - "session_14", - "session_6", - "session_1", - "session_10", - "session_15" - ], - "sample_id": "conv-30" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_3" - ], - "question": "What did Jon want his customers to feel in her store?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_14", - "session_6", - "session_8", - "session_5", - "session_11", - "session_15", - "session_10", - "session_2", - "session_1" - ], - "sample_id": "conv-30" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_3" - ], - "question": "What made Jon choose the furniture and decor for his store?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_16", - "session_7", - "session_8", - "session_5", - "session_2", - "session_6", - "session_10", - "session_15", - "session_11" - ], - "sample_id": "conv-30" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "How is Jon's store doing?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_1", - "session_4", - "session_14", - "session_5", - "session_10", - "session_6", - "session_18", - "session_8", - "session_16" - ], - "sample_id": "conv-30" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_5" - ], - "question": "What does Jon's tattoo symbolize?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_16", - "session_4", - "session_14", - "session_1", - "session_18", - "session_12", - "session_8", - "session_19", - "session_15" - ], - "sample_id": "conv-30" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_8" - ], - "question": "Why did Gina shut down her bank account?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_14", - "session_12", - "session_16", - "session_17", - "session_18", - "session_5", - "session_11", - "session_1", - "session_2" - ], - "sample_id": "conv-30" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_8" - ], - "question": "Why did Jon combine his clothing business with dance?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_14", - "session_10", - "session_2", - "session_5", - "session_3", - "session_11", - "session_1", - "session_6", - "session_15" - ], - "sample_id": "conv-30" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_9" - ], - "question": "What did Gina receive from a dance contest?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_13", - "session_4", - "session_14", - "session_1", - "session_18", - "session_8", - "session_11", - "session_2", - "session_5" - ], - "sample_id": "conv-30" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_12" - ], - "question": "What kind of professional experience did Jon get accepted for on May 23, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_4", - "session_13", - "session_3", - "session_1", - "session_2", - "session_12", - "session_11", - "session_7", - "session_14" - ], - "sample_id": "conv-30" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_12" - ], - "question": "Where is Gina's HR internship?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_11", - "session_4", - "session_19", - "session_15", - "session_16", - "session_13", - "session_14", - "session_17", - "session_1" - ], - "sample_id": "conv-30" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_12" - ], - "question": "Where is Jon's fashion internship?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_11", - "session_7", - "session_13", - "session_6", - "session_17", - "session_5", - "session_8", - "session_14", - "session_4" - ], - "sample_id": "conv-30" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_12" - ], - "question": "What book is Gina currently reading?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_4", - "session_16", - "session_13", - "session_14", - "session_1", - "session_19", - "session_15", - "session_17", - "session_11" - ], - "sample_id": "conv-30" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_13" - ], - "question": "How does Gina use the clipboard with a notepad attached to it?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_14", - "session_4", - "session_19", - "session_17", - "session_11", - "session_12", - "session_15", - "session_16", - "session_1" - ], - "sample_id": "conv-30" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_15" - ], - "question": "What did Jon take a trip to Barcelona for?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_1", - "session_9", - "session_17", - "session_2", - "session_5", - "session_7", - "session_14", - "session_16", - "session_11" - ], - "sample_id": "conv-30" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_16" - ], - "question": "What did Jon make a limited edition line of?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_15", - "session_3", - "session_18", - "session_11", - "session_17", - "session_2", - "session_5", - "session_1", - "session_14" - ], - "sample_id": "conv-30" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "What temporary job did Gina take to cover expenses?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_17", - "session_1", - "session_9", - "session_16", - "session_14", - "session_6", - "session_11", - "session_2", - "session_5" - ], - "sample_id": "conv-30" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "What plans does Gina have after receiving advice at the networking event?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_16", - "session_7", - "session_14", - "session_1", - "session_10", - "session_4", - "session_8", - "session_19", - "session_12" - ], - "sample_id": "conv-30" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_13" - ], - "question": "Who did Maria have dinner with on May 3, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_23", - "session_2", - "session_13", - "session_15", - "session_27", - "session_24", - "session_6", - "session_32", - "session_7" - ], - "sample_id": "conv-41" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_2" - ], - "question": "When did Maria donate her car?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_21", - "session_11", - "session_6", - "session_7", - "session_31", - "session_5", - "session_30", - "session_14", - "session_32" - ], - "sample_id": "conv-41" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_2" - ], - "question": "What martial arts has John done?", - "recall": 0.5, - "retrieved_ids": [ - "session_25", - "session_24", - "session_27", - "session_15", - "session_26", - "session_29", - "session_2", - "session_16", - "session_4", - "session_28" - ], - "sample_id": "conv-41" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_2", - "session_3" - ], - "question": "What type of volunteering have John and Maria both done?", - "recall": 0.0, - "retrieved_ids": [ - "session_13", - "session_27", - "session_17", - "session_24", - "session_25", - "session_15", - "session_26", - "session_29", - "session_4", - "session_10" - ], - "sample_id": "conv-41" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_3" - ], - "question": "When did John join the online support group?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_27", - "session_24", - "session_4", - "session_23", - "session_12", - "session_14", - "session_32", - "session_26", - "session_25" - ], - "sample_id": "conv-41" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_3" - ], - "question": "When did Maria go to the beach?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_19", - "session_16", - "session_13", - "session_15", - "session_1", - "session_7", - "session_9", - "session_12", - "session_27" - ], - "sample_id": "conv-41" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_14", - "session_19", - "session_2", - "session_4" - ], - "question": "Where has Maria made friends?", - "recall": 0.25, - "retrieved_ids": [ - "session_29", - "session_18", - "session_24", - "session_16", - "session_20", - "session_6", - "session_2", - "session_27", - "session_26", - "session_25" - ], - "sample_id": "conv-41" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_3", - "session_5" - ], - "question": "What items des John mention having as a child?", - "recall": 0.0, - "retrieved_ids": [ - "session_13", - "session_26", - "session_9", - "session_2", - "session_19", - "session_31", - "session_1", - "session_20", - "session_23", - "session_14" - ], - "sample_id": "conv-41" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_5" - ], - "question": "What might John's financial status be?", - "recall": 0.0, - "retrieved_ids": [ - "session_16", - "session_21", - "session_25", - "session_9", - "session_1", - "session_15", - "session_4", - "session_2", - "session_13", - "session_28" - ], - "sample_id": "conv-41" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_5", - "session_6" - ], - "question": "Who gave Maria's family money when she was younger and her family was going through tough times?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_29", - "session_21", - "session_5", - "session_7", - "session_6", - "session_18", - "session_31", - "session_30", - "session_27" - ], - "sample_id": "conv-41" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_7" - ], - "question": "When did Maria meet Jean?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_3", - "session_30", - "session_32", - "session_6", - "session_9", - "session_23", - "session_4", - "session_27", - "session_24" - ], - "sample_id": "conv-41" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_21", - "session_27", - "session_6", - "session_7" - ], - "question": "What people has Maria met and helped while volunteering?", - "recall": 0.75, - "retrieved_ids": [ - "session_21", - "session_22", - "session_7", - "session_27", - "session_5", - "session_10", - "session_8", - "session_20", - "session_24", - "session_28" - ], - "sample_id": "conv-41" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_3", - "session_8" - ], - "question": "What test has John taken multiple times?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_19", - "session_30", - "session_3", - "session_15", - "session_18", - "session_2", - "session_13", - "session_31", - "session_25" - ], - "sample_id": "conv-41" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_8" - ], - "question": "When did Maria's grandmother pass away?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_18", - "session_11", - "session_30", - "session_17", - "session_32", - "session_6", - "session_20", - "session_27", - "session_24" - ], - "sample_id": "conv-41" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_8" - ], - "question": "Would John be considered a patriotic person?", - "recall": 0.0, - "retrieved_ids": [ - "session_9", - "session_3", - "session_6", - "session_21", - "session_31", - "session_15", - "session_17", - "session_16", - "session_4", - "session_28" - ], - "sample_id": "conv-41" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_7", - "session_9" - ], - "question": "What writing classes has Maria taken?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_15", - "session_1", - "session_12", - "session_19", - "session_25", - "session_4", - "session_16", - "session_9", - "session_10" - ], - "sample_id": "conv-41" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_9" - ], - "question": "When did John get his degree?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_8", - "session_9", - "session_12", - "session_15", - "session_1", - "session_4", - "session_2", - "session_25", - "session_31" - ], - "sample_id": "conv-41" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_9" - ], - "question": "What might John's degree be in?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_16", - "session_21", - "session_25", - "session_13", - "session_1", - "session_15", - "session_4", - "session_2", - "session_28" - ], - "sample_id": "conv-41" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_7" - ], - "question": "Who did John go to yoga with?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_1", - "session_10", - "session_7", - "session_18", - "session_19", - "session_27", - "session_12", - "session_15", - "session_9" - ], - "sample_id": "conv-41" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_11", - "session_4" - ], - "question": "What damages have happened to John's car?", - "recall": 0.5, - "retrieved_ids": [ - "session_14", - "session_21", - "session_15", - "session_11", - "session_18", - "session_2", - "session_23", - "session_28", - "session_1", - "session_7" - ], - "sample_id": "conv-41" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_11" - ], - "question": "When did John take a road trip to the Pacific Northwest?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_18", - "session_8", - "session_22", - "session_1", - "session_14", - "session_30", - "session_13", - "session_16", - "session_12" - ], - "sample_id": "conv-41" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_11", - "session_12" - ], - "question": "What areas of the U.S. has John been to or is planning to go to?", - "recall": 0.0, - "retrieved_ids": [ - "session_9", - "session_1", - "session_15", - "session_14", - "session_5", - "session_2", - "session_23", - "session_4", - "session_25", - "session_17" - ], - "sample_id": "conv-41" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_12" - ], - "question": "When did John go to a convention with colleagues?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_11", - "session_7", - "session_1", - "session_15", - "session_27", - "session_9", - "session_10", - "session_31", - "session_13" - ], - "sample_id": "conv-41" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_13", - "session_2" - ], - "question": "What desserts has Maria made?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_29", - "session_27", - "session_5", - "session_15", - "session_26", - "session_6", - "session_2", - "session_32", - "session_3" - ], - "sample_id": "conv-41" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_13" - ], - "question": "When did John start boot camp with his family?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_27", - "session_6", - "session_8", - "session_18", - "session_25", - "session_30", - "session_31", - "session_2", - "session_15" - ], - "sample_id": "conv-41" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_13", - "session_8" - ], - "question": "What European countries has Maria been to?", - "recall": 0.5, - "retrieved_ids": [ - "session_27", - "session_6", - "session_8", - "session_19", - "session_9", - "session_7", - "session_4", - "session_14", - "session_12", - "session_20" - ], - "sample_id": "conv-41" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_11", - "session_14" - ], - "question": "What has Maria done to feel closer to her faith?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_14", - "session_27", - "session_31", - "session_5", - "session_25", - "session_21", - "session_30", - "session_2", - "session_7" - ], - "sample_id": "conv-41" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_15" - ], - "question": "When did John have a party with veterans?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_24", - "session_27", - "session_29", - "session_21", - "session_13", - "session_6", - "session_32", - "session_7", - "session_4" - ], - "sample_id": "conv-41" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_12", - "session_15", - "session_9" - ], - "question": "What causes does John feel passionate about supporting?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_16", - "session_10", - "session_30", - "session_12", - "session_1", - "session_15", - "session_9", - "session_2", - "session_32" - ], - "sample_id": "conv-41" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_15", - "session_16" - ], - "question": "What events is Maria planning for the homeless shelter funraiser?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_16", - "session_29", - "session_12", - "session_10", - "session_6", - "session_25", - "session_17", - "session_9", - "session_4" - ], - "sample_id": "conv-41" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_11", - "session_17", - "session_2" - ], - "question": "What shelters does Maria volunteer at?", - "recall": 0.3333, - "retrieved_ids": [ - "session_27", - "session_7", - "session_11", - "session_26", - "session_5", - "session_8", - "session_21", - "session_1", - "session_31", - "session_20" - ], - "sample_id": "conv-41" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_17" - ], - "question": "When did John get his dog Max?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_30", - "session_6", - "session_31", - "session_17", - "session_8", - "session_1", - "session_15", - "session_25", - "session_2" - ], - "sample_id": "conv-41" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_16", - "session_18" - ], - "question": "What outdoor activities has John done with his colleagues?", - "recall": 0.0, - "retrieved_ids": [ - "session_31", - "session_8", - "session_26", - "session_15", - "session_25", - "session_27", - "session_24", - "session_12", - "session_7", - "session_10" - ], - "sample_id": "conv-41" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_18", - "session_19" - ], - "question": "What types of yoga has Maria practiced?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_1", - "session_7", - "session_10", - "session_18", - "session_19", - "session_22", - "session_13", - "session_16", - "session_4" - ], - "sample_id": "conv-41" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_19" - ], - "question": "When did Maria join a gym?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_19", - "session_30", - "session_27", - "session_24", - "session_26", - "session_14", - "session_3", - "session_4", - "session_25" - ], - "sample_id": "conv-41" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_18", - "session_19" - ], - "question": "What states has Maria vacationed at?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_13", - "session_5", - "session_6", - "session_16", - "session_4", - "session_31", - "session_18", - "session_21", - "session_2" - ], - "sample_id": "conv-41" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_20", - "session_8" - ], - "question": "What music events has John attended?", - "recall": 0.5, - "retrieved_ids": [ - "session_20", - "session_5", - "session_16", - "session_22", - "session_29", - "session_24", - "session_6", - "session_10", - "session_13", - "session_3" - ], - "sample_id": "conv-41" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_15", - "session_21", - "session_24", - "session_29" - ], - "question": "What events for veterans has John participated in?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_21", - "session_24", - "session_10", - "session_3", - "session_27", - "session_16", - "session_15", - "session_6", - "session_20" - ], - "sample_id": "conv-41" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_21" - ], - "question": "When did Maria get in a car accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_21", - "session_14", - "session_2", - "session_8", - "session_6", - "session_1", - "session_27", - "session_4", - "session_12" - ], - "sample_id": "conv-41" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_21" - ], - "question": "Around which US holiday did Maria get into a car accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_32", - "session_14", - "session_31", - "session_12", - "session_30", - "session_11", - "session_2", - "session_28", - "session_9" - ], - "sample_id": "conv-41" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_22", - "session_8" - ], - "question": "What are the names of John's children?", - "recall": 0.5, - "retrieved_ids": [ - "session_24", - "session_6", - "session_30", - "session_31", - "session_2", - "session_16", - "session_8", - "session_15", - "session_4", - "session_28" - ], - "sample_id": "conv-41" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_22" - ], - "question": "Does John live close to a beach or the mountains?", - "recall": 0.0, - "retrieved_ids": [ - "session_18", - "session_16", - "session_20", - "session_2", - "session_3", - "session_19", - "session_32", - "session_23", - "session_9", - "session_14" - ], - "sample_id": "conv-41" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_14", - "session_23" - ], - "question": "What area was hit by a flood?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_2", - "session_29", - "session_1", - "session_9", - "session_11", - "session_21", - "session_5", - "session_14" - ], - "sample_id": "conv-41" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_23" - ], - "question": "When was John's old area hit with a flood?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_14", - "session_2", - "session_5", - "session_29", - "session_11", - "session_21", - "session_1", - "session_9", - "session_8" - ], - "sample_id": "conv-41" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_24", - "session_25", - "session_28" - ], - "question": "What activities has Maria done with her church friends?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_24", - "session_31", - "session_27", - "session_7", - "session_28", - "session_30", - "session_5", - "session_21", - "session_14" - ], - "sample_id": "conv-41" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_24", - "session_7" - ], - "question": "Would John be open to moving to another country?", - "recall": 0.5, - "retrieved_ids": [ - "session_27", - "session_31", - "session_6", - "session_8", - "session_26", - "session_3", - "session_17", - "session_24", - "session_1", - "session_12" - ], - "sample_id": "conv-41" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_25" - ], - "question": "When did Maria go hiking with her church friends?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_25", - "session_24", - "session_28", - "session_31", - "session_7", - "session_30", - "session_5", - "session_21", - "session_14" - ], - "sample_id": "conv-41" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_10", - "session_25" - ], - "question": "What exercises has John done?", - "recall": 0.3333, - "retrieved_ids": [ - "session_25", - "session_15", - "session_27", - "session_26", - "session_29", - "session_28", - "session_16", - "session_22", - "session_4", - "session_13" - ], - "sample_id": "conv-41" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_26" - ], - "question": "When did John have his first firefighter call-out?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_6", - "session_8", - "session_13", - "session_2", - "session_32", - "session_25", - "session_31", - "session_9", - "session_15" - ], - "sample_id": "conv-41" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_25", - "session_26" - ], - "question": "What food item did Maria drop off at the homeless shelter?", - "recall": 0.5, - "retrieved_ids": [ - "session_2", - "session_26", - "session_29", - "session_3", - "session_16", - "session_6", - "session_19", - "session_32", - "session_24", - "session_18" - ], - "sample_id": "conv-41" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_2", - "session_26", - "session_3", - "session_4" - ], - "question": "What attributes describe John?", - "recall": 0.0, - "retrieved_ids": [], - "sample_id": "conv-41" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_27" - ], - "question": "When did Maria start volunteering at the homeless shelter?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_1", - "session_16", - "session_12", - "session_10", - "session_25", - "session_6", - "session_17", - "session_13", - "session_23" - ], - "sample_id": "conv-41" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_21", - "session_27" - ], - "question": "Who have written notes of gratitude to Maria?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_21", - "session_27", - "session_13", - "session_7", - "session_4", - "session_6", - "session_19", - "session_31", - "session_16" - ], - "sample_id": "conv-41" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_28" - ], - "question": "When did John help renovate his hometown community center?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_6", - "session_2", - "session_32", - "session_1", - "session_15", - "session_4", - "session_31", - "session_8", - "session_25" - ], - "sample_id": "conv-41" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_28" - ], - "question": "When did Maria take up community work with her church friends?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_24", - "session_28", - "session_25", - "session_14", - "session_7", - "session_31", - "session_30", - "session_5", - "session_21" - ], - "sample_id": "conv-41" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_29" - ], - "question": "When did Maria receive a medal from the homeless shelter?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_27", - "session_15", - "session_16", - "session_1", - "session_7", - "session_26", - "session_11", - "session_3", - "session_8" - ], - "sample_id": "conv-41" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_29" - ], - "question": "When did John participate in a 5K charity run?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_10", - "session_14", - "session_6", - "session_16", - "session_7", - "session_8", - "session_27", - "session_24", - "session_21" - ], - "sample_id": "conv-41" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_29", - "session_3", - "session_6" - ], - "question": "What causes has John done events for?", - "recall": 0.6667, - "retrieved_ids": [ - "session_29", - "session_6", - "session_16", - "session_15", - "session_10", - "session_27", - "session_25", - "session_26", - "session_30", - "session_32" - ], - "sample_id": "conv-41" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_30" - ], - "question": "When did Maria get Coco?", - "recall": 1.0, - "retrieved_ids": [ - "session_30", - "session_7", - "session_6", - "session_1", - "session_27", - "session_4", - "session_32", - "session_12", - "session_8", - "session_13" - ], - "sample_id": "conv-41" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_30" - ], - "question": "When did John go on a camping trip with Max?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_30", - "session_8", - "session_22", - "session_16", - "session_1", - "session_11", - "session_12", - "session_13", - "session_17" - ], - "sample_id": "conv-41" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_30", - "session_31" - ], - "question": "What are Maria's dogs' names?", - "recall": 1.0, - "retrieved_ids": [ - "session_31", - "session_24", - "session_6", - "session_30", - "session_17", - "session_2", - "session_16", - "session_4", - "session_8", - "session_13" - ], - "sample_id": "conv-41" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_31" - ], - "question": "When did Maria adopt Shadow?", - "recall": 1.0, - "retrieved_ids": [ - "session_31", - "session_13", - "session_6", - "session_27", - "session_24", - "session_15", - "session_32", - "session_7", - "session_4", - "session_17" - ], - "sample_id": "conv-41" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_30", - "session_31" - ], - "question": "How many dogs has Maria adopted from the dog shelter she volunteers at?", - "recall": 1.0, - "retrieved_ids": [ - "session_31", - "session_27", - "session_7", - "session_5", - "session_20", - "session_29", - "session_9", - "session_21", - "session_17", - "session_30" - ], - "sample_id": "conv-41" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_30", - "session_31" - ], - "question": "How many weeks passed between Maria adopting Coco and Shadow?", - "recall": 1.0, - "retrieved_ids": [ - "session_31", - "session_30", - "session_27", - "session_20", - "session_9", - "session_8", - "session_17", - "session_6", - "session_29", - "session_11" - ], - "sample_id": "conv-41" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_11", - "session_27", - "session_32", - "session_5" - ], - "question": "What job might Maria pursue in the future?", - "recall": 0.25, - "retrieved_ids": [ - "session_25", - "session_7", - "session_6", - "session_5", - "session_3", - "session_16", - "session_1", - "session_9", - "session_21", - "session_28" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_1" - ], - "question": "What is John's main focus in local politics?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_4", - "session_30", - "session_29", - "session_7", - "session_18", - "session_6", - "session_12", - "session_32", - "session_17" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_1" - ], - "question": "What sparked John's interest in improving education and infrastructure in the community?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_1", - "session_22", - "session_4", - "session_12", - "session_14", - "session_5", - "session_23", - "session_6", - "session_2" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_1" - ], - "question": "How did the extra funding help the school shown in the photo shared by John?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_10", - "session_19", - "session_5", - "session_6", - "session_29", - "session_9", - "session_31", - "session_30", - "session_17" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_1" - ], - "question": "What type of workout class did Maria start doing in December 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_1", - "session_10", - "session_12", - "session_27", - "session_6", - "session_13", - "session_9", - "session_7", - "session_19" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What did Maria donate to a homeless shelter in December 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_16", - "session_27", - "session_1", - "session_7", - "session_26", - "session_2", - "session_8", - "session_3", - "session_32" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What kind of meal did John and his family make together in the photo shared by John?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_6", - "session_11", - "session_10", - "session_8", - "session_17", - "session_15", - "session_31", - "session_30", - "session_25" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "What kind of online group did John join?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_27", - "session_24", - "session_26", - "session_12", - "session_14", - "session_4", - "session_23", - "session_32", - "session_25" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "What kind of activities did John and his mates from the online group do as part of their service efforts?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_6", - "session_8", - "session_12", - "session_2", - "session_3", - "session_25", - "session_32", - "session_27", - "session_31" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_5" - ], - "question": "Who inspired Maria to start volunteering?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_12", - "session_27", - "session_1", - "session_16", - "session_6", - "session_25", - "session_13", - "session_17", - "session_23" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_5" - ], - "question": "Why did Maria sit with the little girl at the shelter event in February 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_29", - "session_16", - "session_21", - "session_8", - "session_20", - "session_24", - "session_10", - "session_6", - "session_19" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What did Jean go through before meeting Maria?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_4", - "session_1", - "session_9", - "session_18", - "session_14", - "session_19", - "session_23", - "session_3", - "session_30" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "Why did John decide to run for office again?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_14", - "session_15", - "session_29", - "session_6", - "session_25", - "session_28", - "session_5", - "session_10", - "session_8" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What activity did John's colleague, Rob, invite him to?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_24", - "session_6", - "session_31", - "session_10", - "session_30", - "session_21", - "session_17", - "session_15", - "session_12" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What is the name of John's one-year-old child?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_31", - "session_13", - "session_1", - "session_24", - "session_6", - "session_14", - "session_30", - "session_2", - "session_23" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "How often does John take his kids to the park?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_22", - "session_18", - "session_13", - "session_31", - "session_25", - "session_15", - "session_6", - "session_2", - "session_11" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What did Maria make for her home to remind her of a trip to England?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_19", - "session_30", - "session_21", - "session_7", - "session_31", - "session_5", - "session_17", - "session_18", - "session_23" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "Where did Maria get the idea for the castle shadow box in her home?", - "recall": 1.0, - "retrieved_ids": [ - "session_31", - "session_30", - "session_1", - "session_8", - "session_4", - "session_21", - "session_17", - "session_7", - "session_5", - "session_12" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_9" - ], - "question": "What did John receive a certificate for?", - "recall": 0.0, - "retrieved_ids": [ - "session_27", - "session_12", - "session_24", - "session_15", - "session_11", - "session_7", - "session_6", - "session_4", - "session_13", - "session_32" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_9" - ], - "question": "What areas is John particularly interested in for policymaking?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_1", - "session_30", - "session_16", - "session_5", - "session_2", - "session_14", - "session_23", - "session_25", - "session_26" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "What did Maria participate in last weekend before April 10, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_24", - "session_29", - "session_25", - "session_10", - "session_4", - "session_14", - "session_8", - "session_21", - "session_17" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "What event did John volunteer at last weekend?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_29", - "session_8", - "session_24", - "session_6", - "session_25", - "session_3", - "session_18", - "session_20", - "session_16" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_11" - ], - "question": "What did John do that put a strain on his wallet?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_31", - "session_9", - "session_6", - "session_8", - "session_19", - "session_24", - "session_25", - "session_15", - "session_2" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_11" - ], - "question": "Where did John explore on a road trip last year?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_1", - "session_8", - "session_18", - "session_22", - "session_30", - "session_13", - "session_27", - "session_25", - "session_14" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_12" - ], - "question": "What topic has John been blogging about recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_1", - "session_11", - "session_2", - "session_8", - "session_3", - "session_7", - "session_30", - "session_6", - "session_21" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_12" - ], - "question": "Why did John start blogging about politics and policies?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_1", - "session_6", - "session_17", - "session_27", - "session_23", - "session_16", - "session_4", - "session_13", - "session_7" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_12" - ], - "question": "What was the focus of John's recent research and writing on his blog?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_30", - "session_8", - "session_6", - "session_7", - "session_18", - "session_25", - "session_31", - "session_15", - "session_2" - ], - "sample_id": "conv-41" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_12" - ], - "question": "What did John attend with his colleagues in March 2023?", - "recall": 0.0, - "retrieved_ids": [], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "How often does John work out with his family?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_8", - "session_19", - "session_25", - "session_31", - "session_6", - "session_13", - "session_24", - "session_15", - "session_2" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "How has John's fitness improved since starting boot camps with his family?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_14", - "session_19", - "session_31", - "session_2", - "session_1", - "session_18", - "session_6", - "session_25", - "session_30" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "What kind of food did Maria have on her dinner spread iwth her mother?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_13", - "session_30", - "session_5", - "session_7", - "session_31", - "session_23", - "session_3", - "session_24", - "session_32" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "What activity did Maria and her mom do together in May 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_13", - "session_31", - "session_7", - "session_8", - "session_28", - "session_24", - "session_30", - "session_21", - "session_5" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_14" - ], - "question": "What did Maria do to feel closer to a community and her faith?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_14", - "session_5", - "session_6", - "session_2", - "session_32", - "session_31", - "session_30", - "session_7", - "session_21" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_14" - ], - "question": "Why did Maria join a nearby church recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_24", - "session_3", - "session_21", - "session_12", - "session_6", - "session_25", - "session_28", - "session_22", - "session_18" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "What did John host for the veterans in May 2023 as part of the project?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_12", - "session_27", - "session_24", - "session_29", - "session_21", - "session_14", - "session_32", - "session_3", - "session_28" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "What did John and the veterans do during the small party?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_21", - "session_24", - "session_4", - "session_13", - "session_19", - "session_27", - "session_29", - "session_5", - "session_11" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "What emotions did John feel during the small party with the veterans?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_21", - "session_4", - "session_5", - "session_27", - "session_24", - "session_11", - "session_13", - "session_3", - "session_29" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "What event is Maria getting ready for at the shelter on May 25, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_29", - "session_11", - "session_5", - "session_10", - "session_20", - "session_3", - "session_24", - "session_6", - "session_28" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "What does Maria need to spread the word about for the fundraiser for the volunteer shelter?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_3", - "session_29", - "session_15", - "session_7", - "session_30", - "session_23", - "session_32", - "session_20", - "session_24" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What was the name of the pet that John had to say goodbye to on 3 June, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_25", - "session_27", - "session_31", - "session_30", - "session_10", - "session_18", - "session_8", - "session_24", - "session_6" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "How long was Max a part of John's family?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_27", - "session_30", - "session_18", - "session_1", - "session_6", - "session_32", - "session_9", - "session_7", - "session_12" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "How does John plan to honor the memories of his beloved pet?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_30", - "session_6", - "session_18", - "session_8", - "session_31", - "session_2", - "session_25", - "session_29", - "session_15" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What important values does John want to teach his kids through adopting a rescue dog?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_21", - "session_27", - "session_7", - "session_8", - "session_13", - "session_25", - "session_15", - "session_31", - "session_26" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What new activity did Maria start recently, as mentioned on 3 June, 2023?", - "recall": 0.0, - "retrieved_ids": [ - "session_13", - "session_24", - "session_21", - "session_31", - "session_12", - "session_19", - "session_3", - "session_26", - "session_9", - "session_14" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What did Maria say it was like being at the waterfall in Oregon?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_10", - "session_4", - "session_27", - "session_12", - "session_13", - "session_32", - "session_17", - "session_24", - "session_8" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What does Maria say she feels when doing upside-down yoga poses?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_10", - "session_27", - "session_31", - "session_21", - "session_7", - "session_5", - "session_30", - "session_8", - "session_17" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "What exciting news did Maria share on 16 June, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_15", - "session_12", - "session_13", - "session_19", - "session_1", - "session_10", - "session_31", - "session_21", - "session_32" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "What yoga activity has Maria been trying to improve her strength and endurance?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_13", - "session_1", - "session_10", - "session_7", - "session_31", - "session_25", - "session_14", - "session_2", - "session_21" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "What did John recently get promoted to?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_7", - "session_12", - "session_11", - "session_21", - "session_2", - "session_3", - "session_4", - "session_1", - "session_6" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "What was one of the biggest challenges John faced in his journey to becoming assistant manager?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_6", - "session_13", - "session_9", - "session_8", - "session_11", - "session_26", - "session_31", - "session_21", - "session_3" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "How does John describe the support he received during his journey to becoming assistant manager?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_13", - "session_26", - "session_6", - "session_4", - "session_21", - "session_8", - "session_11", - "session_27", - "session_17" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_20" - ], - "question": "What kind of event did John and his family attend in June 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_8", - "session_16", - "session_29", - "session_10", - "session_2", - "session_20", - "session_15", - "session_31", - "session_25" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_21" - ], - "question": "Why did Maria need to help her cousin find a new place to live?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_16", - "session_30", - "session_1", - "session_25", - "session_2", - "session_7", - "session_27", - "session_31", - "session_5" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_21" - ], - "question": "What event did John participate in to show support for veterans' rights?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_15", - "session_3", - "session_27", - "session_29", - "session_24", - "session_10", - "session_5", - "session_6", - "session_20" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_21" - ], - "question": "What inspired John to join the marching event for veterans' rights?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_24", - "session_5", - "session_30", - "session_3", - "session_27", - "session_15", - "session_29", - "session_10", - "session_19" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_22" - ], - "question": "How often does John get to see sunsets like the one he shared with Maria?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_19", - "session_13", - "session_4", - "session_6", - "session_8", - "session_30", - "session_25", - "session_3", - "session_17" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "What natural disaster affected John's old area on 7 July, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_1", - "session_14", - "session_5", - "session_2", - "session_8", - "session_30", - "session_18", - "session_25", - "session_3" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "How did the flood impact the homes in John's old area?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_2", - "session_9", - "session_14", - "session_5", - "session_8", - "session_19", - "session_1", - "session_31", - "session_17" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "What motivated Maria and John to discuss potential solutions for their community on 7 July, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_15", - "session_12", - "session_2", - "session_28", - "session_32", - "session_9", - "session_11", - "session_4", - "session_8" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "What did Maria plan to do later on the evening of 7 July, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_12", - "session_29", - "session_9", - "session_25", - "session_30", - "session_4", - "session_14", - "session_23", - "session_17" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "What kind of activities did Maria do at the picnic with her church friends?", - "recall": 1.0, - "retrieved_ids": [ - "session_31", - "session_24", - "session_8", - "session_25", - "session_7", - "session_5", - "session_30", - "session_14", - "session_28", - "session_21" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "What does John appreciate about the veteran's hospital visit?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_27", - "session_4", - "session_16", - "session_15", - "session_29", - "session_21", - "session_22", - "session_28", - "session_3" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "What did John take away from visiting the veteran's hospital?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_27", - "session_15", - "session_4", - "session_18", - "session_30", - "session_8", - "session_29", - "session_21", - "session_13" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "Why did John feel inspired to join the military after the visit to the hospital?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_27", - "session_26", - "session_3", - "session_21", - "session_13", - "session_10", - "session_1", - "session_15", - "session_7" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_25" - ], - "question": "In what activity did Maria and her church friends participate in July 2023?", - "recall": 0.0, - "retrieved_ids": [ - "session_24", - "session_21", - "session_7", - "session_31", - "session_28", - "session_29", - "session_5", - "session_10", - "session_30", - "session_14" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_25" - ], - "question": "What does John think about trying new classes at the yoga studio?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_13", - "session_1", - "session_15", - "session_19", - "session_10", - "session_18", - "session_21", - "session_7", - "session_9" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_25" - ], - "question": "Which activity has John done apart from yoga at the studio?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_1", - "session_19", - "session_27", - "session_24", - "session_15", - "session_26", - "session_29", - "session_10", - "session_18" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_26" - ], - "question": "What community service did Maria mention that she was involved in on 31 July, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_26", - "session_5", - "session_9", - "session_7", - "session_6", - "session_31", - "session_30", - "session_21", - "session_3" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_27" - ], - "question": "How did Maria start volunteering at the homeless shelter?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_1", - "session_12", - "session_16", - "session_10", - "session_17", - "session_25", - "session_6", - "session_13", - "session_23" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_27" - ], - "question": "What did John do the week before August 3, 2023 involving his kids?", - "recall": 0.0, - "retrieved_ids": [ - "session_14", - "session_18", - "session_8", - "session_31", - "session_5", - "session_1", - "session_15", - "session_9", - "session_6", - "session_25" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_27" - ], - "question": "How did John describe his kids' reaction at the military memorial?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_8", - "session_21", - "session_17", - "session_24", - "session_3", - "session_15", - "session_20", - "session_31", - "session_2" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_27" - ], - "question": "Why does Maria think it's important for younger generations to visit military memorials?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_24", - "session_15", - "session_17", - "session_21", - "session_1", - "session_18", - "session_26", - "session_6", - "session_2" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_27" - ], - "question": "What does John believe is important for children regarding veterans?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_21", - "session_27", - "session_15", - "session_7", - "session_4", - "session_24", - "session_29", - "session_16", - "session_19" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_28" - ], - "question": "What happened to John's job in August 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_6", - "session_7", - "session_25", - "session_1", - "session_18", - "session_21", - "session_14", - "session_23", - "session_3" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_28" - ], - "question": "What activity did Maria take up with her friends from church in August 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_31", - "session_24", - "session_25", - "session_30", - "session_14", - "session_28", - "session_21", - "session_5", - "session_7" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_28" - ], - "question": "What did John do to help his community last year in his hometown?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_1", - "session_6", - "session_27", - "session_15", - "session_8", - "session_13", - "session_31", - "session_2", - "session_25" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_29" - ], - "question": "What cause did the 5K charity run organized by John support?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_10", - "session_16", - "session_14", - "session_1", - "session_6", - "session_8", - "session_32", - "session_7", - "session_30" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_29" - ], - "question": "Who did John work with to raise awareness and funds for victims of domestic abuse?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_12", - "session_15", - "session_5", - "session_32", - "session_16", - "session_1", - "session_24", - "session_7", - "session_4" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_29" - ], - "question": "What recognition did Maria receive at the homeless shelter in August 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_15", - "session_27", - "session_16", - "session_1", - "session_7", - "session_26", - "session_11", - "session_8", - "session_3" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_30" - ], - "question": "What is the name of Maria's puppy she got two weeks before August 11, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_30", - "session_31", - "session_14", - "session_8", - "session_24", - "session_18", - "session_5", - "session_32", - "session_7", - "session_21" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_30" - ], - "question": "What activity did John and Max enjoy together last summer?", - "recall": 1.0, - "retrieved_ids": [ - "session_30", - "session_31", - "session_8", - "session_25", - "session_18", - "session_19", - "session_13", - "session_24", - "session_21", - "session_17" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_30" - ], - "question": "How does John describe the camping trip with Max?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_30", - "session_8", - "session_17", - "session_1", - "session_22", - "session_12", - "session_13", - "session_16", - "session_11" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_30" - ], - "question": "Why does John say he feels stuck and questioning his decisions and goals?", - "recall": 1.0, - "retrieved_ids": [ - "session_30", - "session_6", - "session_31", - "session_27", - "session_10", - "session_8", - "session_19", - "session_25", - "session_17", - "session_15" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_31" - ], - "question": "What is the name of Maria's second puppy?", - "recall": 1.0, - "retrieved_ids": [ - "session_30", - "session_31", - "session_8", - "session_18", - "session_16", - "session_24", - "session_6", - "session_4", - "session_2", - "session_13" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_31" - ], - "question": "How is Maria's new puppy adjusting to its new home?", - "recall": 1.0, - "retrieved_ids": [ - "session_30", - "session_31", - "session_4", - "session_21", - "session_19", - "session_13", - "session_28", - "session_8", - "session_17", - "session_23" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_31" - ], - "question": "What is John currently doing as a volunteer in August 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_27", - "session_16", - "session_11", - "session_8", - "session_28", - "session_1", - "session_7", - "session_5", - "session_31" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_31" - ], - "question": "What activities does John's family enjoy doing together?", - "recall": 1.0, - "retrieved_ids": [ - "session_31", - "session_8", - "session_4", - "session_24", - "session_18", - "session_19", - "session_13", - "session_3", - "session_25", - "session_21" - ], - "sample_id": "conv-41" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_32" - ], - "question": "What did the donations help John's community acquire on 16 August, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_32", - "session_3", - "session_15", - "session_1", - "session_9", - "session_4", - "session_6", - "session_29", - "session_2", - "session_12" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_1" - ], - "question": "What is John's main focus in international politics?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_18", - "session_7", - "session_4", - "session_30", - "session_6", - "session_12", - "session_15", - "session_16", - "session_28" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_2" - ], - "question": "What did Maria donate to a luxury store in December 2023?", - "recall": 0.0, - "retrieved_ids": [], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_5" - ], - "question": "Who inspired John to start volunteering?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_12", - "session_27", - "session_16", - "session_1", - "session_6", - "session_25", - "session_13", - "session_17", - "session_23" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_7" - ], - "question": "Why did Maria decide to run for office again?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_14", - "session_15", - "session_29", - "session_6", - "session_25", - "session_28", - "session_5", - "session_10", - "session_8" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_7" - ], - "question": "What activity did Maria's colleague, Rob, invite her to?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_31", - "session_8", - "session_10", - "session_15", - "session_24", - "session_21", - "session_5", - "session_30", - "session_12" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_8" - ], - "question": "What is the name of Maria's one-year-old child?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_31", - "session_13", - "session_1", - "session_24", - "session_14", - "session_6", - "session_30", - "session_2", - "session_23" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_8" - ], - "question": "How often does John take his kids to the library?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_13", - "session_17", - "session_22", - "session_25", - "session_15", - "session_31", - "session_5", - "session_6", - "session_2" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_8" - ], - "question": "What did Maria make for her home to remind her of a trip to France?", - "recall": 1.0, - "retrieved_ids": [ - "session_30", - "session_19", - "session_21", - "session_31", - "session_7", - "session_5", - "session_8", - "session_17", - "session_18", - "session_23" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_8" - ], - "question": "Where did John get the idea for the castle shadow box in his home?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_12", - "session_31", - "session_30", - "session_4", - "session_6", - "session_17", - "session_15", - "session_25", - "session_2" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_9" - ], - "question": "What did Maria receive a certificate for?", - "recall": 0.0, - "retrieved_ids": [ - "session_27", - "session_12", - "session_24", - "session_11", - "session_15", - "session_6", - "session_7", - "session_4", - "session_13", - "session_32" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_9" - ], - "question": "What areas is John particularly interested in for art appreciation?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_1", - "session_26", - "session_25", - "session_24", - "session_16", - "session_5", - "session_14", - "session_2", - "session_23" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_12" - ], - "question": "Why did Maria start blogging about politics and policies?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_1", - "session_6", - "session_17", - "session_27", - "session_23", - "session_16", - "session_4", - "session_13", - "session_7" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_12" - ], - "question": "What was the focus of John's recent travel and photography blog?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_30", - "session_2", - "session_8", - "session_18", - "session_6", - "session_7", - "session_11", - "session_21", - "session_3" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_13" - ], - "question": "How often does Maria work out with her family?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_31", - "session_21", - "session_24", - "session_5", - "session_8", - "session_13", - "session_25", - "session_7", - "session_30" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_13" - ], - "question": "How has John's artistic skills improved since starting boot camps with his family?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_14", - "session_31", - "session_26", - "session_28", - "session_2", - "session_18", - "session_6", - "session_25", - "session_30" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_13" - ], - "question": "What kind of food did Maria have on her dinner spread with her father?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_13", - "session_31", - "session_5", - "session_7", - "session_30", - "session_23", - "session_3", - "session_24", - "session_32" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_14" - ], - "question": "What did John do to feel closer to a community and his faith?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_14", - "session_4", - "session_6", - "session_2", - "session_8", - "session_32", - "session_15", - "session_25", - "session_31" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_14" - ], - "question": "Why did John join a nearby church recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_24", - "session_3", - "session_21", - "session_12", - "session_25", - "session_28", - "session_6", - "session_22", - "session_18" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_17" - ], - "question": "How long was Max a part of Maria's family?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_27", - "session_30", - "session_18", - "session_1", - "session_6", - "session_32", - "session_9", - "session_7", - "session_12" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_17" - ], - "question": "How does Maria plan to honor the memories of her beloved pet?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_30", - "session_2", - "session_31", - "session_18", - "session_29", - "session_7", - "session_5", - "session_21", - "session_8" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_17" - ], - "question": "What important values does Maria want to teach her kids through adopting a rescue dog?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_21", - "session_15", - "session_7", - "session_27", - "session_31", - "session_13", - "session_5", - "session_26", - "session_30" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "What did Maria say it was like being at the desert in Oregon?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_10", - "session_4", - "session_27", - "session_12", - "session_13", - "session_32", - "session_17", - "session_24", - "session_8" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "What does John say she feels when doing upside-down yoga poses?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_10", - "session_27", - "session_31", - "session_21", - "session_7", - "session_5", - "session_30", - "session_8", - "session_17" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_19" - ], - "question": "What did Maria recently get promoted to?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_7", - "session_12", - "session_11", - "session_21", - "session_2", - "session_4", - "session_3", - "session_1", - "session_6" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_19" - ], - "question": "What was one of the biggest challenges Maria faced in her journey to becoming assistant manager?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_6", - "session_13", - "session_11", - "session_9", - "session_31", - "session_26", - "session_21", - "session_8", - "session_3" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_21" - ], - "question": "Why did John need to help his cousin find a new place to live?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_8", - "session_25", - "session_6", - "session_15", - "session_27", - "session_1", - "session_2", - "session_7", - "session_31" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_21" - ], - "question": "What event did Maria participate in to show support for veterans' rights?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_15", - "session_3", - "session_27", - "session_29", - "session_24", - "session_10", - "session_5", - "session_6", - "session_20" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_23" - ], - "question": "How did the drought impact the homes in John's old area?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_2", - "session_9", - "session_14", - "session_5", - "session_8", - "session_19", - "session_1", - "session_31", - "session_17" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_24" - ], - "question": "What does John criticize about the veteran's hospital visit?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_27", - "session_15", - "session_29", - "session_17", - "session_5", - "session_21", - "session_12", - "session_30", - "session_8" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_24" - ], - "question": "What did John take away from visiting the orphanage?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_30", - "session_8", - "session_24", - "session_27", - "session_9", - "session_15", - "session_4", - "session_13", - "session_12" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_24" - ], - "question": "Why did Maria feel inspired to join the military after the visit to the hospital?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_27", - "session_3", - "session_26", - "session_21", - "session_13", - "session_10", - "session_1", - "session_15", - "session_7" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_27" - ], - "question": "How did Maria describe her kids' reaction at the military memorial?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_27", - "session_20", - "session_5", - "session_31", - "session_24", - "session_3", - "session_7", - "session_30", - "session_8" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_27" - ], - "question": "Why does Maria think it's important for younger generations to visit art galleries?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_5", - "session_24", - "session_1", - "session_25", - "session_18", - "session_6", - "session_22", - "session_2", - "session_17" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_28" - ], - "question": "What happened to Maria's job in August 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_6", - "session_1", - "session_7", - "session_25", - "session_18", - "session_21", - "session_14", - "session_23", - "session_3" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_29" - ], - "question": "What cause did the 5K charity run organized by Maria support?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_10", - "session_16", - "session_14", - "session_1", - "session_6", - "session_8", - "session_32", - "session_7", - "session_30" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_29" - ], - "question": "Who did John work with to raise awareness and funds for animal welfare?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_12", - "session_4", - "session_31", - "session_32", - "session_5", - "session_16", - "session_1", - "session_24", - "session_7" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_29" - ], - "question": "What recognition did John receive at the homeless shelter in August 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_15", - "session_27", - "session_16", - "session_1", - "session_7", - "session_26", - "session_11", - "session_8", - "session_3" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_30" - ], - "question": "What is the name of John's puppy he got two weeks before August 11, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_30", - "session_4", - "session_14", - "session_8", - "session_18", - "session_31", - "session_32", - "session_24", - "session_6", - "session_17" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_30" - ], - "question": "How does Maria describe the camping trip with Max?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_30", - "session_8", - "session_17", - "session_1", - "session_22", - "session_12", - "session_13", - "session_16", - "session_11" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_31" - ], - "question": "What is the name of Maria's second kitten?", - "recall": 1.0, - "retrieved_ids": [ - "session_31", - "session_8", - "session_18", - "session_24", - "session_16", - "session_6", - "session_4", - "session_30", - "session_2", - "session_13" - ], - "sample_id": "conv-41" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_31" - ], - "question": "How is John's new puppy adjusting to its new home?", - "recall": 1.0, - "retrieved_ids": [ - "session_30", - "session_31", - "session_4", - "session_21", - "session_19", - "session_13", - "session_28", - "session_8", - "session_17", - "session_23" - ], - "sample_id": "conv-41" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_1" - ], - "question": "Is it likely that Nate has friends besides Joanna?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_16", - "session_15", - "session_1", - "session_6", - "session_10", - "session_4", - "session_12", - "session_14", - "session_27" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_10", - "session_20", - "session_3", - "session_4" - ], - "question": "What kind of interests do Joanna and Nate share?", - "recall": 0.4, - "retrieved_ids": [ - "session_14", - "session_6", - "session_9", - "session_26", - "session_23", - "session_17", - "session_1", - "session_4", - "session_19", - "session_25" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_1" - ], - "question": "When did Joanna first watch \"Eternal Sunshine of the Spotless Mind?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_4", - "session_1", - "session_23", - "session_25", - "session_28", - "session_2", - "session_9", - "session_17", - "session_14" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_1" - ], - "question": "When did Nate win his first video game tournament?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_1", - "session_17", - "session_14", - "session_26", - "session_28", - "session_2", - "session_10", - "session_22", - "session_15" - ], - "sample_id": "conv-42" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_2" - ], - "question": "What pets wouldn't cause any discomfort to Joanna?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_5", - "session_2", - "session_1", - "session_12", - "session_8", - "session_19", - "session_13", - "session_24", - "session_27" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_2" - ], - "question": "What are Joanna's hobbies?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_23", - "session_8", - "session_2", - "session_15", - "session_4", - "session_14", - "session_21", - "session_25", - "session_28" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_2" - ], - "question": "How long has Nate had his first two turtles?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_1", - "session_25", - "session_28", - "session_9", - "session_14", - "session_5", - "session_15", - "session_4", - "session_22" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_2" - ], - "question": "When did Joanna finish her first screenplay?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_9", - "session_12", - "session_4", - "session_14", - "session_13", - "session_3", - "session_24", - "session_22", - "session_27" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_2" - ], - "question": "When did Nate get his first two turtles?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_5", - "session_2", - "session_25", - "session_28", - "session_4", - "session_15", - "session_9", - "session_1", - "session_22" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_2" - ], - "question": "What major achievement did Joanna accomplish in January 2022?", - "recall": 0.0, - "retrieved_ids": [ - "session_14", - "session_21", - "session_22", - "session_19", - "session_10", - "session_7", - "session_11", - "session_4", - "session_16", - "session_17" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_2", - "session_3" - ], - "question": "What emotions is Joanna feeling about the screenplay she submitted?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_27", - "session_3", - "session_4", - "session_29", - "session_24", - "session_22", - "session_23", - "session_16", - "session_6" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_2", - "session_4", - "session_5" - ], - "question": "What is Joanna allergic to?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_24", - "session_5", - "session_28", - "session_21", - "session_1", - "session_15", - "session_4", - "session_14", - "session_25" - ], - "sample_id": "conv-42" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_2", - "session_5" - ], - "question": "What underlying condition might Joanna have based on her allergies?", - "recall": 0.5, - "retrieved_ids": [ - "session_2", - "session_9", - "session_21", - "session_12", - "session_24", - "session_1", - "session_3", - "session_16", - "session_17", - "session_8" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_6" - ], - "question": "When did Joanna have an audition for a writing gig?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_22", - "session_7", - "session_11", - "session_4", - "session_17", - "session_9", - "session_26", - "session_19", - "session_25" - ], - "sample_id": "conv-42" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_7" - ], - "question": "What nickname does Nate use for Joanna?", - "recall": 0.0, - "retrieved_ids": [ - "session_22", - "session_1", - "session_14", - "session_4", - "session_15", - "session_21", - "session_13", - "session_2", - "session_25", - "session_28" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_7" - ], - "question": "When did Nate get purple hair?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_23", - "session_11", - "session_6", - "session_17", - "session_4", - "session_25", - "session_13", - "session_14", - "session_28" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_7" - ], - "question": "What physical transformation did Nate undergo in April 2022?", - "recall": 0.0, - "retrieved_ids": [], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_10" - ], - "question": "What movie did Joanna watch on 1 May, 2022?", - "recall": 0.0, - "retrieved_ids": [ - "session_1", - "session_23", - "session_25", - "session_3", - "session_27", - "session_28", - "session_9", - "session_29", - "session_22", - "session_17" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_11" - ], - "question": "Which outdoor spot did Joanna visit in May?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_11", - "session_8", - "session_25", - "session_7", - "session_16", - "session_4", - "session_10", - "session_17", - "session_21" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_11", - "session_8" - ], - "question": "How many times has Joanna found new hiking trails?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_8", - "session_12", - "session_23", - "session_14", - "session_7", - "session_6", - "session_26", - "session_5", - "session_28" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_12" - ], - "question": "When did Nate adopt Max?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_13", - "session_25", - "session_7", - "session_16", - "session_11", - "session_17", - "session_4", - "session_10", - "session_21" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_12" - ], - "question": "Who was the new addition to Nate's family in May 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_16", - "session_18", - "session_22", - "session_23", - "session_3", - "session_7", - "session_15", - "session_24", - "session_28" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_12" - ], - "question": "When did Joanna start writing her third screenplay?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_28", - "session_11", - "session_4", - "session_12", - "session_27", - "session_15", - "session_9", - "session_25", - "session_24" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_14", - "session_2", - "session_24", - "session_3" - ], - "question": "Which of Joanna's screenplay were rejected from production companies?", - "recall": 0.75, - "retrieved_ids": [ - "session_14", - "session_24", - "session_16", - "session_10", - "session_13", - "session_2", - "session_27", - "session_11", - "session_4", - "session_23" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_14" - ], - "question": "When is Nate hosting a gaming party?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_16", - "session_6", - "session_10", - "session_9", - "session_23", - "session_28", - "session_27", - "session_22", - "session_1" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_14" - ], - "question": "When did Joanna hike with her buddies?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_13", - "session_28", - "session_24", - "session_11", - "session_9", - "session_19", - "session_17", - "session_7", - "session_8" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_14" - ], - "question": "When did Nate win his third tourney?", - "recall": 0.0, - "retrieved_ids": [ - "session_28", - "session_26", - "session_20", - "session_15", - "session_10", - "session_12", - "session_25", - "session_2", - "session_22", - "session_9" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_16", - "session_2" - ], - "question": "What places has Joanna submitted her work to?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_27", - "session_24", - "session_17", - "session_5", - "session_11", - "session_2", - "session_23", - "session_6", - "session_16" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_16" - ], - "question": "When did Nate make vegan icecream and share it with a vegan diet group?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_21", - "session_20", - "session_18", - "session_13", - "session_9", - "session_3", - "session_19", - "session_8", - "session_25" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_16" - ], - "question": "When is Joanna going to make Nate's ice cream for her family?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_21", - "session_8", - "session_28", - "session_18", - "session_29", - "session_4", - "session_9", - "session_24", - "session_15" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_17", - "session_18", - "session_2" - ], - "question": "What kind of writings does Joanna do?", - "recall": 0.6667, - "retrieved_ids": [ - "session_6", - "session_4", - "session_9", - "session_18", - "session_17", - "session_19", - "session_5", - "session_15", - "session_20", - "session_14" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_17" - ], - "question": "When did Nate win his fourth video game tournament?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_26", - "session_1", - "session_22", - "session_10", - "session_14", - "session_9", - "session_2", - "session_19", - "session_15" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_17" - ], - "question": "Where did Joanna travel to in July 2022?", - "recall": 0.0, - "retrieved_ids": [], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_18" - ], - "question": "When did someone write Joanna a touching letter?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_24", - "session_6", - "session_27", - "session_11", - "session_14", - "session_13", - "session_3", - "session_8", - "session_5" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_19", - "session_3" - ], - "question": "What book recommendations has Joanna given to Nate?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_9", - "session_22", - "session_4", - "session_1", - "session_3", - "session_6", - "session_17", - "session_8", - "session_15" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_19" - ], - "question": "When did Nate take time off to chill with his pets?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_5", - "session_23", - "session_2", - "session_15", - "session_29", - "session_8", - "session_9", - "session_12", - "session_22" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_19" - ], - "question": "When did Joanna share her book with her writers group?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_19", - "session_21", - "session_13", - "session_6", - "session_22", - "session_16", - "session_24", - "session_17", - "session_8" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_19" - ], - "question": "When did Nate win an international tournament?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_10", - "session_19", - "session_2", - "session_6", - "session_1", - "session_14", - "session_17", - "session_22", - "session_27" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_21" - ], - "question": "When did Joanna make a desert with almond milk?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_20", - "session_26", - "session_29", - "session_4", - "session_18", - "session_3", - "session_8", - "session_10", - "session_16" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_21" - ], - "question": "When did Nate attend a cooking show?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_16", - "session_15", - "session_10", - "session_4", - "session_29", - "session_20", - "session_3", - "session_28", - "session_7" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_21" - ], - "question": "When did Joanna's laptop crash?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_3", - "session_7", - "session_16", - "session_11", - "session_17", - "session_6", - "session_4", - "session_10", - "session_25" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_22" - ], - "question": "When did Joanna make a chocolate tart with raspberries?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_20", - "session_4", - "session_22", - "session_3", - "session_26", - "session_13", - "session_16", - "session_6", - "session_10" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_22", - "session_3" - ], - "question": "What movies have both Joanna and Nate seen?", - "recall": 0.6667, - "retrieved_ids": [ - "session_1", - "session_23", - "session_3", - "session_22", - "session_19", - "session_24", - "session_26", - "session_2", - "session_27", - "session_12" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_17", - "session_22" - ], - "question": "How long did it take for Joanna to finish writing her book?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_19", - "session_6", - "session_12", - "session_22", - "session_2", - "session_27", - "session_17", - "session_8", - "session_24" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_22" - ], - "question": "When did Nate win a lot of money in a video game tournament?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_22", - "session_26", - "session_28", - "session_1", - "session_10", - "session_14", - "session_2", - "session_27", - "session_19" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_22" - ], - "question": "When did Joanna finish up the writing for her book?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_12", - "session_19", - "session_22", - "session_2", - "session_6", - "session_17", - "session_27", - "session_24", - "session_8" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_16", - "session_23" - ], - "question": "What board games has Nate played?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_23", - "session_27", - "session_15", - "session_17", - "session_1", - "session_10", - "session_13", - "session_28", - "session_16" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_14", - "session_23" - ], - "question": "What places has Nate met new people?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_14", - "session_17", - "session_21", - "session_11", - "session_25", - "session_2", - "session_16", - "session_19", - "session_6" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_23" - ], - "question": "When did Nate go to a convention and meet new people?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_13", - "session_28", - "session_16", - "session_17", - "session_21", - "session_14", - "session_19", - "session_2", - "session_26" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_14", - "session_24" - ], - "question": "How many times has Joanna's scripts been rejected?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_26", - "session_6", - "session_5", - "session_28", - "session_29", - "session_9", - "session_24", - "session_14", - "session_17" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_13", - "session_24" - ], - "question": "What is something Nate gave to Joanna that brings her a lot of joy?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_13", - "session_2", - "session_12", - "session_29", - "session_10", - "session_28", - "session_5", - "session_3", - "session_9" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_13", - "session_24" - ], - "question": "When did Nate get Tilly for Joanna?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_23", - "session_11", - "session_6", - "session_4", - "session_17", - "session_25", - "session_13", - "session_14", - "session_28" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_15", - "session_25" - ], - "question": "How many of Joanna's writing have made it to the big screen?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_25", - "session_22", - "session_28", - "session_6", - "session_2", - "session_15", - "session_3", - "session_4", - "session_5" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_25", - "session_5" - ], - "question": "How many times has Nate taken his turtles on a walk?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_24", - "session_25", - "session_2", - "session_8", - "session_5", - "session_6", - "session_22", - "session_15", - "session_9" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_25" - ], - "question": "When was Joanna's second movie script shown on the big screens?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_5", - "session_27", - "session_3", - "session_28", - "session_29", - "session_15", - "session_17", - "session_4", - "session_10" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_11", - "session_25", - "session_26", - "session_4", - "session_7" - ], - "question": "What is Joanna inspired by?", - "recall": 0.8, - "retrieved_ids": [ - "session_7", - "session_4", - "session_11", - "session_26", - "session_22", - "session_9", - "session_17", - "session_15", - "session_6", - "session_8" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_26", - "session_5" - ], - "question": "What animal do both Nate and Joanna like?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_22", - "session_1", - "session_24", - "session_13", - "session_26", - "session_27", - "session_12", - "session_19", - "session_5" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_26" - ], - "question": "When did Joanna plan to go over to Nate's and share recipes?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_26", - "session_11", - "session_2", - "session_14", - "session_22", - "session_23", - "session_19", - "session_27", - "session_25" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_19", - "session_2", - "session_27", - "session_9" - ], - "question": "What things has Nate reccomended to Joanna?", - "recall": 0.8, - "retrieved_ids": [ - "session_2", - "session_27", - "session_19", - "session_10", - "session_8", - "session_15", - "session_3", - "session_13", - "session_23", - "session_16" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_15", - "session_27" - ], - "question": "What does Joanna do to remember happy memories?", - "recall": 0.5, - "retrieved_ids": [ - "session_27", - "session_24", - "session_13", - "session_9", - "session_29", - "session_1", - "session_22", - "session_26", - "session_23", - "session_8" - ], - "sample_id": "conv-42" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_27" - ], - "question": "What Console does Nate own?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_25", - "session_15", - "session_29", - "session_11", - "session_21", - "session_2", - "session_27", - "session_12", - "session_8" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_22", - "session_27" - ], - "question": "What mediums does Nate use to play games?", - "recall": 0.5, - "retrieved_ids": [ - "session_27", - "session_23", - "session_9", - "session_16", - "session_8", - "session_1", - "session_10", - "session_13", - "session_28", - "session_17" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_14", - "session_18" - ], - "question": "How many letters has Joanna recieved?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_18", - "session_27", - "session_14", - "session_21", - "session_1", - "session_25", - "session_28", - "session_15", - "session_4" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_10", - "session_23", - "session_27" - ], - "question": "What video games does Nate play?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_23", - "session_28", - "session_1", - "session_27", - "session_17", - "session_24", - "session_10", - "session_16", - "session_6" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_27" - ], - "question": "When did Nate win a big Valorant tourney?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_20", - "session_2", - "session_28", - "session_3", - "session_10", - "session_22", - "session_26", - "session_1", - "session_14" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_27" - ], - "question": "Which torunament did Nate win in the beginning of November 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_10", - "session_2", - "session_27", - "session_7", - "session_1", - "session_17", - "session_14", - "session_19", - "session_22" - ], - "sample_id": "conv-42" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_19", - "session_25", - "session_28", - "session_5" - ], - "question": "What alternative career might Nate consider after gaming?", - "recall": 0.5, - "retrieved_ids": [ - "session_9", - "session_19", - "session_16", - "session_21", - "session_5", - "session_27", - "session_8", - "session_12", - "session_22", - "session_13" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_12", - "session_28", - "session_8" - ], - "question": "What pets does Nate have?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_24", - "session_13", - "session_5", - "session_2", - "session_28", - "session_27", - "session_8", - "session_19", - "session_1" - ], - "sample_id": "conv-42" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_11", - "session_14", - "session_28", - "session_7" - ], - "question": "How many hikes has Joanna been on?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_11", - "session_27", - "session_10", - "session_24", - "session_28", - "session_20", - "session_8", - "session_14", - "session_7" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_28", - "session_8" - ], - "question": "How many turtles does Nate have?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_2", - "session_24", - "session_25", - "session_26", - "session_28", - "session_29", - "session_20", - "session_8", - "session_5" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_25", - "session_28" - ], - "question": "What activities does Nate do with his turtles?", - "recall": 0.5, - "retrieved_ids": [ - "session_8", - "session_26", - "session_15", - "session_2", - "session_25", - "session_5", - "session_18", - "session_9", - "session_22", - "session_12" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_11", - "session_28" - ], - "question": "What do both Joanna and Nate appreciate the beauty of?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_26", - "session_28", - "session_12", - "session_11", - "session_23", - "session_1", - "session_2", - "session_8", - "session_19" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_28" - ], - "question": "When did Joanna plan on going to Nate's to watch him play with his turtles?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_8", - "session_27", - "session_22", - "session_2", - "session_25", - "session_15", - "session_12", - "session_9", - "session_23" - ], - "sample_id": "conv-42" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_28" - ], - "question": "What state did Joanna visit in summer 2021?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_28", - "session_25", - "session_7", - "session_11", - "session_4", - "session_10", - "session_21", - "session_16", - "session_17" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_15", - "session_19", - "session_23", - "session_3" - ], - "question": "What recommendations has Nate received from Joanna?", - "recall": 0.8, - "retrieved_ids": [ - "session_1", - "session_3", - "session_9", - "session_22", - "session_19", - "session_2", - "session_12", - "session_4", - "session_15", - "session_14" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_21", - "session_3" - ], - "question": "What are Nate's favorite desserts?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_3", - "session_27", - "session_1", - "session_10", - "session_4", - "session_15", - "session_20", - "session_25", - "session_18" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_18", - "session_21" - ], - "question": "How has Nate tried to disburse his vegan ice-cream recipes?", - "recall": 0.5, - "retrieved_ids": [ - "session_21", - "session_20", - "session_16", - "session_10", - "session_22", - "session_28", - "session_15", - "session_8", - "session_4", - "session_29" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_10" - ], - "question": "When did Nate win his second tournament?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_26", - "session_2", - "session_19", - "session_1", - "session_15", - "session_5", - "session_22", - "session_9", - "session_14" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_10", - "session_14", - "session_17", - "session_19", - "session_20", - "session_22", - "session_27", - "session_6" - ], - "question": "How many video game tournaments has Nate participated in?", - "recall": 0.6667, - "retrieved_ids": [ - "session_6", - "session_1", - "session_24", - "session_22", - "session_26", - "session_28", - "session_17", - "session_14", - "session_9", - "session_20" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_12", - "session_2", - "session_4", - "session_5" - ], - "question": "How many screenplays has Joanna written?", - "recall": 0.5, - "retrieved_ids": [ - "session_6", - "session_2", - "session_16", - "session_10", - "session_4", - "session_11", - "session_15", - "session_13", - "session_3", - "session_14" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_10", - "session_14", - "session_17", - "session_19", - "session_22", - "session_27" - ], - "question": "How many tournaments has Nate won?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_1", - "session_10", - "session_22", - "session_19", - "session_27", - "session_14", - "session_17", - "session_28", - "session_24" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_19", - "session_20", - "session_21", - "session_22" - ], - "question": "What recipes has Joanna made?", - "recall": 0.6, - "retrieved_ids": [ - "session_20", - "session_11", - "session_10", - "session_21", - "session_24", - "session_4", - "session_29", - "session_16", - "session_18", - "session_8" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_3", - "session_4" - ], - "question": "What recipes has Nate made?", - "recall": 0.5, - "retrieved_ids": [ - "session_20", - "session_11", - "session_10", - "session_21", - "session_24", - "session_4", - "session_29", - "session_16", - "session_18", - "session_8" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_14", - "session_18", - "session_26" - ], - "question": "What are the skills that Nate has helped others learn?", - "recall": 0.6667, - "retrieved_ids": [ - "session_14", - "session_26", - "session_20", - "session_10", - "session_3", - "session_13", - "session_2", - "session_19", - "session_9", - "session_17" - ], - "sample_id": "conv-42" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_20", - "session_21" - ], - "question": "Was the first half of September 2022 a good month career-wise for Nate and Joanna? Answer yes or no.", - "recall": 0.5, - "retrieved_ids": [ - "session_22", - "session_14", - "session_9", - "session_16", - "session_1", - "session_4", - "session_21", - "session_3", - "session_27", - "session_19" - ], - "sample_id": "conv-42" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_29" - ], - "question": "What kind of job is Joanna beginning to preform the duties of because of her movie scripts?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_9", - "session_5", - "session_26", - "session_17", - "session_1", - "session_24", - "session_29", - "session_15", - "session_7" - ], - "sample_id": "conv-42" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_29" - ], - "question": "When did Nate take his turtles to the beach?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_28", - "session_26", - "session_5", - "session_15", - "session_24", - "session_2", - "session_9", - "session_25", - "session_22" - ], - "sample_id": "conv-42" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_29" - ], - "question": "What state did Nate visit?", - "recall": 0.0, - "retrieved_ids": [ - "session_3", - "session_6", - "session_7", - "session_11", - "session_4", - "session_10", - "session_21", - "session_25", - "session_16", - "session_17" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_1" - ], - "question": "What is one of Joanna's favorite movies?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_3", - "session_9", - "session_23", - "session_25", - "session_21", - "session_27", - "session_19", - "session_15", - "session_26" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What color did Nate choose for his hair?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_29", - "session_15", - "session_21", - "session_5", - "session_2", - "session_19", - "session_9", - "session_22", - "session_12" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_9" - ], - "question": "What is Nate's favorite movie trilogy?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_1", - "session_3", - "session_25", - "session_26", - "session_23", - "session_27", - "session_15", - "session_10", - "session_21" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_9" - ], - "question": "What is Nate's favorite book series about?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_19", - "session_27", - "session_25", - "session_21", - "session_22", - "session_17", - "session_15", - "session_6", - "session_8" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "What kind of lighting does Nate's gaming room have?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_14", - "session_19", - "session_15", - "session_10", - "session_6", - "session_4", - "session_5", - "session_20", - "session_18" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "What game was the second tournament that Nate won based on?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_1", - "session_14", - "session_22", - "session_27", - "session_17", - "session_19", - "session_6", - "session_24", - "session_5" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_12" - ], - "question": "What is Joanna's third screenplay about?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_13", - "session_11", - "session_4", - "session_10", - "session_25", - "session_3", - "session_16", - "session_12", - "session_15" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_27" - ], - "question": "What is Nate's favorite video game?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_26", - "session_23", - "session_9", - "session_27", - "session_15", - "session_21", - "session_3", - "session_25", - "session_19" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_1" - ], - "question": "What type of movies does Nate enjoy watching the most?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_22", - "session_3", - "session_29", - "session_25", - "session_23", - "session_27", - "session_19", - "session_2", - "session_16" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What did Joanna just finish last Friday on 23 January, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_27", - "session_23", - "session_16", - "session_17", - "session_22", - "session_14", - "session_21", - "session_13", - "session_12" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What genre is Joanna's first screenplay?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_4", - "session_3", - "session_14", - "session_2", - "session_11", - "session_10", - "session_9", - "session_16", - "session_6" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What are Joanna's plans for her finished screenplay in January 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_12", - "session_24", - "session_11", - "session_14", - "session_13", - "session_22", - "session_19", - "session_9", - "session_27" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "For how long has Nate had his turtles?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_28", - "session_15", - "session_24", - "session_9", - "session_25", - "session_29", - "session_22", - "session_26", - "session_5" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "What did Nate think of the coconut milk ice cream he made?", - "recall": 0.0, - "retrieved_ids": [ - "session_29", - "session_4", - "session_21", - "session_8", - "session_28", - "session_10", - "session_16", - "session_13", - "session_27", - "session_12" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "Which dairy-free dessert flavors does Nate enjoy?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_3", - "session_20", - "session_10", - "session_21", - "session_22", - "session_29", - "session_18", - "session_8", - "session_26" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "What did Joanna recently watch and recommend to Nate on February 7, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_3", - "session_5", - "session_10", - "session_23", - "session_1", - "session_15", - "session_24", - "session_19", - "session_9" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "What is \"Little Women\" about according to Joanna?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_22", - "session_5", - "session_13", - "session_2", - "session_23", - "session_27", - "session_19", - "session_15", - "session_8" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What flavor of ice cream did Nate make for his friend on 25 February, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_29", - "session_28", - "session_16", - "session_8", - "session_20", - "session_21", - "session_15", - "session_22", - "session_3" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What inspired Joanna's new screenplay on 25 February, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_11", - "session_10", - "session_13", - "session_3", - "session_14", - "session_2", - "session_26", - "session_16", - "session_15" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_5" - ], - "question": "Why does Nate like turtles as pets?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_8", - "session_2", - "session_28", - "session_27", - "session_5", - "session_19", - "session_12", - "session_13", - "session_25" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_5" - ], - "question": "How does Nate describe the process of taking care of turtles?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_26", - "session_10", - "session_19", - "session_9", - "session_7", - "session_16", - "session_6", - "session_25", - "session_3" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_6" - ], - "question": "What was Joanna's audition for?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_3", - "session_1", - "session_15", - "session_4", - "session_14", - "session_21", - "session_25", - "session_2", - "session_28" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "Why did Nate choose the hair color he did?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_29", - "session_21", - "session_19", - "session_5", - "session_28", - "session_17", - "session_12", - "session_13", - "session_27" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What are the main ingredients of the ice cream recipe shared by Nate?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_16", - "session_29", - "session_21", - "session_8", - "session_20", - "session_28", - "session_1", - "session_10", - "session_27" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_9" - ], - "question": "What is Joanna's project called in the writers group?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_1", - "session_19", - "session_6", - "session_7", - "session_11", - "session_21", - "session_23", - "session_27", - "session_16" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_9" - ], - "question": "What is Nate's favorite genre of movies?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_3", - "session_9", - "session_23", - "session_15", - "session_25", - "session_21", - "session_26", - "session_27", - "session_6" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_9" - ], - "question": "What kind of books does Nate enjoy?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_19", - "session_17", - "session_4", - "session_22", - "session_6", - "session_8", - "session_20", - "session_15", - "session_18" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_9" - ], - "question": "What kind of films does Joanna enjoy?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_4", - "session_3", - "session_2", - "session_15", - "session_29", - "session_24", - "session_16", - "session_6", - "session_18" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_9" - ], - "question": "Which activity helps Nate escape and stimulates his imagination?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_27", - "session_18", - "session_8", - "session_25", - "session_23", - "session_15", - "session_2", - "session_12", - "session_22" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "What filling did Joanna use in the cake she made recently in May 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_26", - "session_22", - "session_24", - "session_20", - "session_3", - "session_29", - "session_21", - "session_23", - "session_12" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "What kind of frosting did Joanna use on the cake she made recently in May 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_20", - "session_22", - "session_24", - "session_3", - "session_26", - "session_17", - "session_18", - "session_29", - "session_21" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_11" - ], - "question": "What does Nate feel he could do when out in cool places like Whispering Falls?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_23", - "session_5", - "session_3", - "session_13", - "session_27", - "session_17", - "session_8", - "session_6", - "session_12" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_11" - ], - "question": "What creative activity does Nate joke about pursuing after being inspired by their hikes with Jo?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_28", - "session_15", - "session_18", - "session_14", - "session_8", - "session_12", - "session_19", - "session_7", - "session_6" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_11" - ], - "question": "Who invited Nate to join her on the trails sometime?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_9", - "session_20", - "session_23", - "session_14", - "session_24", - "session_5", - "session_8", - "session_19", - "session_28" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "What did Nate do for Joanna on 25 May, 2022?", - "recall": 0.0, - "retrieved_ids": [ - "session_14", - "session_17", - "session_7", - "session_11", - "session_4", - "session_10", - "session_21", - "session_25", - "session_16", - "session_6" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "How does Nate describe the stuffed animal he got for Joanna?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_24", - "session_2", - "session_27", - "session_5", - "session_22", - "session_12", - "session_19", - "session_28", - "session_17" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_14" - ], - "question": "What event is Nate organizing in June 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_26", - "session_1", - "session_7", - "session_4", - "session_21", - "session_13", - "session_2", - "session_22", - "session_17" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_14" - ], - "question": "Who did Nate plan to invite to his gaming party in June 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_23", - "session_16", - "session_11", - "session_19", - "session_9", - "session_15", - "session_22", - "session_2", - "session_25" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_14" - ], - "question": "What special items did Nate get for everyone at his gaming party?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_11", - "session_15", - "session_9", - "session_10", - "session_5", - "session_16", - "session_22", - "session_2", - "session_20" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "What did Joanna write yesterday that appeared on the big screen?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_25", - "session_27", - "session_28", - "session_3", - "session_4", - "session_5", - "session_22", - "session_29", - "session_6" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "What superhero is Joanna a fan of?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_4", - "session_28", - "session_27", - "session_8", - "session_2", - "session_1", - "session_21", - "session_14", - "session_25" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "Which superhero toy figure does Nate share a photo of?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_28", - "session_13", - "session_11", - "session_4", - "session_18", - "session_22", - "session_9", - "session_21", - "session_25" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "What is displayed on Joanna's cork board for motivation and creativity?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_9", - "session_5", - "session_12", - "session_18", - "session_11", - "session_10", - "session_20", - "session_26", - "session_6" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "What does the photo on Joanna's cork board remind her of?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_9", - "session_13", - "session_18", - "session_7", - "session_11", - "session_24", - "session_29", - "session_25", - "session_28" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "What did Nate make and share with his vegan diet group?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_20", - "session_21", - "session_13", - "session_9", - "session_25", - "session_22", - "session_19", - "session_15", - "session_2" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "How many people attended the gaming party hosted by Nate in June 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_14", - "session_24", - "session_6", - "session_19", - "session_28", - "session_23", - "session_21", - "session_2", - "session_17" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "What recipe Nate offer to share with Joanna?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_21", - "session_28", - "session_29", - "session_18", - "session_4", - "session_16", - "session_11", - "session_8", - "session_10" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "What did Joanna plan to do with the recipe Nate promised to share?", - "recall": 0.0, - "retrieved_ids": [ - "session_22", - "session_28", - "session_14", - "session_27", - "session_11", - "session_2", - "session_23", - "session_19", - "session_25", - "session_17" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "How many video game tournaments has Nate won by July 10, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_1", - "session_27", - "session_22", - "session_17", - "session_14", - "session_10", - "session_24", - "session_26", - "session_23" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "Where did Joanna go for a road trip for research?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_4", - "session_7", - "session_12", - "session_5", - "session_16", - "session_29", - "session_6", - "session_10", - "session_21" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What did Joanna discover at the library in Woodhaven?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_3", - "session_6", - "session_7", - "session_11", - "session_4", - "session_10", - "session_21", - "session_25", - "session_16" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What specific themes are explored in Joanna's new book?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_17", - "session_9", - "session_19", - "session_26", - "session_2", - "session_6", - "session_8", - "session_12", - "session_1" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What inspired Joanna's new script in July 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_28", - "session_4", - "session_17", - "session_29", - "session_26", - "session_25", - "session_27", - "session_5", - "session_15" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What did Nate do while Joanna was on her road trip?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_4", - "session_7", - "session_24", - "session_10", - "session_8", - "session_9", - "session_14", - "session_27", - "session_29" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What does Nate do that he loves and can make money from?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_4", - "session_17", - "session_12", - "session_20", - "session_13", - "session_23", - "session_27", - "session_16", - "session_18" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "How did Joanna feel when someone wrote her a letter after reading her blog post?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_19", - "session_8", - "session_14", - "session_6", - "session_25", - "session_27", - "session_17", - "session_24", - "session_9" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What kind of impact does Joanna hope to have with her writing?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_4", - "session_9", - "session_27", - "session_18", - "session_12", - "session_24", - "session_5", - "session_26", - "session_16" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What kind of content did Joanna share that someone wrote her a letter about?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_5", - "session_14", - "session_9", - "session_22", - "session_27", - "session_28", - "session_6", - "session_24", - "session_15" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What motivates Joanna to keep writing even on tough days?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_18", - "session_12", - "session_24", - "session_9", - "session_26", - "session_13", - "session_15", - "session_4", - "session_5" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What type of ice cream does Joanna mention that Nate makes and is delicious?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_4", - "session_8", - "session_29", - "session_10", - "session_2", - "session_28", - "session_18", - "session_16", - "session_1" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "How did Nate feel about sharing his love for dairy-free desserts with Joanna?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_21", - "session_10", - "session_3", - "session_4", - "session_20", - "session_22", - "session_9", - "session_15", - "session_8" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "What did Joanna share with her writers group in August 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_21", - "session_14", - "session_13", - "session_19", - "session_24", - "session_16", - "session_6", - "session_3", - "session_25" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "How did Joanna celebrate after sharing her book with her writers group?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_9", - "session_8", - "session_21", - "session_29", - "session_16", - "session_24", - "session_6", - "session_22", - "session_17" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "How did Nate celebrate winning the international tournament?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_26", - "session_1", - "session_10", - "session_22", - "session_27", - "session_6", - "session_17", - "session_14", - "session_2" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_20" - ], - "question": "Why is Joanna experimenting with dairy-free options in her dessert recipes?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_10", - "session_21", - "session_3", - "session_7", - "session_4", - "session_18", - "session_24", - "session_9", - "session_8" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_20" - ], - "question": "What substitution does Nate suggest for butter in dairy-free baking?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_10", - "session_21", - "session_18", - "session_1", - "session_4", - "session_3", - "session_22", - "session_8", - "session_26" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_21" - ], - "question": "What type of show did Nate host where he taught vegan ice cream recipes?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_29", - "session_16", - "session_4", - "session_8", - "session_1", - "session_13", - "session_9", - "session_27", - "session_12" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_21" - ], - "question": "What is Nate's favorite dish from the cooking show he hosted?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_28", - "session_15", - "session_3", - "session_10", - "session_4", - "session_27", - "session_20", - "session_13", - "session_12" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_21" - ], - "question": "What is one of Nate's favorite dairy-free treats besides coconut milk ice cream?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_4", - "session_8", - "session_29", - "session_20", - "session_3", - "session_26", - "session_10", - "session_16", - "session_18" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_21" - ], - "question": "What dessert did Joanna share a photo of that has an almond flour crust, chocolate ganache, and fresh raspberries?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_20", - "session_18", - "session_4", - "session_3", - "session_11", - "session_10", - "session_15", - "session_9", - "session_25" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_21" - ], - "question": "What kind of cake did Joanna share a photo of that she likes making for birthdays and special days?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_18", - "session_9", - "session_25", - "session_29", - "session_15", - "session_22", - "session_20", - "session_11", - "session_24" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_21" - ], - "question": "What two main ingredients are part of the dessert Joanna shared a photo of with blueberries, coconut milk, and a gluten-free crust?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_4", - "session_20", - "session_8", - "session_3", - "session_18", - "session_26", - "session_25", - "session_10", - "session_1" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_22" - ], - "question": "What movie did Nate recently watch and enjoy on October 6, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_3", - "session_27", - "session_1", - "session_10", - "session_23", - "session_25", - "session_5", - "session_17", - "session_24" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_22" - ], - "question": "What did Joanna make for one of the ladies at her writing club?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_9", - "session_17", - "session_4", - "session_25", - "session_28", - "session_19", - "session_24", - "session_26", - "session_6" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "What game did Nate play at the game convention he attended on 9 October, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_17", - "session_16", - "session_10", - "session_27", - "session_28", - "session_13", - "session_9", - "session_1", - "session_12" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "What movie has Nate recently seen that blew his mind?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_3", - "session_27", - "session_1", - "session_24", - "session_4", - "session_15", - "session_22", - "session_9", - "session_2" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "What game has Nate been playing nonstop with a futuristic setting and gameplay on October 9, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_13", - "session_1", - "session_10", - "session_27", - "session_9", - "session_26", - "session_28", - "session_29", - "session_16" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "What did Nate share a photo of when mentioning unwinding at home?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_28", - "session_9", - "session_13", - "session_11", - "session_15", - "session_25", - "session_27", - "session_19", - "session_2" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "How did Joanna describe the classic movie he watched?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_23", - "session_25", - "session_27", - "session_28", - "session_3", - "session_29", - "session_22", - "session_12", - "session_13" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "What does Joanna recommend to make a living room comfy like hers?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_19", - "session_22", - "session_10", - "session_15", - "session_9", - "session_3", - "session_24", - "session_18", - "session_1" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "What helps Joanna stay focused and brings her joy?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_13", - "session_2", - "session_27", - "session_5", - "session_9", - "session_20", - "session_28", - "session_12", - "session_6" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "What does Joanna do while she writes?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_24", - "session_8", - "session_7", - "session_29", - "session_10", - "session_4", - "session_14", - "session_27", - "session_17" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "Why did Joanna name the stuffed animal dog Tilly?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_13", - "session_27", - "session_25", - "session_5", - "session_16", - "session_2", - "session_7", - "session_11", - "session_17" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "What does Joanna do after receiving a rejection from a production company?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_14", - "session_5", - "session_29", - "session_4", - "session_18", - "session_19", - "session_8", - "session_27", - "session_12" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "How does Nate feel about Joanna's ability to bounce back from setbacks?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_10", - "session_28", - "session_13", - "session_22", - "session_27", - "session_20", - "session_4", - "session_9", - "session_3" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "What encouragement does Nate give to Joanna after her setback?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_14", - "session_28", - "session_29", - "session_9", - "session_5", - "session_22", - "session_27", - "session_20", - "session_18" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "What does Nate rely on for cheer and joy?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_2", - "session_25", - "session_7", - "session_29", - "session_13", - "session_21", - "session_5", - "session_28", - "session_12" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "What does Joanna use to remember her dog from Michigan?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_13", - "session_9", - "session_22", - "session_26", - "session_27", - "session_14", - "session_25", - "session_8", - "session_15" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_25" - ], - "question": "What did Joanna contribute to that was shown on the big screen on the Sunday before October 25, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_27", - "session_7", - "session_15", - "session_3", - "session_5", - "session_28", - "session_4", - "session_22", - "session_2" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_25" - ], - "question": "What inspires Joanna to create drawings of her characters?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_9", - "session_2", - "session_26", - "session_4", - "session_24", - "session_28", - "session_27", - "session_19", - "session_15" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_25" - ], - "question": "Where does Joanna get her ideas for the characters from?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_9", - "session_25", - "session_27", - "session_24", - "session_4", - "session_19", - "session_5", - "session_28", - "session_1" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_25" - ], - "question": "How did Joanna feel on October 25, 2022 about seeing her characters come alive on the big screen?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_27", - "session_15", - "session_5", - "session_2", - "session_29", - "session_4", - "session_9", - "session_3", - "session_24" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_25" - ], - "question": "What type of diet do Nate's turtles have?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_25", - "session_8", - "session_1", - "session_29", - "session_24", - "session_5", - "session_26", - "session_28", - "session_16" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_26" - ], - "question": "What ingredient did Nate use to make the ice cream lactose-free?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_16", - "session_4", - "session_29", - "session_26", - "session_28", - "session_8", - "session_10", - "session_20", - "session_18" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_26" - ], - "question": "What did Joanna find in old notebooks last week that prompted her to reflect on her progress as a writer?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_21", - "session_26", - "session_24", - "session_14", - "session_17", - "session_19", - "session_4", - "session_28", - "session_20" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_27" - ], - "question": "What game is Nate currently playing and recommends to others on November 7, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_27", - "session_6", - "session_23", - "session_3", - "session_1", - "session_22", - "session_15", - "session_9", - "session_19" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_27" - ], - "question": "What did Joanna receive from her brother that brought back childhood memories?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_10", - "session_24", - "session_9", - "session_12", - "session_1", - "session_22", - "session_28", - "session_18", - "session_19" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_27" - ], - "question": "What is the type of game \"Xenoblade Chronicles\" that Nate is playing?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_27", - "session_10", - "session_23", - "session_9", - "session_8", - "session_28", - "session_13", - "session_17", - "session_16" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_28" - ], - "question": "What dish did Nate make on 9 November, 2022?", - "recall": 0.0, - "retrieved_ids": [ - "session_21", - "session_26", - "session_4", - "session_10", - "session_13", - "session_25", - "session_16", - "session_6", - "session_17", - "session_19" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_28" - ], - "question": "What project is Joanna working on in her notebook on November 9, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_10", - "session_9", - "session_26", - "session_28", - "session_24", - "session_1", - "session_6", - "session_7", - "session_18" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_28" - ], - "question": "What is Nate creating for YouTube on 9 November, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_27", - "session_22", - "session_7", - "session_1", - "session_4", - "session_2", - "session_25", - "session_14", - "session_21" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_28" - ], - "question": "What inspired Nate to start making gaming videos?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_9", - "session_27", - "session_17", - "session_6", - "session_11", - "session_4", - "session_18", - "session_15", - "session_2" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_28" - ], - "question": "What new content is Nate creating for YouTube?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_22", - "session_27", - "session_11", - "session_7", - "session_10", - "session_3", - "session_2", - "session_23", - "session_25" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_28" - ], - "question": "What advice does Joanna give to Nate about making YouTube videos?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_21", - "session_9", - "session_14", - "session_3", - "session_22", - "session_1", - "session_29", - "session_6", - "session_16" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_28" - ], - "question": "What did Joanna take a picture of near Fort Wayne last summer?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_11", - "session_6", - "session_7", - "session_15", - "session_4", - "session_27", - "session_16", - "session_19", - "session_10" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_28" - ], - "question": "What inspired Joanna to take a picture of the sunset in the field near Fort Wayne?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_7", - "session_11", - "session_8", - "session_4", - "session_26", - "session_15", - "session_22", - "session_27", - "session_17" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_28" - ], - "question": "Why did Nate get a third turtle?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_12", - "session_25", - "session_2", - "session_29", - "session_24", - "session_8", - "session_26", - "session_5", - "session_20" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_28" - ], - "question": "What does Nate want to do when he goes over to Joanna's place?", - "recall": 0.0, - "retrieved_ids": [ - "session_23", - "session_7", - "session_26", - "session_6", - "session_13", - "session_11", - "session_12", - "session_18", - "session_17", - "session_27" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_29" - ], - "question": "What did Nate take to the beach in Tampa?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_3", - "session_7", - "session_4", - "session_11", - "session_26", - "session_16", - "session_6", - "session_19", - "session_10" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_29" - ], - "question": "What does Nate love most about having turtles?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_2", - "session_22", - "session_25", - "session_28", - "session_26", - "session_5", - "session_16", - "session_24", - "session_19" - ], - "sample_id": "conv-42" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_29" - ], - "question": "What did Nate share a photo of as a part of his experimentation in November 2022?", - "recall": 0.0, - "retrieved_ids": [ - "session_25", - "session_28", - "session_15", - "session_20", - "session_26", - "session_18", - "session_9", - "session_11", - "session_22", - "session_2" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_7" - ], - "question": "What color did Joanna choose for her hair?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_21", - "session_24", - "session_29", - "session_9", - "session_5", - "session_28", - "session_17", - "session_12", - "session_19" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_9" - ], - "question": "What is Joanna's favorite movie trilogy?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_1", - "session_3", - "session_25", - "session_26", - "session_23", - "session_27", - "session_15", - "session_10", - "session_21" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_9" - ], - "question": "What is Joanna's favorite book series about?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_19", - "session_27", - "session_25", - "session_21", - "session_22", - "session_17", - "session_15", - "session_6", - "session_8" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_10" - ], - "question": "What kind of lighting does Joanna's gaming room have?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_14", - "session_19", - "session_15", - "session_10", - "session_6", - "session_4", - "session_5", - "session_20", - "session_18" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_10" - ], - "question": "What game was the second tournament that Joanna won based on?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_1", - "session_14", - "session_22", - "session_27", - "session_17", - "session_6", - "session_19", - "session_24", - "session_5" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_12" - ], - "question": "What is Nate's third screenplay about?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_13", - "session_4", - "session_11", - "session_10", - "session_25", - "session_3", - "session_16", - "session_12", - "session_15" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_1" - ], - "question": "What type of movies does Nate hate watching the most?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_3", - "session_22", - "session_23", - "session_29", - "session_25", - "session_2", - "session_27", - "session_16", - "session_19" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_2" - ], - "question": "What genre is Joanna's first novella?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_28", - "session_4", - "session_9", - "session_3", - "session_14", - "session_25", - "session_2", - "session_17", - "session_6" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_2" - ], - "question": "What are Nate's plans for his finished screenplay in January 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_15", - "session_22", - "session_11", - "session_14", - "session_12", - "session_19", - "session_13", - "session_27", - "session_9" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_2" - ], - "question": "For how long has Nate had his snakes?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_20", - "session_1", - "session_6", - "session_9", - "session_18", - "session_2", - "session_16", - "session_22", - "session_19" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_3" - ], - "question": "What did Nate think of the caramel ice cream he made?", - "recall": 0.0, - "retrieved_ids": [ - "session_21", - "session_29", - "session_4", - "session_10", - "session_28", - "session_16", - "session_8", - "session_13", - "session_27", - "session_12" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What flavor of cake did Nate make for his friend on 25 February, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_29", - "session_4", - "session_10", - "session_20", - "session_15", - "session_22", - "session_21", - "session_9", - "session_2" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_6" - ], - "question": "What was Nate's audition for?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_17", - "session_1", - "session_14", - "session_7", - "session_2", - "session_4", - "session_21", - "session_22", - "session_13" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_7" - ], - "question": "Why did Joanna choose the hair color she did?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_29", - "session_17", - "session_24", - "session_22", - "session_21", - "session_19", - "session_5", - "session_28", - "session_12" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_8" - ], - "question": "What are the main ingredients of the ice cream recipe shared by Joanna?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_16", - "session_29", - "session_21", - "session_8", - "session_20", - "session_28", - "session_1", - "session_10", - "session_27" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_9" - ], - "question": "What is Nate's project called in the writers group?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_1", - "session_6", - "session_19", - "session_7", - "session_11", - "session_21", - "session_23", - "session_27", - "session_16" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_9" - ], - "question": "Which activity helps Nate escape and numbs his mind?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_23", - "session_2", - "session_9", - "session_8", - "session_4", - "session_15", - "session_3", - "session_12", - "session_22" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_10" - ], - "question": "What filling did Nate use in the cake he made recently in May 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_10", - "session_12", - "session_20", - "session_3", - "session_26", - "session_21", - "session_23", - "session_27", - "session_13" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_14" - ], - "question": "Who did Joanna plan to invite to her gaming party in June 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_23", - "session_9", - "session_16", - "session_11", - "session_19", - "session_2", - "session_24", - "session_22", - "session_25" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_14" - ], - "question": "What special items did Joanna get for everyone at her gaming party?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_9", - "session_10", - "session_24", - "session_11", - "session_15", - "session_5", - "session_18", - "session_16", - "session_20" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_15" - ], - "question": "What supervillain is Joanna a fan of?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_28", - "session_27", - "session_8", - "session_2", - "session_1", - "session_15", - "session_14", - "session_21", - "session_25" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_15" - ], - "question": "Which superhero toy figure does Joanna share a photo of?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_13", - "session_28", - "session_11", - "session_4", - "session_18", - "session_22", - "session_9", - "session_21", - "session_25" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_16" - ], - "question": "What did Joanna make and share with her vegan diet group?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_21", - "session_23", - "session_9", - "session_20", - "session_13", - "session_25", - "session_6", - "session_19", - "session_24" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_16" - ], - "question": "How many people attended the gaming party hosted by Joanna in June 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_24", - "session_14", - "session_6", - "session_19", - "session_28", - "session_23", - "session_21", - "session_2", - "session_17" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_17" - ], - "question": "Where did Nate go for a road trip for research?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_4", - "session_7", - "session_12", - "session_5", - "session_16", - "session_29", - "session_6", - "session_10", - "session_21" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_17" - ], - "question": "What did Joanna discover at the museum in Woodhaven?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_3", - "session_6", - "session_7", - "session_11", - "session_4", - "session_10", - "session_21", - "session_25", - "session_16" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_17" - ], - "question": "What specific themes are explored in Nate's new book?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_17", - "session_9", - "session_19", - "session_26", - "session_2", - "session_6", - "session_8", - "session_12", - "session_1" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "How did Nate feel when someone wrote him a letter after reading his blog post?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_28", - "session_12", - "session_15", - "session_6", - "session_8", - "session_14", - "session_25", - "session_27", - "session_9" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "What kind of impact does Joanna hope to have with her painting?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_24", - "session_27", - "session_4", - "session_9", - "session_5", - "session_18", - "session_12", - "session_16", - "session_26" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_19" - ], - "question": "What did Nate share with his writers group in August 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_21", - "session_19", - "session_25", - "session_15", - "session_13", - "session_22", - "session_16", - "session_6", - "session_2" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_19" - ], - "question": "How did Nate celebrate after sharing his book with a writers group?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_6", - "session_9", - "session_8", - "session_16", - "session_21", - "session_22", - "session_15", - "session_17", - "session_2" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_19" - ], - "question": "How did Joanna celebrate winning the international tournament?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_26", - "session_1", - "session_10", - "session_6", - "session_22", - "session_27", - "session_17", - "session_14", - "session_2" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_20" - ], - "question": "What substitution does Nate suggest for sugar in dairy-free baking?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_10", - "session_20", - "session_21", - "session_18", - "session_1", - "session_4", - "session_3", - "session_22", - "session_26" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_21" - ], - "question": "What type of show did Joanna host where she taught vegan ice cream recipes?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_29", - "session_28", - "session_16", - "session_4", - "session_8", - "session_1", - "session_24", - "session_9", - "session_22" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_21" - ], - "question": "What is Joanna's favorite dish from the cooking show she hosted?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_28", - "session_29", - "session_15", - "session_3", - "session_24", - "session_10", - "session_4", - "session_22", - "session_20" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_21" - ], - "question": "What dessert did Nate share a photo of that has an almond flour crust, chocolate ganache, and fresh raspberries?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_20", - "session_18", - "session_4", - "session_3", - "session_11", - "session_10", - "session_15", - "session_9", - "session_25" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_21" - ], - "question": "What two main ingredients are part of the dessert Nate shared a photo of with blueberries, coconut milk, and a gluten-free crust?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_4", - "session_20", - "session_8", - "session_3", - "session_26", - "session_25", - "session_18", - "session_10", - "session_1" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_22" - ], - "question": "What movie did Joanna recently watch and enjoy on October 6, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_27", - "session_3", - "session_1", - "session_10", - "session_23", - "session_25", - "session_5", - "session_17", - "session_24" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_22" - ], - "question": "What did Nate make for one of the ladies at his writing club?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_28", - "session_4", - "session_9", - "session_25", - "session_19", - "session_2", - "session_26", - "session_6", - "session_15" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_23" - ], - "question": "What game has Joanna been playing nonstop with a futuristic setting and gameplay on October 9, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_13", - "session_1", - "session_10", - "session_27", - "session_9", - "session_26", - "session_28", - "session_29", - "session_16" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_23" - ], - "question": "How did Nate describe the classic movie he watched?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_23", - "session_25", - "session_27", - "session_28", - "session_3", - "session_29", - "session_22", - "session_12", - "session_13" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_23" - ], - "question": "What does Nate recommend to make a living room comfy like his?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_3", - "session_19", - "session_15", - "session_10", - "session_22", - "session_9", - "session_18", - "session_1", - "session_2" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_24" - ], - "question": "What helps Joanna stay distracted and brings her sadness?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_27", - "session_2", - "session_9", - "session_20", - "session_23", - "session_12", - "session_28", - "session_6", - "session_18" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_24" - ], - "question": "What does Nate do while he writes?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_7", - "session_10", - "session_4", - "session_24", - "session_14", - "session_27", - "session_17", - "session_12", - "session_13" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_24" - ], - "question": "What does Nate do after receiving a rejection from a production company?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_14", - "session_5", - "session_29", - "session_4", - "session_18", - "session_19", - "session_8", - "session_27", - "session_12" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_24" - ], - "question": "What does Joanna rely on for cheer and joy?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_2", - "session_25", - "session_7", - "session_29", - "session_13", - "session_21", - "session_5", - "session_28", - "session_12" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_24" - ], - "question": "What does Nate use to remember his dog from Michigan?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_27", - "session_13", - "session_22", - "session_26", - "session_15", - "session_9", - "session_25", - "session_8", - "session_2" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_25" - ], - "question": "What inspires Joanna to create music for her characters?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_25", - "session_2", - "session_26", - "session_4", - "session_24", - "session_28", - "session_27", - "session_19", - "session_15" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_25" - ], - "question": "What type of diet do Joanna's turtles have?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_25", - "session_8", - "session_1", - "session_29", - "session_24", - "session_26", - "session_5", - "session_28", - "session_16" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_26" - ], - "question": "What did Nate find in old notebooks last week that prompted him to reflect on her progress as a writer?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_21", - "session_26", - "session_20", - "session_24", - "session_14", - "session_28", - "session_4", - "session_12", - "session_19" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_27" - ], - "question": "What game is Joanna currently playing and recommends to others on November 7, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_27", - "session_6", - "session_23", - "session_3", - "session_1", - "session_22", - "session_15", - "session_9", - "session_19" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_27" - ], - "question": "What did Nate receive from his brother that brought back childhood memories?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_24", - "session_1", - "session_15", - "session_9", - "session_22", - "session_12", - "session_18", - "session_19", - "session_2" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_27" - ], - "question": "What is the type of game \"Xenoblade Chronicles\" that Joanna is playing?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_27", - "session_10", - "session_23", - "session_9", - "session_8", - "session_28", - "session_13", - "session_17", - "session_16" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_28" - ], - "question": "What project is Nate working on in his notebook on November 9, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_10", - "session_28", - "session_1", - "session_15", - "session_9", - "session_2", - "session_6", - "session_7", - "session_22" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_28" - ], - "question": "What is Joanna creating for YouTube on 9 November, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_3", - "session_27", - "session_1", - "session_15", - "session_4", - "session_2", - "session_25", - "session_14", - "session_21" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_28" - ], - "question": "What inspired Joanna to start making gaming videos?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_9", - "session_27", - "session_6", - "session_17", - "session_11", - "session_4", - "session_18", - "session_15", - "session_2" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_28" - ], - "question": "What new content is Nate creating for television?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_11", - "session_22", - "session_27", - "session_7", - "session_10", - "session_3", - "session_2", - "session_23", - "session_25" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_28" - ], - "question": "What did Nate take a picture of near Fort Wayne last summer?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_6", - "session_11", - "session_7", - "session_15", - "session_4", - "session_27", - "session_16", - "session_19", - "session_10" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_28" - ], - "question": "Why did Joanna get a third turtle?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_12", - "session_25", - "session_2", - "session_29", - "session_24", - "session_8", - "session_26", - "session_5", - "session_20" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_29" - ], - "question": "What did Joanna take to the beach in Tampa?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_6", - "session_7", - "session_4", - "session_11", - "session_3", - "session_26", - "session_16", - "session_19", - "session_10" - ], - "sample_id": "conv-42" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_29" - ], - "question": "What does Joanna love most about having turtles?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_2", - "session_28", - "session_22", - "session_25", - "session_26", - "session_5", - "session_16", - "session_24", - "session_19" - ], - "sample_id": "conv-42" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_11", - "session_6" - ], - "question": "what are John's goals with regards to his basketball career?", - "recall": 0.6667, - "retrieved_ids": [ - "session_11", - "session_6", - "session_19", - "session_23", - "session_7", - "session_12", - "session_21", - "session_27", - "session_5", - "session_20" - ], - "sample_id": "conv-43" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_11", - "session_6" - ], - "question": "What are John's goals for his career that are not related to his basketball skills?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_21", - "session_12", - "session_11", - "session_1", - "session_27", - "session_23", - "session_5", - "session_2", - "session_20" - ], - "sample_id": "conv-43" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_12", - "session_27" - ], - "question": "What items does John collect?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_27", - "session_24", - "session_1", - "session_2", - "session_20", - "session_8", - "session_26", - "session_9", - "session_28" - ], - "sample_id": "conv-43" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_1" - ], - "question": "Would Tim enjoy reading books by C. S. Lewis or John Greene?", - "recall": 0.0, - "retrieved_ids": [ - "session_19", - "session_4", - "session_14", - "session_22", - "session_2", - "session_11", - "session_3", - "session_26", - "session_16", - "session_7" - ], - "sample_id": "conv-43" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_11", - "session_2", - "session_20", - "session_22", - "session_26", - "session_6" - ], - "question": "What books has Tim read?", - "recall": 0.4286, - "retrieved_ids": [ - "session_22", - "session_5", - "session_14", - "session_19", - "session_11", - "session_26", - "session_15", - "session_4", - "session_28", - "session_27" - ], - "sample_id": "conv-43" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_2" - ], - "question": "Based on Tim's collections, what is a shop that he would enjoy visiting in New York city?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_2", - "session_1", - "session_11", - "session_29", - "session_16", - "session_26", - "session_27", - "session_12", - "session_3" - ], - "sample_id": "conv-43" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_3" - ], - "question": "In which month's game did John achieve a career-high score in points?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_21", - "session_23", - "session_5", - "session_27", - "session_11", - "session_20", - "session_26", - "session_9", - "session_6" - ], - "sample_id": "conv-43" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_14", - "session_3" - ], - "question": "Which geographical locations has Tim been to?", - "recall": 0.6667, - "retrieved_ids": [ - "session_25", - "session_9", - "session_1", - "session_4", - "session_18", - "session_21", - "session_10", - "session_24", - "session_14", - "session_6" - ], - "sample_id": "conv-43" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_25", - "session_3" - ], - "question": "Which outdoor gear company likely signed up John for an endorsement deal?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_3", - "session_29", - "session_21", - "session_11", - "session_1", - "session_2", - "session_16", - "session_7", - "session_23" - ], - "sample_id": "conv-43" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_25", - "session_29", - "session_3" - ], - "question": "Which endorsement deals has John been offered?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_29", - "session_3", - "session_25", - "session_11", - "session_1", - "session_2", - "session_4", - "session_23", - "session_24" - ], - "sample_id": "conv-43" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_3", - "session_5" - ], - "question": "When was John in Seattle for a game?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_8", - "session_23", - "session_6", - "session_24", - "session_28", - "session_13", - "session_5", - "session_16", - "session_12" - ], - "sample_id": "conv-43" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_2", - "session_3" - ], - "question": "What sports does John like besides basketball?", - "recall": 0.6667, - "retrieved_ids": [ - "session_16", - "session_26", - "session_3", - "session_23", - "session_6", - "session_15", - "session_5", - "session_2", - "session_28", - "session_9" - ], - "sample_id": "conv-43" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_3" - ], - "question": "What year did John start surfing?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_21", - "session_8", - "session_14", - "session_1", - "session_16", - "session_11", - "session_26", - "session_9", - "session_7" - ], - "sample_id": "conv-43" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_2", - "session_3" - ], - "question": "What does Tim do to escape reality?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_17", - "session_2", - "session_26", - "session_29", - "session_14", - "session_20", - "session_3", - "session_15", - "session_9" - ], - "sample_id": "conv-43" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_15", - "session_2", - "session_4" - ], - "question": "What kind of writing does Tim do?", - "recall": 0.6667, - "retrieved_ids": [ - "session_4", - "session_16", - "session_6", - "session_22", - "session_1", - "session_29", - "session_25", - "session_19", - "session_15", - "session_9" - ], - "sample_id": "conv-43" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_4" - ], - "question": "Who is Anthony?", - "recall": 1.0, - "retrieved_ids": [ - "session_4" - ], - "sample_id": "conv-43" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_3", - "session_5" - ], - "question": "After how many weeks did Tim reconnect with the fellow Harry Potter fan from California?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_5", - "session_11", - "session_13", - "session_27", - "session_1", - "session_26", - "session_2", - "session_6", - "session_17" - ], - "sample_id": "conv-43" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_22", - "session_23", - "session_24", - "session_3", - "session_5" - ], - "question": "How many games has John mentioned winning?", - "recall": 0.4, - "retrieved_ids": [ - "session_3", - "session_6", - "session_24", - "session_16", - "session_13", - "session_29", - "session_26", - "session_27", - "session_17", - "session_2" - ], - "sample_id": "conv-43" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_", - "session_1", - "session_2", - "session_20", - "session_26", - "session_4", - "session_5" - ], - "question": "What authors has Tim read books from?", - "recall": 0.4286, - "retrieved_ids": [ - "session_15", - "session_17", - "session_5", - "session_26", - "session_22", - "session_14", - "session_11", - "session_19", - "session_7", - "session_4" - ], - "sample_id": "conv-43" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_3", - "session_6" - ], - "question": "What is a prominent charity organization that John might want to work with and why?", - "recall": 0.5, - "retrieved_ids": [ - "session_6", - "session_11", - "session_15", - "session_28", - "session_4", - "session_29", - "session_22", - "session_16", - "session_21", - "session_26" - ], - "sample_id": "conv-43" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_3", - "session_5", - "session_6" - ], - "question": "Which city was John in before traveling to Chicago?", - "recall": 0.6667, - "retrieved_ids": [ - "session_6", - "session_27", - "session_9", - "session_11", - "session_10", - "session_29", - "session_14", - "session_21", - "session_3", - "session_26" - ], - "sample_id": "conv-43" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_3", - "session_6", - "session_9" - ], - "question": "Which US cities does John mention visiting to Tim?", - "recall": 0.6667, - "retrieved_ids": [ - "session_3", - "session_29", - "session_16", - "session_15", - "session_2", - "session_9", - "session_14", - "session_11", - "session_26", - "session_28" - ], - "sample_id": "conv-43" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_7" - ], - "question": "When did John meet with his teammates after returning from Chicago?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_12", - "session_6", - "session_7", - "session_3", - "session_1", - "session_5", - "session_27", - "session_22", - "session_20" - ], - "sample_id": "conv-43" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_7" - ], - "question": "When is Tim attending a book conference?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_14", - "session_11", - "session_19", - "session_13", - "session_5", - "session_22", - "session_27", - "session_26", - "session_4" - ], - "sample_id": "conv-43" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_6", - "session_7" - ], - "question": "Where was John between August 11 and August 15 2023?", - "recall": 0.0, - "retrieved_ids": [ - "session_22", - "session_15", - "session_13", - "session_20", - "session_3", - "session_28", - "session_1", - "session_24", - "session_26", - "session_27" - ], - "sample_id": "conv-43" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_16", - "session_7" - ], - "question": "What similar sports collectible do Tim and John own?", - "recall": 0.0, - "retrieved_ids": [ - "session_19", - "session_3", - "session_15", - "session_23", - "session_22", - "session_2", - "session_21", - "session_9", - "session_1", - "session_27" - ], - "sample_id": "conv-43" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_17", - "session_26" - ], - "question": "Which TV series does Tim mention watching?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_16", - "session_8", - "session_2", - "session_17", - "session_23", - "session_22", - "session_3", - "session_20", - "session_27" - ], - "sample_id": "conv-43" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_18" - ], - "question": "Which popular time management technique does Tim use to prepare for exams?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_9", - "session_24", - "session_18", - "session_29", - "session_5", - "session_14", - "session_12", - "session_3", - "session_15" - ], - "sample_id": "conv-43" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_8" - ], - "question": "Which popular music composer's tunes does Tim enjoy playing on the piano?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_21", - "session_28", - "session_20", - "session_12", - "session_11", - "session_19", - "session_2", - "session_22", - "session_29" - ], - "sample_id": "conv-43" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_6", - "session_9" - ], - "question": "What schools did John play basketball in and how many years was he with his team during high school?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_6", - "session_12", - "session_8", - "session_9", - "session_14", - "session_23", - "session_16", - "session_20", - "session_5" - ], - "sample_id": "conv-43" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_27", - "session_3", - "session_6", - "session_9" - ], - "question": "Which cities has John been to?", - "recall": 0.5, - "retrieved_ids": [ - "session_9", - "session_29", - "session_14", - "session_11", - "session_3", - "session_25", - "session_1", - "session_4", - "session_24", - "session_21" - ], - "sample_id": "conv-43" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_10" - ], - "question": "What month did Tim plan on going to Universal Studios?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_27", - "session_11", - "session_26", - "session_8", - "session_28", - "session_7", - "session_1", - "session_3", - "session_6" - ], - "sample_id": "conv-43" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_10" - ], - "question": "Which US states might Tim be in during September 2023 based on his plans of visiting Universal Studios?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_8", - "session_1", - "session_16", - "session_26", - "session_12", - "session_19", - "session_27", - "session_20", - "session_5" - ], - "sample_id": "conv-43" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_11" - ], - "question": "When does John plan on traveling with his team on a team trip?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_27", - "session_9", - "session_21", - "session_12", - "session_26", - "session_10", - "session_5", - "session_20", - "session_29" - ], - "sample_id": "conv-43" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_11", - "session_26", - "session_27" - ], - "question": "What could John do after his basketball career?", - "recall": 0.6667, - "retrieved_ids": [ - "session_6", - "session_11", - "session_21", - "session_14", - "session_28", - "session_5", - "session_12", - "session_27", - "session_23", - "session_20" - ], - "sample_id": "conv-43" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_12", - "session_3" - ], - "question": "What outdoor activities does John enjoy?", - "recall": 0.5, - "retrieved_ids": [ - "session_15", - "session_25", - "session_4", - "session_28", - "session_14", - "session_20", - "session_12", - "session_11", - "session_8", - "session_24" - ], - "sample_id": "conv-43" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_12", - "session_16" - ], - "question": "Who is Tim and John's favorite basketball player?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_12", - "session_3", - "session_23", - "session_7", - "session_21", - "session_19", - "session_27", - "session_8", - "session_14" - ], - "sample_id": "conv-43" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_13" - ], - "question": "Which week did Tim visit the UK for the Harry Potter Conference?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_26", - "session_1", - "session_2", - "session_27", - "session_19", - "session_29", - "session_18", - "session_7", - "session_15" - ], - "sample_id": "conv-43" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_13", - "session_18" - ], - "question": "which country has Tim visited most frequently in his travels?", - "recall": 0.0, - "retrieved_ids": [ - "session_29", - "session_27", - "session_9", - "session_21", - "session_26", - "session_25", - "session_11", - "session_12", - "session_20", - "session_5" - ], - "sample_id": "conv-43" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_14" - ], - "question": "What year did Tim go to the Smoky Mountains?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_20", - "session_1", - "session_16", - "session_21", - "session_3", - "session_8", - "session_9", - "session_13", - "session_25" - ], - "sample_id": "conv-43" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_14" - ], - "question": "Has Tim been to North Carolina and/or Tennesee states in the US?", - "recall": 0.0, - "retrieved_ids": [], - "sample_id": "conv-43" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_15", - "session_16" - ], - "question": "What kind of fiction stories does Tim write?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_22", - "session_16", - "session_15", - "session_19", - "session_6", - "session_9", - "session_29", - "session_27", - "session_26" - ], - "sample_id": "conv-43" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_15" - ], - "question": "What has John cooked?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_13", - "session_1", - "session_24", - "session_27", - "session_20", - "session_26", - "session_3", - "session_10", - "session_28" - ], - "sample_id": "conv-43" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_12", - "session_16" - ], - "question": "What does John like about Lebron James?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_16", - "session_29", - "session_2", - "session_11", - "session_15", - "session_18", - "session_10", - "session_7", - "session_4" - ], - "sample_id": "conv-43" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_16" - ], - "question": "When did John and his wife go on a European vacation?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_12", - "session_17", - "session_24", - "session_27", - "session_29", - "session_25", - "session_5", - "session_13", - "session_20" - ], - "sample_id": "conv-43" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_18" - ], - "question": "Which country was Tim visiting in the second week of November?", - "recall": 0.0, - "retrieved_ids": [ - "session_29", - "session_16", - "session_1", - "session_20", - "session_3", - "session_15", - "session_2", - "session_28", - "session_9", - "session_21" - ], - "sample_id": "conv-43" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_18" - ], - "question": "Where was Tim in the week before 16 November 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_12", - "session_25", - "session_6", - "session_18", - "session_10", - "session_1", - "session_9", - "session_21", - "session_27" - ], - "sample_id": "conv-43" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_12" - ], - "question": "When did John get married at a greenhouse?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_1", - "session_6", - "session_19", - "session_25", - "session_13", - "session_29", - "session_14", - "session_3", - "session_8" - ], - "sample_id": "conv-43" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_18" - ], - "question": "When did John get an ankle injury in 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_8", - "session_1", - "session_6", - "session_18", - "session_25", - "session_13", - "session_29", - "session_14", - "session_3" - ], - "sample_id": "conv-43" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_18", - "session_19" - ], - "question": "How many times has John injured his ankle?", - "recall": 0.5, - "retrieved_ids": [ - "session_27", - "session_2", - "session_24", - "session_29", - "session_13", - "session_12", - "session_19", - "session_17", - "session_20", - "session_5" - ], - "sample_id": "conv-43" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_18", - "session_19" - ], - "question": "Which book was John reading during his recovery from an ankle injury?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_8", - "session_15", - "session_27", - "session_12", - "session_18", - "session_5", - "session_20", - "session_1", - "session_25" - ], - "sample_id": "conv-43" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_20" - ], - "question": "What kind of yoga for building core strength might John benefit from?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_16", - "session_25", - "session_8", - "session_7", - "session_22", - "session_11", - "session_5", - "session_28", - "session_6" - ], - "sample_id": "conv-43" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_20", - "session_8" - ], - "question": "What does John do to supplement his basketball training?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_12", - "session_8", - "session_6", - "session_1", - "session_25", - "session_29", - "session_11", - "session_5", - "session_20" - ], - "sample_id": "conv-43" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_20", - "session_8" - ], - "question": "What other exercises can help John with his basketball performance?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_8", - "session_24", - "session_11", - "session_6", - "session_23", - "session_25", - "session_5", - "session_12", - "session_20" - ], - "sample_id": "conv-43" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_20" - ], - "question": "When did John take a trip to the Rocky Mountains?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_14", - "session_11", - "session_18", - "session_10", - "session_27", - "session_6", - "session_17", - "session_9", - "session_7" - ], - "sample_id": "conv-43" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_1", - "session_21" - ], - "question": "When did John start playing professionally?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_11", - "session_7", - "session_8", - "session_6", - "session_23", - "session_1", - "session_27", - "session_3", - "session_26" - ], - "sample_id": "conv-43" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_21" - ], - "question": "When did Tim start playing the violin?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_11", - "session_7", - "session_8", - "session_23", - "session_6", - "session_1", - "session_26", - "session_3", - "session_9" - ], - "sample_id": "conv-43" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_21", - "session_8" - ], - "question": "What instruments does Tim play?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_8", - "session_1", - "session_2", - "session_12", - "session_19", - "session_23", - "session_9", - "session_11", - "session_6" - ], - "sample_id": "conv-43" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_22", - "session_4" - ], - "question": "When did John attend the Harry Potter trivia?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_22", - "session_4", - "session_5", - "session_1", - "session_10", - "session_13", - "session_26", - "session_8", - "session_7" - ], - "sample_id": "conv-43" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_23", - "session_3" - ], - "question": "Which career-high performances did John achieve in 2023?", - "recall": 0.5, - "retrieved_ids": [ - "session_29", - "session_25", - "session_23", - "session_21", - "session_11", - "session_27", - "session_9", - "session_6", - "session_26", - "session_8" - ], - "sample_id": "conv-43" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_23" - ], - "question": "When did John achieve a career-high assist performance?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_21", - "session_13", - "session_8", - "session_26", - "session_11", - "session_6", - "session_25", - "session_9", - "session_27" - ], - "sample_id": "conv-43" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_11", - "session_17", - "session_19", - "session_22", - "session_4" - ], - "question": "What books has John read?", - "recall": 0.8, - "retrieved_ids": [ - "session_22", - "session_5", - "session_14", - "session_19", - "session_11", - "session_26", - "session_15", - "session_28", - "session_4", - "session_27" - ], - "sample_id": "conv-43" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_14", - "session_26" - ], - "question": "What does John do to share his knowledge?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_26", - "session_14", - "session_20", - "session_27", - "session_11", - "session_7", - "session_6", - "session_4", - "session_5" - ], - "sample_id": "conv-43" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_26" - ], - "question": "When did John organize a basketball camp for kids?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_6", - "session_22", - "session_23", - "session_11", - "session_19", - "session_28", - "session_16", - "session_8", - "session_20" - ], - "sample_id": "conv-43" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_27" - ], - "question": "Which month was John in Italy?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_28", - "session_10", - "session_7", - "session_21", - "session_3", - "session_11", - "session_26", - "session_24", - "session_20" - ], - "sample_id": "conv-43" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_26", - "session_27", - "session_8" - ], - "question": "What fantasy movies does Tim like?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_26", - "session_12", - "session_27", - "session_20", - "session_17", - "session_15", - "session_9", - "session_5", - "session_1" - ], - "sample_id": "conv-43" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_27" - ], - "question": "What is a Star Wars book that Tim might enjoy?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_19", - "session_14", - "session_22", - "session_16", - "session_4", - "session_11", - "session_20", - "session_15", - "session_2" - ], - "sample_id": "conv-43" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_15", - "session_27", - "session_4", - "session_6" - ], - "question": "What would be a good hobby related to his travel dreams for Tim to pick up?", - "recall": 0.5, - "retrieved_ids": [ - "session_11", - "session_26", - "session_6", - "session_2", - "session_27", - "session_9", - "session_1", - "session_12", - "session_21", - "session_5" - ], - "sample_id": "conv-43" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_28" - ], - "question": "What day did Tim get into his study abroad program?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_21", - "session_28", - "session_12", - "session_20", - "session_26", - "session_18", - "session_9", - "session_5", - "session_4" - ], - "sample_id": "conv-43" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_28" - ], - "question": "When will Tim leave for Ireland?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_11", - "session_27", - "session_18", - "session_10", - "session_12", - "session_20", - "session_3", - "session_26", - "session_17" - ], - "sample_id": "conv-43" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_1", - "session_27", - "session_28" - ], - "question": "Which Star Wars-related locations would Tim enjoy during his visit to Ireland?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_28", - "session_26", - "session_12", - "session_2", - "session_25", - "session_16", - "session_20", - "session_1", - "session_6" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_1" - ], - "question": "Which team did John sign with on 21 May, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_1", - "session_6", - "session_5", - "session_25", - "session_23", - "session_22", - "session_16", - "session_3", - "session_29" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_1" - ], - "question": "What is John's position on the team he signed with?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_7", - "session_6", - "session_16", - "session_14", - "session_12", - "session_3", - "session_29", - "session_27", - "session_18" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_1" - ], - "question": "What challenge did John encounter during pre-season training?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_25", - "session_8", - "session_19", - "session_18", - "session_29", - "session_14", - "session_27", - "session_28", - "session_13" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_1" - ], - "question": "What aspects of the Harry Potter universe will be discussed in John's fan project collaborations?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_5", - "session_27", - "session_11", - "session_26", - "session_3", - "session_19", - "session_8", - "session_2", - "session_10" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What forum did Tim join recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_21", - "session_29", - "session_12", - "session_27", - "session_19", - "session_4", - "session_17", - "session_10", - "session_22" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What kind of picture did Tim share as part of their Harry Potter book collection?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_27", - "session_26", - "session_1", - "session_22", - "session_7", - "session_15", - "session_20", - "session_9", - "session_25" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "What was the highest number of points John scored in a game recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_21", - "session_6", - "session_19", - "session_22", - "session_29", - "session_20", - "session_17", - "session_10", - "session_5" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "What did John celebrate at a restaurant with teammates?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_12", - "session_21", - "session_11", - "session_9", - "session_15", - "session_24", - "session_23", - "session_7", - "session_19" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "What kind of deals did John sign with Nike and Gatorade?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_2", - "session_29", - "session_1", - "session_25", - "session_16", - "session_21", - "session_7", - "session_6", - "session_23" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "Which city is John excited to have a game at?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_29", - "session_9", - "session_2", - "session_15", - "session_1", - "session_14", - "session_7", - "session_11", - "session_26" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "How long has John been surfing?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_16", - "session_24", - "session_9", - "session_14", - "session_21", - "session_15", - "session_5", - "session_22", - "session_28" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "How does John feel while surfing?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_23", - "session_11", - "session_19", - "session_15", - "session_29", - "session_28", - "session_25", - "session_24", - "session_27" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What kind of articles has Tim been writing about for the online magazine?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_25", - "session_16", - "session_6", - "session_1", - "session_29", - "session_19", - "session_15", - "session_9", - "session_13" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "Which two fantasy novels does Tim particularly enjoy writing about?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_12", - "session_4", - "session_15", - "session_19", - "session_7", - "session_28", - "session_9", - "session_6", - "session_26" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What did Anthony and John end up playing during the charity event?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_6", - "session_4", - "session_25", - "session_22", - "session_16", - "session_21", - "session_26", - "session_28", - "session_7" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_5" - ], - "question": "What did John share with the person he skyped about?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_12", - "session_11", - "session_19", - "session_26", - "session_27", - "session_7", - "session_6", - "session_5", - "session_16" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_5" - ], - "question": "How did John describe the team bond?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_5", - "session_22", - "session_9", - "session_17", - "session_25", - "session_1", - "session_23", - "session_13", - "session_3" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_6" - ], - "question": "How did John get introduced to basketball?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_3", - "session_6", - "session_19", - "session_8", - "session_22", - "session_7", - "session_16", - "session_28", - "session_23" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_6" - ], - "question": "What is John's number one goal in his basketball career?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_11", - "session_19", - "session_12", - "session_21", - "session_27", - "session_23", - "session_7", - "session_5", - "session_20" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_6" - ], - "question": "What organization is John teaming up with for his charity work?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_11", - "session_22", - "session_12", - "session_26", - "session_28", - "session_5", - "session_27", - "session_20", - "session_4" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "When did John meet back up with his teammates after his trip in August 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_7", - "session_19", - "session_21", - "session_12", - "session_3", - "session_27", - "session_1", - "session_5", - "session_20" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What did John's teammates give him when they met on Aug 15th?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_16", - "session_3", - "session_5", - "session_12", - "session_22", - "session_27", - "session_11", - "session_21", - "session_9" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "Why did John's teammates sign the basketball they gave him?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_16", - "session_12", - "session_3", - "session_15", - "session_27", - "session_8", - "session_1", - "session_10", - "session_25" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What is the main intention behind Tim wanting to attend the book conference?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_11", - "session_26", - "session_4", - "session_6", - "session_12", - "session_15", - "session_13", - "session_29", - "session_21" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What new activity has Tim started learning in August 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_27", - "session_15", - "session_8", - "session_22", - "session_11", - "session_26", - "session_7", - "session_3", - "session_24" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "Which movie's theme is Tim's favorite to play on the piano?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_27", - "session_12", - "session_20", - "session_21", - "session_26", - "session_15", - "session_17", - "session_1", - "session_4" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What special memory does \"Harry Potter and the Philosopher's Stone\" bring to Tim?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_2", - "session_22", - "session_3", - "session_13", - "session_12", - "session_15", - "session_17", - "session_28", - "session_25" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "Which movie does Tim mention they enjoy watching during Thanksgiving?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_16", - "session_12", - "session_26", - "session_20", - "session_15", - "session_1", - "session_3", - "session_27", - "session_17" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What tradition does Tim mention they love during Thanksgiving?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_16", - "session_26", - "session_15", - "session_1", - "session_25", - "session_24", - "session_14", - "session_3", - "session_18" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_9" - ], - "question": "How long did John and his high school basketball teammates play together?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_27", - "session_8", - "session_23", - "session_12", - "session_6", - "session_15", - "session_20", - "session_14", - "session_5" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_9" - ], - "question": "How was John's experience in New York City?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_1", - "session_29", - "session_14", - "session_11", - "session_3", - "session_6", - "session_26", - "session_21", - "session_10" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_9" - ], - "question": "What did John say about NYC, enticing Tim to visit?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_26", - "session_10", - "session_29", - "session_2", - "session_15", - "session_1", - "session_16", - "session_28", - "session_13" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "What kind of soup did John make recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_6", - "session_29", - "session_19", - "session_4", - "session_22", - "session_2", - "session_14", - "session_21", - "session_17" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "What spice did John add to the soup for flavor?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_1", - "session_29", - "session_22", - "session_15", - "session_8", - "session_28", - "session_23", - "session_7", - "session_13" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "What is Tim excited to see at Universal Studios?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_28", - "session_1", - "session_2", - "session_3", - "session_26", - "session_15", - "session_6", - "session_29", - "session_7" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_11" - ], - "question": "Where are John and his teammates planning to explore on a team trip?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_9", - "session_7", - "session_10", - "session_27", - "session_6", - "session_12", - "session_20", - "session_5", - "session_19" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_11" - ], - "question": "What city did Tim suggest to John for the team trip next month?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_14", - "session_7", - "session_3", - "session_26", - "session_10", - "session_9", - "session_28", - "session_6", - "session_29" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_11" - ], - "question": "What does John want to do after his basketball career?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_3", - "session_6", - "session_21", - "session_12", - "session_27", - "session_28", - "session_23", - "session_5", - "session_20" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_11" - ], - "question": "What advice did Tim give John about picking endorsements?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_2", - "session_21", - "session_29", - "session_14", - "session_12", - "session_3", - "session_19", - "session_17", - "session_7" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_11" - ], - "question": "What book recommendation did Tim give to John for the trip?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_20", - "session_5", - "session_16", - "session_4", - "session_17", - "session_28", - "session_22", - "session_13", - "session_29" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_12" - ], - "question": "What type of venue did John and his girlfriend choose for their wedding ceremony?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_15", - "session_2", - "session_27", - "session_1", - "session_14", - "session_26", - "session_5", - "session_19", - "session_20" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_12" - ], - "question": "What was the setting for John and his wife's first dance?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_27", - "session_16", - "session_22", - "session_21", - "session_8", - "session_10", - "session_24", - "session_5", - "session_17" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_12" - ], - "question": "Which basketball team does Tim support?", - "recall": 0.0, - "retrieved_ids": [ - "session_11", - "session_23", - "session_7", - "session_6", - "session_19", - "session_3", - "session_22", - "session_8", - "session_28", - "session_16" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "What passion does Tim mention connects him with people from all over the world?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_16", - "session_13", - "session_26", - "session_6", - "session_27", - "session_11", - "session_22", - "session_21", - "session_12" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "How does John describe the game season for his team?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_13", - "session_12", - "session_1", - "session_17", - "session_28", - "session_14", - "session_27", - "session_5", - "session_20" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "How does John say his team handles tough opponents?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_12", - "session_22", - "session_24", - "session_27", - "session_19", - "session_8", - "session_5", - "session_10", - "session_20" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "What motivates John's team to get better, according to John?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_13", - "session_7", - "session_21", - "session_11", - "session_26", - "session_15", - "session_16", - "session_20", - "session_18" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "What did John's team win at the end of the season?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_24", - "session_13", - "session_1", - "session_25", - "session_21", - "session_19", - "session_14", - "session_4", - "session_28" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_14" - ], - "question": "Where did Tim capture the photography of the sunset over the mountain range?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_11", - "session_13", - "session_20", - "session_8", - "session_10", - "session_5", - "session_22", - "session_16", - "session_21" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_14" - ], - "question": "How does John feel about being seen as a mentor by some of the younger players?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_23", - "session_21", - "session_19", - "session_11", - "session_25", - "session_16", - "session_12", - "session_26", - "session_17" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_14" - ], - "question": "What does John find rewarding about mentoring the younger players?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_2", - "session_21", - "session_16", - "session_12", - "session_19", - "session_29", - "session_27", - "session_4", - "session_23" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_14" - ], - "question": "What has John been able to help the younger players achieve?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_19", - "session_25", - "session_24", - "session_29", - "session_21", - "session_16", - "session_12", - "session_23", - "session_28" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "What genre is the novel that Tim is writing?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_15", - "session_6", - "session_7", - "session_16", - "session_26", - "session_19", - "session_28", - "session_9", - "session_21" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "Who is one of Tim's sources of inspiration for writing?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_16", - "session_19", - "session_6", - "session_27", - "session_26", - "session_4", - "session_22", - "session_7", - "session_21" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "What J.K. Rowling quote does Tim resonate with?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_1", - "session_12", - "session_20", - "session_3", - "session_27", - "session_26", - "session_28", - "session_18", - "session_10" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "What does John write on the whiteboard to help him stay motivated?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_7", - "session_27", - "session_19", - "session_18", - "session_13", - "session_16", - "session_4", - "session_12", - "session_6" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "What hobby is a therapy for John when away from the court?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_15", - "session_6", - "session_5", - "session_22", - "session_14", - "session_11", - "session_24", - "session_9", - "session_17" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "What type of meal does John often cook using a slow cooker?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_19", - "session_27", - "session_1", - "session_26", - "session_24", - "session_20", - "session_3", - "session_28", - "session_10" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "How will John share the honey garlic chicken recipe with the other person?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_12", - "session_10", - "session_14", - "session_19", - "session_11", - "session_26", - "session_7", - "session_27", - "session_16" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "What was Tim's huge writing issue last week,as mentioned on November 6, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_26", - "session_24", - "session_3", - "session_19", - "session_15", - "session_17", - "session_8", - "session_6", - "session_4" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "What does Tim have that serves as a reminder of hard work and is his prized possession?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_29", - "session_2", - "session_15", - "session_5", - "session_12", - "session_20", - "session_27", - "session_7", - "session_4" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "Why do Tim and John find LeBron inspiring?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_12", - "session_19", - "session_8", - "session_10", - "session_9", - "session_7", - "session_3", - "session_14", - "session_17" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "How did John describe the views during their road trip out on the European coastline?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_15", - "session_14", - "session_7", - "session_1", - "session_27", - "session_26", - "session_28", - "session_16", - "session_3" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What is one of Tim's favorite fantasy TV shows, as mentioned on November 11, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_16", - "session_3", - "session_20", - "session_15", - "session_17", - "session_27", - "session_19", - "session_28", - "session_7" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "How does Tim stay motivated during difficult study sessions?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_19", - "session_14", - "session_15", - "session_28", - "session_27", - "session_21", - "session_20", - "session_9", - "session_4" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What did Tim say about his injury on 16 November, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_12", - "session_27", - "session_19", - "session_8", - "session_4", - "session_18", - "session_16", - "session_5", - "session_20" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "What was the setback Tim faced in his writing project on 21 November, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_16", - "session_1", - "session_12", - "session_27", - "session_15", - "session_8", - "session_4", - "session_5", - "session_20" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "How did John overcome his ankle injury from last season?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_8", - "session_20", - "session_18", - "session_28", - "session_1", - "session_12", - "session_27", - "session_5", - "session_16" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "What motivated Tim to keep pushing himself to get better in writing and reading?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_26", - "session_16", - "session_15", - "session_13", - "session_20", - "session_6", - "session_21", - "session_11", - "session_4" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "How did John overcome a mistake he made during a big game in basketball?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_12", - "session_6", - "session_8", - "session_23", - "session_3", - "session_26", - "session_27", - "session_16", - "session_15" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "What book did John recently finish rereading that left him feeling inspired and hopeful about following dreams?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_29", - "session_16", - "session_11", - "session_27", - "session_6", - "session_17", - "session_22", - "session_12", - "session_23" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "How did \"The Alchemist\" impact John's perspective on following dreams?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_11", - "session_23", - "session_6", - "session_8", - "session_4", - "session_14", - "session_28", - "session_12", - "session_17" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_20" - ], - "question": "What is John trying out to improve his strength and flexibility after recovery from ankle injury?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_20", - "session_8", - "session_5", - "session_14", - "session_7", - "session_25", - "session_12", - "session_27", - "session_18" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_20" - ], - "question": "How long does John usually hold the yoga pose he shared with Tim?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_21", - "session_15", - "session_8", - "session_24", - "session_14", - "session_16", - "session_27", - "session_12", - "session_17" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_20" - ], - "question": "Where was the forest picture shared by John on December 1,2023 taken?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_20", - "session_4", - "session_11", - "session_2", - "session_7", - "session_26", - "session_27", - "session_6", - "session_12" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_21" - ], - "question": "What did Tim recently start learning in addition to being part of a travel club and working on studies?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_19", - "session_20", - "session_26", - "session_27", - "session_9", - "session_7", - "session_29", - "session_11", - "session_12" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_21" - ], - "question": "What instrument is Tim learning to play in December 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_8", - "session_15", - "session_27", - "session_12", - "session_19", - "session_20", - "session_16", - "session_26", - "session_7" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_21" - ], - "question": "How long has Tim been playing the piano for, as of December 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_21", - "session_16", - "session_15", - "session_24", - "session_9", - "session_14", - "session_5", - "session_22", - "session_28" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_22" - ], - "question": "What book did Tim just finish reading on 8th December, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_8", - "session_22", - "session_5", - "session_19", - "session_3", - "session_26", - "session_15", - "session_6", - "session_20" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_22" - ], - "question": "Which book did Tim recommend to John as a good story on 8th December, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_27", - "session_11", - "session_6", - "session_28", - "session_13", - "session_29", - "session_16", - "session_4", - "session_5" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "What is the topic of discussion between John and Tim on 11 December, 2023?", - "recall": 0.0, - "retrieved_ids": [], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "What kind of game did John have a career-high in assists in?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_4", - "session_25", - "session_13", - "session_6", - "session_9", - "session_21", - "session_27", - "session_11", - "session_26" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "What was John's way of dealing with doubts and stress when he was younger?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_8", - "session_14", - "session_9", - "session_25", - "session_16", - "session_27", - "session_20", - "session_3", - "session_12" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "How did John feel about the atmosphere during the big game against the rival team?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_12", - "session_19", - "session_24", - "session_21", - "session_6", - "session_3", - "session_16", - "session_7", - "session_28" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "How did John feel after being able to jog without pain?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_19", - "session_13", - "session_14", - "session_21", - "session_3", - "session_28", - "session_10", - "session_11", - "session_27" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_25" - ], - "question": "What kind of deal did John get in December?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_29", - "session_3", - "session_2", - "session_1", - "session_22", - "session_21", - "session_4", - "session_9", - "session_23" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_25" - ], - "question": "Where was the photoshoot done for John's gear deal?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_3", - "session_23", - "session_29", - "session_5", - "session_21", - "session_27", - "session_20", - "session_28", - "session_26" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_25" - ], - "question": "In which area has John's team seen the most growth during training?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_1", - "session_21", - "session_14", - "session_11", - "session_19", - "session_29", - "session_26", - "session_8", - "session_17" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_26" - ], - "question": "What type of seminars is John conducting?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_15", - "session_1", - "session_24", - "session_19", - "session_20", - "session_27", - "session_3", - "session_28", - "session_13" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_26" - ], - "question": "What activity did Tim do after reading the stories about the Himalayan trek?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_19", - "session_15", - "session_22", - "session_21", - "session_27", - "session_23", - "session_6", - "session_24", - "session_4" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_26" - ], - "question": "What is one cause that John supports with his influence and resources?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_27", - "session_15", - "session_23", - "session_5", - "session_12", - "session_20", - "session_11", - "session_7", - "session_2" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_26" - ], - "question": "What new fantasy TV series is Tim excited about?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_8", - "session_2", - "session_17", - "session_1", - "session_22", - "session_29", - "session_20", - "session_3", - "session_23" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_27" - ], - "question": "Which language is Tim learning?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_15", - "session_21", - "session_20", - "session_8", - "session_7", - "session_16", - "session_12", - "session_19", - "session_26" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_27" - ], - "question": "What language does Tim know besides German?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_11", - "session_5", - "session_29", - "session_22", - "session_21", - "session_16", - "session_15", - "session_28", - "session_9" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_27" - ], - "question": "What book did Tim get in Italy that inspired him to cook?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_11", - "session_15", - "session_12", - "session_22", - "session_14", - "session_16", - "session_19", - "session_6", - "session_10" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_27" - ], - "question": "What is John's favorite book series?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_20", - "session_8", - "session_17", - "session_7", - "session_22", - "session_12", - "session_27", - "session_15", - "session_23" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_27" - ], - "question": "According to John, who is his favorite character from Lord of the Rings?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_26", - "session_1", - "session_12", - "session_4", - "session_17", - "session_5", - "session_22", - "session_20", - "session_6" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_27" - ], - "question": "Why does John like Aragorn from Lord of the Rings?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_26", - "session_29", - "session_13", - "session_5", - "session_7", - "session_3", - "session_11", - "session_12", - "session_20" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_27" - ], - "question": "What kind of painting does John have in his room as a reminder?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_16", - "session_29", - "session_2", - "session_7", - "session_12", - "session_4", - "session_9", - "session_5", - "session_20" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_27" - ], - "question": "What is the painting of Aragorn a reminder for John to be in everything he does?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_29", - "session_16", - "session_2", - "session_20", - "session_3", - "session_15", - "session_17", - "session_7", - "session_12" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_27" - ], - "question": "What map does Tim show to his friend John?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_12", - "session_20", - "session_26", - "session_15", - "session_9", - "session_7", - "session_6", - "session_5", - "session_11" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_28" - ], - "question": "Where will Tim be going for a semester abroad?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_24", - "session_13", - "session_14", - "session_1", - "session_29", - "session_25", - "session_9", - "session_27", - "session_15" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_28" - ], - "question": "Which city in Ireland will Tim be staying in during his semester abroad?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_29", - "session_12", - "session_14", - "session_11", - "session_27", - "session_9", - "session_20", - "session_3", - "session_5" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_28" - ], - "question": "What charity event did John organize recently in 2024?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_22", - "session_26", - "session_10", - "session_28", - "session_29", - "session_11", - "session_19", - "session_4", - "session_7" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_29" - ], - "question": "What achievement did John share with Tim in January 2024?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_3", - "session_11", - "session_12", - "session_4", - "session_7", - "session_6", - "session_29", - "session_25", - "session_27" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_29" - ], - "question": "What was Johns's reaction to sealing the deal with the beverage company?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_21", - "session_25", - "session_3", - "session_11", - "session_27", - "session_20", - "session_26", - "session_23", - "session_28" - ], - "sample_id": "conv-43" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_29" - ], - "question": "Which city did John recommend to Tim in January 2024?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_29", - "session_28", - "session_9", - "session_13", - "session_14", - "session_22", - "session_16", - "session_3", - "session_4" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_1" - ], - "question": "Which team did Tim sign with on 21 May, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_1", - "session_6", - "session_5", - "session_25", - "session_23", - "session_22", - "session_16", - "session_3", - "session_29" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_1" - ], - "question": "What is Tim's position on the team he signed with?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_16", - "session_14", - "session_7", - "session_6", - "session_12", - "session_3", - "session_27", - "session_29", - "session_18" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_1" - ], - "question": "What challenge did Tim encounter during pre-season training?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_25", - "session_8", - "session_19", - "session_18", - "session_14", - "session_29", - "session_27", - "session_28", - "session_13" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_2" - ], - "question": "What cult did Tim join recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_2", - "session_29", - "session_12", - "session_27", - "session_19", - "session_14", - "session_17", - "session_10", - "session_22" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_3" - ], - "question": "What was the highest number of points Tim scored in a game recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_21", - "session_6", - "session_22", - "session_19", - "session_29", - "session_20", - "session_17", - "session_10", - "session_5" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_3" - ], - "question": "What did Tim celebrate at a restaurant with teammates?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_12", - "session_21", - "session_11", - "session_15", - "session_9", - "session_24", - "session_23", - "session_7", - "session_19" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_3" - ], - "question": "What kind of deals did Tim sign with Nike and Gatorade?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_2", - "session_21", - "session_29", - "session_1", - "session_25", - "session_16", - "session_7", - "session_6", - "session_23" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_3" - ], - "question": "How does Tim feel while surfing?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_23", - "session_11", - "session_19", - "session_15", - "session_29", - "session_28", - "session_25", - "session_24", - "session_27" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What kind of articles has John been writing about for the online magazine?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_25", - "session_16", - "session_6", - "session_1", - "session_29", - "session_19", - "session_15", - "session_9", - "session_13" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "Which two mystery novels does Tim particularly enjoy writing about?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_12", - "session_15", - "session_4", - "session_7", - "session_19", - "session_28", - "session_9", - "session_6", - "session_26" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What did Anthony and Tim end up playing during the charity event?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_4", - "session_6", - "session_25", - "session_22", - "session_16", - "session_21", - "session_26", - "session_28", - "session_7" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_6" - ], - "question": "How did Tim get introduced to basketball?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_3", - "session_6", - "session_19", - "session_8", - "session_22", - "session_7", - "session_16", - "session_28", - "session_23" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_6" - ], - "question": "What is Tim's number one goal in his basketball career?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_19", - "session_11", - "session_12", - "session_21", - "session_27", - "session_23", - "session_7", - "session_5", - "session_20" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_6" - ], - "question": "What organization is Tim teaming up with for his charity work?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_11", - "session_22", - "session_12", - "session_26", - "session_28", - "session_5", - "session_27", - "session_20", - "session_4" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_7" - ], - "question": "What did Tim's teammates give him when they met on Aug 15th?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_16", - "session_3", - "session_5", - "session_12", - "session_22", - "session_27", - "session_11", - "session_21", - "session_9" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_7" - ], - "question": "Why did John's teammates sign the football they gave him?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_16", - "session_15", - "session_1", - "session_12", - "session_3", - "session_27", - "session_8", - "session_10", - "session_25" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_7" - ], - "question": "What is the main intention behind John wanting to attend the book conference?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_11", - "session_4", - "session_26", - "session_6", - "session_12", - "session_15", - "session_13", - "session_29", - "session_21" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_8" - ], - "question": "What new activity has John started learning in August 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_22", - "session_21", - "session_15", - "session_8", - "session_11", - "session_26", - "session_7", - "session_3", - "session_24" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_8" - ], - "question": "What special memory does \"Fifty Shades of Grey\" bring to Tim?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_12", - "session_16", - "session_2", - "session_3", - "session_15", - "session_17", - "session_28", - "session_13", - "session_25" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_8" - ], - "question": "Which movie does John mention they enjoy watching during Thanksgiving?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_20", - "session_16", - "session_12", - "session_26", - "session_15", - "session_1", - "session_3", - "session_27", - "session_17" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_8" - ], - "question": "What tradition does Tim mention they love during Halloween?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_16", - "session_26", - "session_15", - "session_14", - "session_1", - "session_25", - "session_24", - "session_3", - "session_18" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_9" - ], - "question": "How long did Tim and his high school basketball teammates play together?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_27", - "session_8", - "session_23", - "session_12", - "session_6", - "session_15", - "session_20", - "session_14", - "session_5" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_9" - ], - "question": "How was Tim's experience in New York City?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_1", - "session_29", - "session_14", - "session_11", - "session_3", - "session_6", - "session_26", - "session_21", - "session_10" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_10" - ], - "question": "What spice did Tim add to the soup for flavor?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_1", - "session_29", - "session_22", - "session_15", - "session_8", - "session_13", - "session_28", - "session_7", - "session_25" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_10" - ], - "question": "What is Tim excited to see at Disneyland?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_2", - "session_3", - "session_26", - "session_15", - "session_28", - "session_6", - "session_10", - "session_29", - "session_7" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_11" - ], - "question": "Where are John and his teammates planning to avoid on a team trip?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_6", - "session_7", - "session_9", - "session_10", - "session_12", - "session_27", - "session_20", - "session_5", - "session_19" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_11" - ], - "question": "What does Tim want to do after his basketball career?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_3", - "session_6", - "session_21", - "session_12", - "session_27", - "session_28", - "session_23", - "session_5", - "session_20" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_12" - ], - "question": "What type of venue did John and his girlfriend choose for their breakup?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_15", - "session_2", - "session_27", - "session_1", - "session_14", - "session_26", - "session_5", - "session_19", - "session_20" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_13" - ], - "question": "What passion does John mention connects him with people from all over the world?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_16", - "session_13", - "session_26", - "session_6", - "session_27", - "session_11", - "session_22", - "session_21", - "session_12" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_13" - ], - "question": "How does Tim say his team handles tough opponents?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_12", - "session_22", - "session_24", - "session_27", - "session_19", - "session_8", - "session_5", - "session_10", - "session_20" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_14" - ], - "question": "Where did Tim capture the painting of the sunset over the mountain range?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_11", - "session_16", - "session_13", - "session_20", - "session_8", - "session_5", - "session_22", - "session_21", - "session_27" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_14" - ], - "question": "What does Tim find rewarding about mentoring the younger players?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_2", - "session_21", - "session_16", - "session_12", - "session_19", - "session_29", - "session_27", - "session_4", - "session_23" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_14" - ], - "question": "What has Tim been able to help the younger players achieve?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_19", - "session_25", - "session_24", - "session_29", - "session_21", - "session_16", - "session_12", - "session_23", - "session_28" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_15" - ], - "question": "What genre is the novel that John is writing?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_15", - "session_6", - "session_7", - "session_16", - "session_26", - "session_19", - "session_28", - "session_9", - "session_21" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_15" - ], - "question": "Who is one of Tim's sources of inspiration for painting?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_16", - "session_26", - "session_15", - "session_7", - "session_22", - "session_21", - "session_6", - "session_19", - "session_4" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_15" - ], - "question": "What does Tim write on the whiteboard to help him stay motivated?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_7", - "session_27", - "session_19", - "session_18", - "session_13", - "session_16", - "session_4", - "session_12", - "session_6" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_15" - ], - "question": "What hobby is a therapy for Tim when away from the court?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_15", - "session_6", - "session_5", - "session_22", - "session_14", - "session_11", - "session_24", - "session_9", - "session_17" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_15" - ], - "question": "What type of meal does Tim often cook using a slow cooker?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_26", - "session_19", - "session_27", - "session_12", - "session_20", - "session_3", - "session_28", - "session_18", - "session_10" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_15" - ], - "question": "How will Tim share the honey garlic chicken recipe with the other person?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_12", - "session_10", - "session_14", - "session_19", - "session_11", - "session_26", - "session_7", - "session_27", - "session_16" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_17" - ], - "question": "What is one of Tim's favorite crime TV shows, as mentioned on November 11, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_16", - "session_3", - "session_20", - "session_15", - "session_17", - "session_7", - "session_28", - "session_27", - "session_19" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_19" - ], - "question": "What was the setback Tim faced in his coding project on 21 November, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_13", - "session_1", - "session_12", - "session_24", - "session_27", - "session_26", - "session_8", - "session_5", - "session_20" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_19" - ], - "question": "How did Tim overcome his ankle injury from last season?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_8", - "session_20", - "session_18", - "session_28", - "session_1", - "session_12", - "session_27", - "session_5", - "session_16" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_19" - ], - "question": "What motivated John to keep pushing himself to get better in writing and reading?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_26", - "session_16", - "session_15", - "session_20", - "session_13", - "session_6", - "session_21", - "session_11", - "session_4" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_19" - ], - "question": "How did Tim overcome a mistake he made during a big game in basketball?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_12", - "session_6", - "session_8", - "session_23", - "session_3", - "session_26", - "session_27", - "session_16", - "session_15" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_20" - ], - "question": "What is Tim trying out to improve his strength and flexibility after recovery from ankle injury?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_20", - "session_8", - "session_14", - "session_5", - "session_7", - "session_25", - "session_12", - "session_27", - "session_18" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_21" - ], - "question": "What did John recently start learning in addition to being part of a travel club and working on studies?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_20", - "session_19", - "session_26", - "session_27", - "session_9", - "session_7", - "session_29", - "session_11", - "session_12" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_21" - ], - "question": "What instrument is John learning to play in December 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_8", - "session_15", - "session_27", - "session_12", - "session_19", - "session_20", - "session_16", - "session_26", - "session_7" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_21" - ], - "question": "How long has John been playing the piano for, as of December 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_21", - "session_16", - "session_15", - "session_24", - "session_9", - "session_14", - "session_5", - "session_22", - "session_28" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_22" - ], - "question": "What movie did Tim just finish watching on 8th December, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_26", - "session_8", - "session_20", - "session_1", - "session_22", - "session_12", - "session_19", - "session_27", - "session_15" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_23" - ], - "question": "What kind of game did Tim have a career-high in assists in?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_4", - "session_25", - "session_13", - "session_6", - "session_9", - "session_21", - "session_27", - "session_11", - "session_26" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_23" - ], - "question": "What was Tim's way of dealing with doubts and stress when he was younger?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_8", - "session_14", - "session_9", - "session_25", - "session_16", - "session_27", - "session_20", - "session_3", - "session_12" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_25" - ], - "question": "Where was the photoshoot done for John's fragrance deal?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_3", - "session_27", - "session_23", - "session_5", - "session_29", - "session_21", - "session_20", - "session_28", - "session_26" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_25" - ], - "question": "In which area has Tim's team seen the most growth during training?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_1", - "session_21", - "session_14", - "session_11", - "session_19", - "session_29", - "session_26", - "session_8", - "session_17" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_26" - ], - "question": "What type of seminars is Tim conducting?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_1", - "session_19", - "session_12", - "session_20", - "session_3", - "session_27", - "session_28", - "session_18", - "session_10" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_26" - ], - "question": "What is one cause that John opposes with his influence and resources?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_27", - "session_8", - "session_12", - "session_15", - "session_5", - "session_20", - "session_11", - "session_19", - "session_18" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_26" - ], - "question": "What new fantasy TV series is John excited about?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_2", - "session_8", - "session_17", - "session_1", - "session_29", - "session_22", - "session_20", - "session_3", - "session_23" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_27" - ], - "question": "Which language is John learning?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_15", - "session_21", - "session_20", - "session_8", - "session_7", - "session_16", - "session_12", - "session_19", - "session_26" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_27" - ], - "question": "According to John, who is his least favorite character from Lord of the Rings?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_26", - "session_1", - "session_12", - "session_4", - "session_17", - "session_5", - "session_22", - "session_20", - "session_6" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_27" - ], - "question": "Why does Tim like Aragorn from Lord of the Rings?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_26", - "session_7", - "session_13", - "session_29", - "session_5", - "session_3", - "session_12", - "session_11", - "session_20" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_27" - ], - "question": "What kind of painting does Tim have in his room as a reminder?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_16", - "session_29", - "session_2", - "session_7", - "session_12", - "session_4", - "session_9", - "session_5", - "session_20" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_27" - ], - "question": "What is the sculpture of Aragorn a reminder for John to be in everything he does?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_29", - "session_16", - "session_20", - "session_2", - "session_3", - "session_15", - "session_17", - "session_7", - "session_12" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_28" - ], - "question": "Which city in Ireland will John be staying in during his semester abroad?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_29", - "session_14", - "session_12", - "session_11", - "session_27", - "session_9", - "session_20", - "session_3", - "session_5" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_28" - ], - "question": "What charity event did Tim organize recently in 2024?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_22", - "session_26", - "session_10", - "session_28", - "session_29", - "session_11", - "session_19", - "session_4", - "session_7" - ], - "sample_id": "conv-43" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_29" - ], - "question": "What was Tims's reaction to sealing the deal with the beverage company?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_21", - "session_25", - "session_3", - "session_11", - "session_12", - "session_20", - "session_23", - "session_28", - "session_18" - ], - "sample_id": "conv-43" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_1" - ], - "question": "Which year did Audrey adopt the first three of her dogs?", - "recall": 0.0, - "retrieved_ids": [ - "session_17", - "session_2", - "session_28", - "session_26", - "session_27", - "session_11", - "session_24", - "session_15", - "session_5", - "session_19" - ], - "sample_id": "conv-44" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_1" - ], - "question": "When did Andrew start his new job as a financial analyst?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_28", - "session_26", - "session_14", - "session_4", - "session_16", - "session_25", - "session_22", - "session_21", - "session_17" - ], - "sample_id": "conv-44" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_13", - "session_19", - "session_23", - "session_25" - ], - "question": "What kind of indoor activities has Andrew pursued with his girlfriend?", - "recall": 0.5, - "retrieved_ids": [ - "session_26", - "session_24", - "session_15", - "session_27", - "session_3", - "session_14", - "session_9", - "session_6", - "session_23", - "session_13" - ], - "sample_id": "conv-44" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_13", - "session_23", - "session_25", - "session_27", - "session_3", - "session_4", - "session_6" - ], - "question": "What kind of places have Andrew and his girlfriend checked out around the city?", - "recall": 0.7143, - "retrieved_ids": [ - "session_3", - "session_26", - "session_23", - "session_15", - "session_24", - "session_13", - "session_21", - "session_25", - "session_22", - "session_6" - ], - "sample_id": "conv-44" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_3" - ], - "question": "When did Audrey make muffins for herself?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_19", - "session_12", - "session_21", - "session_6", - "session_2", - "session_18", - "session_26", - "session_16", - "session_7" - ], - "sample_id": "conv-44" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_4" - ], - "question": "When did Audrey see a hummingbird?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_21", - "session_16", - "session_7", - "session_2", - "session_3", - "session_26", - "session_18", - "session_12", - "session_6" - ], - "sample_id": "conv-44" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_2" - ], - "question": "When did Audrey adopt Pixie?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_15", - "session_26", - "session_24", - "session_5", - "session_7", - "session_12", - "session_21", - "session_16", - "session_3" - ], - "sample_id": "conv-44" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_1", - "session_2" - ], - "question": "How many years passed between Audrey adopting Pixie and her other three dogs?", - "recall": 0.5, - "retrieved_ids": [ - "session_15", - "session_2", - "session_28", - "session_11", - "session_19", - "session_20", - "session_23", - "session_24", - "session_25", - "session_5" - ], - "sample_id": "conv-44" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_2" - ], - "question": "Did Andrew have a pet dog during March 2023?", - "recall": 0.0, - "retrieved_ids": [ - "session_18", - "session_7", - "session_8", - "session_14", - "session_6", - "session_12", - "session_21", - "session_3", - "session_26", - "session_16" - ], - "sample_id": "conv-44" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_14", - "session_16", - "session_27", - "session_6" - ], - "question": "What kind of classes or groups has Audrey joined to take better care of her dogs?", - "recall": 0.6, - "retrieved_ids": [ - "session_27", - "session_19", - "session_26", - "session_6", - "session_11", - "session_13", - "session_8", - "session_14", - "session_15", - "session_2" - ], - "sample_id": "conv-44" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_6" - ], - "question": "When did Audrey's positive reinforcement training course for dogs take place?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_21", - "session_16", - "session_26", - "session_8", - "session_27", - "session_7", - "session_20", - "session_19", - "session_10" - ], - "sample_id": "conv-44" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_8" - ], - "question": "When did Andrew go rock climbing?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_17", - "session_2", - "session_12", - "session_21", - "session_16", - "session_7", - "session_6", - "session_3", - "session_26" - ], - "sample_id": "conv-44" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_14", - "session_17", - "session_8" - ], - "question": "What outdoor activities has Andrew done other than hiking in nature?", - "recall": 0.6667, - "retrieved_ids": [ - "session_14", - "session_23", - "session_24", - "session_13", - "session_27", - "session_18", - "session_8", - "session_19", - "session_25", - "session_5" - ], - "sample_id": "conv-44" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_9" - ], - "question": "When did Audrey move to a new place?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_4", - "session_7", - "session_2", - "session_28", - "session_18", - "session_12", - "session_21", - "session_3", - "session_11" - ], - "sample_id": "conv-44" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_3", - "session_9" - ], - "question": "What is something that Andrew really misses while working in the city?", - "recall": 0.5, - "retrieved_ids": [ - "session_3", - "session_18", - "session_21", - "session_10", - "session_27", - "session_23", - "session_20", - "session_12", - "session_2", - "session_22" - ], - "sample_id": "conv-44" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_7" - ], - "question": "What is a shared frustration regarding dog ownership for Audrey and Andrew?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_19", - "session_10", - "session_13", - "session_20", - "session_8", - "session_9", - "session_4", - "session_14", - "session_7" - ], - "sample_id": "conv-44" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_11" - ], - "question": "When is Andrew going to go hiking with Audrey?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_12", - "session_4", - "session_7", - "session_11", - "session_18", - "session_23", - "session_14", - "session_20", - "session_1" - ], - "sample_id": "conv-44" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_11", - "session_24", - "session_26" - ], - "question": "How many times did Audrey and Andew plan to hike together?", - "recall": 0.3333, - "retrieved_ids": [ - "session_20", - "session_23", - "session_3", - "session_16", - "session_12", - "session_18", - "session_26", - "session_2", - "session_25", - "session_28" - ], - "sample_id": "conv-44" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_11", - "session_2" - ], - "question": "Where did Audrey get Pixie from?", - "recall": 0.5, - "retrieved_ids": [ - "session_2", - "session_26", - "session_15", - "session_18", - "session_21", - "session_12", - "session_16", - "session_3", - "session_6", - "session_7" - ], - "sample_id": "conv-44" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_10", - "session_12" - ], - "question": "What is an indoor activity that Andrew would enjoy doing while make his dog happy?", - "recall": 0.0, - "retrieved_ids": [ - "session_26", - "session_24", - "session_27", - "session_23", - "session_18", - "session_9", - "session_21", - "session_8", - "session_5", - "session_25" - ], - "sample_id": "conv-44" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_10" - ], - "question": "Which meat does Audrey prefer eating more than others?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_13", - "session_10", - "session_19", - "session_15", - "session_27", - "session_22", - "session_4", - "session_2", - "session_3" - ], - "sample_id": "conv-44" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_14", - "session_6" - ], - "question": "What are the classes that Audrey took for her dogs to?", - "recall": 0.6667, - "retrieved_ids": [ - "session_6", - "session_15", - "session_21", - "session_28", - "session_14", - "session_2", - "session_19", - "session_23", - "session_7", - "session_8" - ], - "sample_id": "conv-44" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_14" - ], - "question": "Where did Andrew go during the first weekend of August 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_27", - "session_25", - "session_18", - "session_17", - "session_14", - "session_28", - "session_26", - "session_7", - "session_8" - ], - "sample_id": "conv-44" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_2", - "session_5" - ], - "question": "What are some problems that Andrew faces before he adopted Toby?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_17", - "session_27", - "session_23", - "session_5", - "session_7", - "session_2", - "session_13", - "session_14", - "session_26" - ], - "sample_id": "conv-44" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_13", - "session_2" - ], - "question": "Did Audrey and Andrew grow up with a pet dog?", - "recall": 0.5, - "retrieved_ids": [ - "session_14", - "session_12", - "session_6", - "session_19", - "session_2", - "session_21", - "session_7", - "session_3", - "session_26", - "session_16" - ], - "sample_id": "conv-44" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_17" - ], - "question": "When did Andrew and his girlfriend go fishing?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_15", - "session_26", - "session_3", - "session_6", - "session_25", - "session_24", - "session_13", - "session_21", - "session_16" - ], - "sample_id": "conv-44" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_12", - "session_16", - "session_18" - ], - "question": "What is the biggest stressor in Andrew's life besides not being able to hike frequently?", - "recall": 0.5, - "retrieved_ids": [ - "session_14", - "session_22", - "session_9", - "session_4", - "session_3", - "session_11", - "session_10", - "session_23", - "session_18", - "session_27" - ], - "sample_id": "conv-44" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_12", - "session_16", - "session_18" - ], - "question": "How does Andrew feel about his current work?", - "recall": 0.5, - "retrieved_ids": [ - "session_22", - "session_12", - "session_13", - "session_26", - "session_21", - "session_15", - "session_6", - "session_16", - "session_24", - "session_25" - ], - "sample_id": "conv-44" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_19", - "session_4" - ], - "question": "What is something that Audrey often dresses up her dogs with?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_5", - "session_27", - "session_4", - "session_14", - "session_9", - "session_18", - "session_1", - "session_15", - "session_2" - ], - "sample_id": "conv-44" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_19", - "session_2" - ], - "question": "What are the names of Audrey's dogs?", - "recall": 0.6667, - "retrieved_ids": [ - "session_1", - "session_24", - "session_26", - "session_6", - "session_28", - "session_27", - "session_5", - "session_11", - "session_23", - "session_2" - ], - "sample_id": "conv-44" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_20" - ], - "question": "When is Andrew planning to go to the beach with his girlfriend?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_26", - "session_11", - "session_21", - "session_13", - "session_6", - "session_25", - "session_24", - "session_20", - "session_16" - ], - "sample_id": "conv-44" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_14", - "session_24" - ], - "question": "What has Andrew done with his dogs?", - "recall": 0.5, - "retrieved_ids": [ - "session_13", - "session_24", - "session_15", - "session_26", - "session_25", - "session_11", - "session_6", - "session_16", - "session_21", - "session_2" - ], - "sample_id": "conv-44" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_15", - "session_23", - "session_3" - ], - "question": "What kind of tattoo does Audrey have on her arm?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_3", - "session_19", - "session_23", - "session_26", - "session_22", - "session_25", - "session_2", - "session_13", - "session_1" - ], - "sample_id": "conv-44" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_12", - "session_18", - "session_21" - ], - "question": "What can Andrew potentially do to improve his stress and accomodate his living situation with his dogs?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_24", - "session_16", - "session_21", - "session_18", - "session_13", - "session_6", - "session_15", - "session_10", - "session_25" - ], - "sample_id": "conv-44" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_12", - "session_24" - ], - "question": "How many months passed between Andrew adopting Toby and Buddy?", - "recall": 0.5, - "retrieved_ids": [ - "session_24", - "session_20", - "session_28", - "session_14", - "session_23", - "session_19", - "session_15", - "session_25", - "session_2", - "session_5" - ], - "sample_id": "conv-44" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_12", - "session_24", - "session_28" - ], - "question": "What are the names of Andrew's dogs?", - "recall": 0.6667, - "retrieved_ids": [ - "session_1", - "session_24", - "session_26", - "session_6", - "session_28", - "session_27", - "session_5", - "session_11", - "session_23", - "session_2" - ], - "sample_id": "conv-44" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_25", - "session_3" - ], - "question": "What are some foods that Audrey likes eating?", - "recall": 0.6667, - "retrieved_ids": [ - "session_10", - "session_3", - "session_18", - "session_24", - "session_9", - "session_20", - "session_19", - "session_23", - "session_26", - "session_21" - ], - "sample_id": "conv-44" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_25" - ], - "question": "When did Audrey get into an accident in the park?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_4", - "session_21", - "session_3", - "session_11", - "session_19", - "session_22", - "session_12", - "session_10", - "session_6" - ], - "sample_id": "conv-44" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_25" - ], - "question": "When did Andrew and his girlfriend go on a wine tasting trip?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_15", - "session_26", - "session_17", - "session_20", - "session_3", - "session_10", - "session_21", - "session_6", - "session_5" - ], - "sample_id": "conv-44" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_2", - "session_23" - ], - "question": "What did Audrey get wtih having so many dogs?", - "recall": 0.5, - "retrieved_ids": [ - "session_28", - "session_3", - "session_20", - "session_23", - "session_26", - "session_25", - "session_16", - "session_21", - "session_12", - "session_7" - ], - "sample_id": "conv-44" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_14", - "session_23", - "session_4" - ], - "question": "What is a good place for dogs to run around freely and meet new friends?", - "recall": 0.6667, - "retrieved_ids": [ - "session_11", - "session_5", - "session_4", - "session_27", - "session_9", - "session_23", - "session_8", - "session_28", - "session_7", - "session_12" - ], - "sample_id": "conv-44" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_19", - "session_26" - ], - "question": "What are the breeds of Audrey's dogs?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_26", - "session_5", - "session_6", - "session_27", - "session_14", - "session_11", - "session_23", - "session_10", - "session_2" - ], - "sample_id": "conv-44" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_26", - "session_6" - ], - "question": "What technique is Audrey using to discipline her dogs?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_19", - "session_16", - "session_15", - "session_27", - "session_11", - "session_6", - "session_2", - "session_5", - "session_23" - ], - "sample_id": "conv-44" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_11" - ], - "question": "Which US state do Audrey and Andrew potentially live in?", - "recall": 0.0, - "retrieved_ids": [ - "session_5", - "session_1", - "session_18", - "session_27", - "session_21", - "session_24", - "session_12", - "session_13", - "session_3", - "session_19" - ], - "sample_id": "conv-44" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_11", - "session_5" - ], - "question": "Which national park could Audrey and Andrew be referring to in their conversations?", - "recall": 0.5, - "retrieved_ids": [ - "session_7", - "session_26", - "session_5", - "session_23", - "session_14", - "session_12", - "session_27", - "session_22", - "session_24", - "session_10" - ], - "sample_id": "conv-44" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_12", - "session_24", - "session_28" - ], - "question": "How many pets will Andrew have, as of December 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_25", - "session_12", - "session_1", - "session_24", - "session_6", - "session_13", - "session_20", - "session_15", - "session_23" - ], - "sample_id": "conv-44" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_12", - "session_24" - ], - "question": "How many pets did Andrew have, as of September 2023?", - "recall": 0.5, - "retrieved_ids": [ - "session_28", - "session_25", - "session_12", - "session_20", - "session_23", - "session_7", - "session_21", - "session_16", - "session_26", - "session_3" - ], - "sample_id": "conv-44" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_24", - "session_28" - ], - "question": "How many months passed between Andrew adopting Buddy and Scout", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_20", - "session_24", - "session_14", - "session_23", - "session_25", - "session_19", - "session_2", - "session_15", - "session_5" - ], - "sample_id": "conv-44" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_15", - "session_28" - ], - "question": "What does Andrew view his pets as?", - "recall": 0.5, - "retrieved_ids": [ - "session_26", - "session_14", - "session_1", - "session_24", - "session_15", - "session_4", - "session_8", - "session_13", - "session_11", - "session_21" - ], - "sample_id": "conv-44" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_15", - "session_23" - ], - "question": "What does Audrey view her pets as?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_15", - "session_11", - "session_4", - "session_14", - "session_26", - "session_8", - "session_19", - "session_23", - "session_2" - ], - "sample_id": "conv-44" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_16", - "session_17" - ], - "question": "What is a skill that Audrey learned to take care of her dogs?", - "recall": 0.5, - "retrieved_ids": [ - "session_16", - "session_27", - "session_6", - "session_18", - "session_19", - "session_20", - "session_25", - "session_14", - "session_15", - "session_2" - ], - "sample_id": "conv-44" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_18", - "session_24", - "session_9" - ], - "question": "What items has Audrey bought or made for her dogs?", - "recall": 0.5, - "retrieved_ids": [ - "session_2", - "session_18", - "session_27", - "session_24", - "session_15", - "session_3", - "session_19", - "session_7", - "session_12", - "session_8" - ], - "sample_id": "conv-44" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_1", - "session_20", - "session_23" - ], - "question": "What is something that Andrew could do to make birdwatching hobby to fit in his city schedule?", - "recall": 0.3333, - "retrieved_ids": [ - "session_20", - "session_24", - "session_26", - "session_12", - "session_15", - "session_10", - "session_21", - "session_22", - "session_28", - "session_2" - ], - "sample_id": "conv-44" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_2", - "session_3", - "session_5", - "session_8" - ], - "question": "What is a career that Andrew could potentially pursue with his love for animals and nature?", - "recall": 0.0, - "retrieved_ids": [ - "session_26", - "session_24", - "session_14", - "session_23", - "session_7", - "session_22", - "session_6", - "session_13", - "session_15", - "session_21" - ], - "sample_id": "conv-44" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_13", - "session_23", - "session_27", - "session_4" - ], - "question": "What activity do Audrey's dogs like to do in the dog park?", - "recall": 0.6, - "retrieved_ids": [ - "session_14", - "session_8", - "session_27", - "session_28", - "session_24", - "session_5", - "session_11", - "session_23", - "session_4", - "session_18" - ], - "sample_id": "conv-44" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_28" - ], - "question": "When did Andrew make his dogs a fun indoor area?", - "recall": 0.0, - "retrieved_ids": [ - "session_9", - "session_26", - "session_24", - "session_15", - "session_13", - "session_11", - "session_20", - "session_21", - "session_16", - "session_6" - ], - "sample_id": "conv-44" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_28", - "session_5" - ], - "question": "Has Andrew moved into a new apartment for his dogs?", - "recall": 0.5, - "retrieved_ids": [ - "session_2", - "session_24", - "session_6", - "session_5", - "session_13", - "session_9", - "session_16", - "session_21", - "session_15", - "session_25" - ], - "sample_id": "conv-44" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_28" - ], - "question": "When did Andrew adopt Scout?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_2", - "session_24", - "session_5", - "session_7", - "session_12", - "session_21", - "session_16", - "session_3", - "session_26" - ], - "sample_id": "conv-44" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_25" - ], - "question": "What did Audrey eat for dinner on October 24, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_15", - "session_3", - "session_10", - "session_6", - "session_16", - "session_26", - "session_21", - "session_7", - "session_12" - ], - "sample_id": "conv-44" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_12" - ], - "question": "How long has it been since Andrew adopted his first pet, as of November 2023?", - "recall": 0.0, - "retrieved_ids": [ - "session_24", - "session_28", - "session_13", - "session_21", - "session_17", - "session_26", - "session_5", - "session_25", - "session_27", - "session_2" - ], - "sample_id": "conv-44" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_12", - "session_24", - "session_28" - ], - "question": "How many dogs does Andrew have?", - "recall": 0.3333, - "retrieved_ids": [ - "session_28", - "session_20", - "session_23", - "session_25", - "session_26", - "session_6", - "session_27", - "session_11", - "session_2", - "session_5" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_1" - ], - "question": "Which specific type of bird mesmerizes Andrew?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_26", - "session_7", - "session_17", - "session_22", - "session_9", - "session_19", - "session_20", - "session_25", - "session_14" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What did Andrew express missing about exploring nature trails with his family's dog?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_9", - "session_26", - "session_2", - "session_24", - "session_28", - "session_23", - "session_15", - "session_4", - "session_21" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "What kind of pastries did Andrew and his girlfriend have at the cafe?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_26", - "session_15", - "session_23", - "session_19", - "session_16", - "session_6", - "session_25", - "session_13", - "session_22" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "What kind of flowers does Audrey have a tattoo of?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_3", - "session_23", - "session_15", - "session_9", - "session_26", - "session_22", - "session_11", - "session_1", - "session_13" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What does Audrey do during dog playdates in the park?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_14", - "session_7", - "session_8", - "session_11", - "session_18", - "session_27", - "session_5", - "session_10", - "session_23" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_5" - ], - "question": "What type of dog was Andrew looking to adopt based on his living space?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_28", - "session_24", - "session_1", - "session_9", - "session_10", - "session_2", - "session_14", - "session_21", - "session_25" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_5" - ], - "question": "Where does Andrew want to live to give their dog a large, open space to run around?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_9", - "session_4", - "session_11", - "session_23", - "session_7", - "session_27", - "session_19", - "session_28", - "session_21" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_6" - ], - "question": "Why did Audrey sign up for a workshop about bonding with pets?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_17", - "session_12", - "session_19", - "session_14", - "session_16", - "session_2", - "session_23", - "session_21", - "session_3" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_6" - ], - "question": "How did Audrey hear about the workshop on bonding with pets?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_19", - "session_17", - "session_12", - "session_14", - "session_16", - "session_2", - "session_21", - "session_3", - "session_23" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_6" - ], - "question": "What type of training was the workshop Audrey signed up for in May 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_28", - "session_1", - "session_27", - "session_19", - "session_10", - "session_25", - "session_15", - "session_14", - "session_20" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_6" - ], - "question": "How did Audrey describe she dog he met at the pet store?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_24", - "session_21", - "session_12", - "session_13", - "session_19", - "session_14", - "session_2", - "session_26", - "session_23" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_6" - ], - "question": "Why did Audrey think positive reinforcement training is important for pets?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_27", - "session_17", - "session_26", - "session_7", - "session_19", - "session_20", - "session_4", - "session_10", - "session_16" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What challenge is Andrew facing in their search for a pet?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_18", - "session_5", - "session_14", - "session_7", - "session_10", - "session_3", - "session_11", - "session_26", - "session_21" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "How does Andrew feel about their search for a pet-friendly place?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_28", - "session_18", - "session_10", - "session_5", - "session_6", - "session_9", - "session_23", - "session_26", - "session_2" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What outdoor activities does Andrew plan on trying after the rock climbing class?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_20", - "session_16", - "session_17", - "session_14", - "session_11", - "session_23", - "session_18", - "session_6", - "session_5" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "How long does Audrey typically walk her dogs for?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_13", - "session_20", - "session_8", - "session_14", - "session_4", - "session_3", - "session_1", - "session_2", - "session_15" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_9" - ], - "question": "What did Audrey set up in the backyard for their dogs on June 26, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_11", - "session_16", - "session_6", - "session_3", - "session_26", - "session_2", - "session_12", - "session_7", - "session_21" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_9" - ], - "question": "What did Audrey and her friends stumble across during a hike a few years back, as mentioned on June 26, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_17", - "session_11", - "session_18", - "session_15", - "session_7", - "session_2", - "session_10", - "session_27", - "session_19" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "What is Audrey's favorite recipe that she shares with Andrew on 3 July, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_15", - "session_13", - "session_8", - "session_4", - "session_1", - "session_19", - "session_3", - "session_2", - "session_7" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "What dish is one of Audrey's favorite dishes that includes garlic and is shared with Andrew on 3 July, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_15", - "session_13", - "session_24", - "session_1", - "session_8", - "session_3", - "session_14", - "session_7", - "session_4" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "What did Andrew and his GF do on the Monday before July 24, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_24", - "session_25", - "session_17", - "session_27", - "session_19", - "session_7", - "session_21", - "session_20", - "session_23" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "What is the name of Audrey's childhood dog?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_24", - "session_28", - "session_6", - "session_1", - "session_27", - "session_5", - "session_11", - "session_23", - "session_2" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "What special memories does Audrey have with her childhood dog, Max?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_26", - "session_15", - "session_2", - "session_12", - "session_7", - "session_19", - "session_14", - "session_4", - "session_23" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "What are some of the personalities of Audrey's four fur babies?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_15", - "session_9", - "session_19", - "session_26", - "session_6", - "session_27", - "session_4", - "session_5", - "session_12" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_14" - ], - "question": "What type of classes did Audrey start with her pups recently on 4 August, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_16", - "session_25", - "session_6", - "session_22", - "session_1", - "session_15", - "session_19", - "session_2", - "session_8" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_14" - ], - "question": "How often does Audrey take her pups to the park for practice?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_27", - "session_14", - "session_28", - "session_25", - "session_5", - "session_1", - "session_4", - "session_2", - "session_15" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_14" - ], - "question": "How long did the trail hike that Audrey went on with her pups take?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_19", - "session_24", - "session_14", - "session_20", - "session_4", - "session_1", - "session_8", - "session_2", - "session_15" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "What advice did Audrey give to Andrew regarding grooming Toby?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_16", - "session_17", - "session_27", - "session_14", - "session_20", - "session_5", - "session_28", - "session_26", - "session_21" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What is essential to keep the dogs looking good according to Audrey?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_28", - "session_5", - "session_26", - "session_14", - "session_6", - "session_15", - "session_27", - "session_23", - "session_4" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What did Audrey organize with the neighbors' dogs?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_23", - "session_6", - "session_16", - "session_3", - "session_26", - "session_21", - "session_7", - "session_12", - "session_22" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What did Audrey do to give her dogs extra comfort as the weather cooled down?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_15", - "session_28", - "session_11", - "session_19", - "session_13", - "session_6", - "session_12", - "session_2", - "session_9" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "How does Audrey describe the new beds for her dogs?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_18", - "session_15", - "session_6", - "session_10", - "session_2", - "session_1", - "session_19", - "session_9", - "session_4" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "How did Audrey calm down her dog after the leash incident?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_28", - "session_11", - "session_16", - "session_22", - "session_3", - "session_20", - "session_7", - "session_2", - "session_15" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "How often does Audrey take her dogs for walks?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_22", - "session_27", - "session_4", - "session_14", - "session_5", - "session_1", - "session_8", - "session_2", - "session_15" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "What kind of flowers does Audrey take care of?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_20", - "session_16", - "session_9", - "session_27", - "session_3", - "session_26", - "session_13", - "session_22", - "session_23" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_20" - ], - "question": "What did Andrew learn from reading books about ecological systems?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_6", - "session_25", - "session_27", - "session_3", - "session_16", - "session_14", - "session_26", - "session_12", - "session_21" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_20" - ], - "question": "What did Andrew suggest as a way to reduce carbon footprint?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_3", - "session_5", - "session_4", - "session_26", - "session_21", - "session_22", - "session_12", - "session_1", - "session_16" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_20" - ], - "question": "How does Andrew suggest helping the planet while also training the body?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_10", - "session_22", - "session_6", - "session_27", - "session_26", - "session_1", - "session_4", - "session_19", - "session_5" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_21" - ], - "question": "What did Audrey do with her pups over the weekend before 4th October, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_25", - "session_27", - "session_21", - "session_17", - "session_23", - "session_2", - "session_7", - "session_22", - "session_15" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_22" - ], - "question": "What was the reason Audrey couldn't walk her dogs for a period of time?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_18", - "session_4", - "session_22", - "session_24", - "session_15", - "session_2", - "session_12", - "session_8", - "session_5" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_22" - ], - "question": "What type of jewelry does Audrey make?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_1", - "session_2", - "session_25", - "session_17", - "session_11", - "session_14", - "session_10", - "session_12", - "session_13" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_22" - ], - "question": "Why does Audrey make jewelry out of recycled objects?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_8", - "session_2", - "session_25", - "session_27", - "session_20", - "session_3", - "session_14", - "session_10", - "session_4" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_22" - ], - "question": "What organization does Audrey donate a portion of his profits to?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_26", - "session_24", - "session_15", - "session_13", - "session_6", - "session_25", - "session_16", - "session_18", - "session_21" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_22" - ], - "question": "How does Audrey help out the animal shelter?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_18", - "session_9", - "session_26", - "session_14", - "session_7", - "session_5", - "session_22", - "session_1", - "session_24" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "What type of games do Audrey's dogs like to play at the park?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_11", - "session_1", - "session_27", - "session_9", - "session_4", - "session_19", - "session_14", - "session_13", - "session_25" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "What did Audrey make to thank her neighbors?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_18", - "session_15", - "session_23", - "session_26", - "session_3", - "session_21", - "session_2", - "session_16", - "session_12" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "How do Audrey's dogs react to snow?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_28", - "session_6", - "session_27", - "session_11", - "session_8", - "session_4", - "session_26", - "session_2", - "session_5" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "How does Audrey describe her dogs' response to snow?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_26", - "session_27", - "session_11", - "session_19", - "session_2", - "session_15", - "session_23", - "session_6", - "session_5" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "What kind of experiences are Audrey's dogs the best companions for?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_5", - "session_3", - "session_14", - "session_26", - "session_4", - "session_19", - "session_20", - "session_13", - "session_22" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "What activity do Andrew and Buddy enjoy doing together?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_14", - "session_27", - "session_18", - "session_28", - "session_15", - "session_23", - "session_2", - "session_8", - "session_5" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "What do Andrew and Buddy like doing on walks?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_28", - "session_14", - "session_23", - "session_4", - "session_19", - "session_22", - "session_8", - "session_27", - "session_2" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_25" - ], - "question": "What cuisine did Andrew recently try at a new spot in town?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_3", - "session_27", - "session_26", - "session_23", - "session_24", - "session_6", - "session_5", - "session_15", - "session_18" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_25" - ], - "question": "Which type of sushi did Audrey suggest trying first to someone new to sushi?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_26", - "session_1", - "session_5", - "session_27", - "session_28", - "session_14", - "session_17", - "session_4", - "session_20" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_26" - ], - "question": "What type of date is Andrew going on Sunday?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_8", - "session_20", - "session_1", - "session_25", - "session_11", - "session_14", - "session_4", - "session_23", - "session_12" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_26" - ], - "question": "What did Andrew and Audrey plan to do on the Saturday after October 28, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_26", - "session_16", - "session_11", - "session_3", - "session_19", - "session_23", - "session_18", - "session_12", - "session_22" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_26" - ], - "question": "What aspect of autumn does Andrew find beautiful?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_4", - "session_5", - "session_11", - "session_7", - "session_18", - "session_1", - "session_9", - "session_3", - "session_19" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_27" - ], - "question": "What did Audrey do in November 2023 to better take care of her dogs?", - "recall": 0.0, - "retrieved_ids": [ - "session_19", - "session_13", - "session_24", - "session_6", - "session_11", - "session_26", - "session_25", - "session_14", - "session_15", - "session_2" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_27" - ], - "question": "How often does Audrey meet up with other dog owners for tips and playdates?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_27", - "session_6", - "session_14", - "session_19", - "session_28", - "session_10", - "session_5", - "session_1", - "session_18" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_27" - ], - "question": "What did Audrey share to show ways to keep dogs active in the city?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_27", - "session_20", - "session_14", - "session_23", - "session_24", - "session_12", - "session_18", - "session_16", - "session_8" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_27" - ], - "question": "What type of activities does Audrey suggest for mental stimulation of the dogs?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_14", - "session_18", - "session_1", - "session_26", - "session_8", - "session_5", - "session_4", - "session_20", - "session_25" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_28" - ], - "question": "What is Andrew planning to do with Scout, Toby, and Buddy?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_23", - "session_24", - "session_27", - "session_14", - "session_20", - "session_18", - "session_16", - "session_12", - "session_2" - ], - "sample_id": "conv-44" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_28" - ], - "question": "What did Andrew get for Scout to create a safe and fun space for them?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_4", - "session_11", - "session_9", - "session_12", - "session_2", - "session_13", - "session_5", - "session_19", - "session_21" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_1" - ], - "question": "Which specific type of bird mesmerizes Audrey?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_26", - "session_7", - "session_22", - "session_17", - "session_9", - "session_19", - "session_20", - "session_25", - "session_14" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_3" - ], - "question": "What kind of flowers does Andrew have a tattoo of?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_3", - "session_23", - "session_15", - "session_9", - "session_26", - "session_20", - "session_22", - "session_1", - "session_13" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_5" - ], - "question": "What type of dog was Audrey looking to adopt based on her living space?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_2", - "session_21", - "session_1", - "session_19", - "session_10", - "session_24", - "session_15", - "session_14", - "session_25" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_6" - ], - "question": "Why did Audrey sign up for a workshop about car maintenance?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_19", - "session_12", - "session_2", - "session_18", - "session_26", - "session_21", - "session_16", - "session_7", - "session_3" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_6" - ], - "question": "How did Andrew hear about the workshop on bonding with pets?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_19", - "session_16", - "session_17", - "session_12", - "session_14", - "session_2", - "session_21", - "session_3", - "session_23" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_6" - ], - "question": "What type of training was the workshop Andrew signed up for in May 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_28", - "session_1", - "session_27", - "session_19", - "session_10", - "session_25", - "session_15", - "session_14", - "session_20" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_6" - ], - "question": "How did Andrew describe the dog he met at the pet store?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_24", - "session_12", - "session_13", - "session_14", - "session_3", - "session_21", - "session_16", - "session_26", - "session_23" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_7" - ], - "question": "What challenge is Audrey facing in their search for a pet?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_5", - "session_14", - "session_7", - "session_10", - "session_3", - "session_11", - "session_26", - "session_21" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_8" - ], - "question": "What indoor activities does Andrew plan on trying after the rock climbing class?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_20", - "session_16", - "session_9", - "session_6", - "session_17", - "session_5", - "session_14", - "session_23", - "session_18" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_9" - ], - "question": "What did Andrew set up in the backyard for their dogs on June 26, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_11", - "session_16", - "session_6", - "session_3", - "session_26", - "session_2", - "session_12", - "session_7", - "session_21" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_13" - ], - "question": "What did Audrey and her GF do on the Monday before July 24, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_25", - "session_17", - "session_19", - "session_24", - "session_15", - "session_2", - "session_7", - "session_20", - "session_23" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_13" - ], - "question": "What is the name of Andrew's childhood dog?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_24", - "session_28", - "session_6", - "session_1", - "session_27", - "session_5", - "session_11", - "session_23", - "session_2" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_13" - ], - "question": "What special memories does Andrew have with his childhood dog, Max?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_26", - "session_15", - "session_12", - "session_24", - "session_23", - "session_2", - "session_25", - "session_6", - "session_21" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_13" - ], - "question": "What are some of the personalities of Andrew's four fur babies?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_15", - "session_9", - "session_19", - "session_26", - "session_6", - "session_27", - "session_4", - "session_5", - "session_12" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_14" - ], - "question": "What type of classes did Andrew start with his pups recently on 4 August, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_28", - "session_6", - "session_24", - "session_14", - "session_16", - "session_1", - "session_15", - "session_22", - "session_8" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_17" - ], - "question": "What is essential to keep the dogs looking good according to Andrew?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_28", - "session_14", - "session_5", - "session_26", - "session_6", - "session_15", - "session_27", - "session_23", - "session_4" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "What did Audrey organize with the neighbors' cats?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_23", - "session_6", - "session_16", - "session_3", - "session_26", - "session_21", - "session_22", - "session_12", - "session_7" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "What did Andrew do to give his dogs extra comfort as the weather cooled down?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_15", - "session_6", - "session_28", - "session_11", - "session_12", - "session_13", - "session_25", - "session_9", - "session_21" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "How does Andrew describe the new beds for his dogs?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_16", - "session_24", - "session_18", - "session_28", - "session_13", - "session_10", - "session_6", - "session_21", - "session_25" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_19" - ], - "question": "How did Andrew calm down his dog after the leash incident?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_14", - "session_28", - "session_11", - "session_4", - "session_22", - "session_3", - "session_16", - "session_20", - "session_7" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_19" - ], - "question": "How often does Andrew take his dogs for walks?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_24", - "session_13", - "session_4", - "session_14", - "session_5", - "session_1", - "session_27", - "session_15", - "session_21" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_19" - ], - "question": "What kind of vegetables does Audrey take care of?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_17", - "session_22", - "session_16", - "session_27", - "session_3", - "session_26", - "session_20", - "session_13", - "session_23" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_20" - ], - "question": "What did Andrew learn from reading books about economic systems?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_6", - "session_25", - "session_27", - "session_3", - "session_16", - "session_14", - "session_26", - "session_12", - "session_21" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_22" - ], - "question": "What was the reason Andrew couldn't walk his dogs for a period of time?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_4", - "session_22", - "session_16", - "session_13", - "session_15", - "session_12", - "session_6", - "session_25", - "session_21" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_22" - ], - "question": "What type of jewelry does Andrew make?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_1", - "session_2", - "session_25", - "session_17", - "session_11", - "session_14", - "session_10", - "session_12", - "session_13" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_22" - ], - "question": "Why does Andrew make jewelry out of recycled objects?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_8", - "session_2", - "session_25", - "session_27", - "session_20", - "session_3", - "session_14", - "session_10", - "session_4" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_23" - ], - "question": "What type of games do Andrew's dogs like to play at the park?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_11", - "session_1", - "session_27", - "session_9", - "session_4", - "session_19", - "session_14", - "session_13", - "session_25" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_23" - ], - "question": "What did Andrew make to thank his neighbors?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_24", - "session_15", - "session_23", - "session_26", - "session_25", - "session_13", - "session_6", - "session_21", - "session_16" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_23" - ], - "question": "How do Andrew's dogs react to snow?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_28", - "session_6", - "session_27", - "session_11", - "session_8", - "session_4", - "session_26", - "session_2", - "session_5" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_23" - ], - "question": "How does Andrew describe his dogs' response to snow?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_17", - "session_24", - "session_13", - "session_15", - "session_23", - "session_21", - "session_6", - "session_16", - "session_25" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_23" - ], - "question": "What kind of experiences are Audrey's cats the best companions for?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_4", - "session_5", - "session_3", - "session_14", - "session_26", - "session_19", - "session_20", - "session_13", - "session_22" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_24" - ], - "question": "What activity do Audrey and Buddy enjoy doing together?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_14", - "session_27", - "session_18", - "session_28", - "session_15", - "session_23", - "session_2", - "session_8", - "session_5" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_25" - ], - "question": "What type of drink did Andrew recently try at a new spot in town?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_23", - "session_3", - "session_1", - "session_27", - "session_24", - "session_6", - "session_5", - "session_18", - "session_14" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_25" - ], - "question": "Which type of pizza did Audrey suggest trying first to someone new to Italian cuisine?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_26", - "session_1", - "session_5", - "session_27", - "session_28", - "session_14", - "session_17", - "session_4", - "session_20" - ], - "sample_id": "conv-44" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_27" - ], - "question": "How often does Andrew meet up with other dog owners for tips and playdates?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_27", - "session_6", - "session_14", - "session_19", - "session_28", - "session_10", - "session_5", - "session_1", - "session_18" - ], - "sample_id": "conv-44" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_1" - ], - "question": "What are John's suspected health problems?", - "recall": 0.0, - "retrieved_ids": [], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_1" - ], - "question": "Which recreational activity was James pursuing on March 16, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_14", - "session_23", - "session_3", - "session_17", - "session_18", - "session_28", - "session_31", - "session_1", - "session_8" - ], - "sample_id": "conv-47" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_21", - "session_23" - ], - "question": "Which places or events have John and James planned to meet at?", - "recall": 0.3333, - "retrieved_ids": [ - "session_10", - "session_7", - "session_11", - "session_2", - "session_20", - "session_12", - "session_9", - "session_21", - "session_6", - "session_8" - ], - "sample_id": "conv-47" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_2" - ], - "question": "Do both James and John have pets?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_20", - "session_29", - "session_13", - "session_2", - "session_18", - "session_9", - "session_11", - "session_23", - "session_1" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_3" - ], - "question": "When did John resume playing drums in his adulthood?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_4", - "session_14", - "session_21", - "session_3", - "session_26", - "session_23", - "session_30", - "session_17", - "session_28" - ], - "sample_id": "conv-47" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_3", - "session_4" - ], - "question": "What are John and James' favorite games?", - "recall": 0.5, - "retrieved_ids": [ - "session_19", - "session_9", - "session_3", - "session_1", - "session_10", - "session_14", - "session_28", - "session_27", - "session_20", - "session_8" - ], - "sample_id": "conv-47" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_5" - ], - "question": "Does James live in Connecticut?", - "recall": 0.0, - "retrieved_ids": [ - "session_29", - "session_10", - "session_20", - "session_31", - "session_8", - "session_14", - "session_18", - "session_17", - "session_28", - "session_1" - ], - "sample_id": "conv-47" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_5" - ], - "question": "In which state is the shelter from which James adopted the puppy?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_5", - "session_10", - "session_21", - "session_31", - "session_17", - "session_28", - "session_8", - "session_1", - "session_14" - ], - "sample_id": "conv-47" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_5" - ], - "question": "How many pets does James have?", - "recall": 0.5, - "retrieved_ids": [ - "session_2", - "session_18", - "session_9", - "session_7", - "session_28", - "session_12", - "session_31", - "session_1", - "session_6", - "session_17" - ], - "sample_id": "conv-47" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_5" - ], - "question": "What are the names of James's dogs?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_1", - "session_31", - "session_30", - "session_19", - "session_8", - "session_14", - "session_17", - "session_5", - "session_10" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_5" - ], - "question": "When did James adopt Ned?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_5", - "session_15", - "session_12", - "session_13", - "session_4", - "session_30", - "session_10", - "session_28", - "session_23" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_6" - ], - "question": "How was John feeling on April 10, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_2", - "session_14", - "session_1", - "session_23", - "session_6", - "session_11", - "session_31", - "session_17", - "session_12" - ], - "sample_id": "conv-47" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_6" - ], - "question": "Did James have a girlfriend during April 2022?", - "recall": 0.0, - "retrieved_ids": [ - "session_23", - "session_12", - "session_13", - "session_22", - "session_2", - "session_14", - "session_10", - "session_28", - "session_30", - "session_4" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_6" - ], - "question": "When did James visit Italy?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_16", - "session_31", - "session_12", - "session_13", - "session_4", - "session_30", - "session_10", - "session_28", - "session_23" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_8" - ], - "question": "When did James buy himself a new adventure book?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_8", - "session_2", - "session_14", - "session_10", - "session_21", - "session_30", - "session_24", - "session_23", - "session_7" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_8" - ], - "question": "When did James start playing Civilization VI?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_21", - "session_22", - "session_14", - "session_26", - "session_24", - "session_30", - "session_23", - "session_13", - "session_15" - ], - "sample_id": "conv-47" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_8" - ], - "question": "What is the game with different colored cards that was John talking about with James?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_20", - "session_26", - "session_9", - "session_4", - "session_29", - "session_11", - "session_7", - "session_13", - "session_16" - ], - "sample_id": "conv-47" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_8" - ], - "question": "What is the board game where you have to find the imposter that John mentions to James?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_2", - "session_24", - "session_31", - "session_8", - "session_17", - "session_28", - "session_11", - "session_7", - "session_15" - ], - "sample_id": "conv-47" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_14", - "session_8" - ], - "question": "Which books has John recommended to James?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_24", - "session_3", - "session_25", - "session_8", - "session_18", - "session_26", - "session_30", - "session_17", - "session_28" - ], - "sample_id": "conv-47" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_9" - ], - "question": "Was James feeling lonely before meeting Samantha?", - "recall": 0.0, - "retrieved_ids": [ - "session_6", - "session_17", - "session_19", - "session_23", - "session_29", - "session_21", - "session_24", - "session_2", - "session_25", - "session_28" - ], - "sample_id": "conv-47" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_29" - ], - "question": "How many charity tournaments has John organized till date?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_28", - "session_19", - "session_12", - "session_6", - "session_7", - "session_25", - "session_29", - "session_18", - "session_31" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_10" - ], - "question": "When did John first organize a charity tournament with his friends?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_18", - "session_23", - "session_4", - "session_14", - "session_27", - "session_30", - "session_17", - "session_29", - "session_21" - ], - "sample_id": "conv-47" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_29" - ], - "question": "Who or which organizations have been the beneficiaries of John's charity tournaments?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_11", - "session_18", - "session_16", - "session_30", - "session_12", - "session_4", - "session_15", - "session_3", - "session_29" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_13" - ], - "question": "When will John start his new job?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_14", - "session_23", - "session_4", - "session_21", - "session_18", - "session_10", - "session_29", - "session_12", - "session_17" - ], - "sample_id": "conv-47" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_13", - "session_27" - ], - "question": "What kind of games has James tried to develop?", - "recall": 0.3333, - "retrieved_ids": [ - "session_1", - "session_3", - "session_24", - "session_4", - "session_28", - "session_17", - "session_26", - "session_16", - "session_22", - "session_15" - ], - "sample_id": "conv-47" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_13" - ], - "question": "Are John and James fans of the same football team?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_9", - "session_2", - "session_4", - "session_14", - "session_20", - "session_27", - "session_30", - "session_3", - "session_1" - ], - "sample_id": "conv-47" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_16", - "session_17", - "session_6" - ], - "question": "Which countries has James visited?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_16", - "session_31", - "session_17", - "session_3", - "session_18", - "session_28", - "session_14", - "session_8", - "session_1" - ], - "sample_id": "conv-47" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_13", - "session_23" - ], - "question": "What kind of classes has James joined?", - "recall": 0.5, - "retrieved_ids": [ - "session_20", - "session_23", - "session_24", - "session_26", - "session_21", - "session_16", - "session_15", - "session_1", - "session_17", - "session_8" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_15" - ], - "question": "When did James volunteer at an organization?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_11", - "session_26", - "session_28", - "session_20", - "session_10", - "session_27", - "session_19", - "session_29", - "session_18" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_16" - ], - "question": "When did James depart for his trip to Canada?", - "recall": 0.0, - "retrieved_ids": [ - "session_17", - "session_4", - "session_14", - "session_21", - "session_31", - "session_30", - "session_23", - "session_10", - "session_13", - "session_28" - ], - "sample_id": "conv-47" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_16" - ], - "question": "Which country did James book tickets for in July 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_24", - "session_6", - "session_25", - "session_14", - "session_17", - "session_8", - "session_13", - "session_30", - "session_28" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_16" - ], - "question": "How many days did James plan to spend on his trip in Canada?", - "recall": 0.0, - "retrieved_ids": [ - "session_17", - "session_7", - "session_31", - "session_21", - "session_4", - "session_23", - "session_6", - "session_14", - "session_30", - "session_12" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_16" - ], - "question": "Where was James at on July 12, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_9", - "session_14", - "session_18", - "session_17", - "session_31", - "session_3", - "session_28", - "session_8", - "session_1" - ], - "sample_id": "conv-47" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_17" - ], - "question": "Did John and James study together?", - "recall": 1.0, - "retrieved_ids": [ - "session_30", - "session_4", - "session_6", - "session_29", - "session_10", - "session_15", - "session_17", - "session_2", - "session_20", - "session_1" - ], - "sample_id": "conv-47" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_16", - "session_17" - ], - "question": "Which countries did James visit in July 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_6", - "session_31", - "session_17", - "session_10", - "session_13", - "session_30", - "session_4", - "session_23", - "session_28" - ], - "sample_id": "conv-47" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_17" - ], - "question": "What additional country did James visit during his trip to Canada?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_16", - "session_31", - "session_6", - "session_14", - "session_13", - "session_21", - "session_30", - "session_12", - "session_11" - ], - "sample_id": "conv-47" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_17" - ], - "question": "Who is Jill?", - "recall": 1.0, - "retrieved_ids": [ - "session_17" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_17" - ], - "question": "When did John spend time with his sister and dogs?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_21", - "session_23", - "session_7", - "session_31", - "session_4", - "session_14", - "session_30", - "session_9", - "session_29" - ], - "sample_id": "conv-47" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_18", - "session_4" - ], - "question": "What happened to John's job situation in 2022?", - "recall": 0.5, - "retrieved_ids": [ - "session_18", - "session_17", - "session_13", - "session_15", - "session_27", - "session_29", - "session_12", - "session_9", - "session_25", - "session_10" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_18" - ], - "question": "When did John start his job in IT?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_14", - "session_23", - "session_4", - "session_18", - "session_21", - "session_10", - "session_29", - "session_12", - "session_17" - ], - "sample_id": "conv-47" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_14", - "session_17", - "session_2" - ], - "question": "What kind of tricks do James's pets know?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_17", - "session_26", - "session_9", - "session_25", - "session_1", - "session_15", - "session_16", - "session_18", - "session_14" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_19" - ], - "question": "When did James meet Samantha?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_19", - "session_29", - "session_2", - "session_4", - "session_13", - "session_21", - "session_30", - "session_6", - "session_28" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_19" - ], - "question": "When did James take his 3 dogs to the beach?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_23", - "session_2", - "session_4", - "session_19", - "session_21", - "session_31", - "session_17", - "session_1", - "session_10" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_20" - ], - "question": "When did John plan his next meeting with his siblings?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_4", - "session_20", - "session_8", - "session_22", - "session_23", - "session_6", - "session_14", - "session_17", - "session_2" - ], - "sample_id": "conv-47" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_21" - ], - "question": "Why didn't John want to go to Starbucks?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_6", - "session_28", - "session_1", - "session_7", - "session_25", - "session_4", - "session_12", - "session_18", - "session_23" - ], - "sample_id": "conv-47" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_21", - "session_23" - ], - "question": "What kind of beer does McGee's bar serve?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_23", - "session_29", - "session_3", - "session_26", - "session_16", - "session_24", - "session_15", - "session_22", - "session_28" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_21" - ], - "question": "When did John and James meet at McGee's bar?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_23", - "session_21", - "session_2", - "session_10", - "session_13", - "session_4", - "session_30", - "session_6", - "session_28" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_23" - ], - "question": "When did James ask Samantha to be his girlfriend?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_26", - "session_19", - "session_29", - "session_4", - "session_14", - "session_21", - "session_10", - "session_7", - "session_17" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_23" - ], - "question": "When did James, Samantha and John go to the baseball game together?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_29", - "session_4", - "session_19", - "session_20", - "session_10", - "session_6", - "session_2", - "session_1", - "session_15" - ], - "sample_id": "conv-47" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_20", - "session_23" - ], - "question": "What gaming equipments did John buy or refurbish?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_20", - "session_10", - "session_18", - "session_26", - "session_30", - "session_4", - "session_28", - "session_13", - "session_31" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_23" - ], - "question": "When did James start taking cooking classes?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_30", - "session_14", - "session_13", - "session_26", - "session_24", - "session_8", - "session_17", - "session_1", - "session_10" - ], - "sample_id": "conv-47" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_19", - "session_24", - "session_30", - "session_5", - "session_8" - ], - "question": "Which new games did John start play during the course of the conversation with James?", - "recall": 0.0, - "retrieved_ids": [ - "session_13", - "session_21", - "session_12", - "session_15", - "session_23", - "session_2", - "session_6", - "session_25", - "session_14", - "session_22" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_25" - ], - "question": "When did John start working on his 2D Adventure mobile game?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_21", - "session_1", - "session_3", - "session_14", - "session_30", - "session_23", - "session_7", - "session_4", - "session_17" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_27", - "session_6" - ], - "question": "How long did it take for James to complete his Witcher-inspired game?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_4", - "session_5", - "session_1", - "session_13", - "session_21", - "session_6", - "session_27", - "session_19", - "session_22" - ], - "sample_id": "conv-47" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_27", - "session_28" - ], - "question": "What kind of programming-related events has John hosted?", - "recall": 0.0, - "retrieved_ids": [ - "session_20", - "session_15", - "session_26", - "session_24", - "session_16", - "session_18", - "session_22", - "session_1", - "session_10", - "session_9" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_27" - ], - "question": "When did John and his programming friends host an online programming competition?", - "recall": 0.0, - "retrieved_ids": [ - "session_20", - "session_4", - "session_14", - "session_18", - "session_10", - "session_21", - "session_23", - "session_3", - "session_17", - "session_12" - ], - "sample_id": "conv-47" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_17", - "session_28" - ], - "question": "Which of James's family members have visited him in the last year?", - "recall": 0.5, - "retrieved_ids": [ - "session_6", - "session_1", - "session_5", - "session_31", - "session_16", - "session_18", - "session_20", - "session_14", - "session_17", - "session_21" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_28" - ], - "question": "When did James' mother and her friend visit him?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_6", - "session_31", - "session_23", - "session_9", - "session_5", - "session_16", - "session_14", - "session_21", - "session_19" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_28" - ], - "question": "When did James try Cyberpunk 2077 game?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_26", - "session_19", - "session_4", - "session_5", - "session_30", - "session_31", - "session_9", - "session_1", - "session_18" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_29" - ], - "question": "When did John and his gaming friends organize the charity tournament?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_4", - "session_18", - "session_23", - "session_3", - "session_14", - "session_29", - "session_21", - "session_12", - "session_17" - ], - "sample_id": "conv-47" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_29" - ], - "question": "What games has John played with his friends at charity tournaments?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_3", - "session_29", - "session_4", - "session_23", - "session_17", - "session_21", - "session_12", - "session_16", - "session_14" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_29" - ], - "question": "What was James' big moment with Samantha in October 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_24", - "session_25", - "session_3", - "session_19", - "session_23", - "session_18", - "session_12", - "session_17", - "session_1" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_19", - "session_29" - ], - "question": "How long did James and Samantha date for before deciding to move in together?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_8", - "session_24", - "session_6", - "session_19", - "session_25", - "session_17", - "session_1", - "session_23", - "session_30" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_30" - ], - "question": "When did James, his family and his dogs start on a road trip together?", - "recall": 1.0, - "retrieved_ids": [ - "session_30", - "session_17", - "session_31", - "session_1", - "session_20", - "session_14", - "session_21", - "session_23", - "session_2", - "session_4" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_17", - "session_30" - ], - "question": "How long did John practice chess for before winning the chess tournament?", - "recall": 1.0, - "retrieved_ids": [ - "session_30", - "session_16", - "session_17", - "session_24", - "session_3", - "session_28", - "session_8", - "session_6", - "session_13", - "session_12" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_31" - ], - "question": "When did James and his family visit Mark and Josh?", - "recall": 1.0, - "retrieved_ids": [ - "session_31", - "session_4", - "session_16", - "session_20", - "session_21", - "session_30", - "session_17", - "session_6", - "session_23", - "session_2" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_31" - ], - "question": "When did John work with a game developer on a project?", - "recall": 1.0, - "retrieved_ids": [ - "session_31", - "session_26", - "session_22", - "session_3", - "session_25", - "session_28", - "session_1", - "session_4", - "session_27", - "session_17" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_1" - ], - "question": "What programming languages has James worked with?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_11", - "session_6", - "session_20", - "session_28", - "session_26", - "session_21", - "session_12", - "session_22", - "session_2" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_1" - ], - "question": "What type of mobile application does James plan to build with John?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_11", - "session_30", - "session_9", - "session_25", - "session_20", - "session_14", - "session_3", - "session_6", - "session_2" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_1" - ], - "question": "How does James plan to make his dog-sitting app unique?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_4", - "session_23", - "session_8", - "session_21", - "session_31", - "session_10", - "session_2", - "session_14", - "session_17" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What has John mostly found with the metal detector so far?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_25", - "session_23", - "session_29", - "session_11", - "session_22", - "session_20", - "session_17", - "session_8", - "session_10" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What did James offer to do for John regarding pets?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_17", - "session_21", - "session_26", - "session_15", - "session_9", - "session_7", - "session_25", - "session_1", - "session_13" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "What instrument is John learning to play as of 27 March, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_30", - "session_24", - "session_23", - "session_28", - "session_21", - "session_22", - "session_25", - "session_8", - "session_12" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "How long has John been playing the drums as of 27 March, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_3", - "session_8", - "session_4", - "session_28", - "session_5", - "session_14", - "session_16", - "session_6", - "session_13" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "What game did John play in an intense tournament at the gaming convention in March 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_30", - "session_8", - "session_4", - "session_20", - "session_29", - "session_17", - "session_16", - "session_10", - "session_12" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What game was James playing in the online gaming tournament in April 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_30", - "session_4", - "session_27", - "session_16", - "session_3", - "session_20", - "session_29", - "session_17", - "session_10", - "session_12" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "How does James communicate with his gaming team?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_27", - "session_23", - "session_21", - "session_13", - "session_20", - "session_2", - "session_9", - "session_14", - "session_17" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What advice did James receive from the famous players he met at the tournament?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_30", - "session_3", - "session_12", - "session_18", - "session_27", - "session_19", - "session_23", - "session_21", - "session_14" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_5" - ], - "question": "What did James adopt in April 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_21", - "session_31", - "session_30", - "session_12", - "session_4", - "session_23", - "session_28", - "session_10", - "session_13" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_5" - ], - "question": "What is the name of the pup that was adopted by James?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_5", - "session_31", - "session_19", - "session_15", - "session_14", - "session_8", - "session_28", - "session_1", - "session_17" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_6" - ], - "question": "Why did James embody the appearance of the game character from the woman he saw during a walk?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_14", - "session_2", - "session_13", - "session_19", - "session_12", - "session_1", - "session_21", - "session_18", - "session_23" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_6" - ], - "question": "What inspired James to create the game character in the virtual world?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_27", - "session_11", - "session_14", - "session_19", - "session_12", - "session_22", - "session_25", - "session_1", - "session_28" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_6" - ], - "question": "Which country did James visit in 2021?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_16", - "session_31", - "session_17", - "session_10", - "session_30", - "session_13", - "session_4", - "session_23", - "session_28" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_6" - ], - "question": "What impresses John about Japan?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_3", - "session_21", - "session_26", - "session_15", - "session_18", - "session_9", - "session_25", - "session_8", - "session_17" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What kind of assignment was giving John a hard time at work?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_20", - "session_16", - "session_15", - "session_22", - "session_24", - "session_3", - "session_1", - "session_14", - "session_26" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_9" - ], - "question": "What breed is Daisy, one of James' dogs?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_8", - "session_1", - "session_30", - "session_9", - "session_19", - "session_31", - "session_17", - "session_15", - "session_10" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_9" - ], - "question": "What type of pizza is James' favorite?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_19", - "session_3", - "session_14", - "session_1", - "session_8", - "session_10", - "session_18", - "session_17", - "session_28" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_9" - ], - "question": "What type of pizza is John's favorite?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_19", - "session_3", - "session_14", - "session_1", - "session_10", - "session_13", - "session_4", - "session_25", - "session_8" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "What did John organize with his friends on May 8, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_23", - "session_13", - "session_21", - "session_27", - "session_4", - "session_17", - "session_20", - "session_14", - "session_10" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "What did John and his friends do with the remaining money after helping the dog shelter?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_1", - "session_21", - "session_23", - "session_17", - "session_12", - "session_14", - "session_5", - "session_29", - "session_4" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "What was the main goal of the money raised from the charity tournament organized by John and his friends in May 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_12", - "session_18", - "session_29", - "session_3", - "session_4", - "session_9", - "session_20", - "session_1", - "session_13" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_11" - ], - "question": "What did the system John created help the charitable foundation with?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_6", - "session_30", - "session_3", - "session_22", - "session_14", - "session_28", - "session_1", - "session_27", - "session_21" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_11" - ], - "question": "What did John create for the charitable foundation that helped generate reports for analysis?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_6", - "session_30", - "session_22", - "session_4", - "session_1", - "session_27", - "session_21", - "session_24", - "session_3" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_12" - ], - "question": "What did John receive for achieving second place in the tournament?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_16", - "session_15", - "session_30", - "session_3", - "session_11", - "session_7", - "session_4", - "session_25", - "session_9" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "What project is James working on in his game design course?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_9", - "session_12", - "session_25", - "session_13", - "session_6", - "session_23", - "session_4", - "session_14", - "session_17" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "Who does James support in football matches?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_31", - "session_30", - "session_29", - "session_11", - "session_18", - "session_4", - "session_15", - "session_22", - "session_23" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "Which football club does John support?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_17", - "session_14", - "session_30", - "session_29", - "session_18", - "session_15", - "session_22", - "session_31", - "session_23" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "What disagreement do James and John have about their football teams?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_28", - "session_9", - "session_20", - "session_27", - "session_30", - "session_31", - "session_4", - "session_11", - "session_22" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_14" - ], - "question": "What is Max good at doing according to James?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_30", - "session_1", - "session_29", - "session_25", - "session_10", - "session_15", - "session_23", - "session_11", - "session_20" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "What is the main focus of the organization that James volunteered with?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_11", - "session_10", - "session_29", - "session_18", - "session_26", - "session_9", - "session_1", - "session_27", - "session_19" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "Will there be an interview required to volunteer with the organization James volunteered for?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_11", - "session_13", - "session_8", - "session_26", - "session_19", - "session_10", - "session_27", - "session_25", - "session_18" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "How did John relax in his free time on 9 July, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_21", - "session_1", - "session_23", - "session_4", - "session_3", - "session_25", - "session_2", - "session_14", - "session_17" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "What did James enjoy doing on cold winter days?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_7", - "session_26", - "session_29", - "session_30", - "session_17", - "session_5", - "session_14", - "session_2", - "session_27" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "What new hobby did James become interested in on 9 July, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_23", - "session_2", - "session_10", - "session_18", - "session_3", - "session_13", - "session_24", - "session_8", - "session_28" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "Where did James plan to visit after Toronto?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_18", - "session_1", - "session_5", - "session_10", - "session_27", - "session_31", - "session_6", - "session_20", - "session_8" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "When did James plan to return from his trip to Toronto and Vancouver?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_4", - "session_18", - "session_20", - "session_14", - "session_21", - "session_31", - "session_30", - "session_17", - "session_23" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What online game did John start playing recently for improving strategy?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_30", - "session_20", - "session_8", - "session_27", - "session_3", - "session_2", - "session_25", - "session_16", - "session_13" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What made John leave his IT job?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_4", - "session_29", - "session_12", - "session_23", - "session_10", - "session_21", - "session_13", - "session_16", - "session_7" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "Which game tournaments does John plan to organize besides CS:GO?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_10", - "session_3", - "session_12", - "session_11", - "session_4", - "session_16", - "session_23", - "session_6", - "session_29" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What happened to James's puppy during the recent visit to the clinic?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_25", - "session_14", - "session_2", - "session_6", - "session_13", - "session_27", - "session_16", - "session_31", - "session_12" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "What game genre did John start exploring instead of shooters?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_30", - "session_3", - "session_24", - "session_14", - "session_28", - "session_25", - "session_2", - "session_27", - "session_7" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "Which RPG game is John playing and enjoying on 10 August, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_27", - "session_5", - "session_19", - "session_26", - "session_4", - "session_3", - "session_14", - "session_2", - "session_23" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "What aspect of \"The Witcher 3\" does John find immersive?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_19", - "session_14", - "session_28", - "session_2", - "session_27", - "session_6", - "session_26", - "session_5", - "session_11" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "Whose phone number did James receive during the beach outing?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_12", - "session_2", - "session_14", - "session_4", - "session_13", - "session_22", - "session_8", - "session_30", - "session_28" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "What is James planning to do after receiving Samantha's phone number?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_20", - "session_8", - "session_10", - "session_29", - "session_23", - "session_16", - "session_18", - "session_1", - "session_12" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_20" - ], - "question": "What is John organizing with his siblings?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_20", - "session_22", - "session_4", - "session_23", - "session_17", - "session_14", - "session_15", - "session_10", - "session_18" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_21" - ], - "question": "What type of beer does John not like?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_29", - "session_30", - "session_9", - "session_24", - "session_23", - "session_7", - "session_20", - "session_10", - "session_3" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_22" - ], - "question": "What were some difficulties James faced during the development of his game?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_25", - "session_4", - "session_12", - "session_23", - "session_14", - "session_21", - "session_17", - "session_2", - "session_13" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_22" - ], - "question": "What has John been teaching his siblings?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_21", - "session_25", - "session_4", - "session_23", - "session_20", - "session_17", - "session_14", - "session_26", - "session_3" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_22" - ], - "question": "What kind of programs are John's siblings making?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_22", - "session_3", - "session_1", - "session_21", - "session_26", - "session_16", - "session_15", - "session_24", - "session_17" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "Which company's headphones did John choose for gaming?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_18", - "session_13", - "session_10", - "session_26", - "session_30", - "session_4", - "session_28", - "session_31", - "session_17" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "What did James and Samantha discover they both enjoy at McGee's bar?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_23", - "session_14", - "session_7", - "session_17", - "session_21", - "session_11", - "session_13", - "session_19", - "session_20" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "How much does James pay per cooking class?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_16", - "session_8", - "session_31", - "session_18", - "session_17", - "session_12", - "session_1", - "session_10", - "session_25" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "What did James learn to make in the cooking class besides omelette and meringue?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_21", - "session_12", - "session_8", - "session_22", - "session_17", - "session_28", - "session_1", - "session_6", - "session_10" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "Why did James sign up for a cooking class?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_1", - "session_21", - "session_8", - "session_17", - "session_12", - "session_10", - "session_13", - "session_4", - "session_28" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "What did James prepare for the first time in the cooking class?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_6", - "session_30", - "session_18", - "session_15", - "session_8", - "session_17", - "session_1", - "session_27", - "session_10" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "What is the name of the board game John tried in September 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_31", - "session_28", - "session_24", - "session_8", - "session_17", - "session_21", - "session_19", - "session_9", - "session_1", - "session_5" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "Where does James get his ideas from?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_24", - "session_4", - "session_31", - "session_23", - "session_14", - "session_7", - "session_17", - "session_28", - "session_1" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "What kind of dream did James have recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_15", - "session_3", - "session_20", - "session_27", - "session_13", - "session_26", - "session_16", - "session_6", - "session_2" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "What kind of music does John like?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_16", - "session_22", - "session_3", - "session_20", - "session_1", - "session_13", - "session_26", - "session_15", - "session_2" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "What instrument did James used to play when he was younger?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_3", - "session_20", - "session_28", - "session_21", - "session_30", - "session_23", - "session_4", - "session_18", - "session_14" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "What did John use to play when he was younger to let off steam?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_30", - "session_20", - "session_12", - "session_21", - "session_23", - "session_27", - "session_18", - "session_16", - "session_14" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_25" - ], - "question": "What career milestone did John achieve recently in September 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_2", - "session_15", - "session_18", - "session_11", - "session_12", - "session_26", - "session_9", - "session_3", - "session_14" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_25" - ], - "question": "What type of game is John's upcoming mobile game?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_25", - "session_9", - "session_1", - "session_21", - "session_27", - "session_4", - "session_20", - "session_8", - "session_5" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_25" - ], - "question": "What does John do to stay informed and constantly learn about game design?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_28", - "session_21", - "session_20", - "session_29", - "session_30", - "session_3", - "session_17", - "session_1", - "session_16" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_26" - ], - "question": "What kind of gig was John offered at the game dev non-profit organization?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_15", - "session_24", - "session_25", - "session_11", - "session_20", - "session_2", - "session_16", - "session_7", - "session_13" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_26" - ], - "question": "What does John feel about starting the journey as a programming mentor for game developers?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_14", - "session_13", - "session_3", - "session_28", - "session_8", - "session_1", - "session_16", - "session_27", - "session_22" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_26" - ], - "question": "What kind of games is James excited to play with his new video card?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_1", - "session_24", - "session_20", - "session_21", - "session_22", - "session_4", - "session_8", - "session_14", - "session_23" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_27" - ], - "question": "What inspired James to create his game?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_27", - "session_22", - "session_25", - "session_11", - "session_14", - "session_4", - "session_23", - "session_6", - "session_17" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_28" - ], - "question": "What sparked James' passion for gaming when he was a kid?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_6", - "session_11", - "session_29", - "session_18", - "session_21", - "session_12", - "session_23", - "session_17", - "session_14" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_28" - ], - "question": "What did James lose progress on due to a power outage?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_1", - "session_11", - "session_12", - "session_3", - "session_7", - "session_8", - "session_5", - "session_25", - "session_20" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_29" - ], - "question": "What games were played at the gaming tournament organized by John on 31 October, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_29", - "session_30", - "session_4", - "session_3", - "session_18", - "session_11", - "session_16", - "session_12", - "session_15" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_29" - ], - "question": "What was the purpose of the gaming tournament organized by John on 31 October, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_29", - "session_16", - "session_18", - "session_30", - "session_4", - "session_3", - "session_10", - "session_15", - "session_12" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_29" - ], - "question": "What decision did James and Samantha make on 31 October, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_19", - "session_22", - "session_23", - "session_18", - "session_8", - "session_31", - "session_11", - "session_4", - "session_17" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_29" - ], - "question": "Where did James and Samantha decide to live together on 31 October, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_6", - "session_10", - "session_20", - "session_1", - "session_23", - "session_19", - "session_15", - "session_9", - "session_22" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_29" - ], - "question": "Why did James and Samantha choose an apartment near McGee's bar?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_29", - "session_30", - "session_19", - "session_21", - "session_13", - "session_20", - "session_17", - "session_28", - "session_1" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_30" - ], - "question": "What game is John hooked on playing on 5 November, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_20", - "session_30", - "session_28", - "session_26", - "session_24", - "session_3", - "session_4", - "session_8", - "session_29" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_30" - ], - "question": "What did John suggest James practice before playing FIFA 23 together?", - "recall": 1.0, - "retrieved_ids": [ - "session_30", - "session_17", - "session_29", - "session_21", - "session_24", - "session_6", - "session_8", - "session_28", - "session_25", - "session_31" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_31" - ], - "question": "What project did John work on with a game developer by 7 November, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_31", - "session_26", - "session_22", - "session_3", - "session_25", - "session_28", - "session_1", - "session_17", - "session_27", - "session_4" - ], - "sample_id": "conv-47" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_31" - ], - "question": "What is the name of John's cousin's dog?", - "recall": 1.0, - "retrieved_ids": [ - "session_31", - "session_21", - "session_19", - "session_14", - "session_8", - "session_30", - "session_17", - "session_1", - "session_10", - "session_5" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_5" - ], - "question": "What did John adopt in April 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_21", - "session_31", - "session_30", - "session_12", - "session_23", - "session_4", - "session_28", - "session_13", - "session_10" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_5" - ], - "question": "What is the name of the kitten that was adopted by James?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_19", - "session_31", - "session_8", - "session_14", - "session_18", - "session_5", - "session_17", - "session_28", - "session_1" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_6" - ], - "question": "What inspired John to create the game character in the virtual world?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_27", - "session_11", - "session_14", - "session_28", - "session_19", - "session_12", - "session_22", - "session_25", - "session_1" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_6" - ], - "question": "Which country did John visit in 2021?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_16", - "session_31", - "session_17", - "session_10", - "session_13", - "session_30", - "session_4", - "session_23", - "session_28" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_7" - ], - "question": "What kind of assignment was giving James a hard time at work?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_20", - "session_16", - "session_15", - "session_22", - "session_24", - "session_3", - "session_1", - "session_14", - "session_26" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_10" - ], - "question": "What did James and his friends do with the remaining money after helping the dog shelter?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_1", - "session_21", - "session_23", - "session_17", - "session_12", - "session_14", - "session_5", - "session_29", - "session_4" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_10" - ], - "question": "What was the main goal of the money raised from the political campaign organized by John and his friends in May 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_21", - "session_12", - "session_29", - "session_19", - "session_9", - "session_20", - "session_1", - "session_15", - "session_13" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_11" - ], - "question": "What did the system John created help the illegal organization with?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_15", - "session_3", - "session_28", - "session_20", - "session_14", - "session_19", - "session_27", - "session_18", - "session_10" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_11" - ], - "question": "What did James create for the charitable foundation that helped generate reports for analysis?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_6", - "session_30", - "session_22", - "session_4", - "session_1", - "session_27", - "session_21", - "session_24", - "session_3" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_13" - ], - "question": "Who does James support in cricket matches?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_14", - "session_29", - "session_11", - "session_4", - "session_18", - "session_15", - "session_31", - "session_22", - "session_23" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_14" - ], - "question": "What is Max good at doing according to John?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_30", - "session_1", - "session_29", - "session_25", - "session_10", - "session_15", - "session_11", - "session_23", - "session_20" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_15" - ], - "question": "Will there be a background check required to volunteer with the organization James volunteered for?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_28", - "session_11", - "session_2", - "session_8", - "session_26", - "session_10", - "session_19", - "session_29", - "session_18" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_16" - ], - "question": "How did James relax in his free time on 9 July, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_16", - "session_1", - "session_4", - "session_23", - "session_3", - "session_25", - "session_2", - "session_14", - "session_17" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_16" - ], - "question": "What new hobby did John become interested in on 9 July, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_10", - "session_23", - "session_2", - "session_18", - "session_3", - "session_13", - "session_24", - "session_8", - "session_28" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_16" - ], - "question": "When did John plan to return from his trip to Toronto and Vancouver?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_4", - "session_18", - "session_20", - "session_14", - "session_21", - "session_31", - "session_30", - "session_17", - "session_23" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "What made James leave his IT job?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_4", - "session_29", - "session_12", - "session_23", - "session_10", - "session_21", - "session_13", - "session_16", - "session_7" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "Which game tournaments does James plan to organize besides CS:GO?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_10", - "session_3", - "session_11", - "session_12", - "session_4", - "session_16", - "session_23", - "session_6", - "session_29" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "What happened to James's kitten during the recent visit to the clinic?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_14", - "session_25", - "session_2", - "session_6", - "session_13", - "session_27", - "session_16", - "session_31", - "session_12" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_19" - ], - "question": "What aspect of \"The Witcher 3\" does John find boring?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_25", - "session_27", - "session_19", - "session_6", - "session_28", - "session_11", - "session_7", - "session_2", - "session_15" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_19" - ], - "question": "What is John planning to do after receiving Samantha's phone number?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_20", - "session_8", - "session_10", - "session_29", - "session_23", - "session_16", - "session_18", - "session_1", - "session_12" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_22" - ], - "question": "What has James been teaching his siblings?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_21", - "session_25", - "session_4", - "session_23", - "session_20", - "session_17", - "session_14", - "session_26", - "session_3" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_23" - ], - "question": "How much does James pay per dance class?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_16", - "session_8", - "session_15", - "session_18", - "session_17", - "session_12", - "session_31", - "session_1", - "session_25" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_23" - ], - "question": "What did James learn to make in the chemistry class besides omelette and meringue?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_12", - "session_8", - "session_21", - "session_22", - "session_17", - "session_28", - "session_1", - "session_3", - "session_6" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_23" - ], - "question": "Why did James sign up for a ballet class?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_1", - "session_21", - "session_8", - "session_17", - "session_12", - "session_10", - "session_13", - "session_4", - "session_28" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_23" - ], - "question": "What did John prepare for the first time in the cooking class?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_6", - "session_30", - "session_18", - "session_15", - "session_8", - "session_17", - "session_1", - "session_27", - "session_10" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_24" - ], - "question": "What is the name of the board game James tried in September 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_31", - "session_28", - "session_24", - "session_8", - "session_17", - "session_21", - "session_19", - "session_9", - "session_1", - "session_5" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_24" - ], - "question": "Where does John get his ideas from?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_24", - "session_4", - "session_31", - "session_23", - "session_14", - "session_7", - "session_17", - "session_28", - "session_1" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_24" - ], - "question": "What did James use to play when he was younger to let off steam?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_30", - "session_20", - "session_12", - "session_21", - "session_23", - "session_27", - "session_18", - "session_16", - "session_14" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_25" - ], - "question": "What does James do to stay informed and constantly learn about game design?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_28", - "session_21", - "session_20", - "session_29", - "session_30", - "session_3", - "session_17", - "session_1", - "session_16" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_26" - ], - "question": "What kind of gig was James offered at the game dev non-profit organization?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_15", - "session_24", - "session_25", - "session_11", - "session_20", - "session_2", - "session_16", - "session_7", - "session_13" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_26" - ], - "question": "What does James feel about starting the journey as a programming mentor for game developers?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_14", - "session_13", - "session_3", - "session_28", - "session_8", - "session_1", - "session_16", - "session_27", - "session_22" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_27" - ], - "question": "What inspired James to create his painting?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_27", - "session_22", - "session_11", - "session_25", - "session_14", - "session_17", - "session_4", - "session_23", - "session_6" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_29" - ], - "question": "What games were played at the gaming tournament organized by James on 31 October, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_29", - "session_30", - "session_4", - "session_3", - "session_18", - "session_11", - "session_16", - "session_12", - "session_15" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_29" - ], - "question": "What was the purpose of the gaming tournament organized by James on 31 October, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_29", - "session_16", - "session_18", - "session_30", - "session_4", - "session_3", - "session_10", - "session_15", - "session_12" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_29" - ], - "question": "What decision did John and Samantha make on 31 October, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_22", - "session_19", - "session_23", - "session_18", - "session_8", - "session_31", - "session_11", - "session_4", - "session_17" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_29" - ], - "question": "Where did John and Samantha decide to live together on 31 October, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_6", - "session_10", - "session_20", - "session_1", - "session_23", - "session_19", - "session_15", - "session_9", - "session_22" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_29" - ], - "question": "Why did John and Samantha choose an apartment near McGee's bar?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_29", - "session_30", - "session_19", - "session_21", - "session_13", - "session_20", - "session_17", - "session_28", - "session_1" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_30" - ], - "question": "What game is James hooked on playing on 5 November, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_20", - "session_30", - "session_28", - "session_26", - "session_24", - "session_3", - "session_4", - "session_8", - "session_29" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_31" - ], - "question": "What project did James work on with a game developer by 7 November, 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_31", - "session_26", - "session_22", - "session_3", - "session_25", - "session_28", - "session_1", - "session_17", - "session_27", - "session_4" - ], - "sample_id": "conv-47" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_31" - ], - "question": "What is the name of James's cousin's dog?", - "recall": 1.0, - "retrieved_ids": [ - "session_31", - "session_21", - "session_19", - "session_8", - "session_14", - "session_30", - "session_17", - "session_1", - "session_10", - "session_5" - ], - "sample_id": "conv-47" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_1" - ], - "question": "What kind of project was Jolene working on in the beginning of January 2023?", - "recall": 0.0, - "retrieved_ids": [ - "session_23", - "session_3", - "session_12", - "session_7", - "session_22", - "session_17", - "session_6", - "session_14", - "session_19", - "session_16" - ], - "sample_id": "conv-48" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_2", - "session_6" - ], - "question": "Which of Deborah`s family and friends have passed away?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_22", - "session_1", - "session_24", - "session_12", - "session_6", - "session_9", - "session_30", - "session_13", - "session_28" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_1" - ], - "question": "When did Deborah`s mother pass away?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_22", - "session_2", - "session_4", - "session_30", - "session_13", - "session_24", - "session_3", - "session_8", - "session_5" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_1" - ], - "question": "When did Jolene`s mother pass away?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_22", - "session_2", - "session_4", - "session_30", - "session_13", - "session_24", - "session_3", - "session_8", - "session_5" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_1" - ], - "question": "When did Jolene's mom gift her a pendant?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_4", - "session_1", - "session_30", - "session_12", - "session_28", - "session_20", - "session_2", - "session_29", - "session_23" - ], - "sample_id": "conv-48" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_1" - ], - "question": "In what country did Jolene's mother buy her the pendant?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_4", - "session_23", - "session_2", - "session_22", - "session_15", - "session_14", - "session_16", - "session_28", - "session_29" - ], - "sample_id": "conv-48" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1" - ], - "question": "What symbolic gifts do Deborah and Jolene have from their mothers?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_4", - "session_1", - "session_5", - "session_11", - "session_23", - "session_19", - "session_2", - "session_28", - "session_14" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_1" - ], - "question": "Which country were Jolene and her mother visiting in 2010?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_1", - "session_22", - "session_2", - "session_4", - "session_12", - "session_29", - "session_19", - "session_14", - "session_6" - ], - "sample_id": "conv-48" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_15", - "session_2", - "session_6" - ], - "question": "What helped Deborah find peace when grieving deaths of her loved ones?", - "recall": 0.75, - "retrieved_ids": [ - "session_1", - "session_28", - "session_4", - "session_12", - "session_2", - "session_29", - "session_19", - "session_15", - "session_16", - "session_14" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_2" - ], - "question": "When did Deborah's father pass away?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_22", - "session_2", - "session_30", - "session_13", - "session_7", - "session_3", - "session_24", - "session_8", - "session_5" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_2" - ], - "question": "When was Deborah's parents' wedding?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_24", - "session_7", - "session_20", - "session_26", - "session_14", - "session_25", - "session_23", - "session_13", - "session_29" - ], - "sample_id": "conv-48" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_19", - "session_2", - "session_23", - "session_28" - ], - "question": "Is Deborah married?", - "recall": 0.75, - "retrieved_ids": [ - "session_7", - "session_28", - "session_20", - "session_14", - "session_25", - "session_26", - "session_23", - "session_13", - "session_2", - "session_29" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_2" - ], - "question": "When did Deborah receive an appreciation letter from her community?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_2", - "session_19", - "session_1", - "session_16", - "session_28", - "session_29", - "session_3", - "session_21", - "session_27" - ], - "sample_id": "conv-48" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_19", - "session_2", - "session_4", - "session_6" - ], - "question": "What places give Deborah peace?", - "recall": 0.75, - "retrieved_ids": [ - "session_6", - "session_2", - "session_23", - "session_1", - "session_28", - "session_22", - "session_4", - "session_17", - "session_18", - "session_20" - ], - "sample_id": "conv-48" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_12", - "session_2", - "session_29" - ], - "question": "What were Deborah's mother's hobbies?", - "recall": 0.6667, - "retrieved_ids": [ - "session_2", - "session_19", - "session_22", - "session_4", - "session_27", - "session_1", - "session_9", - "session_13", - "session_23", - "session_12" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What pets does Jolene have?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_14", - "session_28", - "session_15", - "session_22", - "session_16", - "session_29", - "session_25", - "session_9", - "session_20" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What are the names of Jolene's snakes?", - "recall": 0.0, - "retrieved_ids": [], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_2" - ], - "question": "When did Jolene buy her pet Seraphim?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_2", - "session_28", - "session_8", - "session_22", - "session_16", - "session_15", - "session_4", - "session_1", - "session_29" - ], - "sample_id": "conv-48" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_2" - ], - "question": "In what country did Jolene buy snake Seraphim?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_8", - "session_14", - "session_28", - "session_23", - "session_15", - "session_5", - "session_3", - "session_12", - "session_22" - ], - "sample_id": "conv-48" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_2" - ], - "question": "How many times has Jolene been to France?", - "recall": 0.5, - "retrieved_ids": [ - "session_23", - "session_1", - "session_9", - "session_16", - "session_4", - "session_19", - "session_6", - "session_14", - "session_8", - "session_7" - ], - "sample_id": "conv-48" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_15", - "session_19", - "session_2", - "session_20" - ], - "question": "Which games have Jolene and her partner played together?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_12", - "session_2", - "session_7", - "session_15", - "session_3", - "session_24", - "session_27", - "session_16", - "session_20" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_2" - ], - "question": "When do Jolene and her partner plan to complete the game \"Walking Dead\"?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_22", - "session_19", - "session_12", - "session_15", - "session_16", - "session_24", - "session_20", - "session_27", - "session_25" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_3" - ], - "question": "When did Deborah meet Anna?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_15", - "session_10", - "session_4", - "session_26", - "session_8", - "session_19", - "session_1", - "session_7", - "session_27" - ], - "sample_id": "conv-48" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_2", - "session_3" - ], - "question": "Why did Jolene sometimes put off doing yoga?", - "recall": 0.0, - "retrieved_ids": [ - "session_13", - "session_7", - "session_11", - "session_15", - "session_1", - "session_18", - "session_14", - "session_8", - "session_27", - "session_29" - ], - "sample_id": "conv-48" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_14", - "session_4" - ], - "question": "What new yoga poses did Deborah try?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_16", - "session_2", - "session_22", - "session_14", - "session_8", - "session_20", - "session_23", - "session_28", - "session_3" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What are Jolene's favorite books?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_2", - "session_4", - "session_18", - "session_15", - "session_11", - "session_24", - "session_19", - "session_26", - "session_16" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_4" - ], - "question": "Which book did Jolene read in January 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_4", - "session_26", - "session_18", - "session_2", - "session_24", - "session_3", - "session_5", - "session_8", - "session_19" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_4" - ], - "question": "When was Jolene in Bogota?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_3", - "session_10", - "session_25", - "session_20", - "session_9", - "session_14", - "session_2", - "session_23", - "session_29" - ], - "sample_id": "conv-48" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_4" - ], - "question": "In what country was Jolene during summer 2022?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_23", - "session_16", - "session_6", - "session_25", - "session_11", - "session_1", - "session_12", - "session_18", - "session_13" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_5" - ], - "question": "When did Jolene have a mini-retreat to reflect on her career?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_5", - "session_15", - "session_22", - "session_14", - "session_27", - "session_30", - "session_25", - "session_17", - "session_21" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_6" - ], - "question": "When did Jolene have a dinner and drinks with her friends?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_12", - "session_4", - "session_28", - "session_23", - "session_7", - "session_29", - "session_24", - "session_9", - "session_2" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_6" - ], - "question": "When was the last photo of Deborah and Karlie taken?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_23", - "session_30", - "session_24", - "session_1", - "session_25", - "session_14", - "session_7", - "session_20", - "session_28" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_6" - ], - "question": "When was Deborah in Bali?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_8", - "session_26", - "session_14", - "session_25", - "session_23", - "session_2", - "session_13", - "session_20", - "session_29" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "How long have Jolene and her partner been together?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_7", - "session_19", - "session_30", - "session_9", - "session_20", - "session_13", - "session_28", - "session_2", - "session_11" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_7" - ], - "question": "Which year did Jolene and her partner start dating?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_7", - "session_16", - "session_30", - "session_2", - "session_15", - "session_18", - "session_24", - "session_29", - "session_26" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_7" - ], - "question": "When did Deborah go for her first morning jog in a nearby park?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_4", - "session_19", - "session_15", - "session_3", - "session_12", - "session_1", - "session_13", - "session_23", - "session_5" - ], - "sample_id": "conv-48" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_13", - "session_21", - "session_22", - "session_24", - "session_25", - "session_26", - "session_8" - ], - "question": "How old is Jolene?", - "recall": 0.4286, - "retrieved_ids": [ - "session_22", - "session_1", - "session_24", - "session_28", - "session_23", - "session_29", - "session_4", - "session_2", - "session_25", - "session_20" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_8" - ], - "question": "When did Jolene take Seraphim to the park?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_2", - "session_28", - "session_4", - "session_7", - "session_19", - "session_15", - "session_5", - "session_3", - "session_14" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_9" - ], - "question": "When did Deborah start the yoga class in the neighborhood?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_7", - "session_24", - "session_11", - "session_18", - "session_23", - "session_29", - "session_8", - "session_26", - "session_5" - ], - "sample_id": "conv-48" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_18" - ], - "question": "What time management techniques do Deborah and Jolene use?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_18", - "session_14", - "session_9", - "session_8", - "session_25", - "session_6", - "session_19", - "session_12", - "session_16" - ], - "sample_id": "conv-48" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_10" - ], - "question": "Does Deborah live close to the beach or the mountains?", - "recall": 0.0, - "retrieved_ids": [ - "session_4", - "session_2", - "session_6", - "session_29", - "session_19", - "session_24", - "session_28", - "session_25", - "session_3", - "session_13" - ], - "sample_id": "conv-48" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_11", - "session_28" - ], - "question": "What ways do Deborah and Jolene use to enhance their yoga practice?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_22", - "session_11", - "session_4", - "session_26", - "session_21", - "session_10", - "session_23", - "session_5", - "session_17" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_11" - ], - "question": "What music pieces does Deborah listen to during her yoga practice?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_15", - "session_28", - "session_30", - "session_16", - "session_8", - "session_19", - "session_7", - "session_21", - "session_9" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_12" - ], - "question": "When did Deborah go for a bicycle ride with Anna?", - "recall": 0.0, - "retrieved_ids": [ - "session_27", - "session_3", - "session_10", - "session_4", - "session_8", - "session_19", - "session_15", - "session_7", - "session_2", - "session_23" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_12" - ], - "question": "When did Deborah go to an art show with Anna?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_4", - "session_15", - "session_3", - "session_10", - "session_5", - "session_8", - "session_19", - "session_22", - "session_17" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_13" - ], - "question": "When did Jolene finish her robotics project?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_1", - "session_12", - "session_5", - "session_14", - "session_16", - "session_13", - "session_28", - "session_27", - "session_2" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_12", - "session_13", - "session_3" - ], - "question": "How long did Jolene work on the robotics project given to her by her Professor?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_12", - "session_20", - "session_13", - "session_22", - "session_11", - "session_7", - "session_9", - "session_28", - "session_19" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_13" - ], - "question": "When did Jolene do yoga at Talkeetna?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_8", - "session_3", - "session_26", - "session_24", - "session_22", - "session_30", - "session_7", - "session_2", - "session_23" - ], - "sample_id": "conv-48" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_13" - ], - "question": "Which US state did Jolene visit during her internship?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_1", - "session_13", - "session_12", - "session_16", - "session_23", - "session_2", - "session_29", - "session_6", - "session_21" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "How long has Jolene been doing yoga and meditation?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_13", - "session_7", - "session_28", - "session_11", - "session_19", - "session_22", - "session_16", - "session_9", - "session_12" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_13" - ], - "question": "Which year did Jolene start practicing yoga?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_16", - "session_23", - "session_13", - "session_9", - "session_22", - "session_18", - "session_20", - "session_28", - "session_15" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_14" - ], - "question": "When did Jolene buy a new aquarium for Seraphim?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_8", - "session_2", - "session_28", - "session_24", - "session_3", - "session_25", - "session_5", - "session_16", - "session_23" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_16" - ], - "question": "When did Jolene lose a lot of progress in her work?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_4", - "session_1", - "session_16", - "session_7", - "session_27", - "session_2", - "session_28", - "session_29", - "session_6" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_16", - "session_28" - ], - "question": "When did Jolene adopt her snake Susie?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_2", - "session_28", - "session_8", - "session_15", - "session_4", - "session_12", - "session_22", - "session_1", - "session_29" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_16", - "session_2" - ], - "question": "Which pet did Jolene adopt first - Susie or Seraphim?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_28", - "session_2", - "session_14", - "session_7", - "session_8", - "session_3", - "session_24", - "session_5", - "session_10" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_16", - "session_2" - ], - "question": "Which pet did Jolene adopt more recently - Susie or Seraphim?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_2", - "session_16", - "session_14", - "session_27", - "session_8", - "session_22", - "session_15", - "session_25", - "session_18" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_18" - ], - "question": "When did Deborah lead a meditation session during the sunset?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_22", - "session_18", - "session_4", - "session_25", - "session_27", - "session_13", - "session_1", - "session_10", - "session_19" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_19" - ], - "question": "When did Jolene gift her partner a new console?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_2", - "session_3", - "session_20", - "session_23", - "session_29", - "session_12", - "session_30", - "session_28", - "session_15" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "What games does Jolene recommend for Deborah?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_24", - "session_13", - "session_2", - "session_12", - "session_16", - "session_20", - "session_11", - "session_15", - "session_27" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "What do Deborah and her husband do together?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_2", - "session_15", - "session_1", - "session_23", - "session_29", - "session_4", - "session_16", - "session_14", - "session_28" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_21" - ], - "question": "When did Deborah go to a yoga retreat near her mom's place?", - "recall": 1.0, - "retrieved_ids": [ - "session_30", - "session_4", - "session_1", - "session_21", - "session_19", - "session_14", - "session_26", - "session_25", - "session_5", - "session_27" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_22" - ], - "question": "What projects is Jolene planning for next year?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_1", - "session_2", - "session_22", - "session_30", - "session_16", - "session_14", - "session_21", - "session_29", - "session_26" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_22" - ], - "question": "Where did Deborah get her cats?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_15", - "session_14", - "session_28", - "session_4", - "session_1", - "session_27", - "session_16", - "session_2", - "session_29" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_22" - ], - "question": "How old are Deborah's cats?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_4", - "session_1", - "session_24", - "session_28", - "session_23", - "session_29", - "session_15", - "session_27", - "session_2" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15", - "session_22" - ], - "question": "Does Deborah like cats?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_27", - "session_15", - "session_5", - "session_14", - "session_11", - "session_12", - "session_8", - "session_20", - "session_29" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_23" - ], - "question": "Which country was Jolene located in during the last week of August 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_12", - "session_18", - "session_16", - "session_23", - "session_2", - "session_4", - "session_13", - "session_26", - "session_11" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_23" - ], - "question": "When did Jolene and her partner return home from Rio de Janeiro?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_22", - "session_12", - "session_30", - "session_2", - "session_26", - "session_18", - "session_29", - "session_19", - "session_24" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "What was Jolene doing with her partner in Rio de Janeiro?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_19", - "session_15", - "session_30", - "session_24", - "session_2", - "session_28", - "session_12", - "session_16", - "session_29" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_23" - ], - "question": "When did Deborah visit Brazil?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_6", - "session_1", - "session_14", - "session_29", - "session_24", - "session_3", - "session_5", - "session_8", - "session_2" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "Have Deborah and Jolene been to Rio de Janeiro?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_21", - "session_24", - "session_18", - "session_13", - "session_7", - "session_6", - "session_14", - "session_16", - "session_20" - ], - "sample_id": "conv-48" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_23" - ], - "question": "Is the friend who wrote Deborah the motivational quote no longer alive?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_12", - "session_6", - "session_9", - "session_18", - "session_24", - "session_30", - "session_7", - "session_4", - "session_15" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_24" - ], - "question": "When did Deborah go to a community meetup?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_19", - "session_3", - "session_22", - "session_4", - "session_1", - "session_9", - "session_16", - "session_27", - "session_29" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "When did Jolene's parents give her first console?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_10", - "session_7", - "session_3", - "session_24", - "session_5", - "session_20", - "session_28", - "session_19", - "session_29" - ], - "sample_id": "conv-48" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_2", - "session_24" - ], - "question": "Did Jolene teach herself how to play the console?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_3", - "session_24", - "session_19", - "session_15", - "session_12", - "session_20", - "session_27", - "session_9", - "session_10" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_26" - ], - "question": "What do Deborah and Jolene plan to try when they meet in a new cafe?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_22", - "session_5", - "session_23", - "session_3", - "session_17", - "session_27", - "session_7", - "session_1", - "session_30" - ], - "sample_id": "conv-48" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_27" - ], - "question": "What card game is Deborah talking about?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_24", - "session_20", - "session_2", - "session_19", - "session_15", - "session_3", - "session_12", - "session_16", - "session_7" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_29" - ], - "question": "When did Jolene and her partner try scuba diving lessons?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_10", - "session_2", - "session_16", - "session_12", - "session_30", - "session_23", - "session_19", - "session_15", - "session_28" - ], - "sample_id": "conv-48" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_27", - "session_29" - ], - "question": "Where did Jolene and her partner find a cool diving spot?", - "recall": 0.5, - "retrieved_ids": [ - "session_23", - "session_19", - "session_15", - "session_29", - "session_12", - "session_30", - "session_2", - "session_4", - "session_14", - "session_28" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_2" - ], - "question": "Where did Jolene and her partner spend most of September 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_12", - "session_7", - "session_22", - "session_4", - "session_19", - "session_14", - "session_6", - "session_26", - "session_29" - ], - "sample_id": "conv-48" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_28", - "session_29" - ], - "question": "Has Deborah tried surfing?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_28", - "session_29", - "session_14", - "session_18", - "session_26", - "session_13", - "session_25", - "session_23", - "session_2" - ], - "sample_id": "conv-48" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_29" - ], - "question": "Has Jolene tried surfing?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_28", - "session_29", - "session_14", - "session_18", - "session_23", - "session_25", - "session_9", - "session_20", - "session_2" - ], - "sample_id": "conv-48" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_29" - ], - "question": "When did the Deboran and Jolene agree to go surfing?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_10", - "session_27", - "session_28", - "session_8", - "session_24", - "session_5", - "session_30", - "session_23", - "session_2" - ], - "sample_id": "conv-48" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_2", - "session_3", - "session_4", - "session_6" - ], - "question": "Which locations does Deborah practice her yoga at?", - "recall": 0.0, - "retrieved_ids": [ - "session_16", - "session_20", - "session_28", - "session_15", - "session_13", - "session_17", - "session_26", - "session_11", - "session_7", - "session_27" - ], - "sample_id": "conv-48" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_13", - "session_21" - ], - "question": "What kind of professional activities does Jolene participate in to gain more experience in her field?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_21", - "session_23", - "session_19", - "session_15", - "session_5", - "session_12", - "session_6", - "session_7", - "session_3" - ], - "sample_id": "conv-48" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_17", - "session_3", - "session_4" - ], - "question": "What kind of engineering projects has Jolene worked on?", - "recall": 0.5, - "retrieved_ids": [ - "session_3", - "session_13", - "session_22", - "session_12", - "session_17", - "session_5", - "session_19", - "session_7", - "session_6", - "session_23" - ], - "sample_id": "conv-48" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_15", - "session_4" - ], - "question": "Which community activities have Deborah and Anna participated in?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_19", - "session_3", - "session_22", - "session_4", - "session_12", - "session_8", - "session_10", - "session_26", - "session_11" - ], - "sample_id": "conv-48" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_2", - "session_23", - "session_4" - ], - "question": "What gifts has Deborah received?", - "recall": 0.6667, - "retrieved_ids": [ - "session_19", - "session_21", - "session_2", - "session_3", - "session_26", - "session_25", - "session_14", - "session_13", - "session_23", - "session_29" - ], - "sample_id": "conv-48" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_23", - "session_6" - ], - "question": "Which countries has Deborah traveled to?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_6", - "session_27", - "session_2", - "session_20", - "session_25", - "session_26", - "session_14", - "session_13", - "session_29" - ], - "sample_id": "conv-48" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_12", - "session_15", - "session_28", - "session_29" - ], - "question": "What activities does Deborah pursue besides practicing and teaching yoga?", - "recall": 0.5, - "retrieved_ids": [ - "session_22", - "session_13", - "session_19", - "session_2", - "session_4", - "session_14", - "session_11", - "session_15", - "session_26", - "session_12" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What are the names of Jolene's snakes?", - "recall": 0.0, - "retrieved_ids": [], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What are Jolene's favorite books?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_2", - "session_4", - "session_18", - "session_15", - "session_11", - "session_24", - "session_19", - "session_26", - "session_16" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_11" - ], - "question": "What music pieces does Deborah listen to during her yoga practice?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_15", - "session_28", - "session_30", - "session_16", - "session_8", - "session_19", - "session_7", - "session_21", - "session_9" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "What games does Jolene recommend for Deborah?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_24", - "session_13", - "session_2", - "session_12", - "session_16", - "session_20", - "session_11", - "session_15", - "session_27" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_22" - ], - "question": "What projects is Jolene planning for next year?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_1", - "session_2", - "session_22", - "session_30", - "session_16", - "session_14", - "session_21", - "session_29", - "session_26" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_22" - ], - "question": "Where did Deborah get her cats?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_15", - "session_14", - "session_28", - "session_4", - "session_1", - "session_27", - "session_16", - "session_2", - "session_29" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_22" - ], - "question": "How old are Deborah's cats?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_4", - "session_1", - "session_24", - "session_28", - "session_23", - "session_29", - "session_15", - "session_27", - "session_2" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "What was Jolene doing with her partner in Rio de Janeiro?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_19", - "session_15", - "session_30", - "session_24", - "session_2", - "session_28", - "session_12", - "session_16", - "session_29" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "Have Deborah and Jolene been to Rio de Janeiro?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_21", - "session_24", - "session_18", - "session_13", - "session_7", - "session_6", - "session_14", - "session_16", - "session_20" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "When did Jolene's parents give her first console?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_10", - "session_7", - "session_3", - "session_24", - "session_5", - "session_20", - "session_29", - "session_28", - "session_19" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_26" - ], - "question": "What do Deborah and Jolene plan to try when they meet in a new cafe?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_22", - "session_5", - "session_23", - "session_3", - "session_17", - "session_27", - "session_7", - "session_1", - "session_30" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_1" - ], - "question": "What project did Jolene finish last week before 23 January, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_14", - "session_5", - "session_22", - "session_12", - "session_27", - "session_16", - "session_13", - "session_28", - "session_26" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "When did Jolene buy her pet snake?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_15", - "session_22", - "session_14", - "session_16", - "session_8", - "session_28", - "session_1", - "session_12", - "session_29" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "What project was Jolene working on as of 1 February, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_13", - "session_3", - "session_1", - "session_12", - "session_7", - "session_25", - "session_14", - "session_22", - "session_16" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "Where did Deborah meet her new neighbor Anna?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_19", - "session_4", - "session_29", - "session_1", - "session_12", - "session_7", - "session_9", - "session_26", - "session_27" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "What activity did Jolene and her partner plan to do together instead of resuming yoga?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_22", - "session_15", - "session_3", - "session_30", - "session_2", - "session_26", - "session_12", - "session_13", - "session_11" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What milestone did Jolene achieve recently on 4 February, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_10", - "session_25", - "session_14", - "session_13", - "session_4", - "session_15", - "session_18", - "session_27", - "session_5" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What is Jolene's favorite book which she mentioned on 4 February, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_15", - "session_29", - "session_4", - "session_20", - "session_2", - "session_18", - "session_16", - "session_24", - "session_26" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What does Deborah bring with her whenever she comes to reflect on her mom?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_15", - "session_29", - "session_19", - "session_22", - "session_1", - "session_14", - "session_23", - "session_21", - "session_7" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_5" - ], - "question": "What new outlook did Jolene gain after her mini retreat on 9 February, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_14", - "session_13", - "session_28", - "session_19", - "session_27", - "session_12", - "session_18", - "session_7", - "session_21" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_5" - ], - "question": "What cool stuff did Jolene accomplish at the retreat on 9 February, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_21", - "session_17", - "session_14", - "session_23", - "session_25", - "session_4", - "session_27", - "session_12", - "session_16" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_5" - ], - "question": "What idea did Jolene have to help underprivileged kids learn about STEM subjects on 9 February, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_21", - "session_13", - "session_24", - "session_29", - "session_22", - "session_17", - "session_26", - "session_25", - "session_30" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_5" - ], - "question": "How does Jolene plan to involve local engineers in her idea of teaching STEM to underprivileged kids?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_4", - "session_18", - "session_29", - "session_22", - "session_17", - "session_30", - "session_13", - "session_15", - "session_21" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_6" - ], - "question": "What gave Deborah peace in the garden she visited?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_1", - "session_14", - "session_29", - "session_23", - "session_4", - "session_20", - "session_5", - "session_21", - "session_3" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_6" - ], - "question": "Why did Deborah spend time in the garden?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_7", - "session_4", - "session_30", - "session_26", - "session_14", - "session_29", - "session_8", - "session_2", - "session_19" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "How did Jolene and her partner initially meet?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_22", - "session_12", - "session_3", - "session_1", - "session_26", - "session_30", - "session_19", - "session_2", - "session_27" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What activity does Deborah incorporate into her daily routine after going for a morning jog in the park?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_19", - "session_15", - "session_13", - "session_10", - "session_4", - "session_12", - "session_28", - "session_22", - "session_26" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "According to Jolene, what does exercise help her to feel?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_15", - "session_9", - "session_28", - "session_29", - "session_4", - "session_14", - "session_1", - "session_16", - "session_2" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What did Deb share a photo of, which brought a smile to Jolene's face?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_25", - "session_9", - "session_2", - "session_23", - "session_29", - "session_6", - "session_20", - "session_4", - "session_7" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What is one of Jolene's favorite dishes?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_29", - "session_11", - "session_24", - "session_19", - "session_4", - "session_20", - "session_18", - "session_16", - "session_15" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What picture did Jolene share related to feeling overwhelmed?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_8", - "session_15", - "session_4", - "session_30", - "session_24", - "session_28", - "session_17", - "session_7", - "session_10" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What did Jolene and Deb discuss as a helpful strategy for studying and time management?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_21", - "session_8", - "session_10", - "session_26", - "session_9", - "session_15", - "session_19", - "session_24", - "session_27" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_9" - ], - "question": "What did Jolene ask Deb to help with on 13 March, 2023?", - "recall": 0.0, - "retrieved_ids": [ - "session_13", - "session_12", - "session_25", - "session_8", - "session_5", - "session_21", - "session_20", - "session_26", - "session_30", - "session_7" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "What method does Deb suggest Jolene to try for organizing tasks based on importance and urgency?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_24", - "session_18", - "session_15", - "session_16", - "session_27", - "session_1", - "session_8", - "session_22", - "session_19" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "What did Jolene and Anna discuss while watching the sunset by the sea?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_19", - "session_4", - "session_23", - "session_22", - "session_8", - "session_3", - "session_14", - "session_18", - "session_7" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "How does Jolene plan to pursue her dream of learning to surf?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_23", - "session_19", - "session_29", - "session_22", - "session_4", - "session_13", - "session_26", - "session_28", - "session_2" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_11" - ], - "question": "What did Deborah buy to enhance her yoga practice besides the props?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_28", - "session_22", - "session_14", - "session_20", - "session_15", - "session_26", - "session_16", - "session_21", - "session_17" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_11" - ], - "question": "What type of music does Deborah find helpful during her yoga practice?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_20", - "session_13", - "session_28", - "session_19", - "session_22", - "session_15", - "session_30", - "session_16", - "session_6" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_11" - ], - "question": "Who are the musicians mentioned by Jolene that she enjoys listening to during her yoga practice?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_22", - "session_19", - "session_16", - "session_20", - "session_11", - "session_12", - "session_9", - "session_7", - "session_21" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_11" - ], - "question": "What album does Deborah recommend for meditation and deep relaxation?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_13", - "session_20", - "session_25", - "session_2", - "session_8", - "session_28", - "session_4", - "session_7", - "session_19" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_12" - ], - "question": "Which show did Deborah go to with a friend on 9 April, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_23", - "session_4", - "session_5", - "session_22", - "session_6", - "session_9", - "session_24", - "session_28", - "session_7" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_12" - ], - "question": "What does Deborah find comforting about going to art shows?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_14", - "session_4", - "session_2", - "session_29", - "session_22", - "session_16", - "session_6", - "session_17", - "session_5" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_12" - ], - "question": "How does Jolene describe the time spent with her snakes and partner?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_12", - "session_19", - "session_22", - "session_2", - "session_30", - "session_13", - "session_23", - "session_28", - "session_8" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_12" - ], - "question": "What does Jolene enjoy doing with her partner after a long day?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_12", - "session_28", - "session_7", - "session_20", - "session_13", - "session_9", - "session_10", - "session_18", - "session_11" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "What is Jolene currently doing in June 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_7", - "session_10", - "session_20", - "session_13", - "session_15", - "session_4", - "session_26", - "session_11", - "session_2" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_14" - ], - "question": "For how long has Jolene had Seraphim as a pet?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_14", - "session_2", - "session_11", - "session_8", - "session_16", - "session_13", - "session_20", - "session_15", - "session_22" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_14" - ], - "question": "How does Jolene feel when spending time with Seraphim?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_2", - "session_8", - "session_29", - "session_19", - "session_4", - "session_28", - "session_6", - "session_26", - "session_7" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_14" - ], - "question": "Which new yoga pose did Deborah share a photo of?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_28", - "session_14", - "session_20", - "session_2", - "session_4", - "session_9", - "session_22", - "session_24", - "session_16" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "What group activity did Deborah start with Anna?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_10", - "session_19", - "session_2", - "session_12", - "session_8", - "session_4", - "session_3", - "session_11", - "session_26" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "What made being part of the running group easy for Deborah to stay motivated?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_13", - "session_25", - "session_8", - "session_2", - "session_9", - "session_10", - "session_18", - "session_1", - "session_3" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "Why did Jolene decide to get a snake as a pet?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_16", - "session_7", - "session_14", - "session_2", - "session_8", - "session_22", - "session_6", - "session_12", - "session_29" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "What is the favorite game Jolene plays with her partner?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_2", - "session_15", - "session_24", - "session_7", - "session_20", - "session_27", - "session_3", - "session_12", - "session_16" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "What activity does Deborah do with her cats?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_15", - "session_13", - "session_2", - "session_19", - "session_12", - "session_28", - "session_11", - "session_26", - "session_27" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "How does Jolene describe the feeling of finding her snake snuggled under the bed after it got out?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_10", - "session_22", - "session_12", - "session_18", - "session_8", - "session_28", - "session_19", - "session_4", - "session_2" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "Why does Deborah take her cats out for a run in the park every day?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_7", - "session_19", - "session_4", - "session_1", - "session_14", - "session_8", - "session_22", - "session_3", - "session_27" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "How did Jolene come to have her pet, Susie?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_28", - "session_2", - "session_15", - "session_29", - "session_1", - "session_14", - "session_19", - "session_22", - "session_4" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "What activities have been helping Jolene stay distracted during tough times?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_13", - "session_15", - "session_25", - "session_2", - "session_22", - "session_12", - "session_11", - "session_26", - "session_19" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "What kind of yoga routine does Deborah recommend to Jolene?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_6", - "session_13", - "session_3", - "session_26", - "session_7", - "session_12", - "session_11", - "session_25", - "session_16" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What did Jolene design inspired by their love for space and engines?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_11", - "session_15", - "session_28", - "session_23", - "session_4", - "session_22", - "session_5", - "session_10", - "session_3" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What journal has Jolene been using to help track tasks and stay organized?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_10", - "session_20", - "session_8", - "session_22", - "session_11", - "session_21", - "session_15", - "session_19", - "session_26" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "What game did Jolene recommend for being calming and cute?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_15", - "session_13", - "session_20", - "session_2", - "session_11", - "session_27", - "session_24", - "session_12", - "session_16" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "What game did Jolene suggest as an awesome open-world game for the Nintendo Switch?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_24", - "session_5", - "session_4", - "session_15", - "session_29", - "session_27", - "session_16", - "session_2", - "session_17" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "What did Deborah and her husband use to play to bond and make memories?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_2", - "session_28", - "session_15", - "session_4", - "session_23", - "session_24", - "session_12", - "session_27", - "session_7" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "What is special about the bench at the park near Deborah's house?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_1", - "session_4", - "session_29", - "session_8", - "session_23", - "session_2", - "session_28", - "session_20", - "session_3" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "What did Deborah and her mom chat about at their special bench in the park?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_1", - "session_4", - "session_2", - "session_20", - "session_23", - "session_28", - "session_29", - "session_22", - "session_7" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "What feeling does Deborah get when she thinks about the time spent with her mom at their special spot?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_22", - "session_19", - "session_23", - "session_29", - "session_1", - "session_28", - "session_15", - "session_2", - "session_13" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_20" - ], - "question": "What habits does Jolene practice to feel balanced?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_11", - "session_22", - "session_19", - "session_26", - "session_16", - "session_30", - "session_7", - "session_13", - "session_27" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_20" - ], - "question": "Which yoga pose is Jolene a fan of for rest and calmness?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_28", - "session_4", - "session_22", - "session_11", - "session_14", - "session_24", - "session_16", - "session_19", - "session_25" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_20" - ], - "question": "How long has Jolene been doing yoga?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_13", - "session_7", - "session_28", - "session_19", - "session_22", - "session_9", - "session_11", - "session_12", - "session_16" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_21" - ], - "question": "What did Jolene participate in recently that provided her with a rewarding experience?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_4", - "session_21", - "session_29", - "session_15", - "session_25", - "session_14", - "session_19", - "session_27", - "session_16" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_21" - ], - "question": "How did Jolene feel after receiving positive feedback at the virtual conference?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_30", - "session_18", - "session_10", - "session_13", - "session_28", - "session_19", - "session_2", - "session_3", - "session_12" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_21" - ], - "question": "What kind of event did Jolene present at recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_21", - "session_29", - "session_19", - "session_23", - "session_6", - "session_24", - "session_15", - "session_3", - "session_12" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_22" - ], - "question": "What did Jolene's mom stress the value of, which she wants to keep in mind for her engineering projects?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_13", - "session_20", - "session_2", - "session_8", - "session_23", - "session_30", - "session_17", - "session_19", - "session_15" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_22" - ], - "question": "What type of projects is Jolene interested in getting involved in the future?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_18", - "session_29", - "session_3", - "session_12", - "session_16", - "session_6", - "session_9", - "session_15", - "session_26" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_22" - ], - "question": "How did Deborah get Luna, one of her cats?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_1", - "session_16", - "session_14", - "session_28", - "session_15", - "session_4", - "session_27", - "session_29", - "session_2" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_22" - ], - "question": "How old is Max?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_4", - "session_1", - "session_24", - "session_28", - "session_29", - "session_23", - "session_2" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "What type of classes did Jolene and her partner check out during their trip to Rio de Janeiro on 30 August, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_12", - "session_22", - "session_4", - "session_30", - "session_11", - "session_28", - "session_26", - "session_6", - "session_24" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "What type of place does Jolene visit to meditate?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_23", - "session_28", - "session_1", - "session_29", - "session_11", - "session_14", - "session_20", - "session_22", - "session_17" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "What was the new plant Jolene got used as a reminder for on 30 August, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_25", - "session_8", - "session_29", - "session_4", - "session_3", - "session_14", - "session_28", - "session_22", - "session_30" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "Why did Jolene get the new plant on 30 August, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_28", - "session_4", - "session_3", - "session_14", - "session_24", - "session_8", - "session_16", - "session_29", - "session_22" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "What has Jolene been focusing on lately besides studying?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_24", - "session_21", - "session_19", - "session_25", - "session_20", - "session_8", - "session_15", - "session_22", - "session_27" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "How did Deborah's mom support her yoga practice when she first started?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_24", - "session_16", - "session_23", - "session_15", - "session_7", - "session_10", - "session_29", - "session_5", - "session_3" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "What was the video game console that Jolene's parents got her at age 10?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_2", - "session_15", - "session_20", - "session_19", - "session_29", - "session_3", - "session_12", - "session_16", - "session_7" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "What was one of Jolene's favorite games to play with her mom on the nintendo wii game system?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_2", - "session_24", - "session_7", - "session_20", - "session_12", - "session_15", - "session_16", - "session_27", - "session_17" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_25" - ], - "question": "What course did Jolene sign up for on 6 September 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_7", - "session_8", - "session_5", - "session_9", - "session_24", - "session_2", - "session_3", - "session_26", - "session_30" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_26" - ], - "question": "Why did Jolene have to reschedule their meeting with Deborah on September 8, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_1", - "session_11", - "session_27", - "session_23", - "session_28", - "session_22", - "session_7", - "session_3", - "session_26" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_27" - ], - "question": "Where did Jolene and her partner travel for a few weeks in September 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_2", - "session_27", - "session_4", - "session_8", - "session_1", - "session_12", - "session_20", - "session_19", - "session_5" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_27" - ], - "question": "What was the main focus of the session that stood out to Jolene during the retreat?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_27", - "session_14", - "session_11", - "session_13", - "session_4", - "session_30", - "session_25", - "session_23", - "session_7" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_27" - ], - "question": "How did Jolene feel about her progress in practicing mindfulness and gratitude?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_18", - "session_19", - "session_15", - "session_13", - "session_12", - "session_20", - "session_7", - "session_22", - "session_26" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_27" - ], - "question": "What positive change did Jolene experience during the retreat?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_4", - "session_30", - "session_25", - "session_13", - "session_15", - "session_22", - "session_27", - "session_5", - "session_2" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_27" - ], - "question": "What did Jolene recently play that she described to Deb?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_29", - "session_12", - "session_25", - "session_24", - "session_10", - "session_2", - "session_4", - "session_18", - "session_19" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_28" - ], - "question": "What did Deborah do with their mom's old friends?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_23", - "session_4", - "session_24", - "session_22", - "session_1", - "session_2", - "session_29", - "session_6", - "session_12" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_28" - ], - "question": "Where did Deborah get married?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_28", - "session_12", - "session_14", - "session_30", - "session_3", - "session_9", - "session_8", - "session_24", - "session_22" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_28" - ], - "question": "What does yoga on the beach provide for Deborah?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_6", - "session_24", - "session_28", - "session_19", - "session_4", - "session_11", - "session_25", - "session_29", - "session_7" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_28" - ], - "question": "How does Jolene describe their home room?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_23", - "session_22", - "session_30", - "session_1", - "session_26", - "session_11", - "session_18", - "session_16", - "session_5" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_29" - ], - "question": "What new activity did Deborah and her neighbor organize for the community on 16 September, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_19", - "session_29", - "session_22", - "session_8", - "session_15", - "session_3", - "session_9", - "session_10", - "session_18" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_29" - ], - "question": "What was Deborah's mom passionate about?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_2", - "session_23", - "session_24", - "session_28", - "session_29", - "session_22", - "session_15", - "session_4", - "session_12" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_29" - ], - "question": "What food did Deborah's mom make for her on birthdays?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_19", - "session_20", - "session_8", - "session_28", - "session_2", - "session_23", - "session_4", - "session_12", - "session_1" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_29" - ], - "question": "What kind of cookies did Jolene used to bake with someone close to her?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_15", - "session_6", - "session_19", - "session_4", - "session_12", - "session_2", - "session_23", - "session_13", - "session_3" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_29" - ], - "question": "What outdoor activity did Jolene suggest doing together with Deborah?", - "recall": 0.0, - "retrieved_ids": [ - "session_19", - "session_15", - "session_7", - "session_2", - "session_13", - "session_26", - "session_16", - "session_12", - "session_11", - "session_8" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_30" - ], - "question": "What activity did Deborah enjoy at the music festival with their pals on September 20, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_30", - "session_11", - "session_28", - "session_19", - "session_22", - "session_26", - "session_12", - "session_24", - "session_5", - "session_23" - ], - "sample_id": "conv-48" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_30" - ], - "question": "What did Deborah find freeing at the music festival?", - "recall": 1.0, - "retrieved_ids": [ - "session_30", - "session_11", - "session_28", - "session_19", - "session_9", - "session_12", - "session_24", - "session_8", - "session_29", - "session_26" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_2" - ], - "question": "What are the names of Deborah's snakes?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_2", - "session_15", - "session_12", - "session_29", - "session_25", - "session_14", - "session_8", - "session_13", - "session_23" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What are Deborah's favorite books?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_2", - "session_4", - "session_18", - "session_15", - "session_11", - "session_24", - "session_19", - "session_26", - "session_16" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_22" - ], - "question": "Where did Deborah get her dogs?", - "recall": 0.0, - "retrieved_ids": [ - "session_15", - "session_12", - "session_14", - "session_28", - "session_4", - "session_1", - "session_16", - "session_2", - "session_29", - "session_19" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_22" - ], - "question": "How old are Jolene's cats?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_4", - "session_1", - "session_24", - "session_28", - "session_23", - "session_15", - "session_29", - "session_27", - "session_2" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_24" - ], - "question": "When did Deborah's parents give her first console?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_10", - "session_7", - "session_3", - "session_24", - "session_29", - "session_5", - "session_20", - "session_28", - "session_19" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_2" - ], - "question": "When did Jolene release her pet snake?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_15", - "session_22", - "session_14", - "session_16", - "session_8", - "session_28", - "session_29", - "session_12", - "session_27" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_3" - ], - "question": "Where did Jolene meet her new friend Anna?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_4", - "session_8", - "session_28", - "session_19", - "session_7", - "session_15", - "session_1", - "session_26", - "session_27" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What is Deborah's favorite book which she mentioned on 4 February, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_15", - "session_29", - "session_4", - "session_20", - "session_2", - "session_18", - "session_16", - "session_24", - "session_26" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_5" - ], - "question": "What cool stuff did Deborah accomplish at the retreat on 9 February, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_21", - "session_17", - "session_14", - "session_23", - "session_16", - "session_25", - "session_4", - "session_27", - "session_12" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_5" - ], - "question": "How does Deborah plan to involve local engineers in her idea of teaching STEM to underprivileged kids?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_4", - "session_18", - "session_29", - "session_22", - "session_17", - "session_30", - "session_13", - "session_15", - "session_21" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_6" - ], - "question": "What gave Deborah anxiety in the garden she visited?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_1", - "session_20", - "session_14", - "session_29", - "session_23", - "session_4", - "session_5", - "session_21", - "session_3" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_6" - ], - "question": "Why did Jolene spend time in the garden?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_7", - "session_4", - "session_30", - "session_26", - "session_14", - "session_29", - "session_8", - "session_2", - "session_19" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_7" - ], - "question": "How did Jolene and her rival initially meet?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_1", - "session_16", - "session_7", - "session_3", - "session_26", - "session_28", - "session_29", - "session_2", - "session_27" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_7" - ], - "question": "What activity does Jolene incorporate into her daily routine after going for a morning jog in the park?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_13", - "session_19", - "session_15", - "session_10", - "session_4", - "session_12", - "session_28", - "session_22", - "session_26" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_10" - ], - "question": "What method does Jolene suggest Deborah to try for organizing tasks based on importance and urgency?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_24", - "session_18", - "session_16", - "session_19", - "session_15", - "session_27", - "session_1", - "session_8", - "session_22" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_10" - ], - "question": "How does Jolene plan to pursue her dream of climbing mountains?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_22", - "session_4", - "session_16", - "session_13", - "session_10", - "session_29", - "session_30", - "session_28", - "session_2" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_11" - ], - "question": "Who are the authors mentioned by Jolene that she enjoys reading during her yoga practice?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_22", - "session_16", - "session_20", - "session_15", - "session_2", - "session_11", - "session_12", - "session_4", - "session_29" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_12" - ], - "question": "Which show did Jolene go to with a friend on 9 April, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_23", - "session_5", - "session_4", - "session_22", - "session_6", - "session_9", - "session_24", - "session_28", - "session_7" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_12" - ], - "question": "What does Deborah find comforting about going to horror movie screenings?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_8", - "session_6", - "session_12", - "session_2", - "session_28", - "session_22", - "session_16", - "session_26", - "session_5" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_12" - ], - "question": "How does Deborah describe the time spent with her snakes and partner?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_12", - "session_19", - "session_22", - "session_2", - "session_30", - "session_13", - "session_23", - "session_28", - "session_8" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_14" - ], - "question": "For how long has Jolene had Lucifer as a pet?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_14", - "session_11", - "session_16", - "session_2", - "session_13", - "session_9", - "session_20", - "session_15", - "session_22" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_14" - ], - "question": "How does Deborah feel when spending time with Seraphim?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_2", - "session_8", - "session_29", - "session_19", - "session_4", - "session_28", - "session_6", - "session_26", - "session_7" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_15" - ], - "question": "What made being part of the running group easy for Jolene to stay motivated?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_13", - "session_25", - "session_8", - "session_2", - "session_9", - "session_10", - "session_18", - "session_1", - "session_3" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_15" - ], - "question": "Why did Jolene decide to get a tarantula as a pet?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_7", - "session_14", - "session_8", - "session_16", - "session_28", - "session_22", - "session_6", - "session_2", - "session_29" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_16" - ], - "question": "How did Deborah come to have her pet, Susie?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_28", - "session_2", - "session_15", - "session_29", - "session_1", - "session_14", - "session_19", - "session_22", - "session_4" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_17" - ], - "question": "What did Deborah design inspired by their love for space and engines?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_15", - "session_11", - "session_28", - "session_23", - "session_4", - "session_22", - "session_5", - "session_10", - "session_3" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "What journal has Deborah been using to help track tasks and stay organized?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_10", - "session_20", - "session_8", - "session_22", - "session_11", - "session_21", - "session_15", - "session_19", - "session_26" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_19" - ], - "question": "What game did Jolene recommend to Deborah for being thrilling and intense?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_13", - "session_2", - "session_19", - "session_15", - "session_12", - "session_27", - "session_24", - "session_21", - "session_11" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_19" - ], - "question": "What game did Deborah suggest as an awesome open-world game for the Nintendo Switch?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_24", - "session_5", - "session_4", - "session_15", - "session_29", - "session_27", - "session_16", - "session_2", - "session_17" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_19" - ], - "question": "What is special about the bench at the park near Jolene's house?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_1", - "session_4", - "session_29", - "session_8", - "session_23", - "session_28", - "session_2", - "session_20", - "session_3" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_19" - ], - "question": "What did Jolene and her mom chat about at their special bench in the park?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_1", - "session_4", - "session_2", - "session_20", - "session_29", - "session_23", - "session_28", - "session_22", - "session_7" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_21" - ], - "question": "How did Deborah feel after receiving positive feedback at the virtual conference?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_30", - "session_18", - "session_10", - "session_13", - "session_28", - "session_19", - "session_2", - "session_3", - "session_12" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_21" - ], - "question": "What kind of event did Deborah present at recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_21", - "session_23", - "session_29", - "session_19", - "session_6", - "session_24", - "session_15", - "session_3", - "session_12" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_22" - ], - "question": "What did Deborah's mom stress the value of, which she wants to keep in mind for her engineering projects?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_13", - "session_20", - "session_8", - "session_2", - "session_23", - "session_30", - "session_17", - "session_19", - "session_15" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_22" - ], - "question": "What type of projects is Deborah interested in getting involved in the future?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_18", - "session_29", - "session_3", - "session_12", - "session_16", - "session_6", - "session_9", - "session_15", - "session_26" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_22" - ], - "question": "How did Jolene get Luna, one of her cats?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_14", - "session_1", - "session_16", - "session_28", - "session_15", - "session_4", - "session_27", - "session_29", - "session_2" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_23" - ], - "question": "What type of classes did Deborah and her partner check out during their trip to Rio de Janeiro on 30 August, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_12", - "session_22", - "session_4", - "session_30", - "session_11", - "session_28", - "session_26", - "session_6", - "session_24" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_23" - ], - "question": "Why did Deborah get the new plant on 30 August, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_28", - "session_4", - "session_3", - "session_14", - "session_24", - "session_8", - "session_16", - "session_29", - "session_22" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_24" - ], - "question": "How did Jolene's mom support her yoga practice when she first started?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_24", - "session_16", - "session_23", - "session_15", - "session_7", - "session_10", - "session_29", - "session_5", - "session_3" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_24" - ], - "question": "What was the video game console that Deborah's parents got her at age 10?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_2", - "session_15", - "session_20", - "session_19", - "session_29", - "session_3", - "session_12", - "session_16", - "session_7" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_24" - ], - "question": "What was one of Deborah's favorite games to play with her mom on the PlayStation game system?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_2", - "session_24", - "session_7", - "session_12", - "session_16", - "session_20", - "session_15", - "session_27", - "session_17" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_27" - ], - "question": "Where did Deborah and her partner travel for a few weeks in September 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_2", - "session_27", - "session_8", - "session_4", - "session_1", - "session_12", - "session_20", - "session_19", - "session_5" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_28" - ], - "question": "What did Jolene do with their mom's old friends?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_23", - "session_4", - "session_24", - "session_22", - "session_6", - "session_1", - "session_2", - "session_29", - "session_12" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_28" - ], - "question": "Where did Jolene get married?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_28", - "session_12", - "session_14", - "session_30", - "session_3", - "session_9", - "session_24", - "session_8", - "session_22" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_29" - ], - "question": "What new activity did Jolene and her neighbor organize for the community on 16 September, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_29", - "session_19", - "session_22", - "session_8", - "session_15", - "session_3", - "session_9", - "session_10", - "session_18" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_29" - ], - "question": "What food did Jolene's mom make for her on holidays?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_20", - "session_19", - "session_28", - "session_2", - "session_23", - "session_29", - "session_4", - "session_12", - "session_1" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_29" - ], - "question": "What kind of cookies did Deborah used to bake with someone close to her?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_15", - "session_6", - "session_19", - "session_4", - "session_12", - "session_2", - "session_23", - "session_13", - "session_3" - ], - "sample_id": "conv-48" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_30" - ], - "question": "What activity did Jolene enjoy at the music festival with their pals on September 20, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_30", - "session_11", - "session_28", - "session_19", - "session_22", - "session_26", - "session_12", - "session_24", - "session_5", - "session_23" - ], - "sample_id": "conv-48" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_18", - "session_22" - ], - "question": "What kind of car does Evan drive?", - "recall": 0.6667, - "retrieved_ids": [ - "session_10", - "session_20", - "session_18", - "session_19", - "session_9", - "session_17", - "session_8", - "session_16", - "session_22", - "session_25" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_18" - ], - "question": "What kinds of things did Evan have broken?", - "recall": 0.5, - "retrieved_ids": [ - "session_3", - "session_7", - "session_20", - "session_11", - "session_21", - "session_23", - "session_19", - "session_16", - "session_8", - "session_1" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_2" - ], - "question": "Where has Evan been on roadtrips with his family?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_8", - "session_1", - "session_11", - "session_20", - "session_15", - "session_19", - "session_2", - "session_23", - "session_5" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1" - ], - "question": "How many Prius has Evan owned?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_1", - "session_21", - "session_22", - "session_8", - "session_11", - "session_24", - "session_2", - "session_12", - "session_15" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_1" - ], - "question": "Which hobby did Sam take up in May 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_13", - "session_20", - "session_17", - "session_2", - "session_15", - "session_11", - "session_10", - "session_8", - "session_24" - ], - "sample_id": "conv-49" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_2" - ], - "question": "Which country was Evan visiting in May 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_21", - "session_15", - "session_16", - "session_12", - "session_4", - "session_1", - "session_23", - "session_2", - "session_17" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_2" - ], - "question": "How many roadtrips did Evan take in May 2023?", - "recall": 0.5, - "retrieved_ids": [ - "session_8", - "session_24", - "session_1", - "session_15", - "session_13", - "session_7", - "session_25", - "session_9", - "session_21", - "session_11" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_10", - "session_13", - "session_2", - "session_20", - "session_21", - "session_7" - ], - "question": "What new hobbies did Sam consider trying?", - "recall": 0.7143, - "retrieved_ids": [ - "session_13", - "session_1", - "session_20", - "session_17", - "session_11", - "session_8", - "session_4", - "session_10", - "session_3", - "session_2" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_8" - ], - "question": "What hobby did Evan start practicing a few years ago that he enjoys?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_8", - "session_20", - "session_21", - "session_2", - "session_1", - "session_4", - "session_10", - "session_12", - "session_24" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_2" - ], - "question": "When did Evan go to Jasper with his family?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_5", - "session_1", - "session_15", - "session_11", - "session_8", - "session_7", - "session_20", - "session_19", - "session_23" - ], - "sample_id": "conv-49" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_19", - "session_2" - ], - "question": "Which type of vacation would Evan prefer with his family, walking tours in metropolitan cities or camping trip in the outdoors?", - "recall": 0.5, - "retrieved_ids": [ - "session_11", - "session_25", - "session_8", - "session_13", - "session_19", - "session_15", - "session_6", - "session_7", - "session_20", - "session_24" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_12", - "session_13", - "session_14", - "session_15", - "session_16", - "session_17", - "session_2", - "session_24", - "session_25", - "session_3", - "session_5", - "session_6", - "session_7", - "session_8" - ], - "question": "What health issue did Sam face that motivated him to change his lifestyle?", - "recall": 0.4, - "retrieved_ids": [ - "session_7", - "session_5", - "session_6", - "session_3", - "session_24", - "session_25", - "session_18", - "session_20", - "session_9", - "session_22" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_2" - ], - "question": "When did Sam first go to the doctor and find out he had a weight problem?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_12", - "session_7", - "session_24", - "session_20", - "session_17", - "session_21", - "session_19", - "session_4", - "session_18" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_3" - ], - "question": "When did Evan have his sudden heart palpitation incident that really shocked him up?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_7", - "session_17", - "session_24", - "session_13", - "session_1", - "session_15", - "session_20", - "session_11", - "session_8" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_22", - "session_23", - "session_3", - "session_5" - ], - "question": "What is Evan's favorite food?", - "recall": 0.5, - "retrieved_ids": [ - "session_3", - "session_11", - "session_16", - "session_8", - "session_2", - "session_4", - "session_24", - "session_23", - "session_15", - "session_12" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_3" - ], - "question": "What kind of unhealthy snacks does Sam enjoy eating?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_11", - "session_4", - "session_20", - "session_22", - "session_19", - "session_10", - "session_7", - "session_3", - "session_16" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_22", - "session_3" - ], - "question": "What recurring issue frustrates Sam at the grocery store?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_11", - "session_24", - "session_14", - "session_3", - "session_25", - "session_18", - "session_9", - "session_6", - "session_5" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_4" - ], - "question": "When did Sam's friends mock him for being overweight?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_17", - "session_8", - "session_16", - "session_4", - "session_11", - "session_1", - "session_7", - "session_22", - "session_20" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_22", - "session_24", - "session_3", - "session_4" - ], - "question": "What kind of healthy food suggestions has Evan given to Sam?", - "recall": 0.75, - "retrieved_ids": [ - "session_24", - "session_7", - "session_20", - "session_13", - "session_18", - "session_3", - "session_19", - "session_16", - "session_22", - "session_10" - ], - "sample_id": "conv-49" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_12", - "session_14", - "session_22", - "session_3", - "session_8", - "session_9" - ], - "question": "Considering their conversations and personal growth, what advice might Evan and Sam give to someone facing a major life transition or challenge?", - "recall": 0.6667, - "retrieved_ids": [ - "session_11", - "session_14", - "session_20", - "session_16", - "session_23", - "session_8", - "session_12", - "session_9", - "session_24", - "session_25" - ], - "sample_id": "conv-49" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_14", - "session_2", - "session_3", - "session_4", - "session_5", - "session_7", - "session_8", - "session_9" - ], - "question": "In light of the health and dietary changes discussed, what would be an appropriate gift for both Evan and Sam to encourage their healthy lifestyles?", - "recall": 0.5, - "retrieved_ids": [ - "session_23", - "session_4", - "session_15", - "session_7", - "session_2", - "session_8", - "session_11", - "session_6", - "session_16", - "session_21" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_23", - "session_5" - ], - "question": "How does Evan describe the woman and his feelings for her that he met in Canada?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_21", - "session_7", - "session_20", - "session_11", - "session_23", - "session_2", - "session_1", - "session_6", - "session_24" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_5" - ], - "question": "When Evan did meet his future wife?", - "recall": 0.0, - "retrieved_ids": [ - "session_25", - "session_7", - "session_20", - "session_18", - "session_13", - "session_1", - "session_15", - "session_11", - "session_8", - "session_24" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_4" - ], - "question": "When did Sam start working out at the gym?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_14", - "session_1", - "session_15", - "session_17", - "session_11", - "session_8", - "session_24", - "session_7", - "session_13" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_5" - ], - "question": "What significant event happened in Sam's life towards the end of summer 2023?", - "recall": 0.0, - "retrieved_ids": [ - "session_19", - "session_6", - "session_11", - "session_16", - "session_25", - "session_23", - "session_24", - "session_1", - "session_14", - "session_21" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_5" - ], - "question": "Which year did Evan start taking care of his health seriously?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_12", - "session_4", - "session_11", - "session_5", - "session_17", - "session_2", - "session_8", - "session_1", - "session_20" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_5" - ], - "question": "What motivates Evan to take care of his health?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_5", - "session_6", - "session_2", - "session_17", - "session_12", - "session_14", - "session_20", - "session_16", - "session_4" - ], - "sample_id": "conv-49" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_5" - ], - "question": "What electronic device could Evan gift Sam to help him keep up with his fitness goals?", - "recall": 0.0, - "retrieved_ids": [ - "session_7", - "session_17", - "session_3", - "session_16", - "session_24", - "session_12", - "session_20", - "session_9", - "session_14", - "session_2" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_11", - "session_6" - ], - "question": "What kind of writing does Sam do to relax and cope with his health issues?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_7", - "session_11", - "session_16", - "session_25", - "session_6", - "session_5", - "session_18", - "session_3", - "session_22" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_5", - "session_6" - ], - "question": "Who did Evan meet on his trip to Canada, and who did he come back from Canada with?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_6", - "session_5", - "session_16", - "session_20", - "session_21", - "session_18", - "session_7", - "session_3", - "session_24" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_6" - ], - "question": "When Evan get back from a vacation with his SO?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_19", - "session_17", - "session_1", - "session_15", - "session_7", - "session_20", - "session_6", - "session_22", - "session_8" - ], - "sample_id": "conv-49" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_9" - ], - "question": "How might Evan and Sam's experiences with health and lifestyle changes influence their approach to stress and challenges?", - "recall": 0.0, - "retrieved_ids": [ - "session_2", - "session_8", - "session_14", - "session_18", - "session_25", - "session_11", - "session_5", - "session_21", - "session_23", - "session_22" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_21", - "session_6" - ], - "question": "What recurring frustration does Evan experience?", - "recall": 0.0, - "retrieved_ids": [ - "session_14", - "session_24", - "session_22", - "session_2", - "session_25", - "session_11", - "session_18", - "session_9", - "session_3", - "session_5" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_24", - "session_6" - ], - "question": "What is the recurring dream that Sam keeps having?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_6", - "session_17", - "session_11", - "session_23", - "session_14", - "session_15", - "session_1", - "session_7", - "session_3" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_20", - "session_7" - ], - "question": "What accidents has Evan's son faced lately?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_7", - "session_25", - "session_12", - "session_9", - "session_3", - "session_17", - "session_24", - "session_16", - "session_22" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_7" - ], - "question": "When was Evan's son injured at soccer?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_20", - "session_1", - "session_15", - "session_12", - "session_4", - "session_2", - "session_17", - "session_23", - "session_16" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_23", - "session_7", - "session_8" - ], - "question": "What kind of foods or recipes has Sam recommended to Evan?", - "recall": 0.6667, - "retrieved_ids": [ - "session_7", - "session_11", - "session_16", - "session_3", - "session_10", - "session_8", - "session_9", - "session_22", - "session_2", - "session_5" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_11", - "session_18", - "session_3", - "session_7", - "session_8" - ], - "question": "What kind of healthy meals did Sam start eating after getting a health scare?", - "recall": 0.6667, - "retrieved_ids": [ - "session_7", - "session_4", - "session_8", - "session_24", - "session_3", - "session_14", - "session_11", - "session_17", - "session_5", - "session_22" - ], - "sample_id": "conv-49" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_22" - ], - "question": "What role does nature and the outdoors play in Evan and Sam's mental well-being?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_13", - "session_8", - "session_14", - "session_25", - "session_22", - "session_11", - "session_6", - "session_9", - "session_15" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_2", - "session_7" - ], - "question": "How many months lapsed between Sam's first and second doctor's appointment?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_7", - "session_8", - "session_9", - "session_23", - "session_17", - "session_21", - "session_11", - "session_16", - "session_2" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_8" - ], - "question": "When did Evan start taking painting classes?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_11", - "session_1", - "session_10", - "session_20", - "session_21", - "session_24", - "session_13", - "session_7", - "session_2" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_8" - ], - "question": "Which classes did Evan join in mid-August 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_21", - "session_9", - "session_13", - "session_15", - "session_1", - "session_7", - "session_25", - "session_11", - "session_24" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_8" - ], - "question": "How did Evan get into painting?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_11", - "session_13", - "session_10", - "session_9", - "session_20", - "session_21", - "session_8", - "session_22", - "session_2" - ], - "sample_id": "conv-49" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_12", - "session_2", - "session_7" - ], - "question": "How often does Sam get health checkups?", - "recall": 0.6667, - "retrieved_ids": [ - "session_25", - "session_16", - "session_17", - "session_12", - "session_7", - "session_5", - "session_3", - "session_14", - "session_8", - "session_4" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_20", - "session_21", - "session_8" - ], - "question": "What kind of subjects does Evan enjoy painting?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_21", - "session_11", - "session_13", - "session_8", - "session_25", - "session_1", - "session_19", - "session_10", - "session_16" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_8", - "session_9" - ], - "question": "Which places in Canada was Evan visiting in July 2023?", - "recall": 0.5, - "retrieved_ids": [ - "session_21", - "session_6", - "session_5", - "session_23", - "session_11", - "session_8", - "session_16", - "session_12", - "session_15", - "session_17" - ], - "sample_id": "conv-49" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_21" - ], - "question": "How do Evan and Sam use creative outlets to cope with life's challenges?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_20", - "session_10", - "session_21", - "session_2", - "session_6", - "session_1", - "session_8", - "session_16", - "session_18" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_8" - ], - "question": "When did Evan go skiing in Banff?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_19", - "session_23", - "session_25", - "session_1", - "session_13", - "session_15", - "session_7", - "session_11", - "session_24" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_21", - "session_22", - "session_8", - "session_9" - ], - "question": "What new diet and lifestyle change did Sam adopt over time?", - "recall": 0.5, - "retrieved_ids": [ - "session_4", - "session_9", - "session_24", - "session_6", - "session_3", - "session_5", - "session_14", - "session_2", - "session_18", - "session_22" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_11", - "session_7", - "session_9" - ], - "question": "Who was injured in Evan's family?", - "recall": 0.3333, - "retrieved_ids": [ - "session_19", - "session_15", - "session_1", - "session_23", - "session_2", - "session_5", - "session_11", - "session_8", - "session_4", - "session_12" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_25", - "session_4", - "session_6", - "session_8", - "session_9" - ], - "question": "What kind of hobbies does Evan pursue?", - "recall": 0.1667, - "retrieved_ids": [ - "session_11", - "session_20", - "session_21", - "session_13", - "session_1", - "session_19", - "session_10", - "session_7", - "session_16", - "session_2" - ], - "sample_id": "conv-49" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_14", - "session_4" - ], - "question": "What challenges does Sam face in his quest for a healthier lifestyle, and how does he address them?", - "recall": 0.0, - "retrieved_ids": [ - "session_20", - "session_8", - "session_2", - "session_9", - "session_7", - "session_24", - "session_11", - "session_16", - "session_22", - "session_25" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_10" - ], - "question": "Which activity do Evan and Sam plan on doing together during September 2023?", - "recall": 0.0, - "retrieved_ids": [ - "session_16", - "session_6", - "session_13", - "session_17", - "session_24", - "session_4", - "session_19", - "session_25", - "session_9", - "session_23" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_10" - ], - "question": "When did Evan and Sam decide to paint together?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_1", - "session_11", - "session_16", - "session_10", - "session_5", - "session_8", - "session_21", - "session_20", - "session_2" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_11", - "session_3", - "session_9" - ], - "question": "What personal health incidents does Evan face in 2023?", - "recall": 0.6667, - "retrieved_ids": [ - "session_16", - "session_7", - "session_3", - "session_25", - "session_12", - "session_4", - "session_17", - "session_5", - "session_24", - "session_9" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_11", - "session_14" - ], - "question": "What recurring adventure does Evan have with strangers?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_13", - "session_11", - "session_5", - "session_14", - "session_2", - "session_4", - "session_23", - "session_15", - "session_12" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_11", - "session_14" - ], - "question": "What is Sam's persistent problem with his phone?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_20", - "session_11", - "session_23", - "session_14", - "session_18", - "session_5", - "session_12", - "session_22", - "session_2" - ], - "sample_id": "conv-49" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_13" - ], - "question": "Which US state was Sam travelling in during October 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_17", - "session_25", - "session_24", - "session_16", - "session_22", - "session_18", - "session_13", - "session_5", - "session_23" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_12" - ], - "question": "When did Evan start lifting weights?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_20", - "session_4", - "session_11", - "session_1", - "session_25", - "session_18", - "session_2", - "session_16", - "session_8" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_13" - ], - "question": "When did Sam and his friend decide to try kayaking?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_8", - "session_1", - "session_13", - "session_22", - "session_17", - "session_25", - "session_7", - "session_5", - "session_11" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_13" - ], - "question": "Which new activity does Sam take up in October 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_9", - "session_20", - "session_8", - "session_13", - "session_15", - "session_18", - "session_4", - "session_11", - "session_10" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_13" - ], - "question": "What kind of stress was Sam dealing with in October 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_18", - "session_20", - "session_11", - "session_16", - "session_21", - "session_19", - "session_7", - "session_6", - "session_5" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_14", - "session_17", - "session_3" - ], - "question": "What health scares did Sam and Evan experience?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_25", - "session_17", - "session_3", - "session_7", - "session_5", - "session_15", - "session_14", - "session_22", - "session_1" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_14" - ], - "question": "When was Sam in the ER?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_2", - "session_1", - "session_15", - "session_23", - "session_12", - "session_4", - "session_17", - "session_5", - "session_8" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_12", - "session_14", - "session_2", - "session_7" - ], - "question": "Which ailment does Sam have to face due to his weight?", - "recall": 0.75, - "retrieved_ids": [ - "session_20", - "session_25", - "session_12", - "session_18", - "session_4", - "session_7", - "session_2", - "session_16", - "session_23", - "session_9" - ], - "sample_id": "conv-49" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_16" - ], - "question": "Does Evan live close to a beach or mountains?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_25", - "session_20", - "session_22", - "session_2", - "session_24", - "session_3", - "session_9", - "session_8", - "session_23" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_16" - ], - "question": "When did Evan lose his job?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_7", - "session_20", - "session_13", - "session_1", - "session_15", - "session_6", - "session_11", - "session_8", - "session_24" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_16" - ], - "question": "When did Evan and Sam planned a trip to the beach together?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_1", - "session_13", - "session_9", - "session_24", - "session_3", - "session_19", - "session_4", - "session_17", - "session_23" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_18" - ], - "question": "What was Sam doing on December 4, 2023?", - "recall": 0.0, - "retrieved_ids": [ - "session_10", - "session_2", - "session_7", - "session_20", - "session_11", - "session_17", - "session_16", - "session_8", - "session_9", - "session_21" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_19", - "session_21" - ], - "question": "Which two significant life events occur in Evan's life in December 2023 with his partner?", - "recall": 0.5, - "retrieved_ids": [ - "session_19", - "session_5", - "session_23", - "session_20", - "session_11", - "session_25", - "session_16", - "session_1", - "session_7", - "session_24" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_21", - "session_5" - ], - "question": "How long did Evan and his partner date before getting married?", - "recall": 0.5, - "retrieved_ids": [ - "session_21", - "session_22", - "session_24", - "session_7", - "session_10", - "session_20", - "session_4", - "session_1", - "session_19", - "session_25" - ], - "sample_id": "conv-49" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_21" - ], - "question": "Which major holiday season conincides with Evan's wedding?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_19", - "session_21", - "session_15", - "session_12", - "session_1", - "session_4", - "session_2", - "session_17", - "session_16" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_20", - "session_22" - ], - "question": "Which activity did Sam resume in December 2023 after a long time?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_10", - "session_5", - "session_6", - "session_11", - "session_8", - "session_25", - "session_22", - "session_7", - "session_24" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_19" - ], - "question": "When is Evan planning a big family reunion?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_25", - "session_1", - "session_8", - "session_23", - "session_10", - "session_15", - "session_18", - "session_4", - "session_16" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_20" - ], - "question": "When did Evan's son fall off his bike?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_7", - "session_10", - "session_17", - "session_21", - "session_23", - "session_11", - "session_1", - "session_6", - "session_8" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_23" - ], - "question": "When did Evan announce his marriage to his extended family?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_2", - "session_1", - "session_7", - "session_15", - "session_20", - "session_19", - "session_22", - "session_11", - "session_8" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_20" - ], - "question": "When did Evan finish the painting that's hanging in the exhibit?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_21", - "session_1", - "session_10", - "session_13", - "session_11", - "session_8", - "session_2", - "session_17", - "session_24" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_23", - "session_24" - ], - "question": "How does Evan spend his time with his bride after the wedding?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_20", - "session_25", - "session_8", - "session_24", - "session_10", - "session_5", - "session_23", - "session_2", - "session_22" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_21", - "session_22", - "session_23" - ], - "question": "Who did Evan tell about his marriage?", - "recall": 0.6667, - "retrieved_ids": [ - "session_22", - "session_7", - "session_20", - "session_14", - "session_2", - "session_23", - "session_11", - "session_18", - "session_1", - "session_8" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_23" - ], - "question": "When will Evan and his partner have their honeymoon in Canada?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_24", - "session_6", - "session_25", - "session_5", - "session_21", - "session_7", - "session_20", - "session_19", - "session_11" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_24" - ], - "question": "When did Evan have a drunken night with his friends?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_25", - "session_20", - "session_24", - "session_4", - "session_1", - "session_22", - "session_6", - "session_17", - "session_8" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_10", - "session_11", - "session_16", - "session_18", - "session_2", - "session_24", - "session_8" - ], - "question": "What is a stress reliever for Evan?", - "recall": 0.875, - "retrieved_ids": [ - "session_11", - "session_24", - "session_2", - "session_13", - "session_18", - "session_10", - "session_8", - "session_15", - "session_14", - "session_1" - ], - "sample_id": "conv-49" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_13", - "session_16", - "session_18" - ], - "question": "What is a stress reliever for Sam?", - "recall": 0.75, - "retrieved_ids": [ - "session_11", - "session_24", - "session_2", - "session_13", - "session_18", - "session_10", - "session_8", - "session_15", - "session_14", - "session_1" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_1" - ], - "question": "What type of car did Evan get after his old Prius broke down?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_18", - "session_10", - "session_8", - "session_25", - "session_7", - "session_20", - "session_22", - "session_17", - "session_2" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_1" - ], - "question": "How did Evan get into watercolor painting?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_11", - "session_8", - "session_9", - "session_13", - "session_10", - "session_20", - "session_21", - "session_22", - "session_2" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_1" - ], - "question": "What did Evan start doing a few years back as a stress-buster?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_11", - "session_8", - "session_4", - "session_19", - "session_15", - "session_12", - "session_2", - "session_17", - "session_5" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_1" - ], - "question": "What advice did Evan give Sam about finding a passion?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_21", - "session_20", - "session_24", - "session_8", - "session_13", - "session_7", - "session_12", - "session_11", - "session_15" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "Where did Evan take his family for a road trip on 24 May, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_2", - "session_5", - "session_11", - "session_9", - "session_7", - "session_19", - "session_8", - "session_20", - "session_23" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What did Evan find relaxing about his road trip to Jasper?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_18", - "session_11", - "session_8", - "session_16", - "session_9", - "session_25", - "session_7", - "session_13", - "session_20" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "What habit is Sam trying to change in terms of diet?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_10", - "session_4", - "session_15", - "session_2", - "session_17", - "session_23", - "session_6", - "session_9", - "session_24" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "What new suggestion did Evan give to Sam regarding his soda and candy consumption?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_6", - "session_13", - "session_4", - "session_20", - "session_24", - "session_18", - "session_7", - "session_10", - "session_9" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "What did Sam agree to try instead of soda and candy?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_4", - "session_23", - "session_1", - "session_6", - "session_8", - "session_15", - "session_25", - "session_11", - "session_24" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "What frustrating issue did Sam face at the supermarket?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_18", - "session_9", - "session_8", - "session_25", - "session_1", - "session_11", - "session_5", - "session_6", - "session_22" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What novel is Evan reading that he finds gripping?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_25", - "session_1", - "session_10", - "session_21", - "session_12", - "session_2", - "session_22", - "session_20", - "session_24" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What kind of water does Evan suggest Sam try as an alternative to soda?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_20", - "session_3", - "session_13", - "session_7", - "session_10", - "session_18", - "session_19", - "session_16", - "session_25" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_5" - ], - "question": "What does the smartwatch help Evan with?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_5", - "session_13", - "session_6", - "session_10", - "session_16", - "session_15", - "session_2", - "session_22", - "session_25" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_5" - ], - "question": "What does the bonsai tree symbolize for Evan?", - "recall": 0.0, - "retrieved_ids": [], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_5" - ], - "question": "Why did Evan decide to get the bonsai tree?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_5", - "session_1", - "session_9", - "session_15", - "session_13", - "session_7", - "session_25", - "session_8", - "session_24" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_6" - ], - "question": "According to Sam, what is more important than perfection?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_15", - "session_4", - "session_17", - "session_19", - "session_23", - "session_24", - "session_22", - "session_9", - "session_16" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_6" - ], - "question": "What did Evan suggest Sam to check out for insights into his dream?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_9", - "session_1", - "session_7", - "session_22", - "session_20", - "session_24", - "session_12", - "session_13", - "session_18" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_6" - ], - "question": "What did Evan mention he had been searching for fruitlessly for half an hour?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_14", - "session_13", - "session_20", - "session_24", - "session_11", - "session_8", - "session_1", - "session_9", - "session_15" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What class is Sam taking to learn how to make healthier meals?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_8", - "session_24", - "session_11", - "session_2", - "session_10", - "session_6", - "session_22", - "session_18", - "session_3" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What dish did Sam make on 18 August, 2023 that turned out flavorful?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_10", - "session_23", - "session_11", - "session_2", - "session_1", - "session_8", - "session_14", - "session_4", - "session_3" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What kind of recipe did Evan request from Sam on 19 August, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_1", - "session_19", - "session_20", - "session_8", - "session_11", - "session_21", - "session_16", - "session_23", - "session_22" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What food did Sam share a photo of on 19 August, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_19", - "session_16", - "session_11", - "session_7", - "session_6", - "session_8", - "session_22", - "session_25", - "session_1" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What type of painting classes did Evan start taking in 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_11", - "session_24", - "session_1", - "session_10", - "session_20", - "session_21", - "session_7", - "session_13", - "session_2" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What did Evan start painting years ago due to being inspired by a friend's gift?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_11", - "session_1", - "session_4", - "session_20", - "session_21", - "session_15", - "session_2", - "session_16", - "session_12" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What nature concept do watercolor painting classes emphasize according to Evan?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_1", - "session_13", - "session_20", - "session_11", - "session_25", - "session_10", - "session_21", - "session_9", - "session_7" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What type of landscapes does Evan love painting the most?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_21", - "session_2", - "session_13", - "session_1", - "session_20", - "session_23", - "session_11", - "session_10", - "session_24" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What fun activity did Evan mention doing in July 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_20", - "session_6", - "session_11", - "session_13", - "session_4", - "session_8", - "session_24", - "session_14", - "session_9" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_9" - ], - "question": "What injury did Evan suffer from in August 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_9", - "session_1", - "session_13", - "session_8", - "session_11", - "session_7", - "session_24", - "session_15", - "session_25" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_9" - ], - "question": "What sports activity has Evan been doing to stay active while dealing with the knee injury?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_13", - "session_5", - "session_20", - "session_6", - "session_24", - "session_12", - "session_9", - "session_8", - "session_18" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_9" - ], - "question": "What suggestion did Sam give to Evan to help with his knee issue?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_22", - "session_7", - "session_3", - "session_18", - "session_9", - "session_11", - "session_6", - "session_5", - "session_25" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "What did Evan suggest Sam try as a calming hobby?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_13", - "session_1", - "session_17", - "session_9", - "session_7", - "session_11", - "session_24", - "session_16", - "session_25" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "What did Evan recommend Sam acquire to get started with painting?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_11", - "session_1", - "session_9", - "session_21", - "session_13", - "session_8", - "session_2", - "session_5", - "session_22" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_11" - ], - "question": "What activity does Evan do to keep himself busy while healing his knee?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_20", - "session_23", - "session_8", - "session_19", - "session_7", - "session_9", - "session_24", - "session_12", - "session_13" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_11" - ], - "question": "What painting did Evan share with Sam in October?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_13", - "session_1", - "session_7", - "session_19", - "session_16", - "session_11", - "session_21", - "session_22", - "session_6" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_11" - ], - "question": "What kind of writing does Sam enjoy as a form of expression?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_23", - "session_21", - "session_19", - "session_20", - "session_7", - "session_8", - "session_16", - "session_12", - "session_2" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_11" - ], - "question": "What electronics issue has been frustrating Sam lately?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_9", - "session_18", - "session_17", - "session_7", - "session_6", - "session_11", - "session_5", - "session_25", - "session_22" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_12" - ], - "question": "What activity did Evan start one year ago?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_4", - "session_20", - "session_11", - "session_12", - "session_1", - "session_21", - "session_15", - "session_5", - "session_2" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_12" - ], - "question": "What advice did Evan give to Sam to avoid injuries while starting weightlifting?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_24", - "session_20", - "session_13", - "session_1", - "session_23", - "session_19", - "session_11", - "session_8", - "session_15" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "Where did Sam and his mate plan to try kayaking?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_20", - "session_10", - "session_23", - "session_19", - "session_24", - "session_7", - "session_15", - "session_25", - "session_16" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_14" - ], - "question": "What digestive issue did Sam experience lately?", - "recall": 0.0, - "retrieved_ids": [ - "session_5", - "session_25", - "session_11", - "session_3", - "session_6", - "session_22", - "session_2", - "session_18", - "session_1", - "session_8" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_14" - ], - "question": "What adventurous theme is emerging in Evan's life as mentioned by Sam?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_3", - "session_6", - "session_20", - "session_17", - "session_5", - "session_9", - "session_13", - "session_11", - "session_24" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_14" - ], - "question": "What does Evan mention about his progress at the gym to Sam?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_4", - "session_17", - "session_5", - "session_20", - "session_7", - "session_6", - "session_3", - "session_12", - "session_15" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "How did Evan start his transformation journey two years ago?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_11", - "session_15", - "session_12", - "session_4", - "session_1", - "session_20", - "session_2", - "session_5", - "session_7" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "What gift did Evan receive from a close friend?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_20", - "session_1", - "session_4", - "session_17", - "session_24", - "session_8", - "session_22", - "session_15", - "session_11" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "Why had Evan been going through a tough time lately?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_7", - "session_16", - "session_17", - "session_20", - "session_5", - "session_2", - "session_15", - "session_23", - "session_13" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "How does Evan describe the island he grew up on?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_5", - "session_20", - "session_15", - "session_7", - "session_11", - "session_3", - "session_24", - "session_2", - "session_14" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What was the main reason for Evan's frustration with his new Prius breaking down?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_1", - "session_20", - "session_11", - "session_7", - "session_10", - "session_22", - "session_8", - "session_3", - "session_9" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "How did Sam suggest Evan view the setback with his broken Prius?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_1", - "session_13", - "session_7", - "session_9", - "session_20", - "session_22", - "session_3", - "session_11", - "session_6" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What did Sam suggest Evan try for stress relief and flexibility?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_10", - "session_13", - "session_8", - "session_15", - "session_1", - "session_7", - "session_24", - "session_9", - "session_6" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What did Sam offer Evan regarding yoga?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_18", - "session_6", - "session_7", - "session_2", - "session_1", - "session_13", - "session_11", - "session_8", - "session_15" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "What news did Evan share with Sam on 9th December 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_19", - "session_8", - "session_16", - "session_15", - "session_22", - "session_1", - "session_6", - "session_11", - "session_24" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "What family event is Evan planning for next summer?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_23", - "session_16", - "session_11", - "session_24", - "session_8", - "session_15", - "session_25", - "session_10", - "session_20" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "What is the motto of Evan's family?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_1", - "session_23", - "session_5", - "session_11", - "session_8", - "session_15", - "session_2", - "session_4", - "session_12" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_20" - ], - "question": "According to Evan, what is important for Sam to believe in concerning his weight?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_12", - "session_7", - "session_16", - "session_17", - "session_2", - "session_24", - "session_21", - "session_11", - "session_9" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_20" - ], - "question": "Who helped Evan get the painting published in the exhibition?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_13", - "session_10", - "session_21", - "session_11", - "session_12", - "session_1", - "session_2", - "session_6", - "session_8" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_21" - ], - "question": "What did Sam recently start enjoying to clear his head?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_3", - "session_8", - "session_20", - "session_11", - "session_9", - "session_7", - "session_13", - "session_19", - "session_17" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_21" - ], - "question": "What did Sam suggest Evan should do with his keys?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_9", - "session_20", - "session_6", - "session_12", - "session_18", - "session_1", - "session_21", - "session_11", - "session_16" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_21" - ], - "question": "How did Evan feel when he painted the piece with the bird flying over it?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_24", - "session_10", - "session_13", - "session_5", - "session_6", - "session_20", - "session_11", - "session_1", - "session_8" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_21" - ], - "question": "What did Evan suggest Sam should keep doing to find his own version of love?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_16", - "session_1", - "session_20", - "session_6", - "session_11", - "session_2", - "session_7", - "session_9", - "session_18" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_21" - ], - "question": "How did Evan describe the process of creating the painting with the bird flying over it?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_11", - "session_2", - "session_24", - "session_19", - "session_5", - "session_8", - "session_6", - "session_23", - "session_3" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_22" - ], - "question": "What did Evan want to share with his work friends?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_8", - "session_16", - "session_22", - "session_20", - "session_24", - "session_17", - "session_19", - "session_1", - "session_6" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_22" - ], - "question": "What did Evan share with Sam after their hiking trip?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_16", - "session_1", - "session_7", - "session_20", - "session_23", - "session_11", - "session_5", - "session_8", - "session_6" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_22" - ], - "question": "What did Evan offer to share with Sam after talking about healthy snacks?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_7", - "session_24", - "session_8", - "session_2", - "session_5", - "session_4", - "session_10", - "session_13", - "session_6" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "What did Evan and his partner share with their extended family on January 5, 2024?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_19", - "session_7", - "session_11", - "session_22", - "session_8", - "session_20", - "session_24", - "session_16", - "session_25" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "What was Evan limiting himself to on his new diet?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_3", - "session_4", - "session_20", - "session_17", - "session_23", - "session_9", - "session_24", - "session_15", - "session_10" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "What sports activity did Evan and his partner try in a recent weekend?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_23", - "session_20", - "session_24", - "session_14", - "session_22", - "session_7", - "session_19", - "session_2", - "session_25" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "What advice did Evan suggest Sam seek from a doctor?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_7", - "session_1", - "session_18", - "session_20", - "session_17", - "session_9", - "session_12", - "session_2", - "session_13" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "What suggestions did Evan give for low-impact exercises?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_13", - "session_9", - "session_3", - "session_6", - "session_25", - "session_18", - "session_10", - "session_4", - "session_7" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "What movie did Sam watch that motivated him to keep up with his routine?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_24", - "session_12", - "session_5", - "session_16", - "session_20", - "session_17", - "session_9", - "session_15", - "session_25" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "What activity helped Evan with stress and flexibility?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_10", - "session_11", - "session_2", - "session_13", - "session_24", - "session_8", - "session_15", - "session_4", - "session_9" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_25" - ], - "question": "What did Evan share a photo of that was taken on a camping trip?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_19", - "session_25", - "session_22", - "session_24", - "session_1", - "session_7", - "session_6", - "session_11", - "session_8" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_25" - ], - "question": "Why did Evan apologize to his partner?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_19", - "session_7", - "session_20", - "session_24", - "session_23", - "session_15", - "session_1", - "session_11", - "session_8" - ], - "sample_id": "conv-49" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_25" - ], - "question": "How does Evan describe being out on the water while kayaking and watching the sunset?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_13", - "session_11", - "session_16", - "session_21", - "session_24", - "session_3", - "session_19", - "session_7", - "session_4" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_1" - ], - "question": "What type of car did Sam get after his old Prius broke down?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_18", - "session_10", - "session_8", - "session_25", - "session_7", - "session_20", - "session_22", - "session_17", - "session_2" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_1" - ], - "question": "How did Sam get into watercolor painting?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_11", - "session_8", - "session_9", - "session_13", - "session_10", - "session_20", - "session_21", - "session_22", - "session_2" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_1" - ], - "question": "What did Sam start doing a few years back as a stress-buster?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_11", - "session_8", - "session_19", - "session_4", - "session_15", - "session_12", - "session_2", - "session_17", - "session_5" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_2" - ], - "question": "Where did Sam take his family for a road trip on 24 May, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_2", - "session_5", - "session_7", - "session_11", - "session_9", - "session_19", - "session_8", - "session_20", - "session_23" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_2" - ], - "question": "What did Sam find relaxing about his road trip to Jasper?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_18", - "session_11", - "session_8", - "session_16", - "session_9", - "session_25", - "session_7", - "session_13", - "session_20" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_3" - ], - "question": "What habit is Evan trying to change in terms of diet?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_10", - "session_4", - "session_15", - "session_2", - "session_17", - "session_23", - "session_6", - "session_9", - "session_24" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_3" - ], - "question": "What frustrating issue did Evan face at the supermarket?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_18", - "session_9", - "session_8", - "session_25", - "session_1", - "session_11", - "session_5", - "session_6", - "session_22" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What novel is Sam reading that he finds gripping?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_5", - "session_1", - "session_10", - "session_21", - "session_12", - "session_2", - "session_22", - "session_20", - "session_24" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_5" - ], - "question": "What does the smartwatch help Sam with?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_15", - "session_13", - "session_6", - "session_10", - "session_5", - "session_16", - "session_2", - "session_22", - "session_25" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_5" - ], - "question": "Why did Sam decide to get the bonsai tree?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_11", - "session_5", - "session_9", - "session_15", - "session_13", - "session_7", - "session_8", - "session_24", - "session_25" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_6" - ], - "question": "What did Sam mention he had been searching for fruitlessly for half an hour?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_13", - "session_14", - "session_20", - "session_24", - "session_11", - "session_8", - "session_1", - "session_9", - "session_15" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_7" - ], - "question": "What class is Evan taking to learn how to make healthier meals?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_8", - "session_24", - "session_11", - "session_2", - "session_10", - "session_6", - "session_22", - "session_18", - "session_3" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_7" - ], - "question": "What dish did Sam make on 18 August, 2023 that turned out bland?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_10", - "session_23", - "session_11", - "session_2", - "session_24", - "session_15", - "session_14", - "session_1", - "session_8" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_8" - ], - "question": "What food did Evan share a photo of on 19 August, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_19", - "session_16", - "session_11", - "session_7", - "session_6", - "session_8", - "session_22", - "session_25", - "session_1" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_8" - ], - "question": "What did Evan start sculpting years ago due to being inspired by a friend's gift?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_11", - "session_16", - "session_20", - "session_4", - "session_1", - "session_21", - "session_15", - "session_12", - "session_2" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_8" - ], - "question": "What nature concept do watercolor painting classes emphasize according to Sam?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_1", - "session_13", - "session_20", - "session_11", - "session_25", - "session_10", - "session_21", - "session_9", - "session_7" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_8" - ], - "question": "What type of landscapes does Sam love painting the most?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_21", - "session_2", - "session_13", - "session_1", - "session_20", - "session_23", - "session_11", - "session_10", - "session_24" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_9" - ], - "question": "What sports activity has Sam been doing to stay active while dealing with the knee injury?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_13", - "session_5", - "session_20", - "session_6", - "session_24", - "session_12", - "session_9", - "session_8", - "session_18" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_11" - ], - "question": "What activity does Sam do to keep himself busy while healing his knee?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_20", - "session_23", - "session_8", - "session_19", - "session_7", - "session_9", - "session_24", - "session_12", - "session_13" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_11" - ], - "question": "What kind of writing does Evan enjoy as a form of expression?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_23", - "session_21", - "session_19", - "session_20", - "session_7", - "session_8", - "session_16", - "session_12", - "session_2" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_11" - ], - "question": "What electronics issue has been frustrating Evan lately?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_18", - "session_9", - "session_17", - "session_7", - "session_6", - "session_11", - "session_5", - "session_25", - "session_22" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_12" - ], - "question": "What activity did Evan quit one year ago?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_1", - "session_4", - "session_11", - "session_12", - "session_15", - "session_2", - "session_3", - "session_16", - "session_14" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_13" - ], - "question": "Where did Sam and his mate plan to try skydiving?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_20", - "session_19", - "session_10", - "session_23", - "session_24", - "session_7", - "session_15", - "session_16", - "session_25" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_14" - ], - "question": "What digestive issue did Evan experience lately?", - "recall": 0.0, - "retrieved_ids": [ - "session_5", - "session_11", - "session_25", - "session_3", - "session_6", - "session_22", - "session_2", - "session_18", - "session_1", - "session_8" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_15" - ], - "question": "How did Sam start his transformation journey two years ago?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_11", - "session_15", - "session_12", - "session_4", - "session_1", - "session_20", - "session_2", - "session_5", - "session_7" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_16" - ], - "question": "What gift did Sam receive from a close friend?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_20", - "session_1", - "session_4", - "session_17", - "session_24", - "session_8", - "session_22", - "session_15", - "session_11" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_17" - ], - "question": "How does Sam describe the island he grew up on?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_15", - "session_20", - "session_3", - "session_7", - "session_11", - "session_5", - "session_24", - "session_2", - "session_14" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "What was the main reason for Evan's frustration with his new Prius getting stolen?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_7", - "session_12", - "session_22", - "session_1", - "session_20", - "session_11", - "session_13", - "session_3", - "session_9" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_19" - ], - "question": "What family event is Sam planning for next summer?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_23", - "session_16", - "session_11", - "session_24", - "session_8", - "session_15", - "session_25", - "session_10", - "session_20" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_19" - ], - "question": "What is the motto of Sam's family?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_1", - "session_23", - "session_5", - "session_11", - "session_8", - "session_15", - "session_2", - "session_4", - "session_17" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_20" - ], - "question": "Who helped Sam get the painting published in the exhibition?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_12", - "session_13", - "session_10", - "session_21", - "session_11", - "session_1", - "session_2", - "session_6", - "session_8" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_21" - ], - "question": "How did Sam feel when he painted the piece with the bird flying over it?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_24", - "session_10", - "session_13", - "session_5", - "session_6", - "session_20", - "session_11", - "session_1", - "session_8" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_21" - ], - "question": "How did Sam describe the process of creating the painting with the bird flying over it?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_11", - "session_2", - "session_24", - "session_19", - "session_5", - "session_8", - "session_6", - "session_23", - "session_3" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_23" - ], - "question": "What did Evan and his partner keep from their extended family on January 5, 2024?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_8", - "session_19", - "session_11", - "session_1", - "session_7", - "session_20", - "session_24", - "session_15", - "session_25" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_23" - ], - "question": "What was Sam limiting himself to on his new diet?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_3", - "session_4", - "session_20", - "session_17", - "session_23", - "session_9", - "session_24", - "session_15", - "session_10" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_24" - ], - "question": "What dance activity did Evan and his partner try in a recent weekend?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_20", - "session_23", - "session_9", - "session_14", - "session_22", - "session_7", - "session_19", - "session_25", - "session_2" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_24" - ], - "question": "What suggestions did Evan give for high-impact exercises?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_24", - "session_6", - "session_13", - "session_18", - "session_10", - "session_9", - "session_25", - "session_4", - "session_7" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_24" - ], - "question": "What movie did Evan watch that motivated him to keep up with his routine?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_24", - "session_12", - "session_5", - "session_16", - "session_20", - "session_17", - "session_9", - "session_15", - "session_25" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_24" - ], - "question": "What activity hindered Evan's stress and flexibility?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_10", - "session_11", - "session_8", - "session_2", - "session_13", - "session_4", - "session_24", - "session_15", - "session_9" - ], - "sample_id": "conv-49" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_25" - ], - "question": "What did Sam share a photo of that was taken on a camping trip?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_19", - "session_25", - "session_22", - "session_24", - "session_1", - "session_7", - "session_6", - "session_11", - "session_8" - ], - "sample_id": "conv-49" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_3" - ], - "question": "When did Calvin first travel to Tokyo?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_15", - "session_14", - "session_6", - "session_24", - "session_7", - "session_8", - "session_2", - "session_26", - "session_9" - ], - "sample_id": "conv-50" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_2" - ], - "question": "What items did Calvin buy in March 2023?", - "recall": 0.0, - "retrieved_ids": [], - "sample_id": "conv-50" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_2" - ], - "question": "When did Dave see Aerosmith perform live?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_23", - "session_18", - "session_7", - "session_24", - "session_29", - "session_22", - "session_10", - "session_15", - "session_9" - ], - "sample_id": "conv-50" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_2", - "session_23" - ], - "question": "Which bands has Dave enjoyed listening to?", - "recall": 0.5, - "retrieved_ids": [ - "session_28", - "session_2", - "session_15", - "session_19", - "session_3", - "session_9", - "session_1", - "session_8", - "session_25", - "session_18" - ], - "sample_id": "conv-50" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_3" - ], - "question": "Which country do Calvin and Dave want to meet in?", - "recall": 0.0, - "retrieved_ids": [ - "session_15", - "session_8", - "session_13", - "session_17", - "session_1", - "session_27", - "session_26", - "session_21", - "session_24", - "session_25" - ], - "sample_id": "conv-50" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_4", - "session_5" - ], - "question": "What are Dave's dreams?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_19", - "session_12", - "session_2", - "session_26", - "session_15", - "session_22", - "session_24", - "session_5", - "session_11" - ], - "sample_id": "conv-50" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_3", - "session_4" - ], - "question": "Which types of cars does Dave like the most?", - "recall": 0.3333, - "retrieved_ids": [ - "session_13", - "session_9", - "session_2", - "session_17", - "session_22", - "session_12", - "session_30", - "session_4", - "session_7", - "session_25" - ], - "sample_id": "conv-50" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_4" - ], - "question": "Does Dave's shop employ a lot of people?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_9", - "session_23", - "session_27", - "session_28", - "session_6", - "session_3", - "session_19", - "session_12", - "session_18" - ], - "sample_id": "conv-50" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_4" - ], - "question": "When did Dave start his car maintenance shop?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_23", - "session_6", - "session_9", - "session_20", - "session_1", - "session_28", - "session_18", - "session_29", - "session_24" - ], - "sample_id": "conv-50" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_6" - ], - "question": "When did a mishap occur with Calvin's musical gear and favorite mic?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_23", - "session_30", - "session_19", - "session_2", - "session_3", - "session_5", - "session_10", - "session_26", - "session_16" - ], - "sample_id": "conv-50" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_6" - ], - "question": "When did Calvin's place get flooded in Tokyo?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_14", - "session_8", - "session_3", - "session_24", - "session_1", - "session_7", - "session_11", - "session_15", - "session_27" - ], - "sample_id": "conv-50" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_6", - "session_9" - ], - "question": "What mishaps has Calvin run into?", - "recall": 0.5, - "retrieved_ids": [ - "session_7", - "session_19", - "session_22", - "session_27", - "session_26", - "session_3", - "session_25", - "session_28", - "session_14", - "session_9" - ], - "sample_id": "conv-50" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_6", - "session_7" - ], - "question": "When was Calvin's concert in Tokyo?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_15", - "session_28", - "session_6", - "session_3", - "session_19", - "session_14", - "session_7", - "session_5", - "session_25" - ], - "sample_id": "conv-50" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_7" - ], - "question": "Would Calvin enjoy performing at the Hollywood Bowl?", - "recall": 0.0, - "retrieved_ids": [ - "session_24", - "session_2", - "session_25", - "session_14", - "session_1", - "session_10", - "session_27", - "session_18", - "session_20", - "session_19" - ], - "sample_id": "conv-50" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_8" - ], - "question": "When did Calvin meet with the creative team for his new album?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_27", - "session_28", - "session_25", - "session_1", - "session_5", - "session_9", - "session_21", - "session_29", - "session_15" - ], - "sample_id": "conv-50" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_8" - ], - "question": "Why does Dave regularly visit parks?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_3", - "session_1", - "session_18", - "session_26", - "session_10", - "session_6", - "session_24", - "session_30", - "session_28" - ], - "sample_id": "conv-50" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_8" - ], - "question": "When did Dave take a trip to mountainous regions?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_29", - "session_20", - "session_3", - "session_12", - "session_6", - "session_11", - "session_1", - "session_17", - "session_9" - ], - "sample_id": "conv-50" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_9" - ], - "question": "When did Calvin have a car incident?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_21", - "session_28", - "session_14", - "session_9", - "session_3", - "session_12", - "session_2", - "session_26", - "session_4" - ], - "sample_id": "conv-50" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_6", - "session_9" - ], - "question": "How many times has Calvin had to deal with insurance paperwork?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_25", - "session_15", - "session_6", - "session_1", - "session_18", - "session_2", - "session_23", - "session_11", - "session_22" - ], - "sample_id": "conv-50" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_12", - "session_24", - "session_3" - ], - "question": "Which places or events has Calvin visited in Tokyo?", - "recall": 0.6667, - "retrieved_ids": [ - "session_24", - "session_6", - "session_3", - "session_7", - "session_27", - "session_15", - "session_14", - "session_1", - "session_8", - "session_25" - ], - "sample_id": "conv-50" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_12", - "session_26" - ], - "question": "Who inspired Dave's passion for car engineering?", - "recall": 0.5, - "retrieved_ids": [ - "session_26", - "session_13", - "session_4", - "session_5", - "session_25", - "session_11", - "session_22", - "session_3", - "session_20", - "session_7" - ], - "sample_id": "conv-50" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_12", - "session_27" - ], - "question": "Does Calvin wish to become more popular?", - "recall": 0.5, - "retrieved_ids": [ - "session_3", - "session_15", - "session_4", - "session_27", - "session_19", - "session_1", - "session_21", - "session_24", - "session_25", - "session_6" - ], - "sample_id": "conv-50" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_12", - "session_18" - ], - "question": "Does Calvin want to expand his brand?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_18", - "session_17", - "session_13", - "session_1", - "session_9", - "session_26", - "session_6", - "session_29", - "session_24" - ], - "sample_id": "conv-50" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_13", - "session_3", - "session_4", - "session_5" - ], - "question": "What is Dave's main passion?", - "recall": 0.25, - "retrieved_ids": [ - "session_23", - "session_6", - "session_11", - "session_26", - "session_22", - "session_4", - "session_28", - "session_14", - "session_16", - "session_29" - ], - "sample_id": "conv-50" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_13", - "session_20", - "session_22" - ], - "question": "Can Dave work with engines?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_21", - "session_20", - "session_3", - "session_7", - "session_26", - "session_4", - "session_25", - "session_13", - "session_5" - ], - "sample_id": "conv-50" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_15" - ], - "question": "When did Dave host a card-playing night with his friends?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_29", - "session_6", - "session_19", - "session_9", - "session_12", - "session_24", - "session_16", - "session_7", - "session_14" - ], - "sample_id": "conv-50" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_15" - ], - "question": "When did Calvin record a podcast with his friends?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_29", - "session_26", - "session_19", - "session_23", - "session_9", - "session_11", - "session_6", - "session_10", - "session_25" - ], - "sample_id": "conv-50" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_16" - ], - "question": "Which city was Calvin visiting in August 2023?", - "recall": 0.0, - "retrieved_ids": [ - "session_24", - "session_3", - "session_1", - "session_18", - "session_27", - "session_15", - "session_6", - "session_10", - "session_8", - "session_26" - ], - "sample_id": "conv-50" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_5", - "session_7" - ], - "question": "What does Calvin do to relax?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_8", - "session_11", - "session_7", - "session_30", - "session_28", - "session_21", - "session_24", - "session_13", - "session_2" - ], - "sample_id": "conv-50" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_27", - "session_5", - "session_8" - ], - "question": "What are Dave's hobbies other than fixing cars?", - "recall": 0.3333, - "retrieved_ids": [ - "session_7", - "session_19", - "session_12", - "session_5", - "session_17", - "session_21", - "session_13", - "session_22", - "session_18", - "session_10" - ], - "sample_id": "conv-50" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_2", - "session_28" - ], - "question": "What kind of music does Dave listen to?", - "recall": 0.3333, - "retrieved_ids": [ - "session_13", - "session_27", - "session_18", - "session_12", - "session_4", - "session_11", - "session_28", - "session_23", - "session_15", - "session_6" - ], - "sample_id": "conv-50" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_14", - "session_17" - ], - "question": "Where was Dave in the last two weeks of August 2023?", - "recall": 0.5, - "retrieved_ids": [ - "session_24", - "session_30", - "session_6", - "session_9", - "session_28", - "session_17", - "session_8", - "session_2", - "session_23", - "session_7" - ], - "sample_id": "conv-50" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_14", - "session_17" - ], - "question": "Where did Dave return from with new knowledge of different techniques of car restoration?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_1", - "session_28", - "session_25", - "session_17", - "session_5", - "session_4", - "session_9", - "session_12", - "session_22" - ], - "sample_id": "conv-50" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_14", - "session_17" - ], - "question": "What was Dave doing in San Francisco?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_10", - "session_13", - "session_17", - "session_22", - "session_4", - "session_5", - "session_28", - "session_19", - "session_20" - ], - "sample_id": "conv-50" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_17" - ], - "question": "When did Dave return from San Francisco?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_17", - "session_1", - "session_12", - "session_28", - "session_15", - "session_3", - "session_2", - "session_9", - "session_6" - ], - "sample_id": "conv-50" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_17" - ], - "question": "When did Calvin book flight tickets to Boston?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_8", - "session_24", - "session_15", - "session_3", - "session_26", - "session_2", - "session_21", - "session_30", - "session_29" - ], - "sample_id": "conv-50" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_18" - ], - "question": "When was Calvin's album released?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_28", - "session_27", - "session_8", - "session_5", - "session_16", - "session_24", - "session_2", - "session_30", - "session_13" - ], - "sample_id": "conv-50" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_16", - "session_18", - "session_7" - ], - "question": "Does Calvin love music tours?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_15", - "session_18", - "session_16", - "session_25", - "session_11", - "session_24", - "session_3", - "session_14", - "session_9" - ], - "sample_id": "conv-50" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_19" - ], - "question": "When did Dave have a great jam session with his band?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_2", - "session_20", - "session_9", - "session_28", - "session_8", - "session_3", - "session_29", - "session_15", - "session_24" - ], - "sample_id": "conv-50" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [], - "question": "Would Dave prefer working on a Dodge Charger or a Subaru Forester?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_10", - "session_25", - "session_4", - "session_14", - "session_1", - "session_2", - "session_20", - "session_18", - "session_19" - ], - "sample_id": "conv-50" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_20" - ], - "question": "What was the artists Calvin used to listen to when he was a kid?", - "recall": 0.0, - "retrieved_ids": [ - "session_28", - "session_12", - "session_10", - "session_22", - "session_7", - "session_29", - "session_15", - "session_2", - "session_24", - "session_9" - ], - "sample_id": "conv-50" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_12", - "session_20" - ], - "question": "Which of their family member do Calvin and Dave have nostalgic memories about?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_20", - "session_4", - "session_17", - "session_19", - "session_25", - "session_28", - "session_2", - "session_12", - "session_18" - ], - "sample_id": "conv-50" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [], - "question": "Based on the conversation, did Calvin and Dave have a meeting in Boston between August and November 2023? Answer in yes or no.", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_7", - "session_30", - "session_21", - "session_27", - "session_15", - "session_25", - "session_1", - "session_22", - "session_11" - ], - "sample_id": "conv-50" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_21" - ], - "question": "Which city was Calvin at on October 3, 2023?", - "recall": 0.0, - "retrieved_ids": [ - "session_27", - "session_24", - "session_15", - "session_6", - "session_1", - "session_3", - "session_10", - "session_18", - "session_2", - "session_28" - ], - "sample_id": "conv-50" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_21" - ], - "question": "When did Calvin met with local artists in Boston?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_3", - "session_21", - "session_30", - "session_1", - "session_8", - "session_29", - "session_24", - "session_10", - "session_5" - ], - "sample_id": "conv-50" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_21" - ], - "question": "What shared activities do Dave and Calvin have?", - "recall": 0.0, - "retrieved_ids": [ - "session_23", - "session_2", - "session_10", - "session_15", - "session_18", - "session_28", - "session_25", - "session_17", - "session_19", - "session_6" - ], - "sample_id": "conv-50" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_19", - "session_21", - "session_22" - ], - "question": "What is Dave's favorite activity?", - "recall": 0.3333, - "retrieved_ids": [ - "session_23", - "session_30", - "session_10", - "session_2", - "session_3", - "session_5", - "session_19", - "session_6", - "session_26", - "session_16" - ], - "sample_id": "conv-50" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_22", - "session_3" - ], - "question": "How many car shows has Dave attended?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_26", - "session_2", - "session_22", - "session_18", - "session_3", - "session_25", - "session_19", - "session_29", - "session_30" - ], - "sample_id": "conv-50" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_22" - ], - "question": "What was Dave doing in the first weekend of October 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_8", - "session_19", - "session_16", - "session_26", - "session_22", - "session_3", - "session_4", - "session_28", - "session_13" - ], - "sample_id": "conv-50" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_12", - "session_22" - ], - "question": "When Dave was a child, what did he and his father do in the garage?", - "recall": 0.5, - "retrieved_ids": [ - "session_9", - "session_22", - "session_29", - "session_18", - "session_5", - "session_21", - "session_25", - "session_7", - "session_2", - "session_24" - ], - "sample_id": "conv-50" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_23" - ], - "question": "When did Calvin buy his second Ferrari?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_12", - "session_7", - "session_9", - "session_29", - "session_6", - "session_28", - "session_15", - "session_14", - "session_3" - ], - "sample_id": "conv-50" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_15", - "session_24" - ], - "question": "When did Calvin and Frank Ocean start collaborating?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_25", - "session_23", - "session_29", - "session_15", - "session_3", - "session_9", - "session_7", - "session_1", - "session_11" - ], - "sample_id": "conv-50" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_24" - ], - "question": "When did Calvin plan on travelling to Tokyo the second time?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_7", - "session_12", - "session_15", - "session_1", - "session_6", - "session_5", - "session_14", - "session_24", - "session_8" - ], - "sample_id": "conv-50" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_25", - "session_29" - ], - "question": "Who supports Calvin in tough times?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_30", - "session_23", - "session_22", - "session_6", - "session_11", - "session_29", - "session_16", - "session_20", - "session_12" - ], - "sample_id": "conv-50" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_25", - "session_28" - ], - "question": "What does help Calvin stay connected to the creative process?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_30", - "session_5", - "session_25", - "session_6", - "session_27", - "session_11", - "session_7", - "session_9", - "session_8" - ], - "sample_id": "conv-50" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_26" - ], - "question": "When did Calvin visit some of the sights in Boston with a former high school friend?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_29", - "session_8", - "session_7", - "session_18", - "session_11", - "session_6", - "session_24", - "session_5", - "session_25" - ], - "sample_id": "conv-50" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_14", - "session_26" - ], - "question": "Which cities did Dave travel to in 2023?", - "recall": 0.5, - "retrieved_ids": [ - "session_3", - "session_27", - "session_8", - "session_15", - "session_1", - "session_6", - "session_18", - "session_24", - "session_10", - "session_14" - ], - "sample_id": "conv-50" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_27" - ], - "question": "Which hobby did Dave pick up in October 2023?", - "recall": 0.0, - "retrieved_ids": [ - "session_13", - "session_21", - "session_2", - "session_22", - "session_16", - "session_19", - "session_12", - "session_18", - "session_5", - "session_7" - ], - "sample_id": "conv-50" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_12", - "session_25", - "session_26" - ], - "question": "Which events in Dave's life inspired him to take up auto engineering?", - "recall": 0.3333, - "retrieved_ids": [ - "session_25", - "session_9", - "session_13", - "session_1", - "session_3", - "session_5", - "session_24", - "session_30", - "session_23", - "session_27" - ], - "sample_id": "conv-50" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_2", - "session_23" - ], - "question": "How many Ferraris does Calvin own?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_22", - "session_18", - "session_23", - "session_4", - "session_2", - "session_1", - "session_11", - "session_12", - "session_29" - ], - "sample_id": "conv-50" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_16", - "session_4" - ], - "question": "What gifts has Calvin received from his artist friends?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_4", - "session_11", - "session_7", - "session_25", - "session_15", - "session_16", - "session_21", - "session_10", - "session_9" - ], - "sample_id": "conv-50" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_14", - "session_20", - "session_21" - ], - "question": "How long did Dave's work on the Ford Mustang take?", - "recall": 0.6667, - "retrieved_ids": [ - "session_21", - "session_20", - "session_3", - "session_6", - "session_11", - "session_12", - "session_4", - "session_29", - "session_26", - "session_9" - ], - "sample_id": "conv-50" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_14", - "session_17" - ], - "question": "How long was the car modification workshop in San Francisco?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_17", - "session_13", - "session_4", - "session_26", - "session_6", - "session_5", - "session_9", - "session_18", - "session_12" - ], - "sample_id": "conv-50" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_16" - ], - "question": "What style of guitars does Calvin own?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_22", - "session_4", - "session_11", - "session_28", - "session_25", - "session_13", - "session_9", - "session_2", - "session_21" - ], - "sample_id": "conv-50" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_11", - "session_15", - "session_6" - ], - "question": "What activities has Dave participated in with his friends?", - "recall": 1.0, - "retrieved_ids": [ - "session_29", - "session_23", - "session_10", - "session_21", - "session_15", - "session_26", - "session_11", - "session_25", - "session_9", - "session_6" - ], - "sample_id": "conv-50" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_27" - ], - "question": "When did Dave take a photo of a Boston clock tower?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_15", - "session_8", - "session_3", - "session_30", - "session_21", - "session_29", - "session_24", - "session_28", - "session_9" - ], - "sample_id": "conv-50" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_13", - "session_20", - "session_25", - "session_27" - ], - "question": "Do all of Dave's car restoration projects go smoothly?", - "recall": 0.5, - "retrieved_ids": [ - "session_14", - "session_25", - "session_6", - "session_4", - "session_7", - "session_21", - "session_28", - "session_27", - "session_12", - "session_22" - ], - "sample_id": "conv-50" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_28" - ], - "question": "Where was Calvin located in the last week of October 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_30", - "session_6", - "session_9", - "session_28", - "session_17", - "session_8", - "session_2", - "session_23", - "session_7", - "session_16" - ], - "sample_id": "conv-50" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_28" - ], - "question": "When did Dave find the car he repaired and started sharing in his blog?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_1", - "session_9", - "session_6", - "session_29", - "session_25", - "session_7", - "session_11", - "session_23", - "session_24" - ], - "sample_id": "conv-50" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_30" - ], - "question": "When did Dave buy a vintage camera?", - "recall": 1.0, - "retrieved_ids": [ - "session_30", - "session_19", - "session_27", - "session_20", - "session_6", - "session_28", - "session_15", - "session_14", - "session_3", - "session_9" - ], - "sample_id": "conv-50" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_30" - ], - "question": "When did Calvin attend a gala in Boston?", - "recall": 1.0, - "retrieved_ids": [ - "session_30", - "session_19", - "session_1", - "session_26", - "session_24", - "session_15", - "session_3", - "session_2", - "session_21", - "session_29" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_1" - ], - "question": "How long did Calvin plan to stay in Japan?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_10", - "session_3", - "session_5", - "session_12", - "session_6", - "session_14", - "session_9", - "session_11", - "session_20" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "Which band was Dave's favorite at the music festival in April 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_3", - "session_19", - "session_23", - "session_5", - "session_15", - "session_6", - "session_9", - "session_24", - "session_28" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "Where did Calvin attend a music festival in April 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_24", - "session_28", - "session_19", - "session_26", - "session_2", - "session_3", - "session_23", - "session_1", - "session_30" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "What advice did Calvin receive from the producer at the music festival?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_15", - "session_26", - "session_2", - "session_5", - "session_24", - "session_28", - "session_23", - "session_29", - "session_6" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What is Dave's new business venture as of 1 May, 2023?", - "recall": 0.0, - "retrieved_ids": [], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What type of cars does Dave work on at his shop?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_4", - "session_28", - "session_6", - "session_23", - "session_22", - "session_12", - "session_25", - "session_13", - "session_29" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What did Calvin receive as a gift from another artist?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_28", - "session_25", - "session_30", - "session_10", - "session_27", - "session_12", - "session_21", - "session_16", - "session_29" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What was the necklace Calvin received meant to remind him of?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_20", - "session_22", - "session_12", - "session_18", - "session_24", - "session_11", - "session_9", - "session_19", - "session_23" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_5" - ], - "question": "What does Dave do when he feels his creativity is frozen?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_30", - "session_28", - "session_2", - "session_8", - "session_24", - "session_9", - "session_29", - "session_27", - "session_6" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_5" - ], - "question": "How does Calvin plan to jumpstart his inspiration?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_1", - "session_9", - "session_3", - "session_10", - "session_11", - "session_28", - "session_26", - "session_12", - "session_29" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_6" - ], - "question": "What did Calvin manage to save during the flood incident?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_26", - "session_20", - "session_24", - "session_25", - "session_19", - "session_29", - "session_9", - "session_15", - "session_14" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_6" - ], - "question": "What did Dave open in May 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_5", - "session_28", - "session_6", - "session_2", - "session_15", - "session_26", - "session_3", - "session_14", - "session_9" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What gives Dave a sense of achievement and purpose?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_17", - "session_12", - "session_26", - "session_8", - "session_11", - "session_4", - "session_21", - "session_16", - "session_5" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What fuels Calvin's soul?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_7", - "session_28", - "session_16", - "session_20", - "session_2", - "session_18", - "session_24", - "session_30", - "session_13" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What is Dave doing to relax on weekends?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_2", - "session_5", - "session_11", - "session_28", - "session_19", - "session_16", - "session_7", - "session_3", - "session_21" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_9" - ], - "question": "What sports activity is Calvin planning to try after the tour with Frank Ocean?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_24", - "session_9", - "session_25", - "session_15", - "session_7", - "session_23", - "session_5", - "session_1", - "session_10" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "What was Calvin excited to do after getting his car fixed on 7 July, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_18", - "session_10", - "session_24", - "session_20", - "session_2", - "session_3", - "session_29", - "session_19", - "session_21" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "What did Calvin and his friends arrange for in the park?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_11", - "session_29", - "session_21", - "session_15", - "session_1", - "session_8", - "session_3", - "session_30", - "session_9" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_11" - ], - "question": "What kind of music has Calvin been creating lately?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_27", - "session_18", - "session_28", - "session_25", - "session_23", - "session_4", - "session_13", - "session_10", - "session_12" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_11" - ], - "question": "How does Calvin describe his process of adding electronic elements to his songs?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_14", - "session_15", - "session_9", - "session_6", - "session_5", - "session_28", - "session_29", - "session_20", - "session_22" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_12" - ], - "question": "What car brand does Calvin own that he is proud of?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_22", - "session_18", - "session_14", - "session_28", - "session_17", - "session_21", - "session_2", - "session_24", - "session_29" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_12" - ], - "question": "What is Calvin's biggest current goal?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_11", - "session_22", - "session_26", - "session_25", - "session_13", - "session_28", - "session_21", - "session_23", - "session_5" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_12" - ], - "question": "What is Dave's advice to Calvin regarding his dreams?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_5", - "session_24", - "session_9", - "session_3", - "session_2", - "session_4", - "session_12", - "session_28", - "session_29" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "What workshop did Dave get picked for on 11 August, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_24", - "session_2", - "session_3", - "session_14", - "session_11", - "session_28", - "session_16", - "session_9", - "session_6" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "What kind of modifications has Dave been working on in the car mod workshop?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_22", - "session_11", - "session_23", - "session_27", - "session_18", - "session_12", - "session_28", - "session_17", - "session_14" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "What type of car did Dave work on during the workshop?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_20", - "session_13", - "session_28", - "session_14", - "session_19", - "session_12", - "session_4", - "session_24", - "session_29" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "What does Dave say is important for making his custom cars unique?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_4", - "session_13", - "session_23", - "session_22", - "session_5", - "session_19", - "session_29", - "session_16", - "session_9" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_14" - ], - "question": "How did the audience in Tokyo react when Calvin sang one of his songs?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_16", - "session_7", - "session_6", - "session_15", - "session_3", - "session_24", - "session_9", - "session_20", - "session_29" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "How did Calvin meet Frank Ocean?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_3", - "session_24", - "session_9", - "session_23", - "session_25", - "session_7", - "session_27", - "session_1", - "session_21" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "Where did Calvin and Frank Ocean record a song together?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_25", - "session_24", - "session_3", - "session_7", - "session_23", - "session_20", - "session_9", - "session_11", - "session_19" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "What did Calvin and his friends record in August 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_29", - "session_6", - "session_24", - "session_11", - "session_25", - "session_9", - "session_19", - "session_10", - "session_23" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "Where did Calvin start shooting a video for his new album?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_28", - "session_29", - "session_24", - "session_9", - "session_27", - "session_19", - "session_8", - "session_18", - "session_5" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "What design is featured on Calvin's guitar?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_4", - "session_20", - "session_9", - "session_2", - "session_18", - "session_30", - "session_24", - "session_28", - "session_13" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "Why did Calvin get his guitar customized with a shiny finish?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_9", - "session_4", - "session_3", - "session_29", - "session_13", - "session_27", - "session_22", - "session_19", - "session_5" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "What color glow did Calvin customize his guitar with?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_9", - "session_27", - "session_14", - "session_29", - "session_15", - "session_22", - "session_24", - "session_5", - "session_13" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "Where did Dave come back from with insights on car modification on 1st September 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_9", - "session_4", - "session_28", - "session_12", - "session_21", - "session_5", - "session_11", - "session_13", - "session_14" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What emotion does Dave mention feeling when he sees the relief of someone whose car he fixed?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_17", - "session_9", - "session_7", - "session_30", - "session_19", - "session_26", - "session_29", - "session_15", - "session_24" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What did Calvin book a flight ticket for on 1st September 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_12", - "session_8", - "session_15", - "session_3", - "session_2", - "session_28", - "session_14", - "session_9", - "session_6" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What is Calvin excited about after the tour?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_18", - "session_9", - "session_15", - "session_3", - "session_2", - "session_20", - "session_7", - "session_25", - "session_16" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What plans do Calvin and Dave have for when Calvin visits Boston?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_1", - "session_8", - "session_18", - "session_26", - "session_24", - "session_10", - "session_12", - "session_5", - "session_6" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "Which Disney movie did Dave mention as one of his favorites?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_2", - "session_9", - "session_26", - "session_29", - "session_7", - "session_3", - "session_10", - "session_6", - "session_5" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_19" - ], - "question": "How does Dave feel about the reactions of people when they see the finished restoration project?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_25", - "session_19", - "session_4", - "session_27", - "session_21", - "session_12", - "session_3", - "session_16", - "session_13" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_20" - ], - "question": "What activity did Calvin enjoy during his summer drives?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_24", - "session_26", - "session_5", - "session_29", - "session_2", - "session_12", - "session_9", - "session_23", - "session_10" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_20" - ], - "question": "Which song from the childhood of Calvin brings back memories of a road trip with his dad?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_12", - "session_29", - "session_11", - "session_25", - "session_10", - "session_9", - "session_22", - "session_26", - "session_5" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_21" - ], - "question": "What project did Calvin work on to chill out?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_21", - "session_25", - "session_26", - "session_7", - "session_13", - "session_27", - "session_1", - "session_14", - "session_8" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_21" - ], - "question": "What car did Dave work on in the junkyard?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_6", - "session_12", - "session_4", - "session_26", - "session_2", - "session_3", - "session_28", - "session_14", - "session_9" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_21" - ], - "question": "What does Dave find satisfying about restoring old cars?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_28", - "session_22", - "session_12", - "session_17", - "session_19", - "session_4", - "session_11", - "session_25", - "session_29" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_21" - ], - "question": "What do Calvin and Dave use to reach their goals?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_2", - "session_22", - "session_18", - "session_26", - "session_3", - "session_11", - "session_17", - "session_21", - "session_23" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_22" - ], - "question": "What does working on cars represent for Dave?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_22", - "session_4", - "session_12", - "session_5", - "session_25", - "session_13", - "session_21", - "session_16", - "session_28" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_22" - ], - "question": "What does Dave aim to do with his passion for cars?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_9", - "session_26", - "session_4", - "session_11", - "session_28", - "session_14", - "session_29", - "session_23", - "session_16" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "What did Calvin recently get that is a \"masterpiece on wheels\"?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_13", - "session_22", - "session_1", - "session_2", - "session_8", - "session_28", - "session_25", - "session_26", - "session_27" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "Who headlined the music festival that Dave attended in October?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_15", - "session_19", - "session_26", - "session_2", - "session_3", - "session_24", - "session_28", - "session_30", - "session_1" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "How does Calvin stay motivated when faced with setbacks?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_21", - "session_14", - "session_12", - "session_3", - "session_11", - "session_28", - "session_18", - "session_4", - "session_29" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_23" - ], - "question": "What activity does Dave find fulfilling, similar to Calvin's passion for music festivals?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_26", - "session_28", - "session_3", - "session_24", - "session_11", - "session_15", - "session_8", - "session_2", - "session_10" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "Where did Calvin and Dave meet Frank Ocean to start collaborating?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_24", - "session_23", - "session_25", - "session_1", - "session_9", - "session_3", - "session_7", - "session_27", - "session_21" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "Which part of Tokyo is described as Tokyo's Times Square by Calvin?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_25", - "session_15", - "session_7", - "session_14", - "session_3", - "session_6", - "session_26", - "session_10", - "session_13" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "What specific location in Tokyo does Calvin mention being excited to explore?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_8", - "session_14", - "session_15", - "session_7", - "session_3", - "session_6", - "session_16", - "session_5", - "session_19" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_24" - ], - "question": "What dish does Dave recommend Calvin to try in Tokyo?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_24", - "session_1", - "session_9", - "session_15", - "session_7", - "session_14", - "session_6", - "session_5", - "session_25" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_25" - ], - "question": "What does Calvin find energizing during the tour?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_26", - "session_24", - "session_7", - "session_28", - "session_1", - "session_29", - "session_20", - "session_19", - "session_11" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_25" - ], - "question": "How does Calvin balance his job and personal life?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_9", - "session_14", - "session_13", - "session_22", - "session_29", - "session_30", - "session_11", - "session_24", - "session_28" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_25" - ], - "question": "What inspired Calvin's recent music?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_28", - "session_27", - "session_30", - "session_11", - "session_1", - "session_8", - "session_26", - "session_7", - "session_9" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_25" - ], - "question": "How does Calvin describe his music in relation to capturing feelings?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_23", - "session_9", - "session_29", - "session_25", - "session_12", - "session_19", - "session_30", - "session_27", - "session_16" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_25" - ], - "question": "Why did Dave start working on cars?", - "recall": 0.0, - "retrieved_ids": [ - "session_4", - "session_6", - "session_11", - "session_18", - "session_1", - "session_20", - "session_29", - "session_28", - "session_16", - "session_24" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_25" - ], - "question": "What is the toughest part of car restoration according to Dave?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_27", - "session_26", - "session_4", - "session_7", - "session_13", - "session_14", - "session_10", - "session_22", - "session_12" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_25" - ], - "question": "What does Calvin believe makes an artist create something extraordinary?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_12", - "session_4", - "session_7", - "session_13", - "session_28", - "session_10", - "session_2", - "session_11", - "session_23" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_25" - ], - "question": "When did Dave sell the car he restored last year?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_24", - "session_27", - "session_4", - "session_14", - "session_28", - "session_2", - "session_9", - "session_7", - "session_29" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_26" - ], - "question": "When did Calvin first get interested in cars?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_6", - "session_28", - "session_26", - "session_14", - "session_9", - "session_11", - "session_24", - "session_20", - "session_30" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_26" - ], - "question": "How did Calvin feel about performing with someone he admires?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_24", - "session_14", - "session_7", - "session_15", - "session_2", - "session_28", - "session_17", - "session_29", - "session_9" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_26" - ], - "question": "What realization did the nightclub experience bring to Calvin?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_12", - "session_30", - "session_25", - "session_2", - "session_14", - "session_21", - "session_15", - "session_9", - "session_29" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_26" - ], - "question": "What do Dave and Calvin agree on regarding their pursuits?", - "recall": 0.0, - "retrieved_ids": [ - "session_9", - "session_2", - "session_1", - "session_30", - "session_4", - "session_28", - "session_21", - "session_17", - "session_14", - "session_29" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_27" - ], - "question": "Which city is featured in the photograph Dave showed Calvin?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_1", - "session_6", - "session_3", - "session_24", - "session_18", - "session_10", - "session_14", - "session_2", - "session_15" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_28" - ], - "question": "What did Calvin do recently at his Japanese house?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_1", - "session_30", - "session_16", - "session_9", - "session_11", - "session_5", - "session_3", - "session_10", - "session_29" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_28" - ], - "question": "What did Dave recently start a blog about?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_1", - "session_25", - "session_20", - "session_27", - "session_8", - "session_11", - "session_24", - "session_6", - "session_18" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_28" - ], - "question": "What is Dave's way to share his passion with others?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_28", - "session_23", - "session_4", - "session_9", - "session_14", - "session_26", - "session_11", - "session_17", - "session_29" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_28" - ], - "question": "What type of videos does Calvin usually watch on his television?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_24", - "session_5", - "session_9", - "session_19", - "session_16", - "session_29", - "session_2", - "session_30", - "session_13" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_28" - ], - "question": "What type of music has Dave been getting into lately?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_27", - "session_11", - "session_25", - "session_14", - "session_10", - "session_29", - "session_1", - "session_8", - "session_17" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_28" - ], - "question": "What tools does Calvin use to boost his motivation for music?", - "recall": 1.0, - "retrieved_ids": [ - "session_30", - "session_28", - "session_9", - "session_29", - "session_4", - "session_11", - "session_21", - "session_22", - "session_18", - "session_12" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_28" - ], - "question": "What type of content does Dave post on his blog that inspired others to start their own DIY projects?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_25", - "session_14", - "session_4", - "session_11", - "session_18", - "session_6", - "session_29", - "session_9", - "session_2" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_28" - ], - "question": "What kind of impact does Dave's blog on car mods have on people?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_12", - "session_23", - "session_27", - "session_3", - "session_18", - "session_25", - "session_13", - "session_22", - "session_11" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_29" - ], - "question": "Who did Calvin invite to see him perform in Boston on 13 November, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_24", - "session_9", - "session_30", - "session_26", - "session_29", - "session_15", - "session_2", - "session_3", - "session_6" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_30" - ], - "question": "What hobby did Calvin take up recently?", - "recall": 0.0, - "retrieved_ids": [ - "session_7", - "session_25", - "session_22", - "session_27", - "session_12", - "session_18", - "session_5", - "session_13", - "session_19", - "session_21" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_30" - ], - "question": "What new item did Dave buy recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_30", - "session_1", - "session_8", - "session_26", - "session_7", - "session_25", - "session_28", - "session_9", - "session_11" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_30" - ], - "question": "What type of photos does Dave like to capture with his new camera?", - "recall": 1.0, - "retrieved_ids": [ - "session_30", - "session_27", - "session_10", - "session_9", - "session_29", - "session_25", - "session_28", - "session_16", - "session_23", - "session_19" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_30" - ], - "question": "What event did Calvin attend in Boston?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_27", - "session_26", - "session_30", - "session_19", - "session_2", - "session_15", - "session_3", - "session_25", - "session_23" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_30" - ], - "question": "What did Calvin discuss with the cool artist he met at the gala?", - "recall": 1.0, - "retrieved_ids": [ - "session_30", - "session_27", - "session_3", - "session_2", - "session_8", - "session_24", - "session_7", - "session_21", - "session_29", - "session_15" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_30" - ], - "question": "Where did Dave take a stunning photo of a waterfall?", - "recall": 1.0, - "retrieved_ids": [ - "session_30", - "session_4", - "session_9", - "session_28", - "session_27", - "session_10", - "session_15", - "session_6", - "session_18", - "session_11" - ], - "sample_id": "conv-50" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_30" - ], - "question": "What positive impact does Calvin mention nature has on tough times?", - "recall": 1.0, - "retrieved_ids": [ - "session_30", - "session_25", - "session_12", - "session_11", - "session_6", - "session_23", - "session_8", - "session_7", - "session_19", - "session_29" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_2" - ], - "question": "Which DJ was Dave's favorite at the music festival in April 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_2", - "session_10", - "session_3", - "session_15", - "session_19", - "session_6", - "session_5", - "session_24", - "session_26" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_3" - ], - "question": "What advice did Calvin receive from the chef at the music festival?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_15", - "session_26", - "session_2", - "session_5", - "session_24", - "session_28", - "session_29", - "session_23", - "session_6" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What is Calvin's new business venture as of 1 May, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_1", - "session_2", - "session_8", - "session_30", - "session_13", - "session_17", - "session_29", - "session_3", - "session_4" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What type of cars does Calvin work on at his shop?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_4", - "session_28", - "session_6", - "session_23", - "session_22", - "session_12", - "session_25", - "session_13", - "session_29" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What did Dave receive as a gift from another artist?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_25", - "session_28", - "session_30", - "session_10", - "session_27", - "session_12", - "session_21", - "session_29", - "session_16" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What was the necklace Dave received meant to remind him of?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_20", - "session_22", - "session_12", - "session_24", - "session_18", - "session_11", - "session_9", - "session_19", - "session_23" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_6" - ], - "question": "What did Calvin open in May 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_5", - "session_28", - "session_2", - "session_6", - "session_15", - "session_26", - "session_3", - "session_14", - "session_9" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_7" - ], - "question": "What gives Calvin a sense of achievement and purpose?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_17", - "session_12", - "session_4", - "session_26", - "session_8", - "session_11", - "session_21", - "session_16", - "session_5" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_9" - ], - "question": "What sports activity is Dave planning to try after the tour with Frank Ocean?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_24", - "session_9", - "session_25", - "session_15", - "session_7", - "session_23", - "session_5", - "session_1", - "session_10" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_11" - ], - "question": "How does Calvin describe his process of adding acoustic elements to his songs?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_14", - "session_15", - "session_9", - "session_6", - "session_5", - "session_28", - "session_29", - "session_20", - "session_22" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_12" - ], - "question": "What clothing brand does Calvin own that he is proud of?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_22", - "session_18", - "session_14", - "session_28", - "session_17", - "session_21", - "session_2", - "session_24", - "session_29" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_13" - ], - "question": "What workshop did Calvin get picked for on 11 August, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_24", - "session_2", - "session_29", - "session_14", - "session_11", - "session_28", - "session_16", - "session_9", - "session_6" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_13" - ], - "question": "What kind of modifications has Calvin been working on in the car mod workshop?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_22", - "session_11", - "session_23", - "session_27", - "session_18", - "session_12", - "session_28", - "session_17", - "session_14" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_13" - ], - "question": "What type of car did Calvin work on during the workshop?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_26", - "session_20", - "session_28", - "session_14", - "session_19", - "session_12", - "session_4", - "session_24", - "session_29" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_15" - ], - "question": "What did Dave and his friends record in August 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_6", - "session_29", - "session_24", - "session_11", - "session_25", - "session_9", - "session_19", - "session_10", - "session_23" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_16" - ], - "question": "Where did Dave start shooting a video for his new album?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_28", - "session_29", - "session_24", - "session_9", - "session_27", - "session_19", - "session_8", - "session_18", - "session_5" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_16" - ], - "question": "What design is featured on Dave's guitar?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_4", - "session_9", - "session_22", - "session_28", - "session_30", - "session_15", - "session_12", - "session_13", - "session_6" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_16" - ], - "question": "Why did Dave get his guitar customized with a shiny finish?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_4", - "session_9", - "session_3", - "session_29", - "session_13", - "session_27", - "session_22", - "session_19", - "session_5" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_16" - ], - "question": "What color glow did Dave customize his guitar with?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_9", - "session_27", - "session_14", - "session_29", - "session_15", - "session_22", - "session_24", - "session_5", - "session_13" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_17" - ], - "question": "Where did Calvin come back from with insights on car modification on 1st September 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_9", - "session_4", - "session_28", - "session_12", - "session_21", - "session_5", - "session_11", - "session_13", - "session_14" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_17" - ], - "question": "What emotion does Calvin mention feeling when he sees the relief of someone whose car he fixed?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_25", - "session_9", - "session_7", - "session_30", - "session_19", - "session_26", - "session_29", - "session_15", - "session_24" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_17" - ], - "question": "What did Dave book a flight ticket for on 1st September 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_12", - "session_8", - "session_15", - "session_3", - "session_2", - "session_28", - "session_14", - "session_9", - "session_6" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_19" - ], - "question": "Which horror movie did Dave mention as one of his favorites?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_2", - "session_9", - "session_26", - "session_29", - "session_7", - "session_3", - "session_10", - "session_6", - "session_5" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_20" - ], - "question": "Which song from the childhood of Dave brings back memories of a road trip with his dad?", - "recall": 1.0, - "retrieved_ids": [ - "session_20", - "session_12", - "session_29", - "session_11", - "session_25", - "session_10", - "session_9", - "session_22", - "session_26", - "session_5" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_21" - ], - "question": "What car did Calvin work on in the junkyard?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_6", - "session_12", - "session_4", - "session_26", - "session_2", - "session_3", - "session_28", - "session_14", - "session_9" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_21" - ], - "question": "What does Dave find satisfying about destroying old cars?", - "recall": 1.0, - "retrieved_ids": [ - "session_21", - "session_4", - "session_28", - "session_17", - "session_1", - "session_12", - "session_7", - "session_25", - "session_29", - "session_11" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_22" - ], - "question": "What does working on boats represent for Dave?", - "recall": 0.0, - "retrieved_ids": [ - "session_25", - "session_16", - "session_4", - "session_12", - "session_5", - "session_13", - "session_21", - "session_20", - "session_28", - "session_19" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_22" - ], - "question": "What does Dave aim to do with his passion for cooking?", - "recall": 1.0, - "retrieved_ids": [ - "session_22", - "session_9", - "session_29", - "session_26", - "session_11", - "session_4", - "session_28", - "session_14", - "session_16", - "session_23" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_23" - ], - "question": "What did Calvin recently get that is a \"masterpiece on canvas\"?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_9", - "session_22", - "session_1", - "session_8", - "session_28", - "session_25", - "session_13", - "session_27", - "session_26" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_23" - ], - "question": "Who headlined the music festival that Calvin attended in October?", - "recall": 1.0, - "retrieved_ids": [ - "session_23", - "session_15", - "session_28", - "session_19", - "session_26", - "session_2", - "session_3", - "session_24", - "session_30", - "session_1" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_24" - ], - "question": "Which part of Tokyo is described as Tokyo's Times Square by Dave?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_25", - "session_15", - "session_7", - "session_14", - "session_3", - "session_6", - "session_26", - "session_10", - "session_13" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_24" - ], - "question": "What specific location in Tokyo does Calvin mention being excited to avoid?", - "recall": 1.0, - "retrieved_ids": [ - "session_24", - "session_15", - "session_7", - "session_3", - "session_18", - "session_14", - "session_6", - "session_16", - "session_1", - "session_19" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_25" - ], - "question": "When did Calvin sell the car he restored last year?", - "recall": 1.0, - "retrieved_ids": [ - "session_25", - "session_24", - "session_27", - "session_4", - "session_14", - "session_28", - "session_2", - "session_9", - "session_7", - "session_29" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_26" - ], - "question": "When did Calvin first get interested in motorcycles?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_24", - "session_14", - "session_28", - "session_26", - "session_6", - "session_9", - "session_11", - "session_20", - "session_30" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_26" - ], - "question": "What realization did the nightclub experience bring to Dave?", - "recall": 1.0, - "retrieved_ids": [ - "session_26", - "session_8", - "session_12", - "session_30", - "session_25", - "session_14", - "session_21", - "session_15", - "session_9", - "session_29" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_28" - ], - "question": "What did Dave do recently at his Japanese house?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_1", - "session_30", - "session_16", - "session_9", - "session_11", - "session_5", - "session_3", - "session_10", - "session_29" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_28" - ], - "question": "What did Calvin recently start a blog about?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_1", - "session_25", - "session_20", - "session_27", - "session_8", - "session_11", - "session_24", - "session_6", - "session_18" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_28" - ], - "question": "What type of videos does Dave usually watch on his television?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_13", - "session_5", - "session_9", - "session_19", - "session_16", - "session_29", - "session_4", - "session_30", - "session_6" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_28" - ], - "question": "What type of art has Dave been getting into lately?", - "recall": 0.0, - "retrieved_ids": [ - "session_27", - "session_14", - "session_25", - "session_29", - "session_19", - "session_1", - "session_30", - "session_13", - "session_17", - "session_16" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_28" - ], - "question": "What type of content does Dave post on his blog that inspired others to start their own cooking projects?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_25", - "session_14", - "session_4", - "session_11", - "session_18", - "session_6", - "session_29", - "session_2", - "session_9" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_28" - ], - "question": "What kind of impact does Dave's blog on vegan recipes have on people?", - "recall": 1.0, - "retrieved_ids": [ - "session_28", - "session_4", - "session_12", - "session_27", - "session_3", - "session_18", - "session_25", - "session_23", - "session_11", - "session_13" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_29" - ], - "question": "Who did Dave invite to see him perform in Boston on 13 November, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_24", - "session_9", - "session_30", - "session_26", - "session_29", - "session_15", - "session_3", - "session_2", - "session_6" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_30" - ], - "question": "What new item did Calvin buy recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_27", - "session_30", - "session_1", - "session_8", - "session_26", - "session_7", - "session_25", - "session_28", - "session_9", - "session_11" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_30" - ], - "question": "What type of photos does Calvin like to capture with his new camera?", - "recall": 1.0, - "retrieved_ids": [ - "session_30", - "session_27", - "session_10", - "session_9", - "session_29", - "session_25", - "session_28", - "session_16", - "session_23", - "session_19" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_30" - ], - "question": "What did Dave discuss with the cool artist he met at the gala?", - "recall": 1.0, - "retrieved_ids": [ - "session_30", - "session_27", - "session_3", - "session_2", - "session_8", - "session_24", - "session_7", - "session_21", - "session_29", - "session_15" - ], - "sample_id": "conv-50" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_30" - ], - "question": "Where did Calvin take a stunning photo of a waterfall?", - "recall": 1.0, - "retrieved_ids": [ - "session_30", - "session_4", - "session_9", - "session_28", - "session_27", - "session_10", - "session_15", - "session_6", - "session_18", - "session_11" - ], - "sample_id": "conv-50" - } - ], - "runtime_seconds": 157.701, - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/locomo_old_brainctl_cmd_session.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/locomo_old_brainctl_cmd_session.json deleted file mode 100644 index 57a2a2f..0000000 --- a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/locomo_old_brainctl_cmd_session.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\locomo_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\locomo_old_brainctl_cmd_session.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" - }, - "benchmark": "locomo", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "measured": false, - "metadata": {}, - "metrics": { - "avg_recall": 0.9217, - "cat_1_recall": 0.7614, - "cat_2_recall": 0.92, - "cat_3_recall": 0.6616, - "cat_4_recall": 0.9738, - "cat_5_recall": 0.9821, - "perfect_rate": 0.8817, - "top_k": 10, - "zero_rate": 0.0438 - }, - "mode": "cmd_session", - "notes": [ - "granularity=session", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": 0.9217, - "reference_kind": "historical", - "runtime_seconds": 445.74, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_mempalace_raw_session.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_mempalace_raw_session.json deleted file mode 100644 index 291cd74..0000000 --- a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_mempalace_raw_session.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\longmemeval_mempalace_raw_session.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "measured": false, - "metadata": {}, - "metrics": { - "ndcg_at_10": 0.8948, - "ndcg_at_5": 0.893, - "r_at_10": 0.983, - "r_at_5": 0.966 - }, - "mode": "raw_session", - "notes": [ - "top_k=10", - "Runs MemPalace benchmark module raw session retrieval logic directly." - ], - "primary_metric": "r_at_5", - "primary_metric_value": 0.966, - "reference_kind": "historical", - "runtime_seconds": 695.36, - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "mempalace" -} \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_new_brainctl_brain.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_new_brainctl_brain.json deleted file mode 100644 index ba42b30..0000000 --- a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_new_brainctl_brain.json +++ /dev/null @@ -1,12197 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\longmemeval_new_brainctl_brain.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "measured": true, - "metadata": {}, - "metrics": { - "ndcg_at_10": 0.872, - "ndcg_at_5": 0.8678, - "r_at_10": 0.9894, - "r_at_5": 0.9574 - }, - "mode": "brain", - "notes": [ - "top_k=10", - "Legacy 470-question session-level slice." - ], - "primary_metric": "r_at_5", - "primary_metric_value": 0.9574, - "reference_kind": "measured", - "rows": [ - { - "answer_session_ids": [ - "answer_280352e9" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e47becba", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_280352e9", - "02bd2b90_3", - "ultrachat_214101", - "5dac7cc2_1", - "sharegpt_Cr2tc1f_0", - "f6859b48_2" - ] - }, - { - "answer_session_ids": [ - "answer_40a90d51" - ], - "ndcg_at_10": 0.4307, - "ndcg_at_5": 0.4307, - "question_id": "118b2229", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "ultrachat_392094", - "db73b7e4_4", - "afe04238_1", - "answer_40a90d51", - "5dce60dd", - "a0b8b0ad_6", - "db50c0f6", - "3ea9f765", - "d600c646", - "sharegpt_i1iuP70_0" - ] - }, - { - "answer_session_ids": [ - "answer_d61669c7" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "51a45a95", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_d61669c7", - "98c198fb", - "217debf7", - "6e0b1800_2", - "5c44d9fe_1", - "07ba9acd_2", - "sharegpt_CyJ3dal_0", - "sharegpt_hChsWOp_128", - "sharegpt_3vxz2Zr_0", - "ultrachat_282235" - ] - }, - { - "answer_session_ids": [ - "answer_355c48bb" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "58bf7951", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_355c48bb", - "ultrachat_304942", - "ultrachat_268434", - "ultrachat_155546", - "sharegpt_gmLF3PE_17", - "7e17ae56_5", - "sharegpt_M84blrA_53", - "sharegpt_RkFlOeC_19", - "sharegpt_OFiElWf_37", - "sharegpt_CvVLg2J_18" - ] - }, - { - "answer_session_ids": [ - "answer_3e012175" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "1e043500", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_3e012175", - "01bd9def", - "ultrachat_227993", - "7b1d1f43_1", - "8020d945_2", - "e132259f", - "sharegpt_bph7DTk_7", - "ultrachat_271928", - "4d0d7984", - "sharegpt_6QUDIXG_24" - ] - }, - { - "answer_session_ids": [ - "answer_f6168136" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "c5e8278d", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_f6168136", - "89b1028d_1", - "64767b7f", - "aaf71ce2_2", - "d777e7da_1", - "fc6549b2", - "sharegpt_WPGrlAK_0", - "sharegpt_EXJAjmw_0", - "ultrachat_474903", - "3fa35683_7" - ] - }, - { - "answer_session_ids": [ - "answer_9398da02" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "6ade9755", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_9398da02", - "7c5ab02a_2", - "ec8fa65d_2", - "ultrachat_84599", - "ultrachat_393203", - "910ed479_3", - "ultrachat_270925", - "ultrachat_462", - "28621d6a_2", - "sharegpt_YsgacQi_0" - ] - }, - { - "answer_session_ids": [ - "answer_feb5200f" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "6f9b354f", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_feb5200f", - "a79f1a04_1", - "0c0f2449_2", - "ultrachat_549317", - "60207b41_1", - "54d06eeb", - "sharegpt_IuLT3VI_9", - "sharegpt_VAELSB3_0", - "sharegpt_kRjBUsA_79", - "ultrachat_224845" - ] - }, - { - "answer_session_ids": [ - "answer_59547700" - ], - "ndcg_at_10": 0.6309, - "ndcg_at_5": 0.6309, - "question_id": "58ef2f1c", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "13f3da23_3", - "answer_59547700", - "e6ab6a7b_2", - "ultrachat_5212", - "e82ce7cd_1", - "ultrachat_246324", - "ultrachat_477344", - "dbcfab21", - "b546cb39_2", - "403ee298_1" - ] - }, - { - "answer_session_ids": [ - "answer_c3567066" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "f8c5f88b", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c3567066", - "68ace657_1", - "sharegpt_TKj3yO8_0", - "91880423", - "15232887_1", - "ultrachat_360085", - "ultrachat_412973", - "7ff89315_3", - "58de8f9b", - "3b5205b6_1" - ] - }, - { - "answer_session_ids": [ - "answer_235eb6fb" - ], - "ndcg_at_10": 0.6309, - "ndcg_at_5": 0.6309, - "question_id": "5d3d2817", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "ultrachat_384575", - "answer_235eb6fb", - "670ca40e", - "sharegpt_ipLglky_48", - "sharegpt_8zsFRKh_0", - "e93efd9a_1", - "cef33b28_2", - "ultrachat_106114", - "sharegpt_bTDMt3m_64", - "25d1830a_1" - ] - }, - { - "answer_session_ids": [ - "answer_7cb94507" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "7527f7e2", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_7cb94507", - "864a563d_3", - "sharegpt_cRrfdsc_0", - "sharegpt_rraH1Q7_0", - "44bbd8cd_2", - "d81d9846", - "sharegpt_bTDMt3m_0", - "bee872da_2", - "ultrachat_62919", - "18a06652_3" - ] - }, - { - "answer_session_ids": [ - "answer_e05e4612" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "c960da58", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e05e4612", - "89d3053f", - "ultrachat_228940", - "ultrachat_36718", - "ultrachat_109577", - "ultrachat_129096", - "sharegpt_9iUZNsL_0", - "47db3b56_1", - "b76006cb_3", - "f01feb31" - ] - }, - { - "answer_session_ids": [ - "answer_94030872" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "3b6f954b", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_94030872", - "c271c071", - "6d3f5c68_4", - "ultrachat_475284", - "c566edbb_3", - "90b1c6d4_2", - "sharegpt_kpnwrGe_12", - "ultrachat_298929", - "4ed9cd1b_1", - "f999b05c_2" - ] - }, - { - "answer_session_ids": [ - "answer_f38f679b" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "726462e0", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_f38f679b", - "9c05acfe_2", - "741c6781_1", - "082889ba_1", - "5b83c26e_1", - "6ef99651_2", - "ce144f08", - "cdd9704d_1", - "7e4dab66_1", - "322a96e8" - ] - }, - { - "answer_session_ids": [ - "answer_c63c0458" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "94f70d80", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c63c0458", - "67045503", - "58de8f9b", - "sharegpt_6fTnCnw_76", - "ultrachat_380138", - "ultrachat_172828", - "sharegpt_mJ7MnQS_31", - "ec628e8d_2", - "sharegpt_ZvjRGRN_0", - "9a59023e_1" - ] - }, - { - "answer_session_ids": [ - "answer_fea2e4d3" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "66f24dbb", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_fea2e4d3", - "89452526_2", - "8d46b8fa_1", - "798e4ba3_2", - "21c6c41d", - "ca3a4e4f_1", - "sharegpt_83fkMsQ_0", - "ultrachat_445260", - "sharegpt_ZcfatzD_0", - "76299a49_1" - ] - }, - { - "answer_session_ids": [ - "answer_679840f8" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "ad7109d1", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_679840f8", - "22aa7cca_3", - "3dc02d26_2", - "d92cc997", - "ultrachat_74974", - "495cf518_1", - "sharegpt_Ck9R9HX_0", - "1d6744b5_1", - "fb721812_2", - "ultrachat_365256" - ] - }, - { - "answer_session_ids": [ - "answer_82a04f59" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "af8d2e46", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_82a04f59", - "33c3b457", - "73d5f10c", - "ultrachat_560731", - "266d604c", - "32a67b35", - "ultrachat_284576", - "13047a02", - "d5a6f455_2", - "ultrachat_210554" - ] - }, - { - "answer_session_ids": [ - "answer_8f276838" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "dccbc061", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_8f276838", - "0f417163", - "sharegpt_nc62Spr_7", - "ultrachat_278202", - "959e6cb0_3", - "b2a885b9", - "3c614701", - "8d74c5f6", - "sharegpt_MKMWjX0_25", - "ultrachat_519037" - ] - }, - { - "answer_session_ids": [ - "answer_761acef8" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "c8c3f81d", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_761acef8", - "66e94928", - "19ec83c5_1", - "eb5d5a93_1", - "sharegpt_pRqHb1o_0", - "eb403c80_2", - "9c5f1314_1", - "2b16f3d6_1", - "25432e10", - "998616b4_4" - ] - }, - { - "answer_session_ids": [ - "answer_8ad8a34f" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "8ebdbe50", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_8ad8a34f", - "b53f447a_1", - "27d378b2_3", - "90f16f14_2", - "01e806f0_1", - "39911f94", - "90a5e1f3_1", - "sharegpt_QxC97cL_0", - "d4f7a065_2", - "326f41d8" - ] - }, - { - "answer_session_ids": [ - "answer_e623ae87" - ], - "ndcg_at_10": 0.301, - "ndcg_at_5": 0.0, - "question_id": "6b168ec8", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 0.0, - "top_session_ids": [ - "ultrachat_360618", - "ultrachat_377741", - "33dff20c_3", - "ultrachat_249928", - "ultrachat_499691", - "dded1725_3", - "sharegpt_D9kTAIU_0", - "02f0738e_1", - "answer_e623ae87", - "f5b33470_abs" - ] - }, - { - "answer_session_ids": [ - "answer_723bf11f" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "75499fd8", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_723bf11f", - "5328c3c2_2", - "990b60df_1", - "18b0d357_1", - "eb0d8dc1_7", - "3e7df334_1", - "ddf0f116_3", - "sharegpt_1kE6haW_276", - "sharegpt_yPCVYhX_0", - "168feef7_1" - ] - }, - { - "answer_session_ids": [ - "answer_1e6d4567" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "21436231", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_1e6d4567", - "0d0a89dc_1", - "7a7a4bf0_4", - "ultrachat_172276", - "dc378711_1", - "sharegpt_6byuqlt_0", - "sharegpt_DGiqvVv_169", - "d0785ee9_1", - "3197d998_6", - "6bd0b406_1" - ] - }, - { - "answer_session_ids": [ - "answer_cb742a61" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "95bcc1c8", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_cb742a61", - "3919227f", - "ultrachat_320154", - "c9763bff", - "396238f9", - "abeda028", - "99e5e380", - "56266c38_1", - "sharegpt_uATA5m5_41", - "ultrachat_203404" - ] - }, - { - "answer_session_ids": [ - "answer_c6fd8ebd" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "0862e8bf", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c6fd8ebd", - "0566c21b", - "2442d5b9", - "sharegpt_3BkB8g5_0", - "sharegpt_2I1TxKW_0", - "ultrachat_47999", - "sharegpt_A7KJApk_21", - "sharegpt_KOaA4jf_0", - "sharegpt_isQJkCz_17", - "sharegpt_sCFErnY_22" - ] - }, - { - "answer_session_ids": [ - "answer_69811d4a" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "853b0a1d", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_69811d4a", - "sharegpt_n3MgsgL_0", - "c70f9f9c", - "3fa35683_3", - "a8b4bfb3_2", - "f71bf532_1", - "66a6a097_2", - "ebb5bc7c_2", - "451120d3_2", - "sharegpt_B7mSlfQ_55" - ] - }, - { - "answer_session_ids": [ - "answer_6fe9fb49" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "a06e4cfe", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_6fe9fb49", - "ee43db4a", - "3c2e6725", - "84fb50bb_3", - "de64539a_2", - "8dd2d3cc", - "af257b0b_2", - "0c891c0b_1", - "ultrachat_442201", - "ultrachat_231287" - ] - }, - { - "answer_session_ids": [ - "answer_55161935" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "37d43f65", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_55161935", - "3dcf1e0b_2", - "63587769", - "ultrachat_370801", - "ultrachat_525148", - "sharegpt_MOpCbrr_0", - "sharegpt_brc2wJS_264", - "4050ebff_5", - "sharegpt_ErOTMZ3_35", - "sharegpt_afbMhMS_45" - ] - }, - { - "answer_session_ids": [ - "answer_645b0329" - ], - "ndcg_at_10": 0.5, - "ndcg_at_5": 0.5, - "question_id": "b86304ba", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "ea8bb4f8_2", - "sharegpt_xGoJZ6Z_0", - "answer_645b0329", - "ultrachat_328696", - "ultrachat_10525", - "3b73120a_1", - "e831a29f_1", - "fe1e4351_1", - "91074ce7", - "091aa510_1" - ] - }, - { - "answer_session_ids": [ - "answer_0df6aa4b" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "d52b4f67", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_0df6aa4b", - "d31c3ec8_3", - "da1797c4_1", - "ultrachat_258008", - "18a06652_6", - "62a6d083", - "974fe383_2", - "sharegpt_IqVsDp7_89", - "sharegpt_MM7pEKH_0", - "sharegpt_cn3iUbp_7" - ] - }, - { - "answer_session_ids": [ - "answer_986de8c3" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "25e5aa4f", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_986de8c3", - "18807892_4", - "48f804a3", - "ultrachat_381447", - "a459d477", - "ca5ff3b4", - "13ad258b", - "sharegpt_8cFNcVG_0", - "ultrachat_225537", - "087d2b0a_2" - ] - }, - { - "answer_session_ids": [ - "answer_0714183a" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "caf9ead2", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_0714183a", - "f684ac4c_2", - "fc167d00", - "sharegpt_3D3oQC0_213", - "44713827_2", - "9f8bdd23_1", - "48a72204_1", - "ultrachat_30245", - "fcc6d66d_1", - "ultrachat_532107" - ] - }, - { - "answer_session_ids": [ - "answer_c8354ae9" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "8550ddae", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c8354ae9", - "a7189b75", - "71cf5aeb_3", - "e10dfffb_1", - "4f234e2c", - "9deebbc2_2", - "8ec23b2c", - "295dc1ab_1", - "c7a15bdc", - "sharegpt_ipLglky_42" - ] - }, - { - "answer_session_ids": [ - "answer_9cddca88" - ], - "ndcg_at_10": 0.4307, - "ndcg_at_5": 0.4307, - "question_id": "60d45044", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "fcc6d66d_1", - "4ebf9095_1", - "af4d4ecd_2", - "answer_9cddca88", - "26bc645b_5", - "a76e7e3c_2", - "fec059c5", - "bda4a421", - "0c0f2449_1", - "446173bb_1" - ] - }, - { - "answer_session_ids": [ - "answer_57fc1954" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "3f1e9474", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_57fc1954", - "sharegpt_wWUI4SO_7", - "ultrachat_177477", - "e7497c07_2", - "ultrachat_462827", - "3aac691d_2", - "cdf068b1_1", - "4f23a396_1", - "205f348b_2", - "954d4451_2" - ] - }, - { - "answer_session_ids": [ - "answer_dc11c1eb" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "86b68151", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_dc11c1eb", - "sharegpt_beV15ZO_0", - "d8454317_4", - "3d72c0c0", - "ultrachat_182718", - "0abaca24_1", - "sharegpt_rnL5VQO_0", - "4e671700", - "5ace87df_1", - "eed7b3ac_2" - ] - }, - { - "answer_session_ids": [ - "answer_0dd4d99a" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "577d4d32", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_0dd4d99a", - "ultrachat_125084", - "88925167_1", - "6e2cca63_2", - "449b918c_1", - "2c0d04d6_1", - "2ef53f61_2", - "sharegpt_pcoHTby_85", - "b960cbd3", - "sharegpt_101i5OR_0" - ] - }, - { - "answer_session_ids": [ - "answer_ed1982fc" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "ec81a493", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ed1982fc", - "6f55019d", - "ultrachat_447961", - "d2fc150a_1", - "sharegpt_ST22P5t_0", - "ultrachat_27488", - "ab4643a2_4", - "ultrachat_413665", - "ultrachat_492664", - "616fdd27" - ] - }, - { - "answer_session_ids": [ - "answer_586de428" - ], - "ndcg_at_10": 0.6309, - "ndcg_at_5": 0.6309, - "question_id": "15745da0", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "07d33eaa", - "answer_586de428", - "64a97d9b", - "sharegpt_IzpzBot_0", - "1f5370f7_4", - "1c662b7b_1", - "0d73afc8", - "ultrachat_383425", - "sharegpt_iF4hlNs_5", - "3cec6e2b_2" - ] - }, - { - "answer_session_ids": [ - "answer_5ca6cd28" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e01b8e2f", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_5ca6cd28", - "97338ddd_1", - "8acfa731", - "sharegpt_jyMHY1J_31", - "f9d0bc67", - "ultrachat_433104", - "d31c3ec8_1", - "e74be4c6_4", - "8be2c3f1", - "1aaf8057_3" - ] - }, - { - "answer_session_ids": [ - "answer_e6143162" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "bc8a6e93", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e6143162", - "f62bfe2b_1", - "8c1da8f9_1", - "b869d7ed_2", - "a1b3dd27_1", - "sharegpt_qyWsgp8_13", - "552c7fd0", - "c6bed037_1", - "sharegpt_jl0GWjL_0", - "ultrachat_453950" - ] - }, - { - "answer_session_ids": [ - "answer_f1fbb330" - ], - "ndcg_at_10": 0.5, - "ndcg_at_5": 0.5, - "question_id": "ccb36322", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "bdcee74e_3", - "6d52ee93_1", - "answer_f1fbb330", - "c38b33ae", - "d99bab55_1", - "04a0b385", - "8b1019b8_1", - "0e349efa", - "06d1a1a0", - "f79a48dd_1" - ] - }, - { - "answer_session_ids": [ - "answer_530960c1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "001be529", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_530960c1", - "0f71db75_1", - "9becef17_3", - "sharegpt_YQ4UlQM_83", - "sharegpt_PKXxlMF_0", - "ultrachat_351532", - "sharegpt_ETJwE0s_0", - "sharegpt_o3cfukb_0", - "ultrachat_95472", - "ultrachat_39884" - ] - }, - { - "answer_session_ids": [ - "answer_5cc9b056" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "b320f3f8", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_5cc9b056", - "d3a3f421_2", - "514a7856_2", - "b89f68c4_4", - "8bb8bc0d", - "aae4411b_2", - "40628271_1", - "5e4bb245_2", - "sharegpt_4FCMpJR_0", - "1b71c896_2" - ] - }, - { - "answer_session_ids": [ - "answer_5ff494b9" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "19b5f2b3", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_5ff494b9", - "f379d356_1", - "ultrachat_440262", - "ultrachat_463145", - "ultrachat_302261", - "ultrachat_8325", - "ultrachat_268389", - "ultrachat_68139", - "fbe6fa2c_1", - "ultrachat_138083" - ] - }, - { - "answer_session_ids": [ - "answer_2952aee4" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "4fd1909e", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_2952aee4", - "2479cbae", - "94c582bb_2", - "ultrachat_186237", - "sharegpt_hO4FoVt_0", - "fef185b1", - "ultrachat_472809", - "sharegpt_bTDMt3m_0", - "sharegpt_FNyKOSt_0", - "sharegpt_xX1L0kg_29" - ] - }, - { - "answer_session_ids": [ - "answer_47ffab4c" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "545bd2b5", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_47ffab4c", - "b7f69a7d_1", - "5fb3b5ac", - "a79f1a04_1", - "700c16e2_1", - "cc021f81_2", - "6e672b84_2", - "d1a1b9ea_1", - "4404a52a_1", - "sharegpt_nuYnG53_0" - ] - }, - { - "answer_session_ids": [ - "answer_15d63a22" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "8a137a7f", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_15d63a22", - "77908e43_1", - "fcff2dc4_3", - "20507d15_2", - "ultrachat_90882", - "ultrachat_186116", - "ultrachat_382344", - "sharegpt_sj5ZvVB_0", - "ultrachat_151889", - "ultrachat_192894" - ] - }, - { - "answer_session_ids": [ - "answer_bbdc7b0a" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "76d63226", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_bbdc7b0a", - "6a2f9452_2", - "adc47df1_1", - "a5ab6ba7_3", - "f27e27f9_2", - "cb289226_1", - "sharegpt_sRhcMiu_13", - "ultrachat_328229", - "ultrachat_180710", - "50136b31" - ] - }, - { - "answer_session_ids": [ - "answer_96b8c9ee" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "86f00804", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_96b8c9ee", - "sharegpt_ErOTMZ3_277", - "137d8c6a_2", - "0424a40a_2", - "35c5419d_abs_3", - "6e672b84_3", - "sharegpt_PQ71V9I_7", - "60153a02_1", - "fc41f919", - "sharegpt_sCFErnY_22" - ] - }, - { - "answer_session_ids": [ - "answer_7bdcbd23" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "8e9d538c", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_7bdcbd23", - "ultrachat_549592", - "3cdad629_1", - "0bd928bf_2", - "ultrachat_26072", - "a200b713_4", - "2deed26f", - "ultrachat_232141", - "sharegpt_cADPA4R_0", - "sharegpt_KtWBRv1_4" - ] - }, - { - "answer_session_ids": [ - "answer_e40b054e" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "311778f1", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e40b054e", - "5db56d24_2", - "8e6e058b_3", - "sharegpt_vXNQZ2I_0", - "ultrachat_267100", - "d09360d3_1", - "31903f54", - "95cec590_1", - "27fb983c_2", - "ultrachat_316997" - ] - }, - { - "answer_session_ids": [ - "answer_f442ccbe" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "c19f7a0b", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_f442ccbe", - "58ab5fc5", - "eb336de0_3", - "sharegpt_6ZeW9Vb_0", - "c9292210_2", - "1b0d77b0", - "0e829b8f", - "195c8fe6_3", - "8bb8bc0d", - "81ba81ab" - ] - }, - { - "answer_session_ids": [ - "answer_83c13ff9" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "4100d0a0", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_83c13ff9", - "8d068598", - "ea3db78e_1", - "sharegpt_eV0krsR_0", - "b9890c15_1", - "ultrachat_130655", - "f8ab60d7", - "0f6b9698_1", - "665c09b1_1", - "7fc9c056" - ] - }, - { - "answer_session_ids": [ - "answer_7cc5362f" - ], - "ndcg_at_10": 0.5, - "ndcg_at_5": 0.5, - "question_id": "29f2956b", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "6d6ffac5", - "88361fdd", - "answer_7cc5362f", - "ultrachat_238255", - "c4ed2287_3", - "d50a8a33_1", - "sharegpt_jZOf9E5_0", - "715b1b28", - "eb0d8dc1_5", - "sharegpt_UCVg3jO_0" - ] - }, - { - "answer_session_ids": [ - "answer_d01949bf" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "1faac195", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_d01949bf", - "b2543a58_3", - "8064b6ca", - "9d3d54f7", - "ultrachat_283922", - "f2ccf83b", - "94f460be_1", - "ultrachat_12436", - "sharegpt_NhF2Neg_0", - "48d385f0_1" - ] - }, - { - "answer_session_ids": [ - "answer_39b12014" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "faba32e5", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_39b12014", - "67045503", - "69f3fc12_1", - "c6c5bb8b_4", - "770d11f5", - "sharegpt_FNyKOSt_0", - "sharegpt_benxw0S_9", - "ultrachat_205840", - "d3da4592_2", - "ultrachat_250588" - ] - }, - { - "answer_session_ids": [ - "answer_f5b33470" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "f4f1d8a4", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_f5b33470", - "2bfea7af", - "3aac691d_2", - "9614d065_1", - "eb682e52_2", - "6f051087_2", - "ecb24dd8_1", - "76ea88ba_2", - "ultrachat_67942", - "ultrachat_576354" - ] - }, - { - "answer_session_ids": [ - "answer_304511ce" - ], - "ndcg_at_10": 0.5, - "ndcg_at_5": 0.5, - "question_id": "c14c00dd", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "c77250d2", - "7cd7c296_3", - "answer_304511ce", - "ffd66315_2", - "ultrachat_272210", - "183b6085", - "cd1f5147", - "sharegpt_gU0qvZu_0", - "sharegpt_FfqJ2xI_0", - "sharegpt_vXWOUqx_11" - ] - }, - { - "answer_session_ids": [ - "answer_93e1bd22" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "36580ce8", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_93e1bd22", - "ultrachat_53784", - "9b38bda6_2", - "ultrachat_272552", - "ultrachat_270984", - "ultrachat_32917", - "sharegpt_FpTfRvR_0", - "ultrachat_299779", - "ultrachat_149170", - "sharegpt_ZbUcxUa_0" - ] - }, - { - "answer_session_ids": [ - "answer_19c24c11" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "3d86fd0a", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_19c24c11", - "cf8e352f_1", - "sharegpt_d7bP5fb_0", - "69674f25", - "d1d3fd12_2", - "1c1a2b7f_2", - "06d1a1a0", - "091ba7f5_1", - "5a13d047_1", - "7e4dab66_2" - ] - }, - { - "answer_session_ids": [ - "answer_787e6a6d" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "a82c026e", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_787e6a6d", - "d9727262_4", - "f18ebe36_4", - "3009ba66_2", - "ccaabf0b_1", - "516b0aae_1", - "a6c41826_3", - "daa32134", - "c71e1c7a_2", - "bdf735de_1" - ] - }, - { - "answer_session_ids": [ - "answer_afa9873b_2", - "answer_afa9873b_3", - "answer_afa9873b_1" - ], - "ndcg_at_10": 0.9675, - "ndcg_at_5": 0.9675, - "question_id": "0a995998", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_afa9873b_1", - "answer_afa9873b_2", - "85846900_2", - "answer_afa9873b_3", - "439f9ed8_3", - "e6790684_1", - "fa46a10e", - "3f05c474", - "sharegpt_ZzArUrr_0", - "ultrachat_331531" - ] - }, - { - "answer_session_ids": [ - "answer_ec904b3c_1", - "answer_ec904b3c_4", - "answer_ec904b3c_3", - "answer_ec904b3c_2" - ], - "ndcg_at_10": 0.8772, - "ndcg_at_5": 0.8772, - "question_id": "6d550036", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ec904b3c_1", - "2e4430d8_2", - "0bab76de", - "answer_ec904b3c_4", - "ultrachat_184799", - "a9981dc6_3", - "sharegpt_UbMVpdp_93", - "f6246b5f", - "dae3906e_2", - "sharegpt_J7ZAFLd_0" - ] - }, - { - "answer_session_ids": [ - "answer_593bdffd_4", - "answer_593bdffd_1", - "answer_593bdffd_3", - "answer_593bdffd_2" - ], - "ndcg_at_10": 0.5508, - "ndcg_at_5": 0.5013, - "question_id": "gpt4_59c863d7", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "9ef698bc_2", - "de5f2701_1", - "a7d014e4_1", - "answer_593bdffd_1", - "answer_593bdffd_4", - "answer_593bdffd_2", - "2f09d4c8", - "ultrachat_126967", - "eb47739f_2", - "ultrachat_59708" - ] - }, - { - "answer_session_ids": [ - "answer_a8b4290f_3", - "answer_a8b4290f_1", - "answer_a8b4290f_2" - ], - "ndcg_at_10": 0.6395, - "ndcg_at_5": 0.6509, - "question_id": "b5ef892d", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "ultrachat_565056", - "answer_a8b4290f_3", - "sharegpt_XWqXdom_0", - "answer_a8b4290f_1", - "ultrachat_53385", - "35dcacdc_2", - "ultrachat_367439", - "ultrachat_375427", - "answer_a8b4290f_2", - "e552e1f9_2" - ] - }, - { - "answer_session_ids": [ - "answer_86c505e7_1", - "answer_86c505e7_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e831120c", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_86c505e7_2", - "answer_86c505e7_1", - "sharegpt_sDfu38Q_0", - "8fb3fe3a_1", - "a63ad8e3_2", - "ultrachat_310037", - "7c82a6c3", - "6387e969", - "sharegpt_hvdBqMM_0", - "sharegpt_d8EB6Ze_0" - ] - }, - { - "answer_session_ids": [ - "answer_c2204106_2", - "answer_c2204106_3", - "answer_c2204106_1" - ], - "ndcg_at_10": 0.8529, - "ndcg_at_5": 0.8529, - "question_id": "3a704032", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c2204106_1", - "7712aded", - "86e830d2_3", - "answer_c2204106_2", - "answer_c2204106_3", - "78764b10_2", - "ultrachat_501733", - "sharegpt_MRCLhIj_0", - "ultrachat_357807", - "sharegpt_HrxBbBK_0" - ] - }, - { - "answer_session_ids": [ - "answer_2880eb6c_2", - "answer_2880eb6c_4", - "answer_2880eb6c_1", - "answer_2880eb6c_3" - ], - "ndcg_at_10": 0.7606, - "ndcg_at_5": 0.7606, - "question_id": "gpt4_d84a3211", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "18d68208_1", - "answer_2880eb6c_4", - "answer_2880eb6c_3", - "answer_2880eb6c_2", - "answer_2880eb6c_1", - "4889cd54_1", - "4a32d58c_2", - "ultrachat_83526", - "sharegpt_OHd1RQ7_37", - "13c55c69_1" - ] - }, - { - "answer_session_ids": [ - "answer_526354c8_1", - "answer_526354c8_3", - "answer_526354c8_2" - ], - "ndcg_at_10": 0.9325, - "ndcg_at_5": 1.0, - "question_id": "aae3761f", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_526354c8_2", - "answer_526354c8_1", - "b192ae00_2", - "ultrachat_266173", - "sharegpt_8I9vH7n_7", - "answer_526354c8_3", - "3d85d91e_2", - "sharegpt_ynLoh9N_0", - "2a86d0da_2", - "ultrachat_92484" - ] - }, - { - "answer_session_ids": [ - "answer_55a6940c_3", - "answer_55a6940c_1", - "answer_55a6940c_2" - ], - "ndcg_at_10": 0.4716, - "ndcg_at_5": 0.0, - "question_id": "gpt4_f2262a51", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 0.0, - "top_session_ids": [ - "c0d6fa6f_2", - "dd81b163_1", - "sharegpt_eoEbthf_0", - "ultrachat_268434", - "sharegpt_BWMyoNr_0", - "answer_55a6940c_3", - "answer_55a6940c_2", - "answer_55a6940c_1", - "sharegpt_hChsWOp_128", - "sharegpt_ab6IEma_0" - ] - }, - { - "answer_session_ids": [ - "answer_f9de4602_2", - "answer_f9de4602_1" - ], - "ndcg_at_10": 0.8316, - "ndcg_at_5": 1.0, - "question_id": "dd2973ad", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_f9de4602_2", - "sharegpt_O6NO7Vo_9", - "e5f08e40", - "41743aae_1", - "0f73eee7_2", - "answer_f9de4602_1", - "ultrachat_26627", - "e720dbe2", - "32c0dae1_3", - "b192ae00_2" - ] - }, - { - "answer_session_ids": [ - "answer_56d02cab_3", - "answer_56d02cab_4", - "answer_56d02cab_2", - "answer_56d02cab_1" - ], - "ndcg_at_10": 0.9047, - "ndcg_at_5": 0.9047, - "question_id": "c4a1ceb8", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_56d02cab_3", - "40c77045", - "answer_56d02cab_1", - "answer_56d02cab_4", - "answer_56d02cab_2", - "ba9f938b_2", - "cef33b28_1", - "sharegpt_1sxNkyQ_67", - "0b64c6cb_2", - "sharegpt_WanXxRQ_0" - ] - }, - { - "answer_session_ids": [ - "answer_cf9e3940_2", - "answer_cf9e3940_1", - "answer_cf9e3940_3" - ], - "ndcg_at_10": 0.6462, - "ndcg_at_5": 0.6509, - "question_id": "gpt4_a56e767c", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "88c8df0e_3", - "answer_cf9e3940_3", - "f56e6152_1", - "answer_cf9e3940_2", - "ultrachat_257906", - "ultrachat_104129", - "d75245ea", - "answer_cf9e3940_1", - "dac54d1e_1", - "71621be0" - ] - }, - { - "answer_session_ids": [ - "answer_a4204937_2", - "answer_a4204937_1" - ], - "ndcg_at_10": 0.8772, - "ndcg_at_5": 0.8772, - "question_id": "6cb6f249", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_a4204937_1", - "9b6db1a9_2", - "ultrachat_404221", - "answer_a4204937_2", - "5edfe253_1", - "f584ba36_2", - "67620dbc_2", - "ultrachat_374192", - "cbe70f21", - "sharegpt_IkYYvSd_79" - ] - }, - { - "answer_session_ids": [ - "answer_c65042d7_3", - "answer_c65042d7_2", - "answer_c65042d7_1" - ], - "ndcg_at_10": 0.9066, - "ndcg_at_5": 1.0, - "question_id": "46a3abf7", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c65042d7_3", - "answer_c65042d7_2", - "09f2c6c5", - "28eb86ab", - "a95bedca", - "ultrachat_406179", - "c1e598cd_1", - "4fe91719_2", - "answer_c65042d7_1", - "e224317f_2" - ] - }, - { - "answer_session_ids": [ - "answer_ef74281f_2", - "answer_ef74281f_1", - "answer_ef74281f_3" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "36b9f61e", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ef74281f_2", - "answer_ef74281f_1", - "answer_ef74281f_3", - "e05baf83_2", - "3159942d_2", - "fc6f9c59", - "5829ab38", - "3ceb6783_2", - "fd6f60f0_3", - "85cd56c7_1" - ] - }, - { - "answer_session_ids": [ - "answer_8d015d9d_3", - "answer_8d015d9d_5", - "answer_8d015d9d_2", - "answer_8d015d9d_4", - "answer_8d015d9d_1" - ], - "ndcg_at_10": 0.9609, - "ndcg_at_5": 0.9829, - "question_id": "28dc39ac", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_8d015d9d_2", - "answer_8d015d9d_3", - "answer_8d015d9d_1", - "a7cd1c0f_1", - "answer_8d015d9d_4", - "bfa499f7", - "ultrachat_452868", - "answer_8d015d9d_5", - "5854eebc_1", - "95578ea8_2" - ] - }, - { - "answer_session_ids": [ - "answer_e7b0637e_2", - "answer_e7b0637e_1", - "answer_e7b0637e_3" - ], - "ndcg_at_10": 0.5574, - "ndcg_at_5": 0.5438, - "question_id": "gpt4_2f8be40d", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "ultrachat_217382", - "1d6e01e8", - "answer_e7b0637e_2", - "ultrachat_304474", - "answer_e7b0637e_1", - "sharegpt_L64rHOA_0", - "81b971b8_2", - "cb1f962f_4", - "answer_e7b0637e_3", - "dfe646a7_2" - ] - }, - { - "answer_session_ids": [ - "answer_fa526fc0_4", - "answer_fa526fc0_3", - "answer_fa526fc0_1", - "answer_fa526fc0_2" - ], - "ndcg_at_10": 0.8118, - "ndcg_at_5": 0.8772, - "question_id": "2e6d26dc", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_fa526fc0_4", - "sharegpt_vXNQZ2I_0", - "ultrachat_541982", - "answer_fa526fc0_3", - "7a3cbde3", - "14a520c8_2", - "answer_fa526fc0_1", - "answer_fa526fc0_2", - "sharegpt_IigLRfw_0", - "sharegpt_Wt2YDZs_35" - ] - }, - { - "answer_session_ids": [ - "answer_8858d9dc_3", - "answer_8858d9dc_1", - "answer_8858d9dc_4", - "answer_8858d9dc_2" - ], - "ndcg_at_10": 0.6509, - "ndcg_at_5": 0.6509, - "question_id": "gpt4_15e38248", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "sharegpt_mm9tc5g_0", - "answer_8858d9dc_2", - "1e7f3172_5", - "answer_8858d9dc_4", - "530742d1", - "d4ab49f1", - "ec616e7e_5", - "15acfb99", - "sharegpt_01VDd0u_0", - "8c9bc932_2" - ] - }, - { - "answer_session_ids": [ - "answer_733e443a_3", - "answer_733e443a_4", - "answer_733e443a_2", - "answer_733e443a_1" - ], - "ndcg_at_10": 0.4306, - "ndcg_at_5": 0.3869, - "question_id": "88432d0a", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "f777f641", - "1c8832b4_2", - "c9d35c00_2", - "d1f6080f_1", - "answer_733e443a_2", - "ultrachat_122782", - "ultrachat_351103", - "answer_733e443a_3", - "sharegpt_HpkiAOb_9", - "sharegpt_tESxx1y_0" - ] - }, - { - "answer_session_ids": [ - "answer_990c8992_2", - "answer_990c8992_1", - "answer_990c8992_3" - ], - "ndcg_at_10": 0.8278, - "ndcg_at_5": 0.8772, - "question_id": "80ec1f4f", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_990c8992_2", - "f6ce25ef", - "6884f11d", - "answer_990c8992_1", - "b909542d_1", - "d22c92b2", - "answer_990c8992_3", - "ultrachat_318441", - "ultrachat_497101", - "sharegpt_1kE6haW_52" - ] - }, - { - "answer_session_ids": [ - "answer_5a0d28f8_4", - "answer_5a0d28f8_2", - "answer_5a0d28f8_3", - "answer_5a0d28f8_1" - ], - "ndcg_at_10": 0.9439, - "ndcg_at_5": 0.9675, - "question_id": "d23cf73b", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_5a0d28f8_4", - "answer_5a0d28f8_2", - "66369fde_2", - "answer_5a0d28f8_3", - "b124e4a1_1", - "answer_5a0d28f8_1", - "ultrachat_224488", - "ultrachat_283390", - "ultrachat_456819", - "ultrachat_375734" - ] - }, - { - "answer_session_ids": [ - "answer_a679a86a_5", - "answer_a679a86a_4", - "answer_a679a86a_2", - "answer_a679a86a_3", - "answer_a679a86a_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_7fce9456", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_a679a86a_1", - "answer_a679a86a_5", - "answer_a679a86a_4", - "answer_a679a86a_3", - "541ecc45_3", - "1da409cd_1", - "sharegpt_OjaM9hX_225", - "191e1832", - "67154fb2_1", - "288e7d30" - ] - }, - { - "answer_session_ids": [ - "answer_c008e5df_1", - "answer_c008e5df_2", - "answer_c008e5df_3" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "d682f1a2", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c008e5df_3", - "answer_c008e5df_1", - "sharegpt_E7VUcfj_0", - "7332a779_2", - "9de29219_1", - "d7f9742a_2", - "sharegpt_LFPdgVp_22", - "e30307d8_2", - "58335f60_2", - "ba0ae49c_2" - ] - }, - { - "answer_session_ids": [ - "answer_a21f3697_1", - "answer_a21f3697_2", - "answer_a21f3697_3" - ], - "ndcg_at_10": 0.8711, - "ndcg_at_5": 0.9197, - "question_id": "7024f17c", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_a21f3697_1", - "sharegpt_rshCiS5_1", - "answer_a21f3697_2", - "4f838497_3", - "8c652eb0_1", - "answer_a21f3697_3", - "3c12ff0c", - "f8476198", - "d7281662_2", - "445e06ce_2" - ] - }, - { - "answer_session_ids": [ - "answer_203bf3fa_1", - "answer_203bf3fa_3", - "answer_203bf3fa_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_5501fe77", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_203bf3fa_1", - "answer_203bf3fa_3", - "bc5827b5", - "ultrachat_163385", - "021c0d09_2", - "7a2c618c_1", - "6dfb33f1_2", - "sharegpt_x7apNZ0_0", - "631e4016", - "13b79e39_3" - ] - }, - { - "answer_session_ids": [ - "answer_6a3b5c13_3", - "answer_6a3b5c13_1", - "answer_6a3b5c13_2", - "answer_6a3b5c13_4" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_2ba83207", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_6a3b5c13_1", - "answer_6a3b5c13_2", - "answer_6a3b5c13_4", - "answer_6a3b5c13_3", - "ultrachat_568233", - "cc3c5fa9_2", - "ultrachat_385963", - "ultrachat_224880", - "ea336da0", - "bc6de190_1" - ] - }, - { - "answer_session_ids": [ - "answer_eaa8e3ef_1", - "answer_eaa8e3ef_2" - ], - "ndcg_at_10": 0.8066, - "ndcg_at_5": 1.0, - "question_id": "2318644b", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_eaa8e3ef_2", - "920f808b", - "c4ea8219_1", - "e491f2d4_2", - "7d02ef5c_5", - "6ec23dc5_1", - "fc733b20_2", - "answer_eaa8e3ef_1", - "fcae5c39_2", - "7ff89315_3" - ] - }, - { - "answer_session_ids": [ - "answer_901a6763_2", - "answer_901a6763_4", - "answer_901a6763_1", - "answer_901a6763_3" - ], - "ndcg_at_10": 0.4914, - "ndcg_at_5": 0.4307, - "question_id": "2ce6a0f2", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "a1714d2c_2", - "c1634164_2", - "ultrachat_355740", - "answer_901a6763_4", - "23ddce03_1", - "600077b0_3", - "ultrachat_36844", - "answer_901a6763_2", - "answer_901a6763_1", - "3e90815f" - ] - }, - { - "answer_session_ids": [ - "answer_2504635e_3", - "answer_2504635e_2", - "answer_2504635e_1" - ], - "ndcg_at_10": 0.7929, - "ndcg_at_5": 1.0, - "question_id": "gpt4_d12ceb0e", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_2504635e_2", - "73f4798f", - "sharegpt_dxirwR4_25", - "2fe5510e_3", - "2f23dd1a", - "answer_2504635e_1", - "answer_2504635e_3", - "ultrachat_55863", - "d8e33f5c_abs_2", - "sharegpt_dIgdYPv_13" - ] - }, - { - "answer_session_ids": [ - "answer_39900a0a_3", - "answer_39900a0a_2", - "answer_39900a0a_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "00ca467f", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_39900a0a_3", - "answer_39900a0a_1", - "answer_39900a0a_2", - "e4cb6c56", - "ultrachat_145200", - "ultrachat_239705", - "07942c06", - "4951c0b9_1", - "36828c66_1", - "sharegpt_3cVjZ2b_16" - ] - }, - { - "answer_session_ids": [ - "answer_05d808e6_1", - "answer_05d808e6_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "b3c15d39", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_05d808e6_1", - "answer_05d808e6_2", - "5d4ff8db_1", - "a16a4158", - "sharegpt_H4fpIsr_9", - "sharegpt_inuIr9J_119", - "8d25b813_1", - "d298714d_1", - "sharegpt_V2j1zkI_0", - "sharegpt_5n9NBC8_0" - ] - }, - { - "answer_session_ids": [ - "answer_02b63d04_1", - "answer_02b63d04_5", - "answer_02b63d04_2", - "answer_02b63d04_3", - "answer_02b63d04_4" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_31ff4165", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_02b63d04_1", - "answer_02b63d04_5", - "answer_02b63d04_3", - "35201d43", - "sharegpt_tKmxch5_0", - "e2393b63_2", - "c625cff4_3", - "2d0b80f1", - "6aa87dc3", - "ultrachat_555273" - ] - }, - { - "answer_session_ids": [ - "answer_3e5fea0e_1", - "answer_3e5fea0e_2" - ], - "ndcg_at_10": 0.6053, - "ndcg_at_5": 0.6309, - "question_id": "eeda8a6d", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "sharegpt_PTZ2ime_0", - "answer_3e5fea0e_1", - "ea67b29e_1", - "cf7aad73_3", - "f3745025_1", - "answer_3e5fea0e_2", - "91c71874_1", - "sharegpt_JsCcnH4_0", - "sharegpt_wrXNAYQ_0", - "sharegpt_khNFc2W_0" - ] - }, - { - "answer_session_ids": [ - "answer_8f6b938d_1", - "answer_8f6b938d_3", - "answer_8f6b938d_4", - "answer_8f6b938d_2" - ], - "ndcg_at_10": 0.7397, - "ndcg_at_5": 0.7328, - "question_id": "2788b940", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "544fe66c_2", - "answer_8f6b938d_4", - "answer_8f6b938d_2", - "answer_8f6b938d_1", - "ff6e824b_1", - "sharegpt_vgBnRfE_0", - "answer_8f6b938d_3", - "7033540c_2", - "3f863722", - "sharegpt_xFriTfz_35" - ] - }, - { - "answer_session_ids": [ - "answer_e0956e0a_1", - "answer_e0956e0a_2" - ], - "ndcg_at_10": 0.525, - "ndcg_at_5": 0.5, - "question_id": "60bf93ed", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "a200b713_2", - "388c5534_3", - "answer_e0956e0a_1", - "4fe27c66_4", - "0e297c49", - "answer_e0956e0a_2", - "6414f676_4", - "ultrachat_7648", - "e9590b7d", - "sharegpt_f40Yvlz_0" - ] - }, - { - "answer_session_ids": [ - "answer_fcff2dc4_2", - "answer_fcff2dc4_1", - "answer_fcff2dc4_3" - ], - "ndcg_at_10": 0.8194, - "ndcg_at_5": 0.8772, - "question_id": "9d25d4e0", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_fcff2dc4_3", - "464f3821", - "a4b774c4", - "answer_fcff2dc4_1", - "37beb8da", - "ca555919_3", - "46d9d476", - "answer_fcff2dc4_2", - "39911f94", - "41abc171_1" - ] - }, - { - "answer_session_ids": [ - "answer_1de862d6_1", - "answer_1de862d6_3", - "answer_1de862d6_2" - ], - "ndcg_at_10": 0.906, - "ndcg_at_5": 0.906, - "question_id": "129d1232", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_1de862d6_2", - "69a9211c_2", - "answer_1de862d6_1", - "answer_1de862d6_3", - "5e4bb245_2", - "1ab255ba_1", - "sharegpt_De0lanR_0", - "sharegpt_jBwwg7B_0", - "bf3aebdb_1", - "sharegpt_YfIEYo1_0" - ] - }, - { - "answer_session_ids": [ - "answer_e7fe8c8b_1", - "answer_e7fe8c8b_2", - "answer_e7fe8c8b_3" - ], - "ndcg_at_10": 0.8711, - "ndcg_at_5": 0.9197, - "question_id": "60472f9c", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e7fe8c8b_1", - "b1815b81", - "answer_e7fe8c8b_2", - "7b74d32d_3", - "sharegpt_kF8hHMt_45", - "answer_e7fe8c8b_3", - "ultrachat_334324", - "ultrachat_126217", - "ultrachat_470951", - "sharegpt_5BPO9KJ_0" - ] - }, - { - "answer_session_ids": [ - "answer_3826dc55_1", - "answer_3826dc55_3", - "answer_3826dc55_5", - "answer_3826dc55_2", - "answer_3826dc55_4" - ], - "ndcg_at_10": 0.7157, - "ndcg_at_5": 0.7123, - "question_id": "gpt4_194be4b3", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "ultrachat_58180", - "answer_3826dc55_3", - "answer_3826dc55_5", - "31403c51", - "answer_3826dc55_4", - "ultrachat_385467", - "ultrachat_474217", - "answer_3826dc55_1", - "sharegpt_KwbJJ66_12", - "sharegpt_VO7dTG3_0" - ] - }, - { - "answer_session_ids": [ - "answer_cc021f81_2", - "answer_cc021f81_3", - "answer_cc021f81_1" - ], - "ndcg_at_10": 0.906, - "ndcg_at_5": 0.906, - "question_id": "a9f6b44c", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_cc021f81_3", - "837f258b", - "answer_cc021f81_1", - "answer_cc021f81_2", - "d4912cd9", - "ca896bdc_1", - "sharegpt_AqgYTct_0", - "ec830058_1", - "ultrachat_279504", - "b7e36cc5_2" - ] - }, - { - "answer_session_ids": [ - "answer_5cdf9bd2_2", - "answer_5cdf9bd2_1", - "answer_5cdf9bd2_3", - "answer_5cdf9bd2_4" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "d851d5ba", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_5cdf9bd2_2", - "answer_5cdf9bd2_1", - "answer_5cdf9bd2_4", - "answer_5cdf9bd2_3", - "sharegpt_0gZVK1A_15", - "68ace657_2", - "sharegpt_wduYbsX_0", - "sharegpt_dxirwR4_25", - "sharegpt_x7apNZ0_0", - "d77d4ac9_1" - ] - }, - { - "answer_session_ids": [ - "answer_4cef8a3c_3", - "answer_4cef8a3c_1", - "answer_4cef8a3c_2" - ], - "ndcg_at_10": 0.9675, - "ndcg_at_5": 0.9675, - "question_id": "5a7937c8", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_4cef8a3c_3", - "answer_4cef8a3c_2", - "3b38be11_2", - "answer_4cef8a3c_1", - "ultrachat_288747", - "ultrachat_377263", - "b6bc5b09_2", - "a76e7e3c_4", - "b70ac29f_2", - "07417b21" - ] - }, - { - "answer_session_ids": [ - "answer_728deb4d_5", - "answer_728deb4d_2", - "answer_728deb4d_3", - "answer_728deb4d_1", - "answer_728deb4d_4" - ], - "ndcg_at_10": 0.9709, - "ndcg_at_5": 1.0, - "question_id": "gpt4_ab202e7f", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_728deb4d_4", - "answer_728deb4d_5", - "answer_728deb4d_3", - "answer_728deb4d_2", - "ultrachat_433876", - "sharegpt_q3qJzui_0", - "29bc69b3", - "cdcbdf13", - "answer_728deb4d_1", - "sharegpt_xD2q9ER_14" - ] - }, - { - "answer_session_ids": [ - "answer_6350aa4f_1", - "answer_6350aa4f_2", - "answer_6350aa4f_3", - "answer_6350aa4f_4" - ], - "ndcg_at_10": 0.9709, - "ndcg_at_5": 1.0, - "question_id": "gpt4_e05b82a6", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_6350aa4f_2", - "answer_6350aa4f_4", - "answer_6350aa4f_3", - "88e01130_1", - "b405d3d8_1", - "answer_6350aa4f_1", - "aed6b1b6_1", - "386f6df9", - "ultrachat_43272", - "ultrachat_256330" - ] - }, - { - "answer_session_ids": [ - "answer_826d51da_3", - "answer_826d51da_4", - "answer_826d51da_2", - "answer_826d51da_1" - ], - "ndcg_at_10": 0.7614, - "ndcg_at_5": 1.0, - "question_id": "gpt4_731e37d7", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_826d51da_1", - "07820595_1", - "5365840b_2", - "267a912c", - "c566edbb_3", - "sharegpt_Rwql31f_62", - "answer_826d51da_2", - "7e4aa7c2_1", - "baa5b468_3", - "answer_826d51da_3" - ] - }, - { - "answer_session_ids": [ - "answer_60e8941a_2", - "answer_60e8941a_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "edced276", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_60e8941a_2", - "answer_60e8941a_1", - "1da409cd_1", - "f584ba36_2", - "59a6150d_3", - "726fa34a_1", - "5726dc37_2", - "f5e28561_4", - "93f23301_1", - "e4bbcd5f_4" - ] - }, - { - "answer_session_ids": [ - "answer_e0585cb5_2", - "answer_e0585cb5_1" - ], - "ndcg_at_10": 0.5, - "ndcg_at_5": 0.5, - "question_id": "10d9b85a", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "c51583cd_3", - "cbd1fe79_2", - "answer_e0585cb5_2", - "8dd2fca2", - "84889496_1", - "aa6afba8", - "d576152e_1", - "answer_e0585cb5_1", - "faac3c60", - "5cbfaf3e_4" - ] - }, - { - "answer_session_ids": [ - "answer_b6018747_2", - "answer_b6018747_4", - "answer_b6018747_1", - "answer_b6018747_3" - ], - "ndcg_at_10": 0.7107, - "ndcg_at_5": 0.6934, - "question_id": "e3038f8c", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "sharegpt_hgGAUvu_0", - "answer_b6018747_2", - "answer_b6018747_4", - "85846900_2", - "a3d8e134_2", - "answer_b6018747_3", - "answer_b6018747_1", - "sharegpt_s72sGRc_0", - "4629287b", - "sharegpt_WxCMnMO_5" - ] - }, - { - "answer_session_ids": [ - "answer_23759615_2", - "answer_23759615_3", - "answer_23759615_1" - ], - "ndcg_at_10": 0.9066, - "ndcg_at_5": 1.0, - "question_id": "2b8f3739", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_23759615_3", - "answer_23759615_1", - "dd25eeb5_6", - "c075835c", - "67d6f18f_1", - "285525b2_1", - "ec8691f5", - "sharegpt_ZH6IAa2_11", - "answer_23759615_2", - "ce6cae1b_3" - ] - }, - { - "answer_session_ids": [ - "answer_2bd23659_3", - "answer_2bd23659_2", - "answer_2bd23659_4", - "answer_2bd23659_1" - ], - "ndcg_at_10": 0.906, - "ndcg_at_5": 0.906, - "question_id": "1a8a66a6", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_2bd23659_3", - "54aa4875", - "answer_2bd23659_1", - "answer_2bd23659_2", - "1e91a236_1", - "dc71023e", - "sharegpt_lsiJM0Y_0", - "d6792df8", - "ebb5d262", - "ultrachat_287097" - ] - }, - { - "answer_session_ids": [ - "answer_923c0221_1", - "answer_923c0221_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "c2ac3c61", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_923c0221_2", - "answer_923c0221_1", - "0c483a3c", - "8e4e215b", - "sharegpt_2BSHRQc_25", - "6009ba4e_5", - "9f803df5_1", - "ultrachat_412857", - "ultrachat_38171", - "0b64c6cb_1" - ] - }, - { - "answer_session_ids": [ - "answer_7726e7e9_1", - "answer_7726e7e9_3", - "answer_7726e7e9_2" - ], - "ndcg_at_10": 0.8385, - "ndcg_at_5": 0.8772, - "question_id": "bf659f65", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_7726e7e9_1", - "f2835879_2", - "9d480413_5", - "answer_7726e7e9_2", - "ultrachat_465222", - "answer_7726e7e9_3", - "258645ba_3", - "71619051_1", - "ultrachat_377765", - "sharegpt_SBJezEx_4" - ] - }, - { - "answer_session_ids": [ - "answer_35c5419d_3", - "answer_35c5419d_2", - "answer_35c5419d_1" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "gpt4_372c3eed", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_35c5419d_3", - "4d04b866", - "answer_35c5419d_1", - "ultrachat_400797", - "ultrachat_290513", - "ultrachat_424492", - "0d396995_1", - "bc1f1051_5", - "c95f88fd_1", - "sharegpt_d1pTYei_39" - ] - }, - { - "answer_session_ids": [ - "answer_669318cf_2", - "answer_669318cf_1", - "answer_669318cf_3" - ], - "ndcg_at_10": 0.9469, - "ndcg_at_5": 0.9469, - "question_id": "gpt4_2f91af09", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_669318cf_2", - "answer_669318cf_1", - "22b3af37_1", - "ultrachat_42946", - "answer_669318cf_3", - "51c82c8e_2", - "d75245ea", - "541ecc45_1", - "fd6f60f0_2", - "e7e416f5_2" - ] - }, - { - "answer_session_ids": [ - "answer_da3c1266_3", - "answer_da3c1266_4", - "answer_da3c1266_1", - "answer_da3c1266_5", - "answer_da3c1266_2" - ], - "ndcg_at_10": 0.9829, - "ndcg_at_5": 0.9829, - "question_id": "81507db6", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_da3c1266_2", - "answer_da3c1266_4", - "answer_da3c1266_5", - "5c2f41a0_1", - "answer_da3c1266_1", - "9071ba70_3", - "ultrachat_162129", - "88c09142_2", - "42fa05d3", - "058f17e5_1" - ] - }, - { - "answer_session_ids": [ - "answer_edb03329" - ], - "ndcg_at_10": 0.6309, - "ndcg_at_5": 0.6309, - "question_id": "8a2466db", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "6a5b5a78", - "answer_edb03329", - "6dcf5fa0_1", - "3392c0c7", - "6be54739_3", - "d8b3e1c8_2", - "3d0c9f89_1", - "76967d22", - "sharegpt_MKMWjX0_25", - "sharegpt_osTHjYi_0" - ] - }, - { - "answer_session_ids": [ - "answer_555dfb94" - ], - "ndcg_at_10": 0.3869, - "ndcg_at_5": 0.3869, - "question_id": "06878be2", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "a6074da9_1", - "ca929779_1", - "3e9fce53_1", - "465e7ef9", - "answer_555dfb94", - "8fcfbe43_2", - "sharegpt_GSC090N_6", - "ultrachat_448784", - "79d53fd0_2", - "sharegpt_InRLwN7_0" - ] - }, - { - "answer_session_ids": [ - "answer_d87a6ef8" - ], - "ndcg_at_10": 0.0, - "ndcg_at_5": 0.0, - "question_id": "75832dbd", - "question_type": "single-session-preference", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 0.0, - "r_at_5": 0.0, - "top_session_ids": [ - "d49b1a24_1", - "1ef63c66", - "c884b67a_1", - "sharegpt_xni9yYO_6", - "ultrachat_165893", - "sharegpt_FYqt26U_0", - "47ec1674_2", - "11e9d591_1", - "sharegpt_fK7pfNf_0", - "ultrachat_93325" - ] - }, - { - "answer_session_ids": [ - "answer_d586e9cd" - ], - "ndcg_at_10": 0.5, - "ndcg_at_5": 0.5, - "question_id": "0edc2aef", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "f20e72e4_1", - "6a747f2e", - "answer_d586e9cd", - "08ca1f31_2", - "sharegpt_AsgRx0A_0", - "53dc1394", - "3c3a9042_3", - "8464304d_2", - "c6c3a982_1", - "52939d20" - ] - }, - { - "answer_session_ids": [ - "answer_9b182436" - ], - "ndcg_at_10": 0.5, - "ndcg_at_5": 0.5, - "question_id": "35a27287", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "ultrachat_267380", - "465f0d1b_1", - "answer_9b182436", - "58ae7c4f_3", - "880ee13e", - "e4a1b565_2", - "203bf3fa_2", - "c9b1c309_4", - "ultrachat_422353", - "7128c070_2" - ] - }, - { - "answer_session_ids": [ - "answer_0250ae1c" - ], - "ndcg_at_10": 0.5, - "ndcg_at_5": 0.5, - "question_id": "32260d93", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "c4d370d3_2", - "04bef53b", - "answer_0250ae1c", - "573dc24b_1", - "3f787a78_4", - "ultrachat_26485", - "ultrachat_171173", - "sharegpt_H4jw5s7_39", - "sharegpt_8jxRz9U_0", - "sharegpt_cMHMPNy_29" - ] - }, - { - "answer_session_ids": [ - "answer_6dc4305e" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "195a1a1b", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_6dc4305e", - "ultrachat_396124", - "9f12daa1_1", - "1b71c896_2", - "d03b6a05_2", - "ultrachat_576932", - "9b90460f_3", - "29695e1c_1", - "87fff4b4_1", - "ultrachat_199480" - ] - }, - { - "answer_session_ids": [ - "answer_8549e5e0" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "afdc33df", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_8549e5e0", - "0b0bc8ae_2", - "22f9f163_3", - "b6558bfb", - "afa0c2e7_2", - "d03b6a05_3", - "43b53ac4_1", - "ultrachat_381452", - "ultrachat_512490", - "e58109ed_1" - ] - }, - { - "answer_session_ids": [ - "answer_2fc6aabb" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "caf03d32", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_2fc6aabb", - "ec806228", - "6b137bce_4", - "eb0d8dc1_6", - "4374f9a6_2", - "ultrachat_501938", - "sharegpt_3D7Qpuq_14", - "af631aa3_1", - "bd6d0687", - "147e93c4" - ] - }, - { - "answer_session_ids": [ - "answer_f7b22c66" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "54026fce", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_f7b22c66", - "ba4db050_1", - "fab585c2_1", - "670c49e6", - "ultrachat_166270", - "ef5d4aa7_1", - "1c95e152_1", - "sharegpt_jyHBcl1_0", - "sharegpt_DnGdKtp_0", - "sharegpt_8YTuqXO_0" - ] - }, - { - "answer_session_ids": [ - "answer_92d5f7cd" - ], - "ndcg_at_10": 0.0, - "ndcg_at_5": 0.0, - "question_id": "06f04340", - "question_type": "single-session-preference", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 0.0, - "r_at_5": 0.0, - "top_session_ids": [ - "6e6fbb6b", - "91223fd5_1", - "sharegpt_h4ZC1fl_203", - "66bfa1db", - "0844dea6", - "42924d15", - "8b156015_2", - "728deb4d_4", - "sharegpt_MkLNumZ_0", - "sharegpt_tb5H6IH_3" - ] - }, - { - "answer_session_ids": [ - "answer_f6502d0f" - ], - "ndcg_at_10": 0.3869, - "ndcg_at_5": 0.3869, - "question_id": "6b7dfb22", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "574df152_1", - "9ee69ca6_3", - "377f8fd9_1", - "56859d37", - "answer_f6502d0f", - "be581797", - "ultrachat_443160", - "fc69fd44_3", - "603a8580_2", - "ultrachat_336051" - ] - }, - { - "answer_session_ids": [ - "answer_719502eb" - ], - "ndcg_at_10": 0.6309, - "ndcg_at_5": 0.6309, - "question_id": "1a1907b4", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "9585e0c6_1", - "answer_719502eb", - "4f3b36d4_5", - "3dced3e9_1", - "e42e7876_5", - "4e3da326_1", - "ultrachat_183428", - "ultrachat_490921", - "097c93d8", - "sharegpt_buGcwO5_0" - ] - }, - { - "answer_session_ids": [ - "answer_b10dce5e" - ], - "ndcg_at_10": 0.3562, - "ndcg_at_5": 0.0, - "question_id": "09d032c9", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 0.0, - "top_session_ids": [ - "b21bd3e2", - "3fc2244f", - "612e23f1", - "af631aa3_2", - "d3971322_1", - "answer_b10dce5e", - "26d9aaaf", - "9d1999d7_1", - "e8bfacec_2", - "sharegpt_e9sAtcZ_63" - ] - }, - { - "answer_session_ids": [ - "answer_772472c8" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "38146c39", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_772472c8", - "8064b6ca", - "909d57ca_2", - "381b80a3", - "5263736d_1", - "87aaee77", - "5806662d_1", - "a86b30e4", - "sharegpt_yBVQLaX_15", - "ultrachat_233519" - ] - }, - { - "answer_session_ids": [ - "answer_7c0ade93" - ], - "ndcg_at_10": 0.6309, - "ndcg_at_5": 0.6309, - "question_id": "d24813b1", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "3124bb28_1", - "answer_7c0ade93", - "084802f9_3", - "61a46ff7_3", - "c2a34674_1", - "ultrachat_494027", - "446173bb_1", - "1bc08e44", - "128f4e4d_3", - "sharegpt_tntUB4J_0" - ] - }, - { - "answer_session_ids": [ - "answer_1bde8d3b" - ], - "ndcg_at_10": 0.3562, - "ndcg_at_5": 0.0, - "question_id": "57f827a0", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 0.0, - "top_session_ids": [ - "d0f42e3f", - "1bff1675", - "e85e0eaf_1", - "2ef55f49_1", - "d05d6391_1", - "answer_1bde8d3b", - "1fc5074c_2", - "5150a4e9_2", - "6dd592c3", - "963e896f_1" - ] - }, - { - "answer_session_ids": [ - "answer_5e613445" - ], - "ndcg_at_10": 0.3155, - "ndcg_at_5": 0.0, - "question_id": "95228167", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 0.0, - "top_session_ids": [ - "b916da64_2", - "3d2e66f2_2", - "385a60c8_3", - "99e0fd9f_3", - "084802f9_2", - "5d8c99d3_3", - "7e17ae56_3", - "answer_5e613445", - "e2691068", - "c34b6a1c_1" - ] - }, - { - "answer_session_ids": [ - "answer_f3164f2c" - ], - "ndcg_at_10": 0.5, - "ndcg_at_5": 0.5, - "question_id": "505af2f5", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "7d52ca48_2", - "4f2d6be4_2", - "answer_f3164f2c", - "5e4bb245_1", - "af631aa3_1", - "0e193841_8", - "sharegpt_cjulkGS_15", - "ultrachat_208471", - "sharegpt_kjeGJvK_11", - "9f8fa5e4_1" - ] - }, - { - "answer_session_ids": [ - "answer_8ee04a2e" - ], - "ndcg_at_10": 0.6309, - "ndcg_at_5": 0.6309, - "question_id": "75f70248", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "2d74df23_4", - "answer_8ee04a2e", - "53dc1394", - "75671e3f_1", - "204862e4_2", - "ff67236f_3", - "6f689aee", - "ultrachat_27339", - "sharegpt_ijZ2Bps_0", - "ultrachat_309088" - ] - }, - { - "answer_session_ids": [ - "answer_b0fac439" - ], - "ndcg_at_10": 0.4307, - "ndcg_at_5": 0.4307, - "question_id": "d6233ab6", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "e419b7c3_4", - "94bc18df_3", - "87cfeb28_2", - "answer_b0fac439", - "911f859b_5", - "f916c63a_2", - "ultrachat_457634", - "sharegpt_xni9yYO_6", - "b45e05f6", - "ultrachat_329160" - ] - }, - { - "answer_session_ids": [ - "answer_4d3be2ab" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "1da05512", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_4d3be2ab", - "f2565b3b_2", - "sharegpt_eSTYwMj_14", - "ce919391_2", - "6b93018e_2", - "6dfa3dc6_2", - "ce3ad11f_2", - "5aa88956_2", - "ultrachat_159846", - "6de70f46_1" - ] - }, - { - "answer_session_ids": [ - "answer_a1e169b1" - ], - "ndcg_at_10": 0.3869, - "ndcg_at_5": 0.3869, - "question_id": "fca70973", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "fc005760", - "f504b269", - "5d5e80c5_1", - "99b061c5_1", - "answer_a1e169b1", - "2f2884ad_1", - "ultrachat_3370", - "312bf939", - "sharegpt_lSeiu11_0", - "sharegpt_fPvtTbw_0" - ] - }, - { - "answer_session_ids": [ - "answer_8414cc57" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "b6025781", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_8414cc57", - "f20c73f5_3", - "1a693999_4", - "3a1ea336_1", - "d6956a2e_1", - "1d6744b5_1", - "612c8368_2", - "6446f6e6", - "ebe60348_1", - "2ef53f61_1" - ] - }, - { - "answer_session_ids": [ - "answer_8f15ac24" - ], - "ndcg_at_10": 0.5, - "ndcg_at_5": 0.5, - "question_id": "a89d7624", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "99f2f5b1_1", - "02f70006_1", - "answer_8f15ac24", - "a479a7d6_4", - "fd778d8e", - "39a056ae", - "082fc07f_1", - "168776ed_2", - "bef3247f_1", - "sharegpt_G3RxeFJ_94" - ] - }, - { - "answer_session_ids": [ - "answer_30f63ddb" - ], - "ndcg_at_10": 0.3869, - "ndcg_at_5": 0.3869, - "question_id": "b0479f84", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "6af947bf_1", - "0cf86cbe_2", - "fbfda981", - "457b1adb_1", - "answer_30f63ddb", - "94cffbd2_2", - "9090b1f0_2", - "ddf41a24", - "d1bb5323_1", - "1cde7ee4_1" - ] - }, - { - "answer_session_ids": [ - "answer_e6b6353d" - ], - "ndcg_at_10": 0.3155, - "ndcg_at_5": 0.0, - "question_id": "1d4e3b97", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 0.0, - "top_session_ids": [ - "ecb24dd8_2", - "809cabca_1", - "c51583cd_3", - "03633a51", - "fb2368b8_3", - "1b320137_1", - "12b6f0f1_3", - "answer_e6b6353d", - "5cd6ab1b", - "ultrachat_8041" - ] - }, - { - "answer_session_ids": [ - "answer_d03098f9" - ], - "ndcg_at_10": 0.3333, - "ndcg_at_5": 0.0, - "question_id": "07b6f563", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 0.0, - "top_session_ids": [ - "dcc7f430_1", - "66ab6260", - "5e7b1c98_1", - "13b79e39_2", - "0ac1a6f9_2", - "6af947bf_1", - "answer_d03098f9", - "bbccfe79", - "55c267aa", - "39358a85_2" - ] - }, - { - "answer_session_ids": [ - "answer_8da8c7e0" - ], - "ndcg_at_10": 0.3869, - "ndcg_at_5": 0.3869, - "question_id": "1c0ddc50", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "2aa70c9c_1", - "5439cbf0_2", - "c7ca6dff", - "b33e89b5_1", - "answer_8da8c7e0", - "a0aa5035", - "ultrachat_339938", - "2566382f_2", - "6cd203f7_1", - "0a7d5bf6_2" - ] - }, - { - "answer_session_ids": [ - "answer_cebb7159" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "0a34ad58", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_cebb7159", - "fb048b5d_1", - "9ca2a195_1", - "f849dd04", - "121efb3b_2", - "4b495a04_1", - "f2b8e6a8_1", - "58ad78d6_2", - "ultrachat_122225", - "f7e682c3_5" - ] - }, - { - "answer_session_ids": [ - "answer_5237bb0b_2", - "answer_5237bb0b_1" - ], - "ndcg_at_10": 0.8503, - "ndcg_at_5": 0.8503, - "question_id": "d3ab962e", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_5237bb0b_2", - "ultrachat_564044", - "3329e5e8_1", - "sharegpt_6fTnCnw_49", - "answer_5237bb0b_1", - "aa235649_1", - "ultrachat_164834", - "ultrachat_155769", - "sharegpt_CDacKfJ_0", - "ultrachat_248741" - ] - }, - { - "answer_session_ids": [ - "answer_bf633415_2", - "answer_bf633415_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "2311e44b", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_bf633415_2", - "answer_bf633415_1", - "b5b8f8f9_3", - "5854eebc_2", - "ultrachat_360918", - "c824c711_1", - "84fedd9f_2", - "c6bed037_1", - "sharegpt_YAiYuAD_7", - "2714885f_1" - ] - }, - { - "answer_session_ids": [ - "answer_4fb01417_2", - "answer_4fb01417_1" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "cc06de0d", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_4fb01417_1", - "01551a39_2", - "answer_4fb01417_2", - "55033f6f", - "d5eab084_2", - "1dbe77da_1", - "ultrachat_104440", - "66ffbb8b_1", - "670c49e6", - "92d74e73" - ] - }, - { - "answer_session_ids": [ - "answer_c69ee1f9_2", - "answer_c69ee1f9_1" - ], - "ndcg_at_10": 0.8503, - "ndcg_at_5": 0.8503, - "question_id": "a11281a2", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c69ee1f9_2", - "ultrachat_55839", - "bb2180e9_3", - "c4c4d85a", - "answer_c69ee1f9_1", - "b8adda7f_1", - "28209b6a_5", - "d95e64a7_4", - "9126833b", - "60f10fd3" - ] - }, - { - "answer_session_ids": [ - "answer_50940cb7_2", - "answer_50940cb7_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "4f54b7c9", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_50940cb7_1", - "answer_50940cb7_2", - "bcfd4ab7", - "dde142b2_1", - "ultrachat_518898", - "8a834aee_1", - "b89f68c4_1", - "49a91ed8", - "663337ce_1", - "ultrachat_473718" - ] - }, - { - "answer_session_ids": [ - "answer_dcb18b9b_2", - "answer_dcb18b9b_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "85fa3a3f", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_dcb18b9b_2", - "answer_dcb18b9b_1", - "8f707521_1", - "71110dbd_1", - "0d66562c_1", - "c6c06e6e", - "93ff2f73", - "d31c3ec8_1", - "b7fd3b48", - "sharegpt_3WzI4bW_0" - ] - }, - { - "answer_session_ids": [ - "answer_353d3c6d_2", - "answer_353d3c6d_1" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "9aaed6a3", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_353d3c6d_2", - "c846422a_1", - "answer_353d3c6d_1", - "819dfad7_2", - "3f9f66f1_1", - "2e89bd03_1", - "11e942da_2", - "sharegpt_HpdNUlz_9", - "1bf761b8_2", - "73726c40_1" - ] - }, - { - "answer_session_ids": [ - "answer_d8454588_1", - "answer_d8454588_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "1f2b8d4f", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_d8454588_2", - "answer_d8454588_1", - "d3ee5958_2", - "c9b1c309_4", - "c8c64252", - "b5c147c7_2", - "9e96d5b9_1", - "0bd59f15_3", - "23759615_1", - "sharegpt_OHIxTQm_22" - ] - }, - { - "answer_session_ids": [ - "answer_4eb6d671_2", - "answer_4eb6d671_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e6041065", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_4eb6d671_2", - "answer_4eb6d671_1", - "d79173aa_1", - "b73a4d04_1", - "49f2e396_2", - "4a006b77", - "ec616e7e_4", - "2a86d0da_2", - "bdcee74e_3", - "b7dca8f1_1" - ] - }, - { - "answer_session_ids": [ - "answer_58820c75_1", - "answer_58820c75_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "51c32626", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_58820c75_1", - "answer_58820c75_2", - "c051e740_2", - "sharegpt_hzwc0HI_13", - "84d7650f_1", - "809cbce9_1", - "sharegpt_ErOTMZ3_59", - "932ea3bf_1", - "ultrachat_273746", - "sharegpt_jBafAKm_0" - ] - }, - { - "answer_session_ids": [ - "answer_85a77c48_2", - "answer_85a77c48_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "d905b33f", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_85a77c48_1", - "answer_85a77c48_2", - "8cfa91c3_2", - "2d364fb0", - "a9b7f49e_1", - "368276ab_3", - "dd4a0e15", - "6423d6a4_2", - "7613d5ec_2", - "sharegpt_ektieAO_13" - ] - }, - { - "answer_session_ids": [ - "answer_80323f3f_1", - "answer_80323f3f_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "7405e8b1", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_80323f3f_1", - "answer_80323f3f_2", - "f2f2a606_3", - "a6b00510_4", - "6c1c5bc3_2", - "c578da1f_1", - "6e983235_1", - "3a4012a5_1", - "8f8f2ced_2", - "643d285a_3" - ] - }, - { - "answer_session_ids": [ - "answer_e9bb9500_2", - "answer_e9bb9500_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "f35224e0", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e9bb9500_2", - "answer_e9bb9500_1", - "96da07f9_4", - "e184e4c3_1", - "9393001b_3", - "ultrachat_215582", - "4648f214_2", - "42234f98", - "ultrachat_558221", - "sharegpt_Pqhvz5n_0" - ] - }, - { - "answer_session_ids": [ - "answer_743f03a1_2", - "answer_743f03a1_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "6456829e", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_743f03a1_1", - "answer_743f03a1_2", - "c44d1533_2", - "ultrachat_490336", - "ultrachat_518760", - "a8c73552", - "898cfd46_2", - "ultrachat_555576", - "3304f57b_3", - "ultrachat_247641" - ] - }, - { - "answer_session_ids": [ - "answer_feb5f98a_1", - "answer_feb5f98a_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "a4996e51", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_feb5f98a_2", - "answer_feb5f98a_1", - "252715f8_3", - "e9fba4f8", - "c1dac02f", - "082b7e52_2", - "d1a61aa1_3", - "ultrachat_506025", - "sharegpt_KjF70Iy_0", - "eea3e4be_2" - ] - }, - { - "answer_session_ids": [ - "answer_c8cc60d6_2", - "answer_c8cc60d6_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "3c1045c8", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c8cc60d6_2", - "ultrachat_561782", - "ultrachat_20893", - "b52f03a1_1", - "62541047", - "sharegpt_oIB2JG9_0", - "4f3b36d4_3", - "ultrachat_299649", - "sharegpt_GYqnAhC_190", - "cf422c07" - ] - }, - { - "answer_session_ids": [ - "answer_e552e1f9_1", - "answer_e552e1f9_2" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "60036106", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e552e1f9_2", - "sharegpt_moARQa7_0", - "answer_e552e1f9_1", - "sharegpt_ckvrgF1_99", - "c575291e_2", - "e9cb7125", - "ultrachat_466668", - "23a7f0ec", - "763f028a", - "ultrachat_134557" - ] - }, - { - "answer_session_ids": [ - "answer_3be95d43_1", - "answer_3be95d43_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "681a1674", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_3be95d43_2", - "answer_3be95d43_1", - "ultrachat_64650", - "fdd25ea1", - "50e8f2b5_1", - "d81e853b", - "sharegpt_8Kzmn95_0", - "ultrachat_436833", - "7d33fcb4_2", - "9e411500_1" - ] - }, - { - "answer_session_ids": [ - "answer_6702277b_1", - "answer_6702277b_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e25c3b8d", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_6702277b_2", - "answer_6702277b_1", - "sharegpt_40qcQLS_17", - "sharegpt_WYQSZ3n_0", - "90fea364_1", - "sharegpt_Nk10TjE_29", - "824634d9_1", - "ff201786_1", - "4256f981_2", - "ultrachat_363088" - ] - }, - { - "answer_session_ids": [ - "answer_6efce493_1", - "answer_6efce493_2" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "4adc0475", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_6efce493_1", - "bb27df5e_2", - "answer_6efce493_2", - "fc6f9c59", - "4fc2cf8c_1", - "2880eb6c_3", - "0bde5dac", - "sharegpt_wcTU09Q_31", - "debc34d7_2", - "ee659010_2" - ] - }, - { - "answer_session_ids": [ - "answer_9ef115d4_1", - "answer_9ef115d4_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "4bc144e2", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_9ef115d4_1", - "answer_9ef115d4_2", - "4f23a396_2", - "d75245ea", - "ultrachat_380790", - "42bcee92_2", - "099c1b6c_3", - "60207b41_3", - "26d9aaaf", - "2e2ae792_1" - ] - }, - { - "answer_session_ids": [ - "answer_f7fd1029_2", - "answer_f7fd1029_1" - ], - "ndcg_at_10": 0.8772, - "ndcg_at_5": 0.8772, - "question_id": "ef66a6e5", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_f7fd1029_1", - "48941fec_4", - "74c2d3a1_1", - "answer_f7fd1029_2", - "3f05a03e_1", - "41e778e4_2", - "afb58af0", - "295645f5_1", - "67045503", - "5273d4b8" - ] - }, - { - "answer_session_ids": [ - "answer_688f9a3f_1", - "answer_688f9a3f_2" - ], - "ndcg_at_10": 0.6934, - "ndcg_at_5": 0.6934, - "question_id": "5025383b", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "22f9af5f", - "answer_688f9a3f_2", - "answer_688f9a3f_1", - "f40fa3ed", - "d10f0307_1", - "2d21f921", - "ultrachat_50563", - "ultrachat_150500", - "sharegpt_BmS3AX0_10", - "ultrachat_137351" - ] - }, - { - "answer_session_ids": [ - "answer_17dc2f5b_2", - "answer_17dc2f5b_1" - ], - "ndcg_at_10": 0.5, - "ndcg_at_5": 0.5, - "question_id": "a1cc6108", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "298a07de_5", - "fce56dfb_4", - "answer_17dc2f5b_2", - "7285299a_4", - "6dfc99bf_1", - "001cefa7_2", - "fb328ace_1" - ] - }, - { - "answer_session_ids": [ - "answer_66c23110_1", - "answer_66c23110_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "9ee3ecd6", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_66c23110_2", - "answer_66c23110_1", - "fd56c5bd_2", - "a9eda378", - "6d9d5941", - "ultrachat_282862", - "fa0fa74d_2", - "ultrachat_96128", - "sharegpt_6cz1Sq6_264", - "42b45f6e_1" - ] - }, - { - "answer_session_ids": [ - "answer_419d21d5_2", - "answer_419d21d5_1" - ], - "ndcg_at_10": 0.8316, - "ndcg_at_5": 1.0, - "question_id": "3fdac837", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_419d21d5_2", - "92b06116_1", - "a98ff421_7", - "ultrachat_526748", - "c85ef24e_1", - "answer_419d21d5_1", - "ac8a2cd0", - "ultrachat_293546", - "88c8df0e_3", - "22210456_1" - ] - }, - { - "answer_session_ids": [ - "answer_5404a208_1", - "answer_5404a208_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "91b15a6e", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_5404a208_1", - "answer_5404a208_2", - "ultrachat_15125", - "f9a304e9_2", - "6169ef55_1", - "ultrachat_217922", - "7cfeb89c_2", - "ultrachat_338803", - "ultrachat_326780", - "ultrachat_222016" - ] - }, - { - "answer_session_ids": [ - "answer_a37bdf22_2", - "answer_a37bdf22_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "27016adc", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_a37bdf22_2", - "answer_a37bdf22_1", - "9f554b46_2", - "fa858208_2", - "28ea20bf", - "ultrachat_495725", - "ff5e0db0_1", - "sharegpt_3b5ojYa_39", - "ultrachat_131737", - "ultrachat_284316" - ] - }, - { - "answer_session_ids": [ - "answer_c9dfeaea_1", - "answer_c9dfeaea_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "720133ac", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c9dfeaea_2", - "answer_c9dfeaea_1", - "4c128d97_2", - "e200d96c_3", - "ff2de758", - "ultrachat_479833", - "sharegpt_bZg2XwX_69", - "0cf86cbe_2", - "ultrachat_112896", - "ultrachat_512422" - ] - }, - { - "answer_session_ids": [ - "answer_33c251f0_2", - "answer_33c251f0_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "77eafa52", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_33c251f0_1", - "answer_33c251f0_2", - "bc1f1051_4", - "sharegpt_qEukY6n_14", - "sharegpt_FlvsHKg_4", - "sharegpt_C9GjDsF_0", - "sharegpt_bpxJpAU_0", - "ultrachat_309057", - "9c4358a4", - "7b13b4d6_2" - ] - }, - { - "answer_session_ids": [ - "answer_35e36341_1", - "answer_35e36341_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "8979f9ec", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_35e36341_1", - "answer_35e36341_2", - "91c71874_1", - "ultrachat_466891", - "bcc9382a_2", - "ultrachat_562471", - "4c967baa_1", - "a6b4c4c2_1", - "f18ebe36_7", - "1064ed80_2" - ] - }, - { - "answer_session_ids": [ - "answer_35c9798c_2", - "answer_35c9798c_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "0100672e", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_35c9798c_2", - "answer_35c9798c_1", - "a4b223a7_1", - "091aa510_2", - "326e86e0_3", - "8237de6a_2", - "ultrachat_372868", - "07313722_2", - "b167a4a8", - "f60c9de5" - ] - }, - { - "answer_session_ids": [ - "answer_ef84b994_1", - "answer_ef84b994_2" - ], - "ndcg_at_10": 0.8503, - "ndcg_at_5": 0.8503, - "question_id": "a96c20ee", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ef84b994_1", - "1b51fb71_1", - "dff89d9d_1", - "9ca3534f_1", - "answer_ef84b994_2", - "2aa70c9c_2", - "sharegpt_opQbgPL_7", - "32e292f2", - "sharegpt_lWLBUhQ_539", - "sharegpt_00qHEQ6_0" - ] - }, - { - "answer_session_ids": [ - "answer_6cb8f792_1", - "answer_6cb8f792_2" - ], - "ndcg_at_10": 0.525, - "ndcg_at_5": 0.5, - "question_id": "92a0aa75", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "ultrachat_178564", - "sharegpt_YwmoLVl_0", - "answer_6cb8f792_1", - "a5854a8d_2", - "2bdd78cb_1", - "answer_6cb8f792_2", - "bd00a24d_2", - "sharegpt_xkWMnwQ_0", - "sharegpt_9l7gjYP_17", - "sharegpt_hPax8pw_11" - ] - }, - { - "answer_session_ids": [ - "answer_1bb63ea5_2", - "answer_1bb63ea5_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "3fe836c9", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_1bb63ea5_1", - "answer_1bb63ea5_2", - "e41b78c7_3", - "0a42783a", - "c81897cd_2", - "sharegpt_2xK3jjf_0", - "8129a7a8", - "f0d7d364", - "ba184a34", - "ultrachat_258020" - ] - }, - { - "answer_session_ids": [ - "answer_4cb841a8_1", - "answer_4cb841a8_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "1c549ce4", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_4cb841a8_1", - "answer_4cb841a8_2", - "f1e4daa8_2", - "f999b05c_1", - "d743153b_2", - "ultrachat_441412", - "162ff451", - "sharegpt_miOKGt1_13", - "35f052f9_2", - "sharegpt_yLBm8Vr_23" - ] - }, - { - "answer_session_ids": [ - "answer_cc1ced21_2", - "answer_cc1ced21_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "6c49646a", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_cc1ced21_2", - "answer_cc1ced21_1", - "d4380d3f", - "8acb29f9_2", - "ultrachat_565061", - "fffa6e59", - "9f220c66_1", - "6a2f306a", - "sharegpt_1X61wv2_7", - "edf2a486" - ] - }, - { - "answer_session_ids": [ - "answer_e184e4c3_2", - "answer_e184e4c3_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "1192316e", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e184e4c3_2", - "answer_e184e4c3_1", - "574df152_3", - "c5db849c_1", - "6cd7234e_2", - "ee82ed2c_2", - "21ef2d05_1", - "sharegpt_pZ3I356_11", - "1ba79ea8_1", - "ultrachat_457634" - ] - }, - { - "answer_session_ids": [ - "answer_dc5e537d_1", - "answer_dc5e537d_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "0ea62687", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_dc5e537d_1", - "answer_dc5e537d_2", - "fba28a35_3", - "26eedae1_1", - "0835368c", - "c5e9cbd8_1", - "d3ee5958_1", - "ultrachat_108112", - "sharegpt_HYI3HgX_0", - "sharegpt_Ktz3pBy_0" - ] - }, - { - "answer_session_ids": [ - "answer_3a5010af_2", - "answer_3a5010af_1" - ], - "ndcg_at_10": 0.8772, - "ndcg_at_5": 0.8772, - "question_id": "67e0d0f2", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_3a5010af_2", - "6868c94f", - "98cd882c_1", - "answer_3a5010af_1", - "ultrachat_217333", - "0cde5602_2", - "8c568d75_1", - "2252d3dd", - "sharegpt_tJkPZbz_2", - "sharegpt_GJpf1ou_0" - ] - }, - { - "answer_session_ids": [ - "answer_de64539a_1", - "answer_de64539a_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "bb7c3b45", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_de64539a_2", - "answer_de64539a_1", - "60ff3a8c", - "3891baea_4", - "a51dc9ca", - "64b7d9cc_2", - "ultrachat_305663", - "f3d43a12_3", - "4f2d6be4_1", - "752b10e6_1" - ] - }, - { - "answer_session_ids": [ - "answer_cbd08e3c_1", - "answer_cbd08e3c_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "ba358f49", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_cbd08e3c_1", - "2a2ea005_1", - "20b5caab_2", - "9cc709cb", - "8db0ab95_1", - "4ea6f550", - "385683f0_2", - "648d9cda", - "d964ed3b", - "4f838497_2" - ] - }, - { - "answer_session_ids": [ - "answer_872e8da2_2", - "answer_872e8da2_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "61f8c8f8", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_872e8da2_1", - "answer_872e8da2_2", - "a3107e2a_1", - "6e110a53_1", - "7b88c38b_2", - "2154c810_1", - "7af38385_2", - "e10846c4", - "sharegpt_iL51bzd_0", - "sharegpt_7jGo66i_99" - ] - }, - { - "answer_session_ids": [ - "answer_75eca223_2", - "answer_75eca223_1" - ], - "ndcg_at_10": 0.4825, - "ndcg_at_5": 0.4307, - "question_id": "60159905", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "21f8f481_3", - "d5f4d9fa_2", - "02eae87c_5", - "answer_75eca223_1", - "sharegpt_V7ePzVN_1", - "answer_75eca223_2", - "6ade84f0", - "c351fa3b_1", - "c92b27e2_1", - "c36f4196" - ] - }, - { - "answer_session_ids": [ - "answer_87e3a1cb_1", - "answer_87e3a1cb_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "ef9cf60a", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_87e3a1cb_2", - "answer_87e3a1cb_1", - "2357de36_2", - "b330d6b1", - "7b88c38b_1", - "sharegpt_FNyKOSt_0", - "7a34bc76_3", - "12dd7681_4", - "0babf5b6_2", - "de47d59b_1" - ] - }, - { - "answer_session_ids": [ - "answer_1881e7db_1", - "answer_1881e7db_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "73d42213", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_1881e7db_2", - "answer_1881e7db_1", - "sharegpt_ab6IEma_0", - "9614d065_1", - "sharegpt_G8j6c8J_87", - "sharegpt_ndynIbS_0", - "sharegpt_GKjb2be_0", - "ultrachat_275858", - "a13abca5_1", - "sharegpt_2Na4ASq_0" - ] - }, - { - "answer_session_ids": [ - "answer_92147866_1", - "answer_92147866_2" - ], - "ndcg_at_10": 0.8503, - "ndcg_at_5": 0.8503, - "question_id": "bc149d6b", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_92147866_1", - "7211c69e", - "86935301_2", - "451ecb23_3", - "answer_92147866_2", - "4f435d20_2", - "sharegpt_bmEgv3O_12", - "05be3ff0_3", - "eb5c65e2_1", - "db73b7e4_3" - ] - }, - { - "answer_session_ids": [ - "answer_80d6d664_2", - "answer_80d6d664_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "099778bb", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_80d6d664_2", - "answer_80d6d664_1", - "sharegpt_s1I3UkG_0", - "47103d08", - "0070840d_2", - "b1793bba", - "6bb8af95_1", - "33877350_2", - "95ce0e21_1", - "sharegpt_kshUqZh_6" - ] - }, - { - "answer_session_ids": [ - "answer_96c743d0_2", - "answer_96c743d0_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "09ba9854", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_96c743d0_1", - "answer_96c743d0_2", - "ultrachat_177260", - "32e6c308", - "7a2bdd15_1", - "ae32e467_2", - "8c63238e", - "864a563d_3", - "69a42467_2", - "8682e696_2" - ] - }, - { - "answer_session_ids": [ - "answer_23f3a657_2", - "answer_23f3a657_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "d6062bb9", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_23f3a657_1", - "answer_23f3a657_2", - "sharegpt_xuhzAgG_19", - "b35610f4_4", - "4f8caea3_1", - "5b6aedac", - "59c704ad_2", - "86298245_2", - "sharegpt_afCogMg_13", - "ultrachat_496826" - ] - }, - { - "answer_session_ids": [ - "answer_8de18468_2", - "answer_8de18468_1" - ], - "ndcg_at_10": 0.301, - "ndcg_at_5": 0.0, - "question_id": "157a136e", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 0.0, - "top_session_ids": [ - "ultrachat_255132", - "01049599_1", - "ultrachat_3770", - "sharegpt_qqo6qRj_0", - "a2e2cb72_5", - "sharegpt_6ZXBs1w_0", - "ultrachat_506827", - "298bf7a3_1", - "answer_8de18468_2", - "ultrachat_446084" - ] - }, - { - "answer_session_ids": [ - "answer_2e2085fa_2", - "answer_2e2085fa_1" - ], - "ndcg_at_10": 0.4307, - "ndcg_at_5": 0.4307, - "question_id": "c18a7dc8", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "9d5a389d", - "ultrachat_353377", - "5022812c", - "answer_2e2085fa_1", - "fc6c3985", - "f716720c_3", - "ultrachat_415226", - "37141183", - "ultrachat_463016", - "sharegpt_YPC6Oll_18" - ] - }, - { - "answer_session_ids": [ - "answer_16ece55f_2", - "answer_16ece55f_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "a3332713", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_16ece55f_2", - "answer_16ece55f_1", - "62be624f", - "67388dfc_2", - "3e59ee68_1", - "fe291ebc", - "117098d9_3", - "33994682_3", - "sharegpt_bYymtlW_0", - "sharegpt_8zrM3l4_0" - ] - }, - { - "answer_session_ids": [ - "answer_fa08bf49_1", - "answer_fa08bf49_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "55241a1f", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_fa08bf49_1", - "answer_fa08bf49_2", - "d117e2da", - "2aeb1268", - "76491ee4_1", - "9c9a22d8", - "60153a02_4", - "sharegpt_TzwvePR_0", - "46e4a8eb_2", - "1d105d8d" - ] - }, - { - "answer_session_ids": [ - "answer_47152166_2", - "answer_47152166_1" - ], - "ndcg_at_10": 0.6934, - "ndcg_at_5": 0.6934, - "question_id": "a08a253f", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "6e5130ed_2", - "answer_47152166_2", - "answer_47152166_1", - "798fa5f2", - "ultrachat_242484", - "33bc1102", - "c7fc2bda_1", - "ultrachat_329610", - "a79f1a04_1", - "ultrachat_403700" - ] - }, - { - "answer_session_ids": [ - "answer_cfcf5340_1", - "answer_cfcf5340_2" - ], - "ndcg_at_10": 0.8503, - "ndcg_at_5": 0.8503, - "question_id": "f0e564bc", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_cfcf5340_2", - "65ca99fb_2", - "798e4ba3_2", - "bc1f1051_1", - "answer_cfcf5340_1", - "sharegpt_W3YFpVK_7", - "sharegpt_3hCiKrk_0", - "06fe8cfb_1", - "ultrachat_197576", - "932a97c6" - ] - }, - { - "answer_session_ids": [ - "answer_254d8b09_1", - "answer_254d8b09_2" - ], - "ndcg_at_10": 0.8316, - "ndcg_at_5": 1.0, - "question_id": "078150f1", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_254d8b09_1", - "61b27d67_3", - "203a48bb_1", - "46e4a8eb_1", - "ultrachat_202549", - "answer_254d8b09_2", - "0e218647_2", - "57bf2079", - "ultrachat_276164", - "b27b0097_1" - ] - }, - { - "answer_session_ids": [ - "answer_e2278b24_1", - "answer_e2278b24_2" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "8cf4d046", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e2278b24_2", - "a30f52a4", - "answer_e2278b24_1", - "sharegpt_P1Okivz_9", - "sharegpt_6DlPq48_0", - "7ef465f3_1", - "sharegpt_WjpyexI_15", - "7a5c32b0_3", - "c8b1178f_1", - "c36f4196" - ] - }, - { - "answer_session_ids": [ - "answer_4934b2d7_2", - "answer_4934b2d7_1" - ], - "ndcg_at_10": 0.8316, - "ndcg_at_5": 1.0, - "question_id": "a346bb18", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_4934b2d7_2", - "771ba1d5", - "5726dc37_2", - "fd5448f0", - "84fb50bb_1", - "answer_4934b2d7_1", - "sharegpt_VeexfGM_0", - "sharegpt_cYNuFCK_50", - "sharegpt_ooEuxDg_0", - "e5713180_1" - ] - }, - { - "answer_session_ids": [ - "answer_6b9b2b1e_2", - "answer_6b9b2b1e_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "37f165cf", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_6b9b2b1e_2", - "answer_6b9b2b1e_1", - "20b1b65f_2", - "a41d46df", - "c85d1682_1", - "c41e97c9_2", - "sharegpt_2yaKUmC_0", - "395a2f92", - "c0f6571a_2", - "7e8e790a" - ] - }, - { - "answer_session_ids": [ - "answer_477ae455_1", - "answer_477ae455_2" - ], - "ndcg_at_10": 0.0, - "ndcg_at_5": 0.0, - "question_id": "8e91e7d9", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 0.0, - "r_at_5": 0.0, - "top_session_ids": [ - "sharegpt_PZrBCF2_0", - "ultrachat_147574", - "4e3da326_1", - "bc790dfb", - "2f980ae0_2", - "c3023a45_4", - "06170195_1", - "sharegpt_8lpX5R7_19", - "c2381e1a_2", - "sharegpt_Z35PJ8x_2" - ] - }, - { - "answer_session_ids": [ - "answer_f56e6152_2", - "answer_f56e6152_1" - ], - "ndcg_at_10": 0.4306, - "ndcg_at_5": 0.3869, - "question_id": "87f22b4a", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "02b99ec3_1", - "ultrachat_350147", - "28741edc_1", - "ultrachat_182225", - "answer_f56e6152_2", - "889076d6", - "c4c504d2_6", - "answer_f56e6152_1", - "a8ac3d1d_3", - "ea28bb75" - ] - }, - { - "answer_session_ids": [ - "answer_430d0a87_1", - "answer_430d0a87_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e56a43b9", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_430d0a87_1", - "answer_430d0a87_2", - "83e14506_2", - "0eef411a_2", - "38e81260_7", - "7596ba6a", - "ce21f0e4", - "sharegpt_XaM55h6_28", - "ultrachat_245457", - "ultrachat_12201" - ] - }, - { - "answer_session_ids": [ - "answer_fc81d1a2_1", - "answer_fc81d1a2_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "efc3f7c2", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_fc81d1a2_1", - "answer_fc81d1a2_2", - "ultrachat_38938", - "263f2395", - "ultrachat_166366", - "bbdc7b0a", - "ultrachat_102676", - "ultrachat_552063", - "48cb014c", - "fc005760" - ] - }, - { - "answer_session_ids": [ - "answer_2c637141_2", - "answer_2c637141_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "21d02d0d", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_2c637141_2", - "answer_2c637141_1", - "eb8195a6_1", - "ultrachat_71299", - "sharegpt_adeEi7y_7", - "06a39bb5_1", - "42d9467c_2", - "a15fa6eb", - "sharegpt_B9jlITg_0", - "414b8ddb" - ] - }, - { - "answer_session_ids": [ - "answer_d00ba6d0_1", - "answer_d00ba6d0_2" - ], - "ndcg_at_10": 0.8772, - "ndcg_at_5": 0.8772, - "question_id": "gpt4_59149c77", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_d00ba6d0_2", - "9e2c2a6c_2", - "9242fbe3_1", - "answer_d00ba6d0_1", - "6e672b84_1", - "sharegpt_XlNs8Lz_199", - "148f87a0_2", - "54a05e72_2", - "ultrachat_318178", - "ultrachat_280434" - ] - }, - { - "answer_session_ids": [ - "answer_3e9fce53_1", - "answer_3e9fce53_2", - "answer_3e9fce53_3" - ], - "ndcg_at_10": 0.9469, - "ndcg_at_5": 0.9469, - "question_id": "gpt4_f49edff3", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_3e9fce53_3", - "answer_3e9fce53_2", - "7d33fcb4_1", - "1a29bf1f", - "answer_3e9fce53_1", - "dc7c53fa_4", - "9f8fa5e4_2", - "acfdbdcb", - "4376945f", - "ultrachat_257206" - ] - }, - { - "answer_session_ids": [ - "answer_0b4a8adc_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "71017276", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_0b4a8adc_1", - "683c8358_1", - "58ec258f_2", - "sharegpt_tCLMKZY_0", - "10889cf0_1", - "4d6927c2_2", - "sharegpt_11WxPMZ_0", - "44e4ae7e", - "398f23e7", - "sharegpt_tmFunkX_20" - ] - }, - { - "answer_session_ids": [ - "answer_4bfcc250_4", - "answer_4bfcc250_3", - "answer_4bfcc250_2", - "answer_4bfcc250_1" - ], - "ndcg_at_10": 0.8529, - "ndcg_at_5": 0.8529, - "question_id": "b46e15ed", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_4bfcc250_3", - "aeab3296", - "fa34119a_1", - "answer_4bfcc250_2", - "answer_4bfcc250_1", - "4bf66c38_2", - "a410bab2_3", - "47311a4c_1", - "ultrachat_520419", - "f598d30f_1" - ] - }, - { - "answer_session_ids": [ - "answer_ff201786_1", - "answer_ff201786_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_fa19884c", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ff201786_2", - "answer_ff201786_1", - "a7b1b27c", - "f3745025_1", - "1b066639", - "f4f4b504_2", - "800cfd94", - "ultrachat_423544", - "cbf48918_1", - "bb2180e9_3" - ] - }, - { - "answer_session_ids": [ - "answer_f4ea84fb_3", - "answer_f4ea84fb_2", - "answer_f4ea84fb_1" - ], - "ndcg_at_10": 0.6462, - "ndcg_at_5": 0.6509, - "question_id": "0bc8ad92", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "0868982f_1", - "answer_f4ea84fb_3", - "f58c36b8_1", - "answer_f4ea84fb_2", - "ultrachat_349938", - "fccc9400", - "d3cc5bdc_1", - "answer_f4ea84fb_1", - "eca613fc", - "sharegpt_scMKXPh_9" - ] - }, - { - "answer_session_ids": [ - "answer_b51b6115_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "af082822", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_b51b6115_1", - "bc5827b5", - "facef6e0_1", - "b28990c8_7", - "d048ae4b_2", - "1dfef591_2", - "cae65795_2", - "cdf068b1_2", - "1fcb4134_2", - "54f0d6f9_1" - ] - }, - { - "answer_session_ids": [ - "answer_add9b012_2", - "answer_add9b012_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_4929293a", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_add9b012_2", - "answer_add9b012_1", - "623ea729_2", - "ultrachat_329640", - "sharegpt_oQacgfY_0", - "ultrachat_512282", - "b8bc1464_4", - "73353aee_1", - "c1e598cd_1", - "sharegpt_mWFmQxk_36" - ] - }, - { - "answer_session_ids": [ - "answer_a17423e7_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_b5700ca9", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_a17423e7_1", - "44c62f26", - "5053474c_2", - "a53c7542_1", - "33bdd89a_1", - "sharegpt_FqOC5e9_169", - "d743153b_1", - "ultrachat_366301", - "47382adf_1", - "ultrachat_383774" - ] - }, - { - "answer_session_ids": [ - "answer_dba89487_2", - "answer_dba89487_1" - ], - "ndcg_at_10": 0.8316, - "ndcg_at_5": 1.0, - "question_id": "9a707b81", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_dba89487_2", - "4abbeb2a_2", - "5a13d047_1", - "ultrachat_233519", - "6862e478_2", - "answer_dba89487_1", - "990f3ef9_6", - "3fcbb58c_4", - "e1d11615_1", - "59510ab6" - ] - }, - { - "answer_session_ids": [ - "answer_febde667_1", - "answer_febde667_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_1d4ab0c9", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_febde667_2", - "answer_febde667_1", - "ab855b34_2", - "3f1d1224_3", - "7257bd15_1", - "082b7e52_1", - "ultrachat_201007", - "2dbfff2f", - "ultrachat_310433", - "sharegpt_MIYBRJz_27" - ] - }, - { - "answer_session_ids": [ - "answer_c19bd2bf_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_e072b769", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c19bd2bf_1", - "5558a42e_1", - "4f7b5dc9_1", - "b9e32ff8_1", - "2fc9ce79_2", - "sharegpt_1VhiETr_37", - "66d3fdb7_2", - "ultrachat_371691", - "sharegpt_9Ln5Ood_0", - "sharegpt_VbIxRWJ_33" - ] - }, - { - "answer_session_ids": [ - "answer_b9e32ff8_1", - "answer_b9e32ff8_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "0db4c65d", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_b9e32ff8_2", - "answer_b9e32ff8_1", - "71dc2037_2", - "953d5a4b_1", - "667465d6_1", - "sharegpt_5NoXULS_0", - "e9e19917_1", - "b22a2540", - "d36d11b9_2", - "e6c3a50a" - ] - }, - { - "answer_session_ids": [ - "answer_661b711f_1", - "answer_661b711f_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_1d80365e", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_661b711f_1", - "answer_661b711f_2", - "f7a61595_2", - "ultrachat_293442", - "805e571c_1", - "554b054e_2", - "ultrachat_561552", - "ultrachat_271411", - "3be19aed_1", - "sharegpt_5uZ7IdY_0" - ] - }, - { - "answer_session_ids": [ - "answer_5d8c99d3_1", - "answer_5d8c99d3_2", - "answer_5d8c99d3_3" - ], - "ndcg_at_10": 0.4825, - "ndcg_at_5": 0.4307, - "question_id": "gpt4_7f6b06db", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "39c1bf6a", - "928890e4_3", - "ultrachat_31232", - "answer_5d8c99d3_2", - "631e4016", - "answer_5d8c99d3_3", - "d9ca4aca", - "4bf66c38_2", - "da42526b_1", - "sharegpt_3sKfamn_27" - ] - }, - { - "answer_session_ids": [ - "answer_c4df007f_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_6dc9b45b", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c4df007f_1", - "d8f48b0a", - "ultrachat_384387", - "9adc81d7_2", - "ultrachat_355155", - "ultrachat_35147", - "afa0c2e7_5", - "94ff80d3", - "sharegpt_jyynvvk_0", - "sharegpt_0SDNlJ3_63" - ] - }, - { - "answer_session_ids": [ - "answer_56521e65_1" - ], - "ndcg_at_10": 0.5, - "ndcg_at_5": 0.5, - "question_id": "gpt4_8279ba02", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "b909542d_1", - "85f19a30_1", - "answer_56521e65_1", - "b6018747_2", - "86c5d31d", - "e1c807f9", - "62b40a05_2", - "3a735bfa_1", - "ultrachat_78027", - "1a037873_2" - ] - }, - { - "answer_session_ids": [ - "answer_c862f65a_2", - "answer_c862f65a_3", - "answer_c862f65a_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_18c2b244", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c862f65a_3", - "answer_c862f65a_2", - "answer_c862f65a_1", - "dfbc46bd_1", - "7e2c6065", - "d9e1d5fd", - "aeb58d86_4", - "ultrachat_503659", - "84896d13", - "746ae854_2" - ] - }, - { - "answer_session_ids": [ - "answer_e9ad5914_1", - "answer_e9ad5914_2", - "answer_e9ad5914_3", - "answer_e9ad5914_4", - "answer_e9ad5914_5", - "answer_e9ad5914_6" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_a1b77f9c", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e9ad5914_3", - "answer_e9ad5914_4", - "answer_e9ad5914_6", - "answer_e9ad5914_1", - "answer_e9ad5914_5", - "answer_e9ad5914_2", - "199a5225_1", - "47ec1674_2", - "d52d478b", - "f4a4a28a" - ] - }, - { - "answer_session_ids": [ - "answer_447052a5_2", - "answer_447052a5_1" - ], - "ndcg_at_10": 0.8175, - "ndcg_at_5": 1.0, - "question_id": "gpt4_1916e0ea", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_447052a5_2", - "d3fe25ff", - "280352e9", - "4b72fe4c_1", - "fa1ce406_2", - "987b5d00", - "answer_447052a5_1", - "7a36e820_2", - "dffa3157", - "sharegpt_tPA8329_0" - ] - }, - { - "answer_session_ids": [ - "answer_4d5490f1_1", - "answer_4d5490f1_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_7a0daae1", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_4d5490f1_1", - "answer_4d5490f1_2", - "8c68df0d_1", - "014a5d36", - "819dfad7_2", - "2ff297a8_1", - "bf3aebdb_2", - "1c8832b4_1", - "sharegpt_ZWqMvoL_607", - "99883f38_3" - ] - }, - { - "answer_session_ids": [ - "answer_9b09d95b_1" - ], - "ndcg_at_10": 0.3333, - "ndcg_at_5": 0.0, - "question_id": "gpt4_468eb063", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 0.0, - "top_session_ids": [ - "e60a93ff_2", - "ultrachat_108278", - "sharegpt_Sf51OjE_8", - "d97db962_2", - "sharegpt_gfhUw40_17", - "sharegpt_1rIk8tS_0", - "answer_9b09d95b_1", - "ultrachat_91161", - "sharegpt_8M7HcM3_11", - "sharegpt_ipLglky_42" - ] - }, - { - "answer_session_ids": [ - "answer_7093d898_1", - "answer_7093d898_2", - "answer_7093d898_3", - "answer_7093d898_4", - "answer_7093d898_5", - "answer_7093d898_6" - ], - "ndcg_at_10": 0.6199, - "ndcg_at_5": 0.5706, - "question_id": "gpt4_7abb270c", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "5808284e", - "ultrachat_253954", - "answer_7093d898_2", - "answer_7093d898_1", - "39948bcf_4", - "answer_7093d898_6", - "087d2b0a_2", - "sharegpt_W6t3Ck0_0", - "answer_7093d898_5", - "ultrachat_41971" - ] - }, - { - "answer_session_ids": [ - "answer_16bd5ea5_1", - "answer_16bd5ea5_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_1e4a8aeb", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_16bd5ea5_1", - "answer_16bd5ea5_2", - "28621d6a_2", - "f7838a91_1", - "faad7d7a_1", - "bc5827b5", - "15564b61_2", - "fd18e0dd_1", - "ultrachat_413946", - "ultrachat_292933" - ] - }, - { - "answer_session_ids": [ - "answer_be07688f_1", - "answer_be07688f_2" - ], - "ndcg_at_10": 0.8503, - "ndcg_at_5": 0.8503, - "question_id": "gpt4_4fc4f797", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_be07688f_1", - "39566615_1", - "814d3b22", - "47c5482a", - "answer_be07688f_2", - "660cea1d", - "bdcee74e_3", - "7e02e59b_4", - "86298245_2", - "0c783e4a_1" - ] - }, - { - "answer_session_ids": [ - "answer_4bebc782_1", - "answer_4bebc782_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "4dfccbf7", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_4bebc782_1", - "answer_4bebc782_2", - "a5e69b52_1", - "8fa624b2_6", - "491ebc5d_2", - "36f8b380_1", - "ultrachat_327454", - "sharegpt_e1DTWDa_23", - "sharegpt_cXkL3cR_69", - "ultrachat_373047" - ] - }, - { - "answer_session_ids": [ - "answer_e831a29f_1", - "answer_e831a29f_2" - ], - "ndcg_at_10": 0.8503, - "ndcg_at_5": 0.8503, - "question_id": "gpt4_61e13b3c", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e831a29f_1", - "f3fcf4fd_2", - "a6f2320f_2", - "c0d099e6_2", - "answer_e831a29f_2", - "cc380119_3", - "8ca0085a_2", - "2b5c911e_3", - "8e10bf6b_1", - "b4923852" - ] - }, - { - "answer_session_ids": [ - "answer_e6c20e52_3", - "answer_e6c20e52_2", - "answer_e6c20e52_1" - ], - "ndcg_at_10": 0.5641, - "ndcg_at_5": 0.6309, - "question_id": "gpt4_45189cb4", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "5c16fe0b_1", - "answer_e6c20e52_3", - "ultrachat_307270", - "c9d35c00_2", - "dc3ee1d1_1", - "sharegpt_hgGAUvu_0", - "ebf5b3bc_2", - "6593cb8b_1", - "66ab6260", - "answer_e6c20e52_1" - ] - }, - { - "answer_session_ids": [ - "answer_c9d35c00_1", - "answer_c9d35c00_2" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "2ebe6c90", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c9d35c00_2", - "0f73eee7_2", - "answer_c9d35c00_1", - "sharegpt_jyMHY1J_31", - "8fa64936_1", - "9ccea2f0_2", - "c3c1f84a_1", - "4d1b9b0d_4", - "453d354e_1", - "ultrachat_68168" - ] - }, - { - "answer_session_ids": [ - "answer_8c64ce25_2", - "answer_8c64ce25_1", - "answer_8c64ce25_3" - ], - "ndcg_at_10": 0.3155, - "ndcg_at_5": 0.0, - "question_id": "gpt4_e061b84f", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 0.0, - "top_session_ids": [ - "0a6bf5e4_1", - "3275acf9", - "4cc9270f", - "bab3f409", - "73bc3176_2", - "539c2346_3", - "77974d02", - "answer_8c64ce25_3", - "ultrachat_129606", - "9345f7dc_4" - ] - }, - { - "answer_session_ids": [ - "answer_61d1be50_1", - "answer_61d1be50_2" - ], - "ndcg_at_10": 0.8772, - "ndcg_at_5": 0.8772, - "question_id": "370a8ff4", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_61d1be50_1", - "11c0b1c5", - "cfde748a", - "answer_61d1be50_2", - "sharegpt_UbMVpdp_93", - "2b4eaee0_1", - "sharegpt_gVwWKqg_0", - "sharegpt_JRQtEcd_115", - "31d5ad91_1", - "8c64ce25_1" - ] - }, - { - "answer_session_ids": [ - "answer_f999b05b_1", - "answer_f999b05b_4", - "answer_f999b05b_2", - "answer_f999b05b_5", - "answer_f999b05b_3" - ], - "ndcg_at_10": 0.8401, - "ndcg_at_5": 0.8855, - "question_id": "gpt4_d6585ce8", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_f999b05b_3", - "51c82c8e_2", - "answer_f999b05b_5", - "ultrachat_225491", - "answer_f999b05b_2", - "dc0802a9_1", - "c1e598cd_3", - "6ec9bed9", - "answer_f999b05b_4", - "answer_f999b05b_1" - ] - }, - { - "answer_session_ids": [ - "answer_f964cea3_1", - "answer_f964cea3_2" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "gpt4_4ef30696", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_f964cea3_2", - "80b3f093", - "answer_f964cea3_1", - "d9868305_2", - "sharegpt_ELYlA30_25", - "4b29957c_1", - "4067bede", - "1ccb08a0", - "750b437b_1", - "sharegpt_K8mXaIT_0" - ] - }, - { - "answer_session_ids": [ - "answer_e3aa84c4_2", - "answer_e3aa84c4_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_ec93e27f", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e3aa84c4_1", - "answer_e3aa84c4_2", - "5a994ed3_2", - "sharegpt_Sg0ctDe_14", - "6b63bdc5_2", - "5de5c1ed_1", - "85846900_2", - "ultrachat_357838", - "dd049be8", - "sharegpt_1ZQntAk_0" - ] - }, - { - "answer_session_ids": [ - "answer_88841f26_1", - "answer_88841f26_2" - ], - "ndcg_at_10": 0.5, - "ndcg_at_5": 0.5, - "question_id": "6e984301", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "b38766c1_2", - "535ce50e_2", - "answer_88841f26_1", - "b500d9be", - "55c23a23_2", - "ultrachat_402356", - "954c790c_2", - "answer_88841f26_2", - "2bc4ae7d", - "018426a5" - ] - }, - { - "answer_session_ids": [ - "answer_0dd54b7c_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "8077ef71", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_0dd54b7c_1", - "f7ffddbc", - "fd88019d", - "5365840b_1", - "ultrachat_468198", - "sharegpt_Wi7Op1u_39", - "sharegpt_11WxPMZ_0", - "d87a6ef8", - "a993bf57_2", - "sharegpt_hoQsVGz_51" - ] - }, - { - "answer_session_ids": [ - "answer_d8a1af6b_1", - "answer_d8a1af6b_2", - "answer_d8a1af6b_3", - "answer_d8a1af6b_4", - "answer_d8a1af6b_5" - ], - "ndcg_at_10": 0.6181, - "ndcg_at_5": 0.5013, - "question_id": "gpt4_f420262c", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "sharegpt_jFDrHZ3_0", - "sharegpt_B0riiep_0", - "116bae09", - "answer_d8a1af6b_4", - "answer_d8a1af6b_3", - "answer_d8a1af6b_5", - "answer_d8a1af6b_1", - "answer_d8a1af6b_2", - "ultrachat_446175", - "ultrachat_548517" - ] - }, - { - "answer_session_ids": [ - "answer_d97db962_1", - "answer_d97db962_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_8e165409", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_d97db962_2", - "answer_d97db962_1", - "sharegpt_JRQtEcd_115", - "4c9f6bf5_2", - "09599d45_1", - "ultrachat_572191", - "ultrachat_106695", - "d5b3cf54_4", - "f1fe6512_1", - "ultrachat_577133" - ] - }, - { - "answer_session_ids": [ - "answer_aed8cf17_1", - "answer_aed8cf17_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_74aed68e", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_aed8cf17_2", - "answer_aed8cf17_1", - "318048af_2", - "9523d17c", - "69a9211c_1", - "25b3e36c_2", - "470f29c0_1", - "50d66391_6", - "sharegpt_O3FeoHQ_153", - "sharegpt_DXFHJtN_0" - ] - }, - { - "answer_session_ids": [ - "answer_43ba3965_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "bcbe585f", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_43ba3965_1", - "a158b7e7", - "7fa8099b_1", - "a98ff421_4", - "44f94343_1", - "5dbe11b9", - "6bed7862", - "8166673a", - "896162a2", - "8953c8c8_2" - ] - }, - { - "answer_session_ids": [ - "answer_1e2369c9_1", - "answer_1e2369c9_2" - ], - "ndcg_at_10": 0.8772, - "ndcg_at_5": 0.8772, - "question_id": "gpt4_21adecb5", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_1e2369c9_1", - "ultrachat_555122", - "ultrachat_444790", - "answer_1e2369c9_2", - "e1f37e24", - "sharegpt_ipLglky_42", - "6c574737_2", - "128082f8_2", - "ultrachat_333888", - "19f2e2c5_2" - ] - }, - { - "answer_session_ids": [ - "answer_c18d480b_1" - ], - "ndcg_at_10": 0.4307, - "ndcg_at_5": 0.4307, - "question_id": "5e1b23de", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "1a5344e9_3", - "1380576d_1", - "ultrachat_153798", - "answer_c18d480b_1", - "809021de", - "c3f945f3_1", - "a13abca5_2", - "c28aed33", - "d8e33f5c_2", - "8278312e_1" - ] - }, - { - "answer_session_ids": [ - "answer_5850de18_2", - "answer_5850de18_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_98f46fc6", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_5850de18_2", - "answer_5850de18_1", - "sharegpt_rnL5VQO_0", - "6172cc72_3", - "8e3b479f_1", - "sharegpt_hPTUZia_0", - "ultrachat_238480", - "28897044", - "ultrachat_45511", - "32e292f2" - ] - }, - { - "answer_session_ids": [ - "answer_184c8f56_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_af6db32f", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_184c8f56_1", - "aee13015_6", - "ultrachat_552923", - "sharegpt_1jjEIai_173", - "e0b8c937_1", - "ea834a17_2", - "f26846e1_3", - "b7c70f28_3", - "8930493f_2", - "sharegpt_hDjMGr7_70" - ] - }, - { - "answer_session_ids": [ - "answer_0d4d0347_1", - "answer_0d4d0347_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "eac54adc", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_0d4d0347_2", - "answer_0d4d0347_1", - "b40f0a7a", - "ae051325_3", - "bf234051_2", - "32fc393b_2", - "ultrachat_387180", - "74133043", - "75a306dd_3", - "sharegpt_PgmAp0p_0" - ] - }, - { - "answer_session_ids": [ - "answer_ad8a76cb_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_7ddcf75f", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ad8a76cb_1", - "sharegpt_ZUxBndS_26", - "8c652eb0_2", - "sharegpt_mMdyu3t_0", - "79d122f0", - "7ba84ed3_2", - "9283c152", - "ultrachat_281183", - "07e6a3bd_1", - "sharegpt_5RciDTC_0" - ] - }, - { - "answer_session_ids": [ - "answer_f6d6e33f_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_a2d1d1f6", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_f6d6e33f_1", - "ultrachat_476519", - "f5cf8815_4", - "ultrachat_575969", - "894dd0c2_3", - "2c31d8be_3", - "sharegpt_gI1U1eK_4", - "90f7041a_2", - "a1bfb382_1", - "ultrachat_68856" - ] - }, - { - "answer_session_ids": [ - "answer_581ab834_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_85da3956", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_581ab834_1", - "fcc7fe77_2", - "ultrachat_373754", - "sharegpt_h1UXagU_1", - "14b000d9_1", - "sharegpt_i59HUGf_0", - "sharegpt_ewad5Sr_0", - "8da8c7e0", - "6c184bde_2", - "97f8a545_2" - ] - }, - { - "answer_session_ids": [ - "answer_550bb2d1_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_b0863698", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_550bb2d1_1", - "91e581ad_3", - "4a79d078_2", - "63bc9fa3", - "5b014f9b_1", - "554b054e_2", - "65f6f130_1", - "833750a1", - "bdcee74e_2", - "sharegpt_NITaMP6_0" - ] - }, - { - "answer_session_ids": [ - "answer_9793daa3_2", - "answer_9793daa3_1" - ], - "ndcg_at_10": 0.8316, - "ndcg_at_5": 1.0, - "question_id": "gpt4_68e94287", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_9793daa3_2", - "ultrachat_209526", - "60876def_1", - "50f136f2_3", - "74bcc8f4_1", - "answer_9793daa3_1", - "sharegpt_nh3eDLi_9", - "sharegpt_XzAEFL4_0", - "dadd6379_1", - "afc64a5c_1" - ] - }, - { - "answer_session_ids": [ - "answer_e28c1f0d_1", - "answer_e28c1f0d_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_e414231e", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e28c1f0d_1", - "answer_e28c1f0d_2", - "9f8fc173_3", - "d3065d85", - "1bab001f", - "1980bbfa_3", - "d4439279_2", - "a490e4b8_1", - "7e3fa056_2", - "ultrachat_524205" - ] - }, - { - "answer_session_ids": [ - "answer_cf021b36_1", - "answer_cf021b36_2", - "answer_cf021b36_3" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_7ca326fa", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_cf021b36_2", - "answer_cf021b36_3", - "answer_cf021b36_1", - "8d0ca228", - "b4025421", - "5429e768_1", - "sharegpt_7SCdbX2_0", - "1301fbd1", - "sharegpt_CYVtC0w_0", - "604de0a3_2" - ] - }, - { - "answer_session_ids": [ - "answer_e22d6aef_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_7bc6cf22", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e22d6aef_1", - "e6cc6157_1", - "09d021e7_1", - "ultrachat_215356", - "c82c18dd_1", - "ultrachat_330542", - "ultrachat_465755", - "3e43bf49_1", - "20a1ca39", - "eca3b71c_2" - ] - }, - { - "answer_session_ids": [ - "answer_c9d35c00_1", - "answer_c9d35c00_2" - ], - "ndcg_at_10": 0.3562, - "ndcg_at_5": 0.0, - "question_id": "2ebe6c92", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 0.0, - "top_session_ids": [ - "9d4312f6_3", - "fb303dd2_2", - "ca3a4e4f_1", - "54c5a89c_1", - "16756728_1", - "answer_c9d35c00_2", - "84048eba_5", - "sharegpt_9dKNTQo_0", - "b30a47be_2", - "ultrachat_575902" - ] - }, - { - "answer_session_ids": [ - "answer_8c64ce26_2", - "answer_8c64ce26_1", - "answer_8c64ce26_3" - ], - "ndcg_at_10": 0.3869, - "ndcg_at_5": 0.3869, - "question_id": "gpt4_e061b84g", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "ae15e8b6_1", - "9dac9e37_1", - "68705467_1", - "3d6a2b24_1", - "answer_8c64ce26_3", - "51221cac", - "ce584ba0", - "sharegpt_rB6IP4G_0", - "ultrachat_231231", - "ultrachat_391029" - ] - }, - { - "answer_session_ids": [ - "answer_0b4a8adc_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "71017277", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_0b4a8adc_1", - "85a1be56_2", - "976ca0d9", - "f8de4e92_4", - "5854eebc_2", - "e64f9553_1", - "40a1e681_1", - "ultrachat_557308", - "ultrachat_542251", - "e5b86922_1" - ] - }, - { - "answer_session_ids": [ - "answer_4bfcc251_4", - "answer_4bfcc251_3", - "answer_4bfcc251_2", - "answer_4bfcc251_1" - ], - "ndcg_at_10": 0.8667, - "ndcg_at_5": 0.8855, - "question_id": "b46e15ee", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_4bfcc251_1", - "918c4788_1", - "answer_4bfcc251_2", - "69664c72", - "answer_4bfcc251_3", - "c92239bc", - "answer_4bfcc251_4", - "495cf518_2", - "sharegpt_RhSlNyz_0", - "aa31c397_2" - ] - }, - { - "answer_session_ids": [ - "answer_f999b05c_1", - "answer_f999b05c_4", - "answer_f999b05c_2", - "answer_f999b05c_5", - "answer_f999b05c_3" - ], - "ndcg_at_10": 0.6154, - "ndcg_at_5": 0.5438, - "question_id": "gpt4_d6585ce9", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "a554ed79_4", - "58dd3f35", - "answer_f999b05c_3", - "87f5cb44", - "answer_f999b05c_2", - "answer_f999b05c_4", - "answer_f999b05c_5", - "a7aed4cc_2", - "17d91635_2", - "d8e33f5c_abs_1" - ] - }, - { - "answer_session_ids": [ - "answer_16bd5ea6_1", - "answer_16bd5ea6_2" - ], - "ndcg_at_10": 0.3333, - "ndcg_at_5": 0.0, - "question_id": "gpt4_1e4a8aec", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 0.0, - "top_session_ids": [ - "9f1f3bf4", - "b34a52aa_3", - "a8086137", - "4c8455cb_2", - "b1265fd7_1", - "sharegpt_9JZCfee_0", - "answer_16bd5ea6_2", - "e675c69a_1", - "1c662b7b_1", - "e41b78c7_2" - ] - }, - { - "answer_session_ids": [ - "answer_d8a1af6c_1", - "answer_d8a1af6c_2", - "answer_d8a1af6c_5", - "answer_d8a1af6c_4", - "answer_d8a1af6c_3" - ], - "ndcg_at_10": 0.967, - "ndcg_at_5": 0.9829, - "question_id": "gpt4_f420262d", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_d8a1af6c_5", - "answer_d8a1af6c_2", - "answer_d8a1af6c_1", - "27639dd8_3", - "answer_d8a1af6c_3", - "sharegpt_UUFCRmF_13", - "answer_d8a1af6c_4", - "09e6f061", - "e1023f28", - "00c7b769_1" - ] - }, - { - "answer_session_ids": [ - "answer_d00ba6d1_1", - "answer_d00ba6d1_2" - ], - "ndcg_at_10": 0.4486, - "ndcg_at_5": 0.4307, - "question_id": "gpt4_59149c78", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "sharegpt_GI6737T_2", - "765ce8a7_2", - "23754665", - "answer_d00ba6d1_2", - "a8ac3d1d_1", - "84762061_1", - "sharegpt_YENXhFv_0", - "ultrachat_131385", - "answer_d00ba6d1_1", - "b2341a22" - ] - }, - { - "answer_session_ids": [ - "answer_e28c1f0e_1", - "answer_e28c1f0e_2" - ], - "ndcg_at_10": 0.7904, - "ndcg_at_5": 1.0, - "question_id": "gpt4_e414231f", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e28c1f0e_1", - "ultrachat_150948", - "07a26dfa_5", - "deb5bb60", - "135f761b_1", - "78ef59d0_1", - "sharegpt_K8mXaIT_0", - "752c438c_3", - "fe82a033_2", - "answer_e28c1f0e_2" - ] - }, - { - "answer_session_ids": [ - "answer_add9b013_2", - "answer_add9b013_1" - ], - "ndcg_at_10": 0.0, - "ndcg_at_5": 0.0, - "question_id": "gpt4_4929293b", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 0.0, - "r_at_5": 0.0, - "top_session_ids": [ - "sharegpt_KFhIUCO_0", - "b4ed7223_3", - "cae65795_1", - "ultrachat_511609", - "bda611f6_3", - "ea8c38f8", - "sharegpt_MB6e5CL_23", - "sharegpt_15lfOiQ_39", - "78bab597_2", - "ultrachat_156176" - ] - }, - { - "answer_session_ids": [ - "answer_9b09d95b_1" - ], - "ndcg_at_10": 0.4307, - "ndcg_at_5": 0.4307, - "question_id": "gpt4_468eb064", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "a2f0054f", - "db1aefb6_3", - "4d4df0e0_1", - "answer_9b09d95b_1", - "1e5bd28d_2", - "sharegpt_0PLBHdX_15", - "ultrachat_109577", - "sharegpt_P54kbvt_0", - "ultrachat_337251", - "cc3c5fa9_1" - ] - }, - { - "answer_session_ids": [ - "answer_ff201787_1", - "answer_ff201787_2" - ], - "ndcg_at_10": 0.6509, - "ndcg_at_5": 0.6509, - "question_id": "gpt4_fa19884d", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "ccc87da9_1", - "answer_ff201787_1", - "5f9dd782", - "answer_ff201787_2", - "eb4d1c82_1", - "55da3cbf_2", - "f7a61595_3", - "f910dc31_4", - "ultrachat_525919", - "ultrachat_483034" - ] - }, - { - "answer_session_ids": [ - "answer_dba89488_2", - "answer_dba89488_1" - ], - "ndcg_at_10": 0.3333, - "ndcg_at_5": 0.0, - "question_id": "9a707b82", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 0.0, - "top_session_ids": [ - "8e78fa70_1", - "fab41c07", - "79d122f0", - "sharegpt_bUaSM2v_0", - "990f3ef9_2", - "f8e9d4fa", - "answer_dba89488_1", - "7a4d00b3_2", - "ultrachat_466884", - "09e97c6f_2" - ] - }, - { - "answer_session_ids": [ - "answer_0d4d0348_1", - "answer_0d4d0348_2" - ], - "ndcg_at_10": 0.301, - "ndcg_at_5": 0.0, - "question_id": "eac54add", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 0.0, - "top_session_ids": [ - "2ea6fda4_1", - "7bb01253", - "7966888b_2", - "577da76d_2", - "2946bf12", - "c3023a45_5", - "773aebbd_3", - "369695b4_2", - "answer_0d4d0348_2", - "28269633_2" - ] - }, - { - "answer_session_ids": [ - "answer_4bebc783_1", - "answer_4bebc783_2" - ], - "ndcg_at_10": 0.7977, - "ndcg_at_5": 1.0, - "question_id": "4dfccbf8", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_4bebc783_1", - "sharegpt_3D3oQC0_213", - "a63ad8e3_3", - "1c177942_4", - "b4f63a70_3", - "7db28e68_1", - "1bc87711_1", - "edd89480_1", - "answer_4bebc783_2", - "ultrachat_444490" - ] - }, - { - "answer_session_ids": [ - "answer_f4ea84fc_3", - "answer_f4ea84fc_2", - "answer_f4ea84fc_1" - ], - "ndcg_at_10": 0.9469, - "ndcg_at_5": 0.9469, - "question_id": "0bc8ad93", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_f4ea84fc_1", - "answer_f4ea84fc_2", - "97338ddd_1", - "2c185a2b_1", - "answer_f4ea84fc_3", - "ultrachat_260376", - "dd25eeb5_2", - "7285299a_2", - "d59d1a8d_1", - "0d8f8cc0_2" - ] - }, - { - "answer_session_ids": [ - "answer_88841f27_1", - "answer_88841f27_2" - ], - "ndcg_at_10": 0.3562, - "ndcg_at_5": 0.0, - "question_id": "6e984302", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 0.0, - "top_session_ids": [ - "7285299a_8", - "de1f4aec_1", - "3535dbf0_4", - "7cd7c296_1", - "c9dc9158_1", - "answer_88841f27_1", - "6efebca9", - "96c743d0_2", - "c03fc56c_1", - "sharegpt_4Wjx3pH_0" - ] - }, - { - "answer_session_ids": [ - "answer_56521e66_1" - ], - "ndcg_at_10": 0.0, - "ndcg_at_5": 0.0, - "question_id": "gpt4_8279ba03", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 0.0, - "r_at_5": 0.0, - "top_session_ids": [ - "b357fb8b_2", - "ultrachat_526748", - "2ef55f49_3", - "864a563d_5", - "49b78a55_1", - "ba9f938b_1", - "518d26d3_4", - "bb107057_2", - "570fe405", - "50d66391_4" - ] - }, - { - "answer_session_ids": [ - "answer_a17423e8_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_b5700ca0", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_a17423e8_1", - "93395e5f_3", - "534be93d_1", - "109e07e8_2", - "sharegpt_FpTfRvR_0", - "ultrachat_23298", - "b403d6f0_4", - "e789afdb_2", - "ultrachat_101752", - "ultrachat_160178" - ] - }, - { - "answer_session_ids": [ - "answer_9793daa4_2", - "answer_9793daa4_1" - ], - "ndcg_at_10": 0.4307, - "ndcg_at_5": 0.4307, - "question_id": "gpt4_68e94288", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "4c49e37f", - "128f4e4d_2", - "9d2fdc71", - "answer_9793daa4_1", - "c99dcd81", - "0a7c4f1c_1", - "767b1b89", - "cff08ec1_1", - "ultrachat_396489", - "d2606fe9" - ] - }, - { - "answer_session_ids": [ - "answer_4be1b6b4_1", - "answer_4be1b6b4_2", - "answer_4be1b6b4_3" - ], - "ndcg_at_10": 0.5391, - "ndcg_at_5": 0.5, - "question_id": "gpt4_2655b836", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "d8a1af6b_4", - "9e05d9a0", - "answer_4be1b6b4_2", - "ultrachat_268202", - "e3e66b50", - "0f2434e8", - "answer_4be1b6b4_3", - "answer_4be1b6b4_1", - "b876911c", - "ea8cc87b" - ] - }, - { - "answer_session_ids": [ - "answer_1c6b85ea_2", - "answer_1c6b85ea_1" - ], - "ndcg_at_10": 0.6934, - "ndcg_at_5": 0.6934, - "question_id": "gpt4_2487a7cb", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "4f5880c6_6", - "answer_1c6b85ea_1", - "answer_1c6b85ea_2", - "sharegpt_QN26oUg_0", - "67a8ef5f", - "f87fea58_4", - "sharegpt_CHaU4ax_10", - "sharegpt_XNcyasy_17", - "4c1982d8_2", - "sharegpt_EJC4YOh_0" - ] - }, - { - "answer_session_ids": [ - "answer_b535969f_1", - "answer_b535969f_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_76048e76", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_b535969f_1", - "answer_b535969f_2", - "6a78e959_2", - "33a39aa7_2", - "88e01130_1", - "33630c9e_2", - "ultrachat_273271", - "sharegpt_okVqBzp_13", - "sharegpt_inuIr9J_119", - "sharegpt_vx14orQ_11" - ] - }, - { - "answer_session_ids": [ - "answer_5328c3c2_2", - "answer_5328c3c2_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_2312f94c", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_5328c3c2_2", - "answer_5328c3c2_1", - "a764b677_2", - "869fe9ba_2", - "sharegpt_jE25Ibo_0", - "3bbf4800_1", - "e831a29f_2", - "a1c1739c_3", - "ultrachat_44021", - "sharegpt_XYtuANl_9" - ] - }, - { - "answer_session_ids": [ - "answer_e936197f_1", - "answer_e936197f_2" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "0bb5a684", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e936197f_1", - "1a14d6cb_1", - "answer_e936197f_2", - "3c8e7c0e_3", - "ultrachat_62919", - "565929a2_2", - "b10d0907_2", - "826d51da_3", - "d794dec9_2", - "sharegpt_d3UV0eu_0" - ] - }, - { - "answer_session_ids": [ - "answer_6ea1541e_2", - "answer_6ea1541e_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "08f4fc43", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_6ea1541e_1", - "answer_6ea1541e_2", - "f3bd00ec_2", - "ultrachat_292416", - "sharegpt_JJu53iW_0", - "50303fd0_1", - "9216c4d3_4", - "sharegpt_zqQa6sO_0", - "sharegpt_wXqtiUe_0", - "cdbd6862_1" - ] - }, - { - "answer_session_ids": [ - "answer_d39b7977_1", - "answer_d39b7977_2" - ], - "ndcg_at_10": 0.8772, - "ndcg_at_5": 0.8772, - "question_id": "2c63a862", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_d39b7977_2", - "33baaaab_1", - "93a5cfe6", - "answer_d39b7977_1", - "1854abc7_2", - "ad175dc4_2", - "0dde26d4_3", - "ultrachat_179062", - "ultrachat_149914", - "918c4788_2" - ] - }, - { - "answer_session_ids": [ - "answer_7a4a93f1_2", - "answer_7a4a93f1_1" - ], - "ndcg_at_10": 0.8772, - "ndcg_at_5": 0.8772, - "question_id": "gpt4_385a5000", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_7a4a93f1_2", - "8062f4ab_2", - "80aefc49_1", - "answer_7a4a93f1_1", - "29d27eaa_2", - "sharegpt_hcaZN94_507", - "28f74b09_1", - "ultrachat_480527", - "ultrachat_528011", - "ultrachat_286149" - ] - }, - { - "answer_session_ids": [ - "answer_1cc3cd0c_1", - "answer_1cc3cd0c_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "2a1811e2", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_1cc3cd0c_2", - "answer_1cc3cd0c_1", - "b0e040c1", - "901a6763_2", - "ecfd2047_1", - "e9fb10e7_1", - "ultrachat_187883", - "sharegpt_c9R1uSD_0", - "89e05490", - "f5a4d792_1" - ] - }, - { - "answer_session_ids": [ - "answer_b3763b6b_1", - "answer_b3763b6b_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "bbf86515", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_b3763b6b_1", - "answer_b3763b6b_2", - "df003c93_2", - "592cb8d2", - "75b98d2a_1", - "0c260a71_2", - "e9e19917_1", - "33d0e4b3_2", - "sharegpt_d1Rh99l_63", - "sharegpt_G8j6c8J_38" - ] - }, - { - "answer_session_ids": [ - "answer_099c1b6c_5", - "answer_099c1b6c_2", - "answer_099c1b6c_4", - "answer_099c1b6c_3", - "answer_099c1b6c_1" - ], - "ndcg_at_10": 0.9896, - "ndcg_at_5": 1.0, - "question_id": "gpt4_5dcc0aab", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_099c1b6c_3", - "answer_099c1b6c_1", - "answer_099c1b6c_5", - "answer_099c1b6c_2", - "353d3c6d_1", - "answer_099c1b6c_4", - "3a192d16_1", - "3891baea_3", - "b10c041b_1", - "b31f923a_2" - ] - }, - { - "answer_session_ids": [ - "answer_c4e5d969_2", - "answer_c4e5d969_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_0b2f1d21", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c4e5d969_2", - "answer_c4e5d969_1", - "1cafd864_3", - "f31a0422_1", - "sharegpt_rhWNt54_19", - "ultrachat_455430", - "ultrachat_49268", - "sharegpt_4pDZgCL_14", - "5260c678_2", - "ultrachat_1415" - ] - }, - { - "answer_session_ids": [ - "answer_932ea3bf_2", - "answer_932ea3bf_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "f0853d11", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_932ea3bf_2", - "answer_932ea3bf_1", - "bd8708e2_2", - "sharegpt_9upOSlE_27", - "10c0752c", - "b16a8656_2", - "6e436d54", - "ultrachat_325202", - "ultrachat_114660", - "4c967baa_2" - ] - }, - { - "answer_session_ids": [ - "answer_d50a8a33_1", - "answer_d50a8a33_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_6ed717ea", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_d50a8a33_2", - "answer_d50a8a33_1", - "637bc32a_2", - "15ed03f0_1", - "7e7ecab6", - "d4b6ec0b", - "sharegpt_HQMYIkt_0", - "sharegpt_PuRfIep_7", - "8e657ff7_1", - "e0585cb5_1" - ] - }, - { - "answer_session_ids": [ - "answer_b3070ec4_2", - "answer_b3070ec4_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_70e84552", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_b3070ec4_1", - "answer_b3070ec4_2", - "ultrachat_352927", - "2b9b6224_2", - "30491f6f_1", - "45b155ec_2", - "3f787a78_1", - "ultrachat_573894", - "6ebd9e18_1", - "d5527e62" - ] - }, - { - "answer_session_ids": [ - "answer_4ffa04a2_1", - "answer_4ffa04a2_6", - "answer_4ffa04a2_4", - "answer_4ffa04a2_3", - "answer_4ffa04a2_5", - "answer_4ffa04a2_2" - ], - "ndcg_at_10": 0.9797, - "ndcg_at_5": 1.0, - "question_id": "a3838d2b", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_4ffa04a2_1", - "answer_4ffa04a2_6", - "answer_4ffa04a2_4", - "answer_4ffa04a2_2", - "answer_4ffa04a2_5", - "06317e04", - "8cf0d2a9", - "10305ca9_2", - "638e7094_1", - "answer_4ffa04a2_3" - ] - }, - { - "answer_session_ids": [ - "answer_e5131a1b_2", - "answer_e5131a1b_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_93159ced", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e5131a1b_2", - "ultrachat_387032", - "460be54a", - "ultrachat_365033", - "sharegpt_A83arI9_383", - "sharegpt_9srz3L6_0", - "ultrachat_162445", - "ultrachat_475376", - "ultrachat_304047", - "ultrachat_244924" - ] - }, - { - "answer_session_ids": [ - "answer_3e11e0ae_1", - "answer_3e11e0ae_2" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "gpt4_2d58bcd6", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_3e11e0ae_1", - "0bd0a01c", - "answer_3e11e0ae_2", - "be32ad25_1", - "1bf8b212", - "sharegpt_4tLDGmp_0", - "sharegpt_Xl3kKYf_15", - "3c53154d_1", - "1b205d70_1", - "47e1f8dd_1" - ] - }, - { - "answer_session_ids": [ - "answer_30dfe889_1", - "answer_30dfe889_2" - ], - "ndcg_at_10": 0.6934, - "ndcg_at_5": 0.6934, - "question_id": "gpt4_65aabe59", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "6a7e3aec", - "answer_30dfe889_2", - "answer_30dfe889_1", - "77b8fcd9_1", - "10771408_4", - "31cde680", - "05a37787", - "ultrachat_190669", - "78bcce6a", - "ultrachat_169466" - ] - }, - { - "answer_session_ids": [ - "answer_ab603dd5_1", - "answer_ab603dd5_2" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "982b5123", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ab603dd5_1", - "07ba9acd_2", - "answer_ab603dd5_2", - "e36b0031", - "1bfd5a8b_3", - "359a2a0d", - "f89f184e_1", - "bf8ab267", - "91168bd2_1", - "a200b713_4" - ] - }, - { - "answer_session_ids": [ - "answer_5e3bb940_3", - "answer_5e3bb940_2", - "answer_5e3bb940_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "b9cfe692", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_5e3bb940_1", - "answer_5e3bb940_2", - "answer_5e3bb940_3", - "ed88aded_2", - "sharegpt_oXgiN7q_53", - "sharegpt_JRQtEcd_115", - "2714885f_3", - "da650f3b", - "91d847c1_6", - "ultrachat_265626" - ] - }, - { - "answer_session_ids": [ - "answer_0a00c163_1", - "answer_0a00c163_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_4edbafa2", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_0a00c163_2", - "answer_0a00c163_1", - "22be78fc_2", - "18c0af9c", - "4a006b77", - "725f95e1", - "sharegpt_TQNHiIB_12", - "sharegpt_jZOf9E5_21", - "sharegpt_6MPnc5F_9", - "7533d546_5" - ] - }, - { - "answer_session_ids": [ - "answer_70dc7d08_2", - "answer_70dc7d08_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "c8090214", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_70dc7d08_1", - "answer_70dc7d08_2", - "a4c2a6f7_2", - "sharegpt_xOzzaof_0", - "005ec7cf_1", - "d6c6dadf", - "162ad3bf_1", - "c21c9d97", - "sharegpt_BQ3jrV3_0", - "sharegpt_3cVjZ2b_8" - ] - }, - { - "answer_session_ids": [ - "answer_fb793c87_1", - "answer_fb793c87_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_483dd43c", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_fb793c87_1", - "answer_fb793c87_2", - "3fd56ebf", - "52161c9e_1", - "19c24c11", - "29695e1c_3", - "ultrachat_48790", - "sharegpt_Sg0ctDe_0", - "4243ce01_1", - "ultrachat_374858" - ] - }, - { - "answer_session_ids": [ - "answer_cf425855_1", - "answer_cf425855_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e4e14d04", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_cf425855_2", - "answer_cf425855_1", - "a92165a5", - "ffc627f2", - "5944b36a_1", - "ultrachat_542420", - "f8de4e92_2", - "1c1f5ccc_3", - "ultrachat_325291", - "ultrachat_200098" - ] - }, - { - "answer_session_ids": [ - "answer_cdba3d9f_1", - "answer_cdba3d9f_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "c9f37c46", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_cdba3d9f_1", - "answer_cdba3d9f_2", - "b38766c1_3", - "0cf86cbe_1", - "4ffa04a2_1", - "2ba1710e_1", - "02df1225_1", - "7c9b09ac", - "211bb1c2_1", - "9d4312f6_3" - ] - }, - { - "answer_session_ids": [ - "answer_9af4e346_1", - "answer_9af4e346_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_2c50253f", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_9af4e346_2", - "answer_9af4e346_1", - "ultrachat_120586", - "sharegpt_a6pZtrg_0", - "0138fdec", - "dd81b163_2", - "b6f35f3a_1", - "65bc93cf_2", - "sharegpt_ueZYbCA_15", - "ultrachat_267493" - ] - }, - { - "answer_session_ids": [ - "answer_5e3eeb12_2", - "answer_5e3eeb12_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "dcfa8644", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_5e3eeb12_2", - "answer_5e3eeb12_1", - "b2243528_1", - "0d129c99", - "ultrachat_431004", - "7de51ffe_1", - "sharegpt_NCuLVp2_0", - "b906870f_1", - "4eedac88_1", - "sharegpt_hUAh1Tu_0" - ] - }, - { - "answer_session_ids": [ - "answer_b9d9150e_2", - "answer_b9d9150e_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_b4a80587", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_b9d9150e_2", - "answer_b9d9150e_1", - "ultrachat_389307", - "3cb41ab8_1", - "sharegpt_4tqCoYJ_115", - "9071ba70_2", - "55bcfb77", - "sharegpt_IT52MRw_37", - "sharegpt_HxOhjmq_29", - "9b4f5c73" - ] - }, - { - "answer_session_ids": [ - "answer_8a42fedf_1", - "answer_8a42fedf_3", - "answer_8a42fedf_2" - ], - "ndcg_at_10": 0.9218, - "ndcg_at_5": 1.0, - "question_id": "gpt4_9a159967", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_8a42fedf_2", - "answer_8a42fedf_3", - "ultrachat_534927", - "253742b4_3", - "300cfc4a", - "4ea94f85", - "answer_8a42fedf_1", - "7169e342_2", - "sharegpt_MJrfKtS_0", - "75671e3f_1" - ] - }, - { - "answer_session_ids": [ - "answer_be73098b_2", - "answer_be73098b_1" - ], - "ndcg_at_10": 0.8316, - "ndcg_at_5": 1.0, - "question_id": "cc6d1ec1", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_be73098b_2", - "22db3cc3_4", - "aabf201b", - "9a297400_4", - "ultrachat_316268", - "answer_be73098b_1", - "e549b6b2_2", - "33baaaab_3", - "ultrachat_436945", - "sharegpt_iNlPmuC_9" - ] - }, - { - "answer_session_ids": [ - "answer_72d9aa58_1", - "answer_72d9aa58_2" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "gpt4_8c8961ae", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_72d9aa58_1", - "3b22d17b", - "answer_72d9aa58_2", - "1d14730c_1", - "1cde7ee4_2", - "02f9bedb_2", - "b77d0b58", - "37f0ce6b_1", - "ultrachat_265707", - "d7281662_2" - ] - }, - { - "answer_session_ids": [ - "answer_da704e79_2", - "answer_da704e79_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_d9af6064", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_da704e79_1", - "answer_da704e79_2", - "sharegpt_w9YUs4c_0", - "0d9a6f01", - "sharegpt_VbIxRWJ_33", - "98c83fe4", - "ultrachat_519452", - "sharegpt_brc2wJS_109", - "6cff6108", - "3e3b6d77_2" - ] - }, - { - "answer_session_ids": [ - "answer_6a78e959_2", - "answer_6a78e959_1" - ], - "ndcg_at_10": 0.8772, - "ndcg_at_5": 0.8772, - "question_id": "gpt4_7de946e7", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_6a78e959_2", - "sharegpt_6Yfj9A3_5", - "6155addd_2", - "answer_6a78e959_1", - "caa00337_2", - "sharegpt_mjN4DdI_0", - "f1cf6f00_1", - "c4d370d3_1", - "sharegpt_BacsrkR_0", - "ultrachat_560313" - ] - }, - { - "answer_session_ids": [ - "answer_991d55e5_1", - "answer_991d55e5_2" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "d01c6aa8", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_991d55e5_2", - "2adf224b_2", - "answer_991d55e5_1", - "64460e9d_1", - "f2fa97f5_1", - "fc9a62c0_1", - "7d1f5395", - "ultrachat_333097", - "51fe163e_2", - "sharegpt_YLhXcw8_79" - ] - }, - { - "answer_session_ids": [ - "answer_54f0d6f9_1", - "answer_54f0d6f9_2" - ], - "ndcg_at_10": 0.6509, - "ndcg_at_5": 0.6509, - "question_id": "993da5e2", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "b16a8656_2", - "answer_54f0d6f9_1", - "bcf96d28_3", - "answer_54f0d6f9_2", - "8953c8c8_3", - "a6c0f032_3", - "c69a28f0", - "ultrachat_266485", - "sharegpt_wpbTVG0_0", - "sharegpt_v11Tpg9_0" - ] - }, - { - "answer_session_ids": [ - "answer_016f6bd4_2", - "answer_016f6bd4_1" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "a3045048", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_016f6bd4_1", - "536a334d", - "answer_016f6bd4_2", - "aaf71ce2_2", - "d837b5fa", - "5cb58cc6", - "sharegpt_asoSnhq_0", - "d12d5a98_1", - "ultrachat_406386", - "sharegpt_LC9Pzkr_0" - ] - }, - { - "answer_session_ids": [ - "answer_8464304d_1", - "answer_8464304d_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_d31cdae3", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_8464304d_2", - "answer_8464304d_1", - "41743aae_2", - "ultrachat_74113", - "de695a4e", - "55033f6f", - "809cabca_2", - "ultrachat_324099", - "5ace29d7_1", - "ultrachat_169591" - ] - }, - { - "answer_session_ids": [ - "answer_aa930b56_1", - "answer_aa930b56_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_cd90e484", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_aa930b56_2", - "answer_aa930b56_1", - "9aad36bb_1", - "08c306e2", - "ultrachat_347068", - "4d452cdb_3", - "835e1b84_2", - "4bfcc251_1", - "6043dcf6_1", - "018eac29" - ] - }, - { - "answer_session_ids": [ - "answer_e60a93ff_1", - "answer_e60a93ff_2" - ], - "ndcg_at_10": 0.6509, - "ndcg_at_5": 0.6509, - "question_id": "gpt4_88806d6e", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "ultrachat_562852", - "answer_e60a93ff_1", - "sharegpt_bJh9LPd_0", - "answer_e60a93ff_2", - "20b1b65f_1", - "sharegpt_XYtuANl_11", - "ultrachat_258611", - "c11fb634_1", - "3c11dc79", - "24bfc009_5" - ] - }, - { - "answer_session_ids": [ - "answer_5fcca8bc_2", - "answer_5fcca8bc_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_4cd9eba1", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_5fcca8bc_2", - "answer_5fcca8bc_1", - "f54d04cc", - "456860e7_2", - "6b9b2b1e_1", - "d61f3bb5", - "8242fa3f_1", - "2ef53f61_4", - "0e193841_6", - "sharegpt_2nvkR60_0" - ] - }, - { - "answer_session_ids": [ - "answer_544fe66c_2", - "answer_544fe66c_1", - "answer_544fe66c_3" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_93f6379c", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_544fe66c_2", - "answer_544fe66c_3", - "1b9b2960_3", - "b8a688f6", - "sharegpt_cfESFSo_0", - "e06c6d99_5", - "c07b8034", - "ultrachat_443550", - "d3a34811", - "82e27408_1" - ] - }, - { - "answer_session_ids": [ - "answer_436d4309_2", - "answer_436d4309_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "b29f3365", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_436d4309_2", - "answer_436d4309_1", - "6b7605d1_1", - "6a2f9452_1", - "06ee3665", - "ultrachat_243663", - "95c9666f_5", - "sharegpt_K0i5Zon_0", - "ultrachat_58020", - "4ccc2061" - ] - }, - { - "answer_session_ids": [ - "answer_7a36e820_2", - "answer_7a36e820_3", - "answer_7a36e820_1" - ], - "ndcg_at_10": 0.6871, - "ndcg_at_5": 0.6934, - "question_id": "gpt4_2f56ae70", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "3aac691d_2", - "answer_7a36e820_2", - "answer_7a36e820_1", - "f8e8d445_4", - "647f5851", - "ultrachat_159439", - "answer_7a36e820_3", - "83cd701b_3", - "e61d966a_1", - "sharegpt_KzDwaf1_165" - ] - }, - { - "answer_session_ids": [ - "answer_aaf71ce2_3", - "answer_aaf71ce2_2", - "answer_aaf71ce2_1" - ], - "ndcg_at_10": 0.8772, - "ndcg_at_5": 0.8772, - "question_id": "6613b389", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_aaf71ce2_2", - "e3d8da67", - "5eef2132", - "answer_aaf71ce2_3", - "55bc2e1e", - "2780323d_3", - "f1cc1ddb_1", - "1a65880d", - "2366adbc", - "86c8f348_1" - ] - }, - { - "answer_session_ids": [ - "answer_5a78688d_1", - "answer_5a78688d_2" - ], - "ndcg_at_10": 0.8772, - "ndcg_at_5": 0.8772, - "question_id": "gpt4_78cf46a3", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_5a78688d_1", - "b89a3ec3", - "sharegpt_rx39ipj_0", - "answer_5a78688d_2", - "sharegpt_89M8Aiz_0", - "52b19cba_1", - "ultrachat_305042", - "e8e65a1f", - "a8b4bfb3_1", - "27099532_2" - ] - }, - { - "answer_session_ids": [ - "answer_a68db5db_2", - "answer_a68db5db_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_0a05b494", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_a68db5db_2", - "answer_a68db5db_1", - "0abe3e51", - "sharegpt_24ey7mz_0", - "ultrachat_135150", - "48b68664", - "30677a32", - "ultrachat_273213", - "18687689", - "ultrachat_105014" - ] - }, - { - "answer_session_ids": [ - "answer_faad7d7a_2", - "answer_faad7d7a_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_1a1dc16d", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_faad7d7a_2", - "answer_faad7d7a_1", - "87f1f950_2", - "c12d5181_1", - "14940349_2", - "ultrachat_428346", - "ultrachat_63236", - "ultrachat_92302", - "8bd3814a_2", - "661ee6d9_1" - ] - }, - { - "answer_session_ids": [ - "answer_11a8f823_2", - "answer_11a8f823_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_2f584639", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_11a8f823_2", - "answer_11a8f823_1", - "55ca8b20_3", - "ca555919_3", - "sharegpt_bq9G6bT_2", - "4b3a0ee6_1", - "c6eb17f0_1", - "ultrachat_101401", - "68131948", - "d441f4c5" - ] - }, - { - "answer_session_ids": [ - "answer_53582e7e_2", - "answer_53582e7e_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_213fd887", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_53582e7e_2", - "answer_53582e7e_1", - "9bdbfc62_2", - "10771408_1", - "ultrachat_165091", - "sharegpt_EXJAjmw_7", - "8aa17385", - "dc1f9713_1", - "368276ab_3", - "d10f0307_3" - ] - }, - { - "answer_session_ids": [ - "answer_b10f3828_1", - "answer_b10f3828_2" - ], - "ndcg_at_10": 0.6509, - "ndcg_at_5": 0.6509, - "question_id": "gpt4_5438fa52", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "ultrachat_69599", - "answer_b10f3828_2", - "a8e24e22", - "answer_b10f3828_1", - "290fe09a", - "sharegpt_LVUZee8_35", - "b4f07fef_2", - "ab8dce45", - "d68bab3c_1", - "528f481d_6" - ] - }, - { - "answer_session_ids": [ - "answer_d8e33f5c_1", - "answer_d8e33f5c_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_c27434e8", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_d8e33f5c_2", - "answer_d8e33f5c_1", - "sharegpt_9XlbWvL_0", - "16933961", - "6e2cca63_3", - "4d578deb", - "fd56c5bd_4", - "5d1c7bf7", - "ultrachat_212645", - "sharegpt_DVswHqE_0" - ] - }, - { - "answer_session_ids": [ - "answer_65600ff6_2", - "answer_65600ff6_1" - ], - "ndcg_at_10": 0.6241, - "ndcg_at_5": 0.6241, - "question_id": "gpt4_fe651585", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "e30c4a7b_2", - "answer_65600ff6_1", - "ff6b8624", - "9f220c66_1", - "answer_65600ff6_2", - "ultrachat_576354", - "da828711", - "sharegpt_izuEpAx_20", - "sharegpt_mJ7MnQS_219", - "cee0b8d5" - ] - }, - { - "answer_session_ids": [ - "answer_124f5dc3_2", - "answer_124f5dc3_1" - ], - "ndcg_at_10": 0.6053, - "ndcg_at_5": 0.6309, - "question_id": "8c18457d", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "98cd882c_2", - "answer_124f5dc3_1", - "deb12028_5", - "3c8e7c0e_2", - "2dfc1443_1", - "answer_124f5dc3_2", - "b53f447a_1", - "dc0802a9_3", - "sharegpt_2wY0N2C_0", - "1a5369b9" - ] - }, - { - "answer_session_ids": [ - "answer_a25d4a91_1", - "answer_a25d4a91_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "6a1eabeb", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_a25d4a91_2", - "answer_a25d4a91_1", - "d03995c6", - "4aaf678f_3", - "59a6150d_1", - "6da0bba2_1", - "9f80607d", - "6387e969", - "sharegpt_ZWqMvoL_178", - "sharegpt_9ApEWXK_0" - ] - }, - { - "answer_session_ids": [ - "answer_3f9693b7_1", - "answer_3f9693b7_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "6aeb4375", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_3f9693b7_2", - "answer_3f9693b7_1", - "402e0082_5", - "sharegpt_MWlvrvZ_0", - "1bc87711_1", - "2ed7c45e_2", - "sharegpt_wbbH6cZ_0", - "93bed3d8_1", - "e47d38f4_2", - "b65a30db" - ] - }, - { - "answer_session_ids": [ - "answer_0b1a0942_1", - "answer_0b1a0942_2" - ], - "ndcg_at_10": 0.8772, - "ndcg_at_5": 0.8772, - "question_id": "830ce83f", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_0b1a0942_1", - "8d77be9a_2", - "47c5482a", - "answer_0b1a0942_2", - "639cec58", - "dfbda5e9", - "020149a5_1", - "8bc305e4_2", - "d1e7d11c_2", - "e19c8fd9_1" - ] - }, - { - "answer_session_ids": [ - "answer_3a6f1e82_1", - "answer_3a6f1e82_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "852ce960", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_3a6f1e82_1", - "answer_3a6f1e82_2", - "b0fd1357", - "aaf22693", - "d36d11b9_2", - "a110c4dc_1", - "a76e7e3c_1", - "sharegpt_FYhsZ0Q_0", - "010a28ab_3", - "sharegpt_uiECyCD_66" - ] - }, - { - "answer_session_ids": [ - "answer_6a4f8626_1", - "answer_6a4f8626_2" - ], - "ndcg_at_10": 0.8316, - "ndcg_at_5": 1.0, - "question_id": "945e3d21", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_6a4f8626_2", - "sharegpt_E7VUcfj_0", - "5fcca8bc_1", - "9418c255", - "ultrachat_518638", - "answer_6a4f8626_1", - "dc880372_3", - "sharegpt_zn81iL6_9", - "87252d80_2", - "9bbd38a6" - ] - }, - { - "answer_session_ids": [ - "answer_eecb10d9_1", - "answer_eecb10d9_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "d7c942c3", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_eecb10d9_2", - "answer_eecb10d9_1", - "sharegpt_ezPxUrC_0", - "sharegpt_LOF3smB_25", - "ddf0f116_3", - "sharegpt_vYQdLPe_2", - "sharegpt_OTILwRq_0", - "e5b72a5d_3", - "b2243528_1", - "d15c9567_2" - ] - }, - { - "answer_session_ids": [ - "answer_c44b9df4_1", - "answer_c44b9df4_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "71315a70", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c44b9df4_2", - "answer_c44b9df4_1", - "ultrachat_399278", - "c86a22a2_2", - "b5e94516_2", - "8cfa91c3_1", - "ultrachat_42515", - "sharegpt_ZUxBndS_26", - "ultrachat_119182", - "a08fbe88_1" - ] - }, - { - "answer_session_ids": [ - "answer_e1403127_1", - "answer_e1403127_2" - ], - "ndcg_at_10": 0.8772, - "ndcg_at_5": 0.8772, - "question_id": "89941a93", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e1403127_1", - "8e72316f", - "ultrachat_400703", - "answer_e1403127_2", - "ultrachat_349321", - "sharegpt_bn0XI2B_0", - "ultrachat_352196", - "ultrachat_214088", - "ultrachat_150565", - "ultrachat_151269" - ] - }, - { - "answer_session_ids": [ - "answer_73540165_1", - "answer_73540165_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "ce6d2d27", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_73540165_2", - "answer_73540165_1", - "7a4d00b3_2", - "d8d7eddf_2", - "48ad2a93", - "a46d10a2_1", - "9f6bec4f", - "cd92ac85_1", - "4da24fe5_2", - "sharegpt_jZOf9E5_61" - ] - }, - { - "answer_session_ids": [ - "answer_02e66dec_1", - "answer_02e66dec_2" - ], - "ndcg_at_10": 0.6934, - "ndcg_at_5": 0.6934, - "question_id": "9ea5eabc", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "418c27fa", - "answer_02e66dec_1", - "answer_02e66dec_2", - "62e6593e_1", - "ultrachat_243996", - "d1f30ac6_5", - "6df6fff4", - "8c8b180e", - "7c44bc2d_1", - "6e2cca63_1" - ] - }, - { - "answer_session_ids": [ - "answer_7e9ad7b4_1", - "answer_7e9ad7b4_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "07741c44", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_7e9ad7b4_2", - "answer_7e9ad7b4_1", - "sharegpt_UGg8d44_4", - "af4d4ecd_4", - "sharegpt_CEWG2XW_0", - "d1d6b55b_1", - "3b4c63ba_3", - "aa71db11_1", - "1e0c4b50", - "sharegpt_fH7KHW1_119" - ] - }, - { - "answer_session_ids": [ - "answer_0eb23770_1", - "answer_0eb23770_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "a1eacc2a", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_0eb23770_1", - "answer_0eb23770_2", - "1f035408", - "7daa121d_2", - "521255f7_2", - "fde0ca18", - "93f23301_1", - "80082060_4", - "1ccb1920", - "sharegpt_ePBzYD0_57" - ] - }, - { - "answer_session_ids": [ - "answer_e2f4f947_1", - "answer_e2f4f947_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "184da446", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e2f4f947_2", - "answer_e2f4f947_1", - "bf633415_2", - "82a89303", - "85a49fe0_2", - "341a737e", - "ultrachat_195877", - "sharegpt_w7bjHzU_103", - "30a3c930", - "61ad6074" - ] - }, - { - "answer_session_ids": [ - "answer_8748f791_1", - "answer_8748f791_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "031748ae", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_8748f791_1", - "answer_8748f791_2", - "ultrachat_406392", - "5ac7f9c5_3", - "e0f71050_2", - "39a056ae", - "ultrachat_556006", - "ultrachat_69185", - "ultrachat_219083", - "7a085b95_1" - ] - }, - { - "answer_session_ids": [ - "answer_766ab8da_1", - "answer_766ab8da_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "4d6b87c8", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_766ab8da_2", - "answer_766ab8da_1", - "5c85023a_1", - "cffde796", - "ultrachat_40253", - "60d4332d", - "sharegpt_NUyiNZE_0", - "9af4e346_1", - "sharegpt_mmYyki5_0", - "a0201607_3" - ] - }, - { - "answer_session_ids": [ - "answer_d6d2eba8_1", - "answer_d6d2eba8_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "0f05491a", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_d6d2eba8_1", - "answer_d6d2eba8_2", - "27aba903_2", - "0c260a71_3", - "9760f4ca", - "1b205d70_2", - "e2efa128", - "13905cf6", - "9260a72d", - "sharegpt_DM7idTU_0" - ] - }, - { - "answer_session_ids": [ - "answer_cdbe2250_1", - "answer_cdbe2250_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "08e075c7", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_cdbe2250_2", - "answer_cdbe2250_1", - "b1d9d555_1", - "f7344cf4", - "ae77c245", - "25b3e36c_1", - "77827bdf", - "ultrachat_359377", - "ultrachat_556776", - "ultrachat_269248" - ] - }, - { - "answer_session_ids": [ - "answer_b191df5b_1", - "answer_b191df5b_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "f9e8c073", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_b191df5b_1", - "answer_b191df5b_2", - "0a042aef_2", - "ultrachat_88473", - "ultrachat_219619", - "445e6a7a_1", - "sharegpt_bLaSZdD_0", - "22db3cc3_1", - "50b32eed_2", - "d5283595_1" - ] - }, - { - "answer_session_ids": [ - "answer_c7ddc051_1", - "answer_c7ddc051_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "41698283", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c7ddc051_2", - "answer_c7ddc051_1", - "631e4016", - "47a2bd5a_1", - "431ae25c", - "0e8ba03b_1", - "6554fb34_1", - "28e948f5", - "ultrachat_135408", - "sharegpt_7mlXHfH_41" - ] - }, - { - "answer_session_ids": [ - "answer_9282283d_1", - "answer_9282283d_2" - ], - "ndcg_at_10": 0.5912, - "ndcg_at_5": 0.6309, - "question_id": "2698e78f", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "d76030af_1", - "answer_9282283d_2", - "862218f3_2", - "ebd61d29_2", - "338281a4_1", - "d99bab55_2", - "answer_9282283d_1", - "89c1cdfa_2", - "1cf6c966_2", - "619e96bc" - ] - }, - { - "answer_session_ids": [ - "answer_67074b4b_1", - "answer_67074b4b_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "b6019101", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_67074b4b_1", - "answer_67074b4b_2", - "de43030f_2", - "ultrachat_72569", - "sharegpt_CvVLg2J_18", - "ultrachat_146343", - "9bd36386", - "sharegpt_aVExml0_235", - "ac089c83_4", - "59c6685a_2" - ] - }, - { - "answer_session_ids": [ - "answer_07664d43_1", - "answer_07664d43_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "45dc21b6", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_07664d43_1", - "answer_07664d43_2", - "2f57c190_1", - "5934b91f", - "sharegpt_mMdyu3t_0", - "eb34144a_2", - "0504a710_4", - "ultrachat_318979", - "sharegpt_NSacJOD_0", - "8acb29f9_1" - ] - }, - { - "answer_session_ids": [ - "answer_b0f3dfff_1", - "answer_b0f3dfff_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "5a4f22c0", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_b0f3dfff_2", - "answer_b0f3dfff_1", - "007e7d81_2", - "af778ece_1", - "dba97bb1_3", - "sharegpt_1ZrnYv8_23", - "57a6a404_2", - "sharegpt_cXkL3cR_77", - "sharegpt_IMtsj65_0", - "195f2929" - ] - }, - { - "answer_session_ids": [ - "answer_4dac77cb_1", - "answer_4dac77cb_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "6071bd76", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_4dac77cb_2", - "answer_4dac77cb_1", - "a58eeaf8", - "04616813_2", - "a8899aee_1", - "8d0b4363", - "9de53d68_1", - "ultrachat_551846", - "ultrachat_56513", - "34a230b3" - ] - }, - { - "answer_session_ids": [ - "answer_1a374afa_1", - "answer_1a374afa_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e493bb7c", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_1a374afa_2", - "answer_1a374afa_1", - "9161500f_2", - "7aaa23d7_2", - "sharegpt_jHGL67f_0", - "ultrachat_185447", - "ultrachat_151926", - "ultrachat_575969", - "ultrachat_415114", - "0aacd15f_1" - ] - }, - { - "answer_session_ids": [ - "answer_caf5b52e_1", - "answer_caf5b52e_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "618f13b2", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_caf5b52e_1", - "answer_caf5b52e_2", - "86b56135_2", - "748eb152_2", - "01a120b5", - "dda70510_2", - "a20bc58f", - "ultrachat_47868", - "ultrachat_270398", - "sharegpt_aF6djes_0" - ] - }, - { - "answer_session_ids": [ - "answer_d7de9a6a_1", - "answer_d7de9a6a_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "72e3ee87", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_d7de9a6a_2", - "answer_d7de9a6a_1", - "1c1f5ccc_4", - "ultrachat_523550", - "080a4000_2", - "ultrachat_195333", - "ultrachat_129606", - "sharegpt_KdbXhTW_9", - "ultrachat_233144", - "6f2c01fc_1" - ] - }, - { - "answer_session_ids": [ - "answer_d3bf812b_1", - "answer_d3bf812b_2" - ], - "ndcg_at_10": 0.4218, - "ndcg_at_5": 0.3869, - "question_id": "c4ea545c", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "sharegpt_JJu53iW_0", - "sharegpt_I0uNTjm_16", - "9f9b402a_5", - "ee7f5084_1", - "answer_d3bf812b_1", - "sharegpt_CPzL3ju_0", - "a1f43f8d_2", - "b2d284b1", - "answer_d3bf812b_2", - "ultrachat_369982" - ] - }, - { - "answer_session_ids": [ - "answer_a7b44747_1", - "answer_a7b44747_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "01493427", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_a7b44747_1", - "answer_a7b44747_2", - "55e6ff53", - "077f7bcb_3", - "780b6c7c_1", - "5a840928_1", - "29332a29_2", - "88e73f02", - "c2204106_3", - "sharegpt_kjeGJvK_13" - ] - }, - { - "answer_session_ids": [ - "answer_77f32504_1", - "answer_77f32504_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "6a27ffc2", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_77f32504_1", - "answer_77f32504_2", - "sharegpt_kkqlRlo_13", - "ab483540", - "08effb2b_2", - "sharegpt_KSCqghj_0", - "ultrachat_20656", - "sharegpt_5clkQjb_12", - "0651f7bd_2", - "9ded9e01_2" - ] - }, - { - "answer_session_ids": [ - "answer_52382508_1", - "answer_52382508_2" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "2133c1b5", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_52382508_1", - "ultrachat_354956", - "answer_52382508_2", - "ultrachat_502724", - "25b1afbc", - "8ca0085a_1", - "sharegpt_Jm8wWJN_0", - "cf021b36_1", - "09d14ddf_2", - "1126be1e_2" - ] - }, - { - "answer_session_ids": [ - "answer_fff743f5_1", - "answer_fff743f5_2" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "18bc8abd", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_fff743f5_2", - "ba6b67af_3", - "answer_fff743f5_1", - "ultrachat_379579", - "ultrachat_131871", - "sharegpt_afCogMg_34", - "a35679d5_2", - "sharegpt_vyHqfrX_0", - "sharegpt_hH4KVIN_10" - ] - }, - { - "answer_session_ids": [ - "answer_611b6e83_1", - "answer_611b6e83_2" - ], - "ndcg_at_10": 0.8175, - "ndcg_at_5": 1.0, - "question_id": "db467c8c", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_611b6e83_2", - "6cc835bd", - "1a9f5723_1", - "16ebc8f8_1", - "ecfd2047_1", - "77cf4551_1", - "answer_611b6e83_1", - "ultrachat_137520", - "sharegpt_HBmf6q0_0", - "76967d22" - ] - }, - { - "answer_session_ids": [ - "answer_d08a934d_1", - "answer_d08a934d_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "7a87bd0c", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_d08a934d_1", - "answer_d08a934d_2", - "62369c56_1", - "b98914c6_2", - "2c543c87", - "381227c4_1", - "e250791e_3", - "55a59bc9_4", - "60f9246d", - "15519944_1" - ] - }, - { - "answer_session_ids": [ - "answer_f25c32f5_1", - "answer_f25c32f5_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e61a7584", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_f25c32f5_2", - "answer_f25c32f5_1", - "ultrachat_183954", - "82159da4_1", - "1fd16470_2", - "d02bcc9e_2", - "51add7da_1", - "ultrachat_367590", - "sharegpt_o0MporM_0", - "e1416816_2" - ] - }, - { - "answer_session_ids": [ - "answer_79c395a9_1", - "answer_79c395a9_2" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "1cea1afa", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_79c395a9_2", - "7af38385_1", - "answer_79c395a9_1", - "602ad002_1", - "52d46b4a_1", - "ultrachat_51727", - "ultrachat_67100", - "e9ad5914_5", - "ultrachat_255553", - "sharegpt_X1NXUxZ_0" - ] - }, - { - "answer_session_ids": [ - "answer_babbaccb_1", - "answer_babbaccb_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "ed4ddc30", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_babbaccb_2", - "answer_babbaccb_1", - "d2f4a309", - "8b726b64_1", - "ultrachat_238480", - "4ffa04a2_4", - "499652a0_3", - "ultrachat_342966", - "5d2dc022_1", - "a0c40934_1" - ] - }, - { - "answer_session_ids": [ - "answer_966cecbb_1", - "answer_966cecbb_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "8fb83627", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_966cecbb_1", - "answer_966cecbb_2", - "9315f268_1", - "c15319aa_1", - "02b81ad9", - "0638f6f6_1", - "ultrachat_172298", - "88e01130_1", - "be050ab3_2", - "ultrachat_307166" - ] - }, - { - "answer_session_ids": [ - "answer_8c0712af_1", - "answer_8c0712af_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "b01defab", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_8c0712af_2", - "answer_8c0712af_1", - "b1d9eb66_2", - "sharegpt_VbIxRWJ_33", - "1dcfb9a0_1", - "fc69fd44_7", - "9aa07e25", - "ultrachat_479632", - "sharegpt_fB2ri01_0", - "3891baea_2" - ] - }, - { - "answer_session_ids": [ - "answer_bcce0b73_1", - "answer_bcce0b73_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "22d2cb42", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_bcce0b73_1", - "answer_bcce0b73_2", - "74932466_1", - "060aeced", - "eec72a82_4", - "5a76fadb_2", - "ff898925", - "ultrachat_18212", - "sharegpt_hGQUdgE_10", - "ultrachat_227551" - ] - }, - { - "answer_session_ids": [ - "answer_f2f998c7_1", - "answer_f2f998c7_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "0e4e4c46", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_f2f998c7_1", - "answer_f2f998c7_2", - "b42cbaf0_1", - "60876def_2", - "0773943b", - "71d1e644", - "sharegpt_1YtwybC_15", - "ultrachat_467820", - "ultrachat_180729", - "debc34d7_1" - ] - }, - { - "answer_session_ids": [ - "answer_2cec623b_1", - "answer_2cec623b_2" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "4b24c848", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_2cec623b_2", - "d9e0c03e", - "answer_2cec623b_1", - "1815d1b7", - "sharegpt_iqJr8nh_0", - "d38e5b38", - "ultrachat_198571", - "16e094b3_1", - "d0c1453b_1", - "b0855671_2" - ] - }, - { - "answer_session_ids": [ - "answer_c9f5693c_1", - "answer_c9f5693c_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "7e974930", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c9f5693c_1", - "answer_c9f5693c_2", - "705b5399_3", - "a1e169b1", - "898ce7a5_1", - "bf139883", - "07b2420e_1", - "ultrachat_380252", - "8bd5b3c4_1", - "2cc24b7c_2" - ] - }, - { - "answer_session_ids": [ - "answer_8afdebac_1", - "answer_8afdebac_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "603deb26", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_8afdebac_2", - "answer_8afdebac_1", - "c77250d2", - "433ce12c", - "58e5adf2", - "f379d356_3", - "88611fff", - "7b0dea50_1", - "sharegpt_z6pSZ19_0", - "sharegpt_etklNfN_11" - ] - }, - { - "answer_session_ids": [ - "answer_b28f2c7a_1", - "answer_b28f2c7a_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "59524333", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_b28f2c7a_2", - "answer_b28f2c7a_1", - "ultrachat_458180", - "e4882e09_5", - "9b9ff448_1", - "4507b423", - "sharegpt_KhxXaow_0", - "sharegpt_5FHWj0a_0", - "ultrachat_493944", - "sharegpt_7gIKsqO_104" - ] - }, - { - "answer_session_ids": [ - "answer_8d63a897_1", - "answer_8d63a897_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "5831f84d", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_8d63a897_1", - "answer_8d63a897_2", - "6efce493_2", - "ef84b994_abs_2", - "41c90f4c_2", - "ff67236f_1", - "83dae08b", - "sharegpt_NHidbra_0", - "b818e2a1_1", - "sharegpt_3Rk1i3g_0" - ] - }, - { - "answer_session_ids": [ - "answer_8a791264_1", - "answer_8a791264_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "eace081b", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_8a791264_2", - "answer_8a791264_1", - "4dae77d3", - "4d04b866", - "413b57cb_3", - "9316aae3_1", - "ultrachat_455723", - "164bd5a4", - "29a18eab_3", - "sharegpt_HOO59Ue_137" - ] - }, - { - "answer_session_ids": [ - "answer_90de9b4d_1", - "answer_90de9b4d_2" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "affe2881", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_90de9b4d_1", - "fb5dd87f_7", - "answer_90de9b4d_2", - "ultrachat_443550", - "6c4a5aee", - "f334878c", - "1b71c896_2", - "sharegpt_eM7toF0_0", - "ultrachat_156501", - "ultrachat_541196" - ] - }, - { - "answer_session_ids": [ - "answer_dcd74827_1", - "answer_dcd74827_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "50635ada", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_dcd74827_1", - "answer_dcd74827_2", - "9f9b402a_5", - "42bcee92_2", - "2e53911b_1", - "cb88f886_1", - "add7cc1e", - "1de8082c", - "35bc3132_2", - "b6f35f3a_1" - ] - }, - { - "answer_session_ids": [ - "answer_ac0140ce_1", - "answer_ac0140ce_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e66b632c", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ac0140ce_2", - "answer_ac0140ce_1", - "84098c1b", - "259d58da_2", - "sharegpt_nspc8Nf_199", - "75621e39_1", - "4f51038b", - "805528d6_1", - "487fd03c", - "b3070ec4_1" - ] - }, - { - "answer_session_ids": [ - "answer_a22b654d_1", - "answer_a22b654d_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "0ddfec37", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_a22b654d_2", - "answer_a22b654d_1", - "a8a0db2b_1", - "147ab7e9_7", - "0a699029_2", - "sharegpt_UyBTuTv_0", - "ultrachat_139747", - "sharegpt_cXkL3cR_45", - "f2199726_2", - "60f9246d" - ] - }, - { - "answer_session_ids": [ - "answer_25df025b_1", - "answer_25df025b_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "f685340e", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_25df025b_1", - "answer_25df025b_2", - "a4815e8e_2", - "e862f726_1", - "5035d8d6", - "e7dcd5df", - "4a9eb139_2", - "142e3203_1", - "sharegpt_RuTDId0_29", - "992eb1f3_2" - ] - }, - { - "answer_session_ids": [ - "answer_a5b68517_1", - "answer_a5b68517_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "cc5ded98", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_a5b68517_1", - "answer_a5b68517_2", - "f14a4532_1", - "c63b2c8f", - "ultrachat_323437", - "bd8708e2_2", - "b13d5e4e_1", - "sharegpt_IEsU45Z_23", - "sharegpt_BJmgxmf_0", - "753b7c1f_1" - ] - }, - { - "answer_session_ids": [ - "answer_35d6c0be_1", - "answer_35d6c0be_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "dfde3500", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_35d6c0be_1", - "answer_35d6c0be_2", - "ad1c7ac6", - "86e218e3_2", - "fa68c9d6_2", - "5c18cb32_2", - "59c704ad_1", - "134127c2_2", - "ultrachat_368201", - "7de51ffe_1" - ] - }, - { - "answer_session_ids": [ - "answer_d6028d6e_1", - "answer_d6028d6e_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "69fee5aa", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_d6028d6e_1", - "answer_d6028d6e_2", - "a554ed79_2", - "a0b8b0ad_7", - "8464304d_2", - "ultrachat_152449", - "6685e6cd", - "2af27403_2", - "42adb80d_2", - "5d284d7d_2" - ] - }, - { - "answer_session_ids": [ - "answer_94650bfa_1", - "answer_94650bfa_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "7401057b", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_94650bfa_1", - "answer_94650bfa_2", - "c38a915a_1", - "30f20967_2", - "8de5b7cf_2", - "c3757c5b", - "1450ffad", - "sharegpt_Ah0tRO5_0", - "ultrachat_402967", - "ultrachat_385162" - ] - }, - { - "answer_session_ids": [ - "answer_ae3a122b_1", - "answer_ae3a122b_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "cf22b7bf", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ae3a122b_2", - "answer_ae3a122b_1", - "36d5bbde_1", - "457e26bf", - "8ab0292c_1", - "c8fcdf5a_1", - "8064b6ca", - "22fe1c93", - "e201572a", - "ultrachat_345143" - ] - }, - { - "answer_session_ids": [ - "answer_5126c02d_1", - "answer_5126c02d_2" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "a2f3aa27", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_5126c02d_1", - "b65f51fb_5", - "answer_5126c02d_2", - "57bf4122_1", - "2fa50779_2", - "a8028ce7_2", - "104055de", - "ultrachat_475462", - "a135a911", - "4cf2ea34" - ] - }, - { - "answer_session_ids": [ - "answer_0cdbca92_1", - "answer_0cdbca92_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "c7dc5443", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_0cdbca92_1", - "answer_0cdbca92_2", - "064e1984_1", - "sharegpt_6cz1Sq6_264", - "ultrachat_435372", - "debc34d7_2", - "8a355b36_1", - "728fc45a", - "aefbe381", - "ultrachat_308370" - ] - }, - { - "answer_session_ids": [ - "answer_da72b1b4_1", - "answer_da72b1b4_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "06db6396", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_da72b1b4_1", - "answer_da72b1b4_2", - "6ebdc1fe_2", - "sharegpt_2pwdGxJ_16", - "ultrachat_456942", - "5bd9f1e6_1", - "35038174_4", - "8235b87d", - "824634d9_2", - "sharegpt_C26CwPQ_0" - ] - }, - { - "answer_session_ids": [ - "answer_cd345582_1", - "answer_cd345582_2" - ], - "ndcg_at_10": 0.5912, - "ndcg_at_5": 0.6309, - "question_id": "3ba21379", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "49576fd6", - "answer_cd345582_1", - "67620dbc_4", - "d557e57a", - "ultrachat_224436", - "sharegpt_YGmoq5F_7", - "answer_cd345582_2", - "ultrachat_74850", - "d68838d0_2", - "f64c4793_1" - ] - }, - { - "answer_session_ids": [ - "answer_c6a0c6c2_1", - "answer_c6a0c6c2_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "9bbe84a2", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c6a0c6c2_1", - "answer_c6a0c6c2_2", - "114c1fb7_6", - "ef69c258", - "sharegpt_hcaZN94_546", - "362aa011", - "acad845c_1", - "1fc31387", - "a0f8468c_2", - "sharegpt_hLkmbBL_0" - ] - }, - { - "answer_session_ids": [ - "answer_67be2c38_1", - "answer_67be2c38_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "10e09553", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_67be2c38_2", - "answer_67be2c38_1", - "7d1f5395", - "e5f785de", - "d3da4592_2", - "918c4788_1", - "ff67236f_2", - "ultrachat_196381", - "f4e2933b", - "sharegpt_YPM1YqM_0" - ] - }, - { - "answer_session_ids": [ - "answer_4a97ae40_1", - "answer_4a97ae40_2" - ], - "ndcg_at_10": 0.6934, - "ndcg_at_5": 0.6934, - "question_id": "dad224aa", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "6de8645d_1", - "answer_4a97ae40_2", - "answer_4a97ae40_1", - "c86a22a2_1", - "sharegpt_opQbgPL_13", - "56b5b760", - "dd345e24_2", - "ultrachat_106352", - "ultrachat_468289", - "5c690047_2" - ] - }, - { - "answer_session_ids": [ - "answer_f377cda7_1", - "answer_f377cda7_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "ba61f0b9", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_f377cda7_1", - "answer_f377cda7_2", - "sharegpt_iqJr8nh_0", - "17a1e5c1", - "c9213a03_1", - "85846900_6", - "495cf518_1", - "6b7605d1_2", - "sharegpt_s8Opwwu_0", - "ad578e11_3" - ] - }, - { - "answer_session_ids": [ - "answer_e3892371_1", - "answer_e3892371_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "42ec0761", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e3892371_2", - "answer_e3892371_1", - "27d0d3e7", - "4fb01417_2", - "sharegpt_vWO8y7i_35", - "f849d84c", - "493bd421_2", - "53acd215", - "b37a44e4_2", - "37c3726a_4" - ] - }, - { - "answer_session_ids": [ - "answer_1cb52d0a_1", - "answer_1cb52d0a_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "5c40ec5b", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_1cb52d0a_2", - "answer_1cb52d0a_1", - "ultrachat_333871", - "be050ab3_2", - "a200b713_3", - "9eb7bb35", - "5cc9e1ed_3", - "ultrachat_122152", - "ultrachat_91084", - "6446f6e6" - ] - }, - { - "answer_session_ids": [ - "answer_626e93c4_1", - "answer_626e93c4_2" - ], - "ndcg_at_10": 0.8503, - "ndcg_at_5": 0.8503, - "question_id": "c6853660", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_626e93c4_2", - "6ed1c85e", - "ultrachat_223464", - "33994682_3", - "answer_626e93c4_1", - "71dc2037_2", - "a4863bcb_2", - "ultrachat_544305", - "f8ab60d7", - "bed5fbab" - ] - }, - { - "answer_session_ids": [ - "answer_f762ad8d_1", - "answer_f762ad8d_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "26bdc477", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_f762ad8d_1", - "answer_f762ad8d_2", - "27ce9f13_2", - "sharegpt_EeXTDjc_23", - "ultrachat_438698", - "37b3c75d", - "551eae69_1", - "sharegpt_lWLBUhQ_517", - "sharegpt_pRBsKdE_5", - "sharegpt_bq9G6bT_2" - ] - }, - { - "answer_session_ids": [ - "answer_3bf5b73b_1", - "answer_3bf5b73b_2" - ], - "ndcg_at_10": 0.6309, - "ndcg_at_5": 0.6309, - "question_id": "0977f2af", - "question_type": "knowledge-update", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "b1d9eb66_4", - "answer_3bf5b73b_2", - "b24eddd9_1", - "f115a3db_1", - "ultrachat_554962", - "809cbce9_3", - "357e33b5", - "1692563c_1", - "9b083158_1", - "sharegpt_FqOC5e9_45" - ] - }, - { - "answer_session_ids": [ - "answer_e1403127_1", - "answer_e1403127_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "89941a94", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e1403127_2", - "answer_e1403127_1", - "6e5130ed_1", - "de275215_3", - "74b0227e_2", - "6a7d7055_2", - "f334878c", - "2ec60f51_3", - "522dd987_1", - "sharegpt_CQ3HZO9_71" - ] - }, - { - "answer_session_ids": [ - "answer_7e9ad7b4_1", - "answer_7e9ad7b4_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "07741c45", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_7e9ad7b4_2", - "answer_7e9ad7b4_1", - "37d65d40_3", - "af4a2fd1", - "66c23110_2", - "ce21f0e4", - "ultrachat_162268", - "ultrachat_267176", - "e06c6d99_3", - "ultrachat_27772" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_5Lzox6N_0" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "7161e7e2", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_5Lzox6N_0", - "3c3fee41", - "5850de18_2", - "ultrachat_68050", - "sharegpt_yguSguz_0", - "6b17c99b_2", - "dba97bb1_4", - "5aed819f", - "sharegpt_PjFSCFK_9", - "sharegpt_6kDZTZ2_0" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_234453" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "c4f10528", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_234453", - "107d40f0", - "51c89181_1", - "29116b50_1", - "689fec3d_1", - "11248f50", - "6cf1848e_2", - "sharegpt_yMwEvl7_557", - "21ff9566_2", - "e12cfbcd" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_YkWn1Ne_0" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "89527b6b", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_YkWn1Ne_0", - "ultrachat_117600", - "48d385f0_1", - "644e1d00", - "51c262b6_2", - "4b637e89_1", - "ultrachat_84388", - "sharegpt_htSCqmh_171", - "9c722816", - "07e363d9_2" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_480665" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e9327a54", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_480665", - "ba944cdc", - "11decc61", - "c58fdf1e_2", - "c0d099e6_2", - "28e0ea23_2", - "7613d5ec_2", - "b65f51fb_5", - "sharegpt_NP5eyFN_5", - "sharegpt_Q7EQHIJ_0" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_448704" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "4c36ccef", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_448704", - "3dc8ba5f_2", - "07e6a3bd_1", - "5fc83435_2", - "1ba83815_4", - "89749c78_1", - "7b1d1f43_2", - "sharegpt_m8F4NxL_0", - "940629b2_1", - "sharegpt_1PRjwYK_11" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_IUWQYGQ_0" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "6ae235be", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_IUWQYGQ_0", - "0a1e90d1_2", - "99188307", - "809cabca_1", - "31cde680", - "8e608aad", - "39358a85_1", - "61a91bc9_2", - "cb692a80_3", - "sharegpt_KzDwaf1_211" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_370515" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "7e00a6cb", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_370515", - "4dd1ba8d_2", - "b7dca8f1_1", - "24f78a31_2", - "2fa50779_5", - "956ce238", - "39358a85_3", - "9998814e_2", - "ultrachat_188131", - "a9981dc6_2" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_hA7AkP3_0" - ], - "ndcg_at_10": 0.4307, - "ndcg_at_5": 0.4307, - "question_id": "1903aded", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "66369fde_1", - "184efb91_3", - "9b4c070c", - "answer_sharegpt_hA7AkP3_0", - "ultrachat_110523", - "sharegpt_64onasJ_0", - "201df030", - "8adb232f_1", - "f8246a66_1", - "sharegpt_emE20LI_0" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_cGdjmYo_0" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "ceb54acb", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_cGdjmYo_0", - "511bdbe3_2", - "197d99cc", - "7aa2a9af_2", - "de293134_1", - "f6fd00cf", - "0bbd7094_1", - "ultrachat_150417", - "63b72857", - "c60d5c8a_1" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_m2xJfjo_0" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "f523d9fe", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_m2xJfjo_0", - "193c23bd_1", - "22aa7cca_4", - "debc34d7_1", - "9ccea2f0_1", - "73f75ab4", - "a07f9d5c_1", - "ultrachat_391251", - "661b711f_1", - "fbfda981" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_113156" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "0e5e2d1a", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_113156", - "f84b4266", - "ultrachat_201585", - "c756c34e", - "5d79fd65_2", - "22f9f163_1", - "sharegpt_DcVIUDF_0", - "sharegpt_GPluPmJ_7", - "sharegpt_btv4SkQ_0", - "c4e5d969_2" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_187684" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "fea54f57", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_187684", - "34590cab_1", - "0aa99e2e_1", - "c8854b28", - "712e923c", - "ee659010_2", - "ultrachat_435757", - "ultrachat_129340", - "ultrachat_562162", - "a1e90def_1" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_374124" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "cc539528", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_374124", - "18807892_4", - "8bd3814a_1", - "15a8ffb7_2", - "faa194ea", - "20507d15_1", - "fe1faff9_1", - "sharegpt_0z277Yx_0", - "2a394047", - "sharegpt_1SSsoox_0" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_459954" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "dc439ea3", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_459954", - "1bfd5a8b_2", - "eb7366f2_1", - "6ef99651_1", - "5a81277c", - "ultrachat_186512", - "49d8a382", - "cb5637b3", - "sharegpt_pttddrt_5", - "ultrachat_201433" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_hn3IS1q_0" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "18dcd5a5", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_hn3IS1q_0", - "6ea1541e_1", - "ultrachat_425586", - "959e6cb0_2", - "33d16079_2", - "d9fb1588_1", - "c00077f5_2", - "sharegpt_d1tE544_5", - "sharegpt_MW0whoh_79", - "ultrachat_39471" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_275993" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "488d3006", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_275993", - "ce19df54_3", - "a68090b0_2", - "9a57d65b", - "82b82901", - "a394f6b5_1", - "d53b6bfe_1", - "1af2c0fb_1", - "sharegpt_d9F8HCC_0", - "7e4dab66_1" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_U4oCSfU_7" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "58470ed2", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_U4oCSfU_7", - "sharegpt_oPOTyid_141", - "ff49b5a5_3", - "4050ebff_6", - "a49a1438_2", - "2f2884ad_3", - "b6018747_4", - "ultrachat_458590", - "e45db473_2", - "sharegpt_e8sgAJX_0" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_HFMn2ZX_0" - ], - "ndcg_at_10": 0.6309, - "ndcg_at_5": 0.6309, - "question_id": "8cf51dda", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "daa09295", - "answer_sharegpt_HFMn2ZX_0", - "sharegpt_ftoCcOn_0", - "507514d9_1", - "27639dd8_2", - "ultrachat_365938", - "3358b5ca_2", - "9f0f59d1_1", - "sharegpt_jpiaPoJ_738", - "sharegpt_yoSUjfa_0" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_348449" - ], - "ndcg_at_10": 0.5, - "ndcg_at_5": 0.5, - "question_id": "1d4da289", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "09f1c793", - "ultrachat_213759", - "answer_ultrachat_348449", - "4cd929f8_1", - "517ad0f0", - "sharegpt_eVmxjQZ_0", - "bf85791d", - "a05c7d7d", - "ultrachat_322093", - "ultrachat_329521" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_467053" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "8464fc84", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_467053", - "21d86ebb_2", - "6ac43c5b_4", - "bdc804df_2", - "602b4074_3", - "1d5998ca_1", - "d43b63a6", - "7a0abbe2_2", - "98d39a6d", - "9582aa59_2" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_563222" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "8aef76bc", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_563222", - "146dfabe", - "5209e813_2", - "27d378b2_2", - "08e084f1_1", - "sharegpt_n3Hy63g_0", - "ultrachat_351310", - "ultrachat_368669", - "44b1595a", - "c7f6643e_2" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_417348" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "71a3fd6b", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_417348", - "8419b385_3", - "7f7e26d2_2", - "878271ae_1", - "36743359_3", - "sharegpt_1L5GXZJ_0", - "sharegpt_n3kgf14_17", - "sharegpt_asilfXk_321", - "ultrachat_168420", - "b7146673_1" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_2kpncbX_13" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "2bf43736", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_2kpncbX_13", - "16c3be68_1", - "8897a2e5", - "661ee6d9_1", - "2e89bd03_1", - "7e4cd916_2", - "d79173aa_3", - "445e6a7a_1", - "1a5fb2cd", - "a93750ef_2" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_334948" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "70b3e69b", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_334948", - "09e1a929_2", - "ultrachat_47234", - "sharegpt_oTNStu2_51", - "dd92723f_1", - "ultrachat_478639", - "1388b204_2", - "sharegpt_O8qJclh_0", - "sharegpt_s0wsteT_0", - "c2204106_2" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_6pWK9yx_0" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "8752c811", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_6pWK9yx_0", - "d0a41222_1", - "93f23301_2", - "06b5b860", - "9db30f1c_3", - "5c6a5b7c_1", - "3e48ed93", - "02b63d04_1", - "ultrachat_160178", - "9272a6a2_2" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_CaxTGYP_0" - ], - "ndcg_at_10": 0.6309, - "ndcg_at_5": 0.6309, - "question_id": "3249768e", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "ed1d68f1_1", - "answer_sharegpt_CaxTGYP_0", - "0868982f_1", - "71f2b8ba", - "ultrachat_77515", - "56cf2973", - "172b47cb_2", - "e0bd5f14_2", - "23450b93_3", - "sharegpt_M5zaN8c_0" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_115151" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "1b9b7252", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_115151", - "3829d412_2", - "d3cc5bdc_4", - "53ab4734", - "af4d4ecd_6", - "0e726047", - "330c05e7_1", - "b2e5a019", - "4f6e1b77_1", - "sharegpt_1AhFMpw_5" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_d6JJiqH_76" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "1568498a", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_d6JJiqH_76", - "sharegpt_kRkQrRx_0", - "38de4c2a_1", - "6ad46850_2", - "70fed904", - "4022d960_2", - "sharegpt_trljpZ5_27", - "sharegpt_69agvDU_27", - "983b1954", - "f8e8d445_2" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_H9PiM5G_0" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "6222b6eb", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_H9PiM5G_0", - "caf7480d_1", - "sharegpt_cyc2xbb_0", - "40a90d51", - "2fd5e208", - "544c1e7b", - "b35fadaa", - "facb94a8_3", - "2a500dce_1", - "5209e813_2" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_13075" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e8a79c70", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_13075", - "sharegpt_ha8kUEr_0", - "c0d099e6_2", - "sharegpt_vKSMO9J_69", - "742ea86b_1", - "89aed484_2", - "c2b2b2ea_1", - "87252d80_2", - "ultrachat_225061", - "sharegpt_tfIfm00_14" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_252214" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "d596882b", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_252214", - "dc9bf721_1", - "71888aff_2", - "456807b7_1", - "bef3abcd", - "0dc2efcc_2", - "ultrachat_299672", - "087d2b0a_1", - "5fb3b5ac", - "ultrachat_306390" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_5m7gg5F_0" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e3fc4d6e", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_5m7gg5F_0", - "4d261642", - "sharegpt_TVK4gjw_24", - "d31c3ec8_1", - "ultrachat_161238", - "ultrachat_325941", - "sharegpt_sXKNzPE_0", - "ultrachat_495326", - "ultrachat_443354", - "sharegpt_nAacldL_21" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_427265" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "51b23612", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_427265", - "255c753b_1", - "8672f398_2", - "409015ef_1", - "17f444f3", - "ultrachat_343", - "ultrachat_522170", - "ultrachat_555062", - "ultrachat_221034", - "848f9a52" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_94624" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "3e321797", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_94624", - "9252145e_2", - "11302335_4", - "57144028_2", - "00305dd2_1", - "193c23bd_1", - "385a60c8_1", - "sharegpt_OzK1xD9_0", - "ultrachat_557903", - "sharegpt_EZmhz8p_0" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_195444" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e982271f", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_195444", - "eb0d8dc1_4", - "8a7c65c1_1", - "f72c924e", - "db65997e", - "ultrachat_247528", - "ultrachat_383774", - "sharegpt_Xlpa70t_0", - "934419a6_1", - "3e8f07c9_1" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_NoDZzot_7" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "352ab8bd", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_NoDZzot_7", - "6593cb8b_6", - "337cbfc8_2", - "d600c646", - "55a59bc9_2", - "4a42c62b", - "ultrachat_194309", - "27971ce6_5", - "ultrachat_101751", - "sharegpt_SM8NnLU_13" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_39395" - ], - "ndcg_at_10": 0.3869, - "ndcg_at_5": 0.3869, - "question_id": "fca762bc", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "06a34d82_1", - "d0cdfddb_2", - "8fcc3543_3", - "2bc4ae7d", - "answer_ultrachat_39395", - "42d9467c_1", - "sharegpt_wrN9uUo_77", - "3b7a6ecd_1", - "136dfd61_2", - "sharegpt_ZgtoeGU_0" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_i0tMT9q_9" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "7a8d0b71", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_i0tMT9q_9", - "f07752be_1", - "f8169a6d", - "7a8f5003", - "d5eab084_2", - "c14e3f09_2", - "147ab7e9_5", - "f20e72e4_2", - "sharegpt_P0Qe2Gx_0", - "881d8e9b_1" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_269020" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "a40e080f", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_269020", - "4d32ba1b_1", - "85736a07_2", - "sharegpt_fyAkcGP_81", - "89c7a48a_2", - "4fd6129f_1", - "cd38b6ee", - "ed109bc3", - "sharegpt_oXgiN7q_327", - "c9489af0_1" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_289157" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "8b9d4367", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_289157", - "sharegpt_WZzri9J_0", - "7babf05b", - "031061e6_4", - "sharegpt_iHP3XDi_39", - "5534efcb_1", - "ultrachat_79945", - "36743359_1", - "007e7d81_3", - "sharegpt_MHiC4J8_0" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_4aJsGCH_0" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "5809eb10", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_4aJsGCH_0", - "fc3d6c80", - "sharegpt_GDFyxeT_0", - "9b820206", - "sharegpt_obCC1BP_103", - "d3069305_2", - "22ce6625_1", - "ultrachat_536416", - "ultrachat_275993", - "sharegpt_9M4WbfX_0" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_81riySf_0" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "41275add", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_81riySf_0", - "5b83c26e_3", - "33a39aa7_1", - "b3619c2c", - "a68db5db_1", - "4d84cbfa", - "ultrachat_36085", - "ultrachat_556585", - "sharegpt_dUoAhvI_71", - "sharegpt_fV5wNsl_0" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_qTi81nS_0" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "4388e9dd", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_qTi81nS_0", - "cfd21744_2", - "3197d998_7", - "1ebd753f_2", - "ultrachat_412706", - "1b8f0ba5_4", - "fc2d70fe_1", - "0826def6_1", - "cf9eda76", - "9ff53e00_1" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_i9adwQn_0" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "4baee567", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_i9adwQn_0", - "sharegpt_wduYbsX_0", - "sharegpt_NxG2nGm_91", - "385683f0_1", - "sharegpt_kt5ZCxz_10", - "ec830058_5", - "sharegpt_hRVKLa2_0", - "ultrachat_427430", - "d5415748", - "ultrachat_210474" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_hChsWOp_97" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "561fabcd", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_hChsWOp_97", - "864a563d_4", - "34b38398_2", - "sharegpt_QaiMtpK_0", - "aee13015_3", - "3be19aed_1", - "83fb74bf_1", - "d75f4fb5_1", - "33da50d0_5", - "ultrachat_194831" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_2BSXlAr_0" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "b759caee", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_2BSXlAr_0", - "sharegpt_6gBVFdg_1", - "b41aae7f_1", - "bab41bb6_5", - "d3575920", - "ultrachat_554750", - "f55fee5b_4", - "sharegpt_I9veIBa_1", - "ultrachat_281823", - "0bad887e_2" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_GYqnAhC_190" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "ac031881", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_GYqnAhC_190", - "eb95f93c", - "6608c1da", - "sharegpt_LgJ688S_0", - "e7989082_1", - "c986f83a", - "ultrachat_58138", - "2dbe975c", - "sharegpt_cIj2dFl_4", - "19c24c11" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_446979" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "28bcfaac", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_446979", - "8d410160", - "be05680b_2", - "17cb8d2f_5", - "135a62d7_2", - "a27a2811", - "044e200f_2", - "ultrachat_433665", - "989ad9e6_3", - "sharegpt_MO9Xxet_71" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_294807" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "16c90bf4", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_294807", - "8464304d_2", - "5053474c_2", - "b8770374_2", - "sharegpt_Ou9rQ6U_8", - "b0da5097_2", - "ad72ce15", - "d576152e_4", - "d4230511_1", - "32449a42" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_519486" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "c8f1aeed", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_519486", - "sharegpt_ebYKddM_87", - "aa7a2ec0_1", - "a53fdd02_2", - "af631aa3_3", - "48a72204_3", - "f7cd21e5", - "sharegpt_fH5kcER_0", - "12cd736c", - "c3023a45_5" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_SS141vi_0" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "eaca4986", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_SS141vi_0", - "3477c37c_1", - "931c521e_1", - "ultrachat_311235", - "7bc2c7a3_1", - "1c1a2b7f_2", - "564761aa", - "24ca2090_2", - "sharegpt_NelPQLT_0", - "0e543e9e_2" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_456407" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "c7cf7dfd", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_456407", - "de5a59bf", - "95cec590_1", - "285a29e7_1", - "sharegpt_f28uI6i_31", - "780b6c7c_1", - "c1e681e7_2", - "e27fe923_5", - "1382d6fb_1", - "9db6d988_2" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_174360" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e48988bc", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_174360", - "6e110a53_1", - "ultrachat_494373", - "e388463b_1", - "3fd56ebf", - "ff6bccaa_3", - "476b1cbb", - "sharegpt_TzwvePR_21", - "ultrachat_172828", - "ultrachat_307163" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_440262" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "1de5cff2", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_440262", - "4c6775ed_1", - "848d721f_1", - "8fcfbe43_3", - "ultrachat_58031", - "64048e35", - "28031c43_2", - "c14dbe17", - "42d9467c_1", - "4cb33063" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_403752" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "65240037", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_403752", - "37141183", - "46f07bd0_2", - "f8dc1273", - "fce669f5_3", - "0984a772", - "7df4c735_1", - "3c3a9042_1", - "dba5a924", - "sharegpt_Caxpnd3_6" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_399000" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "778164c6", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_399000", - "sharegpt_ki9IVDq_6", - "ultrachat_346016", - "483ddd90", - "490bb46d_1", - "05b551b6", - "b605741a_1", - "ultrachat_144598", - "c15dadce_4", - "35201d43" - ] - } - ], - "runtime_seconds": 403.852, - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_new_brainctl_cmd.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_new_brainctl_cmd.json deleted file mode 100644 index 50ef218..0000000 --- a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_new_brainctl_cmd.json +++ /dev/null @@ -1,12197 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\longmemeval_new_brainctl_cmd.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "measured": true, - "metadata": {}, - "metrics": { - "ndcg_at_10": 0.8753, - "ndcg_at_5": 0.8715, - "r_at_10": 0.9894, - "r_at_5": 0.9596 - }, - "mode": "cmd", - "notes": [ - "top_k=10", - "Legacy 470-question session-level slice." - ], - "primary_metric": "r_at_5", - "primary_metric_value": 0.9596, - "reference_kind": "measured", - "rows": [ - { - "answer_session_ids": [ - "answer_280352e9" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e47becba", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_280352e9", - "02bd2b90_3", - "ultrachat_214101", - "5dac7cc2_1", - "sharegpt_Cr2tc1f_0", - "f6859b48_2" - ] - }, - { - "answer_session_ids": [ - "answer_40a90d51" - ], - "ndcg_at_10": 0.4307, - "ndcg_at_5": 0.4307, - "question_id": "118b2229", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "ultrachat_392094", - "db73b7e4_4", - "afe04238_1", - "answer_40a90d51", - "5dce60dd", - "a0b8b0ad_6", - "db50c0f6", - "3ea9f765", - "d600c646", - "sharegpt_i1iuP70_0" - ] - }, - { - "answer_session_ids": [ - "answer_d61669c7" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "51a45a95", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_d61669c7", - "98c198fb", - "217debf7", - "6e0b1800_2", - "5c44d9fe_1", - "07ba9acd_2", - "sharegpt_CyJ3dal_0", - "sharegpt_hChsWOp_128", - "sharegpt_3vxz2Zr_0", - "ultrachat_282235" - ] - }, - { - "answer_session_ids": [ - "answer_355c48bb" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "58bf7951", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_355c48bb", - "ultrachat_304942", - "ultrachat_268434", - "ultrachat_155546", - "sharegpt_gmLF3PE_17", - "7e17ae56_5", - "sharegpt_M84blrA_53", - "sharegpt_RkFlOeC_19", - "sharegpt_OFiElWf_37", - "sharegpt_CvVLg2J_18" - ] - }, - { - "answer_session_ids": [ - "answer_3e012175" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "1e043500", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_3e012175", - "01bd9def", - "ultrachat_227993", - "7b1d1f43_1", - "8020d945_2", - "e132259f", - "sharegpt_bph7DTk_7", - "ultrachat_271928", - "4d0d7984", - "sharegpt_6QUDIXG_24" - ] - }, - { - "answer_session_ids": [ - "answer_f6168136" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "c5e8278d", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_f6168136", - "89b1028d_1", - "64767b7f", - "aaf71ce2_2", - "d777e7da_1", - "fc6549b2", - "sharegpt_WPGrlAK_0", - "sharegpt_EXJAjmw_0", - "ultrachat_474903", - "3fa35683_7" - ] - }, - { - "answer_session_ids": [ - "answer_9398da02" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "6ade9755", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_9398da02", - "7c5ab02a_2", - "ec8fa65d_2", - "ultrachat_84599", - "ultrachat_393203", - "910ed479_3", - "ultrachat_270925", - "ultrachat_462", - "28621d6a_2", - "sharegpt_YsgacQi_0" - ] - }, - { - "answer_session_ids": [ - "answer_feb5200f" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "6f9b354f", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_feb5200f", - "a79f1a04_1", - "0c0f2449_2", - "ultrachat_549317", - "60207b41_1", - "54d06eeb", - "sharegpt_IuLT3VI_9", - "sharegpt_VAELSB3_0", - "sharegpt_kRjBUsA_79", - "ultrachat_224845" - ] - }, - { - "answer_session_ids": [ - "answer_59547700" - ], - "ndcg_at_10": 0.6309, - "ndcg_at_5": 0.6309, - "question_id": "58ef2f1c", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "13f3da23_3", - "answer_59547700", - "e6ab6a7b_2", - "ultrachat_5212", - "e82ce7cd_1", - "ultrachat_246324", - "ultrachat_477344", - "dbcfab21", - "b546cb39_2", - "403ee298_1" - ] - }, - { - "answer_session_ids": [ - "answer_c3567066" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "f8c5f88b", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c3567066", - "68ace657_1", - "sharegpt_TKj3yO8_0", - "91880423", - "15232887_1", - "ultrachat_360085", - "ultrachat_412973", - "7ff89315_3", - "58de8f9b", - "3b5205b6_1" - ] - }, - { - "answer_session_ids": [ - "answer_235eb6fb" - ], - "ndcg_at_10": 0.6309, - "ndcg_at_5": 0.6309, - "question_id": "5d3d2817", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "ultrachat_384575", - "answer_235eb6fb", - "670ca40e", - "sharegpt_ipLglky_48", - "sharegpt_8zsFRKh_0", - "e93efd9a_1", - "cef33b28_2", - "ultrachat_106114", - "sharegpt_bTDMt3m_64", - "25d1830a_1" - ] - }, - { - "answer_session_ids": [ - "answer_7cb94507" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "7527f7e2", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_7cb94507", - "864a563d_3", - "sharegpt_cRrfdsc_0", - "sharegpt_rraH1Q7_0", - "44bbd8cd_2", - "d81d9846", - "sharegpt_bTDMt3m_0", - "bee872da_2", - "ultrachat_62919", - "18a06652_3" - ] - }, - { - "answer_session_ids": [ - "answer_e05e4612" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "c960da58", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e05e4612", - "89d3053f", - "ultrachat_228940", - "ultrachat_36718", - "ultrachat_109577", - "ultrachat_129096", - "sharegpt_9iUZNsL_0", - "47db3b56_1", - "b76006cb_3", - "f01feb31" - ] - }, - { - "answer_session_ids": [ - "answer_94030872" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "3b6f954b", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_94030872", - "c271c071", - "6d3f5c68_4", - "ultrachat_475284", - "c566edbb_3", - "90b1c6d4_2", - "sharegpt_kpnwrGe_12", - "ultrachat_298929", - "4ed9cd1b_1", - "f999b05c_2" - ] - }, - { - "answer_session_ids": [ - "answer_f38f679b" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "726462e0", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_f38f679b", - "9c05acfe_2", - "741c6781_1", - "082889ba_1", - "5b83c26e_1", - "6ef99651_2", - "ce144f08", - "cdd9704d_1", - "7e4dab66_1", - "322a96e8" - ] - }, - { - "answer_session_ids": [ - "answer_c63c0458" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "94f70d80", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c63c0458", - "67045503", - "58de8f9b", - "sharegpt_6fTnCnw_76", - "ultrachat_380138", - "ultrachat_172828", - "sharegpt_mJ7MnQS_31", - "ec628e8d_2", - "sharegpt_ZvjRGRN_0", - "9a59023e_1" - ] - }, - { - "answer_session_ids": [ - "answer_fea2e4d3" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "66f24dbb", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_fea2e4d3", - "89452526_2", - "8d46b8fa_1", - "798e4ba3_2", - "21c6c41d", - "ca3a4e4f_1", - "sharegpt_83fkMsQ_0", - "ultrachat_445260", - "sharegpt_ZcfatzD_0", - "76299a49_1" - ] - }, - { - "answer_session_ids": [ - "answer_679840f8" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "ad7109d1", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_679840f8", - "22aa7cca_3", - "3dc02d26_2", - "d92cc997", - "ultrachat_74974", - "495cf518_1", - "sharegpt_Ck9R9HX_0", - "1d6744b5_1", - "fb721812_2", - "ultrachat_365256" - ] - }, - { - "answer_session_ids": [ - "answer_82a04f59" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "af8d2e46", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_82a04f59", - "33c3b457", - "73d5f10c", - "ultrachat_560731", - "266d604c", - "32a67b35", - "ultrachat_284576", - "13047a02", - "d5a6f455_2", - "ultrachat_210554" - ] - }, - { - "answer_session_ids": [ - "answer_8f276838" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "dccbc061", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_8f276838", - "0f417163", - "sharegpt_nc62Spr_7", - "ultrachat_278202", - "959e6cb0_3", - "b2a885b9", - "3c614701", - "8d74c5f6", - "sharegpt_MKMWjX0_25", - "ultrachat_519037" - ] - }, - { - "answer_session_ids": [ - "answer_761acef8" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "c8c3f81d", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_761acef8", - "eb5d5a93_1", - "66e94928", - "19ec83c5_1", - "sharegpt_pRqHb1o_0", - "eb403c80_2", - "9c5f1314_1", - "2b16f3d6_1", - "998616b4_4", - "25432e10" - ] - }, - { - "answer_session_ids": [ - "answer_8ad8a34f" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "8ebdbe50", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_8ad8a34f", - "b53f447a_1", - "90f16f14_2", - "01e806f0_1", - "27d378b2_3", - "39911f94", - "90a5e1f3_1", - "sharegpt_QxC97cL_0", - "d4f7a065_2", - "326f41d8" - ] - }, - { - "answer_session_ids": [ - "answer_e623ae87" - ], - "ndcg_at_10": 0.301, - "ndcg_at_5": 0.0, - "question_id": "6b168ec8", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 0.0, - "top_session_ids": [ - "ultrachat_360618", - "ultrachat_377741", - "33dff20c_3", - "ultrachat_249928", - "ultrachat_499691", - "dded1725_3", - "sharegpt_D9kTAIU_0", - "02f0738e_1", - "answer_e623ae87", - "f5b33470_abs" - ] - }, - { - "answer_session_ids": [ - "answer_723bf11f" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "75499fd8", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_723bf11f", - "5328c3c2_2", - "990b60df_1", - "18b0d357_1", - "eb0d8dc1_7", - "3e7df334_1", - "ddf0f116_3", - "sharegpt_1kE6haW_276", - "sharegpt_yPCVYhX_0", - "168feef7_1" - ] - }, - { - "answer_session_ids": [ - "answer_1e6d4567" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "21436231", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_1e6d4567", - "0d0a89dc_1", - "7a7a4bf0_4", - "ultrachat_172276", - "dc378711_1", - "sharegpt_6byuqlt_0", - "sharegpt_DGiqvVv_169", - "d0785ee9_1", - "3197d998_6", - "6bd0b406_1" - ] - }, - { - "answer_session_ids": [ - "answer_cb742a61" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "95bcc1c8", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_cb742a61", - "3919227f", - "ultrachat_320154", - "c9763bff", - "396238f9", - "abeda028", - "56266c38_1", - "99e5e380", - "sharegpt_uATA5m5_41", - "ultrachat_203404" - ] - }, - { - "answer_session_ids": [ - "answer_c6fd8ebd" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "0862e8bf", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c6fd8ebd", - "0566c21b", - "2442d5b9", - "sharegpt_3BkB8g5_0", - "sharegpt_2I1TxKW_0", - "ultrachat_47999", - "sharegpt_A7KJApk_21", - "sharegpt_KOaA4jf_0", - "sharegpt_isQJkCz_17", - "sharegpt_sCFErnY_22" - ] - }, - { - "answer_session_ids": [ - "answer_69811d4a" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "853b0a1d", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_69811d4a", - "sharegpt_n3MgsgL_0", - "c70f9f9c", - "3fa35683_3", - "a8b4bfb3_2", - "f71bf532_1", - "66a6a097_2", - "ebb5bc7c_2", - "451120d3_2", - "sharegpt_B7mSlfQ_55" - ] - }, - { - "answer_session_ids": [ - "answer_6fe9fb49" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "a06e4cfe", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_6fe9fb49", - "ee43db4a", - "3c2e6725", - "84fb50bb_3", - "de64539a_2", - "8dd2d3cc", - "af257b0b_2", - "0c891c0b_1", - "ultrachat_442201", - "ultrachat_231287" - ] - }, - { - "answer_session_ids": [ - "answer_55161935" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "37d43f65", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_55161935", - "3dcf1e0b_2", - "63587769", - "ultrachat_370801", - "ultrachat_525148", - "sharegpt_MOpCbrr_0", - "sharegpt_brc2wJS_264", - "4050ebff_5", - "sharegpt_ErOTMZ3_35", - "sharegpt_afbMhMS_45" - ] - }, - { - "answer_session_ids": [ - "answer_645b0329" - ], - "ndcg_at_10": 0.5, - "ndcg_at_5": 0.5, - "question_id": "b86304ba", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "ea8bb4f8_2", - "sharegpt_xGoJZ6Z_0", - "answer_645b0329", - "ultrachat_328696", - "ultrachat_10525", - "3b73120a_1", - "e831a29f_1", - "fe1e4351_1", - "91074ce7", - "091aa510_1" - ] - }, - { - "answer_session_ids": [ - "answer_0df6aa4b" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "d52b4f67", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_0df6aa4b", - "d31c3ec8_3", - "da1797c4_1", - "ultrachat_258008", - "18a06652_6", - "62a6d083", - "974fe383_2", - "sharegpt_IqVsDp7_89", - "sharegpt_MM7pEKH_0", - "sharegpt_cn3iUbp_7" - ] - }, - { - "answer_session_ids": [ - "answer_986de8c3" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "25e5aa4f", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_986de8c3", - "18807892_4", - "48f804a3", - "ultrachat_381447", - "a459d477", - "ca5ff3b4", - "13ad258b", - "sharegpt_8cFNcVG_0", - "ultrachat_225537", - "087d2b0a_2" - ] - }, - { - "answer_session_ids": [ - "answer_0714183a" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "caf9ead2", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_0714183a", - "f684ac4c_2", - "fc167d00", - "sharegpt_3D3oQC0_213", - "44713827_2", - "9f8bdd23_1", - "48a72204_1", - "ultrachat_30245", - "fcc6d66d_1", - "ultrachat_532107" - ] - }, - { - "answer_session_ids": [ - "answer_c8354ae9" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "8550ddae", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c8354ae9", - "a7189b75", - "71cf5aeb_3", - "e10dfffb_1", - "4f234e2c", - "9deebbc2_2", - "8ec23b2c", - "295dc1ab_1", - "c7a15bdc", - "sharegpt_ipLglky_42" - ] - }, - { - "answer_session_ids": [ - "answer_9cddca88" - ], - "ndcg_at_10": 0.4307, - "ndcg_at_5": 0.4307, - "question_id": "60d45044", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "fcc6d66d_1", - "4ebf9095_1", - "af4d4ecd_2", - "answer_9cddca88", - "26bc645b_5", - "a76e7e3c_2", - "fec059c5", - "bda4a421", - "0c0f2449_1", - "446173bb_1" - ] - }, - { - "answer_session_ids": [ - "answer_57fc1954" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "3f1e9474", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_57fc1954", - "sharegpt_wWUI4SO_7", - "ultrachat_177477", - "e7497c07_2", - "ultrachat_462827", - "3aac691d_2", - "cdf068b1_1", - "4f23a396_1", - "205f348b_2", - "954d4451_2" - ] - }, - { - "answer_session_ids": [ - "answer_dc11c1eb" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "86b68151", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_dc11c1eb", - "sharegpt_beV15ZO_0", - "d8454317_4", - "3d72c0c0", - "ultrachat_182718", - "0abaca24_1", - "sharegpt_rnL5VQO_0", - "4e671700", - "5ace87df_1", - "eed7b3ac_2" - ] - }, - { - "answer_session_ids": [ - "answer_0dd4d99a" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "577d4d32", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_0dd4d99a", - "ultrachat_125084", - "88925167_1", - "6e2cca63_2", - "449b918c_1", - "2c0d04d6_1", - "2ef53f61_2", - "sharegpt_pcoHTby_85", - "b960cbd3", - "sharegpt_101i5OR_0" - ] - }, - { - "answer_session_ids": [ - "answer_ed1982fc" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "ec81a493", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ed1982fc", - "6f55019d", - "ultrachat_447961", - "d2fc150a_1", - "sharegpt_ST22P5t_0", - "ultrachat_27488", - "ab4643a2_4", - "ultrachat_413665", - "ultrachat_492664", - "616fdd27" - ] - }, - { - "answer_session_ids": [ - "answer_586de428" - ], - "ndcg_at_10": 0.6309, - "ndcg_at_5": 0.6309, - "question_id": "15745da0", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "07d33eaa", - "answer_586de428", - "64a97d9b", - "sharegpt_IzpzBot_0", - "1f5370f7_4", - "1c662b7b_1", - "0d73afc8", - "ultrachat_383425", - "sharegpt_iF4hlNs_5", - "3cec6e2b_2" - ] - }, - { - "answer_session_ids": [ - "answer_5ca6cd28" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e01b8e2f", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_5ca6cd28", - "97338ddd_1", - "8acfa731", - "sharegpt_jyMHY1J_31", - "f9d0bc67", - "ultrachat_433104", - "d31c3ec8_1", - "e74be4c6_4", - "8be2c3f1", - "1aaf8057_3" - ] - }, - { - "answer_session_ids": [ - "answer_e6143162" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "bc8a6e93", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e6143162", - "f62bfe2b_1", - "8c1da8f9_1", - "b869d7ed_2", - "a1b3dd27_1", - "sharegpt_qyWsgp8_13", - "552c7fd0", - "c6bed037_1", - "sharegpt_jl0GWjL_0", - "ultrachat_453950" - ] - }, - { - "answer_session_ids": [ - "answer_f1fbb330" - ], - "ndcg_at_10": 0.5, - "ndcg_at_5": 0.5, - "question_id": "ccb36322", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "bdcee74e_3", - "6d52ee93_1", - "answer_f1fbb330", - "c38b33ae", - "d99bab55_1", - "04a0b385", - "8b1019b8_1", - "0e349efa", - "06d1a1a0", - "f79a48dd_1" - ] - }, - { - "answer_session_ids": [ - "answer_530960c1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "001be529", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_530960c1", - "0f71db75_1", - "9becef17_3", - "sharegpt_YQ4UlQM_83", - "sharegpt_PKXxlMF_0", - "ultrachat_351532", - "sharegpt_ETJwE0s_0", - "sharegpt_o3cfukb_0", - "ultrachat_95472", - "ultrachat_39884" - ] - }, - { - "answer_session_ids": [ - "answer_5cc9b056" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "b320f3f8", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_5cc9b056", - "d3a3f421_2", - "514a7856_2", - "b89f68c4_4", - "8bb8bc0d", - "aae4411b_2", - "40628271_1", - "5e4bb245_2", - "sharegpt_4FCMpJR_0", - "1b71c896_2" - ] - }, - { - "answer_session_ids": [ - "answer_5ff494b9" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "19b5f2b3", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_5ff494b9", - "f379d356_1", - "ultrachat_440262", - "ultrachat_463145", - "ultrachat_302261", - "ultrachat_8325", - "ultrachat_268389", - "ultrachat_68139", - "fbe6fa2c_1", - "ultrachat_138083" - ] - }, - { - "answer_session_ids": [ - "answer_2952aee4" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "4fd1909e", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_2952aee4", - "2479cbae", - "94c582bb_2", - "ultrachat_186237", - "sharegpt_hO4FoVt_0", - "fef185b1", - "ultrachat_472809", - "sharegpt_bTDMt3m_0", - "sharegpt_FNyKOSt_0", - "sharegpt_xX1L0kg_29" - ] - }, - { - "answer_session_ids": [ - "answer_47ffab4c" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "545bd2b5", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_47ffab4c", - "b7f69a7d_1", - "5fb3b5ac", - "a79f1a04_1", - "700c16e2_1", - "cc021f81_2", - "6e672b84_2", - "d1a1b9ea_1", - "4404a52a_1", - "sharegpt_nuYnG53_0" - ] - }, - { - "answer_session_ids": [ - "answer_15d63a22" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "8a137a7f", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_15d63a22", - "77908e43_1", - "fcff2dc4_3", - "20507d15_2", - "ultrachat_90882", - "ultrachat_186116", - "ultrachat_382344", - "sharegpt_sj5ZvVB_0", - "ultrachat_151889", - "ultrachat_192894" - ] - }, - { - "answer_session_ids": [ - "answer_bbdc7b0a" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "76d63226", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_bbdc7b0a", - "6a2f9452_2", - "adc47df1_1", - "a5ab6ba7_3", - "f27e27f9_2", - "cb289226_1", - "sharegpt_sRhcMiu_13", - "ultrachat_328229", - "ultrachat_180710", - "50136b31" - ] - }, - { - "answer_session_ids": [ - "answer_96b8c9ee" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "86f00804", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_96b8c9ee", - "sharegpt_ErOTMZ3_277", - "137d8c6a_2", - "0424a40a_2", - "35c5419d_abs_3", - "6e672b84_3", - "sharegpt_PQ71V9I_7", - "60153a02_1", - "fc41f919", - "sharegpt_sCFErnY_22" - ] - }, - { - "answer_session_ids": [ - "answer_7bdcbd23" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "8e9d538c", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_7bdcbd23", - "ultrachat_549592", - "3cdad629_1", - "0bd928bf_2", - "ultrachat_26072", - "a200b713_4", - "2deed26f", - "ultrachat_232141", - "sharegpt_KtWBRv1_4", - "sharegpt_cADPA4R_0" - ] - }, - { - "answer_session_ids": [ - "answer_e40b054e" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "311778f1", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e40b054e", - "5db56d24_2", - "8e6e058b_3", - "sharegpt_vXNQZ2I_0", - "ultrachat_267100", - "d09360d3_1", - "31903f54", - "95cec590_1", - "27fb983c_2", - "ultrachat_316997" - ] - }, - { - "answer_session_ids": [ - "answer_f442ccbe" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "c19f7a0b", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_f442ccbe", - "58ab5fc5", - "eb336de0_3", - "sharegpt_6ZeW9Vb_0", - "c9292210_2", - "1b0d77b0", - "0e829b8f", - "195c8fe6_3", - "8bb8bc0d", - "81ba81ab" - ] - }, - { - "answer_session_ids": [ - "answer_83c13ff9" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "4100d0a0", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_83c13ff9", - "8d068598", - "ea3db78e_1", - "sharegpt_eV0krsR_0", - "b9890c15_1", - "ultrachat_130655", - "f8ab60d7", - "0f6b9698_1", - "665c09b1_1", - "7fc9c056" - ] - }, - { - "answer_session_ids": [ - "answer_7cc5362f" - ], - "ndcg_at_10": 0.5, - "ndcg_at_5": 0.5, - "question_id": "29f2956b", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "6d6ffac5", - "88361fdd", - "answer_7cc5362f", - "ultrachat_238255", - "c4ed2287_3", - "d50a8a33_1", - "sharegpt_jZOf9E5_0", - "715b1b28", - "eb0d8dc1_5", - "sharegpt_UCVg3jO_0" - ] - }, - { - "answer_session_ids": [ - "answer_d01949bf" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "1faac195", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_d01949bf", - "b2543a58_3", - "8064b6ca", - "9d3d54f7", - "ultrachat_283922", - "f2ccf83b", - "94f460be_1", - "ultrachat_12436", - "sharegpt_NhF2Neg_0", - "48d385f0_1" - ] - }, - { - "answer_session_ids": [ - "answer_39b12014" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "faba32e5", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_39b12014", - "67045503", - "69f3fc12_1", - "c6c5bb8b_4", - "770d11f5", - "sharegpt_FNyKOSt_0", - "sharegpt_benxw0S_9", - "ultrachat_205840", - "d3da4592_2", - "ultrachat_250588" - ] - }, - { - "answer_session_ids": [ - "answer_f5b33470" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "f4f1d8a4", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_f5b33470", - "2bfea7af", - "3aac691d_2", - "9614d065_1", - "eb682e52_2", - "6f051087_2", - "ecb24dd8_1", - "76ea88ba_2", - "ultrachat_67942", - "ultrachat_576354" - ] - }, - { - "answer_session_ids": [ - "answer_304511ce" - ], - "ndcg_at_10": 0.5, - "ndcg_at_5": 0.5, - "question_id": "c14c00dd", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "c77250d2", - "7cd7c296_3", - "answer_304511ce", - "ffd66315_2", - "cd1f5147", - "ultrachat_272210", - "183b6085", - "sharegpt_gU0qvZu_0", - "sharegpt_FfqJ2xI_0", - "sharegpt_vXWOUqx_11" - ] - }, - { - "answer_session_ids": [ - "answer_93e1bd22" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "36580ce8", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_93e1bd22", - "ultrachat_53784", - "9b38bda6_2", - "ultrachat_272552", - "ultrachat_270984", - "ultrachat_32917", - "sharegpt_FpTfRvR_0", - "ultrachat_299779", - "ultrachat_149170", - "sharegpt_ZbUcxUa_0" - ] - }, - { - "answer_session_ids": [ - "answer_19c24c11" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "3d86fd0a", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_19c24c11", - "cf8e352f_1", - "sharegpt_d7bP5fb_0", - "69674f25", - "d1d3fd12_2", - "1c1a2b7f_2", - "06d1a1a0", - "091ba7f5_1", - "5a13d047_1", - "7e4dab66_2" - ] - }, - { - "answer_session_ids": [ - "answer_787e6a6d" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "a82c026e", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_787e6a6d", - "d9727262_4", - "f18ebe36_4", - "3009ba66_2", - "ccaabf0b_1", - "516b0aae_1", - "a6c41826_3", - "daa32134", - "c71e1c7a_2", - "bdf735de_1" - ] - }, - { - "answer_session_ids": [ - "answer_afa9873b_2", - "answer_afa9873b_3", - "answer_afa9873b_1" - ], - "ndcg_at_10": 0.9675, - "ndcg_at_5": 0.9675, - "question_id": "0a995998", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_afa9873b_1", - "answer_afa9873b_2", - "85846900_2", - "answer_afa9873b_3", - "439f9ed8_3", - "3f05c474", - "e6790684_1", - "fa46a10e", - "sharegpt_ZzArUrr_0", - "ultrachat_331531" - ] - }, - { - "answer_session_ids": [ - "answer_ec904b3c_1", - "answer_ec904b3c_4", - "answer_ec904b3c_3", - "answer_ec904b3c_2" - ], - "ndcg_at_10": 0.8772, - "ndcg_at_5": 0.8772, - "question_id": "6d550036", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ec904b3c_1", - "2e4430d8_2", - "0bab76de", - "answer_ec904b3c_4", - "ultrachat_184799", - "a9981dc6_3", - "sharegpt_UbMVpdp_93", - "f6246b5f", - "dae3906e_2", - "sharegpt_J7ZAFLd_0" - ] - }, - { - "answer_session_ids": [ - "answer_593bdffd_4", - "answer_593bdffd_1", - "answer_593bdffd_3", - "answer_593bdffd_2" - ], - "ndcg_at_10": 0.5508, - "ndcg_at_5": 0.5013, - "question_id": "gpt4_59c863d7", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "9ef698bc_2", - "de5f2701_1", - "a7d014e4_1", - "answer_593bdffd_1", - "answer_593bdffd_4", - "answer_593bdffd_2", - "2f09d4c8", - "ultrachat_126967", - "eb47739f_2", - "ultrachat_59708" - ] - }, - { - "answer_session_ids": [ - "answer_a8b4290f_3", - "answer_a8b4290f_1", - "answer_a8b4290f_2" - ], - "ndcg_at_10": 0.6395, - "ndcg_at_5": 0.6509, - "question_id": "b5ef892d", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "ultrachat_565056", - "answer_a8b4290f_3", - "sharegpt_XWqXdom_0", - "answer_a8b4290f_1", - "ultrachat_53385", - "35dcacdc_2", - "ultrachat_367439", - "ultrachat_375427", - "answer_a8b4290f_2", - "e552e1f9_2" - ] - }, - { - "answer_session_ids": [ - "answer_86c505e7_1", - "answer_86c505e7_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e831120c", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_86c505e7_2", - "answer_86c505e7_1", - "sharegpt_sDfu38Q_0", - "8fb3fe3a_1", - "a63ad8e3_2", - "ultrachat_310037", - "7c82a6c3", - "6387e969", - "sharegpt_hvdBqMM_0", - "sharegpt_d8EB6Ze_0" - ] - }, - { - "answer_session_ids": [ - "answer_c2204106_2", - "answer_c2204106_3", - "answer_c2204106_1" - ], - "ndcg_at_10": 0.8529, - "ndcg_at_5": 0.8529, - "question_id": "3a704032", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c2204106_1", - "7712aded", - "86e830d2_3", - "answer_c2204106_2", - "answer_c2204106_3", - "78764b10_2", - "ultrachat_501733", - "sharegpt_MRCLhIj_0", - "ultrachat_357807", - "sharegpt_HrxBbBK_0" - ] - }, - { - "answer_session_ids": [ - "answer_2880eb6c_2", - "answer_2880eb6c_4", - "answer_2880eb6c_1", - "answer_2880eb6c_3" - ], - "ndcg_at_10": 0.7606, - "ndcg_at_5": 0.7606, - "question_id": "gpt4_d84a3211", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "18d68208_1", - "answer_2880eb6c_4", - "answer_2880eb6c_2", - "answer_2880eb6c_1", - "answer_2880eb6c_3", - "4889cd54_1", - "4a32d58c_2", - "sharegpt_OHd1RQ7_37", - "ultrachat_83526", - "13c55c69_1" - ] - }, - { - "answer_session_ids": [ - "answer_526354c8_1", - "answer_526354c8_3", - "answer_526354c8_2" - ], - "ndcg_at_10": 0.9218, - "ndcg_at_5": 1.0, - "question_id": "aae3761f", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_526354c8_2", - "answer_526354c8_1", - "b192ae00_2", - "ultrachat_266173", - "sharegpt_8I9vH7n_7", - "3d85d91e_2", - "answer_526354c8_3", - "sharegpt_ynLoh9N_0", - "2a86d0da_2", - "ultrachat_92484" - ] - }, - { - "answer_session_ids": [ - "answer_55a6940c_3", - "answer_55a6940c_1", - "answer_55a6940c_2" - ], - "ndcg_at_10": 0.4716, - "ndcg_at_5": 0.0, - "question_id": "gpt4_f2262a51", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 0.0, - "top_session_ids": [ - "c0d6fa6f_2", - "dd81b163_1", - "sharegpt_eoEbthf_0", - "ultrachat_268434", - "sharegpt_BWMyoNr_0", - "answer_55a6940c_3", - "answer_55a6940c_2", - "answer_55a6940c_1", - "sharegpt_hChsWOp_128", - "sharegpt_ab6IEma_0" - ] - }, - { - "answer_session_ids": [ - "answer_f9de4602_2", - "answer_f9de4602_1" - ], - "ndcg_at_10": 0.8316, - "ndcg_at_5": 1.0, - "question_id": "dd2973ad", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_f9de4602_2", - "sharegpt_O6NO7Vo_9", - "e5f08e40", - "41743aae_1", - "0f73eee7_2", - "answer_f9de4602_1", - "ultrachat_26627", - "e720dbe2", - "32c0dae1_3", - "b192ae00_2" - ] - }, - { - "answer_session_ids": [ - "answer_56d02cab_3", - "answer_56d02cab_4", - "answer_56d02cab_2", - "answer_56d02cab_1" - ], - "ndcg_at_10": 0.9047, - "ndcg_at_5": 0.9047, - "question_id": "c4a1ceb8", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_56d02cab_3", - "40c77045", - "answer_56d02cab_1", - "answer_56d02cab_4", - "answer_56d02cab_2", - "ba9f938b_2", - "cef33b28_1", - "sharegpt_1sxNkyQ_67", - "0b64c6cb_2", - "sharegpt_WanXxRQ_0" - ] - }, - { - "answer_session_ids": [ - "answer_cf9e3940_2", - "answer_cf9e3940_1", - "answer_cf9e3940_3" - ], - "ndcg_at_10": 0.6462, - "ndcg_at_5": 0.6509, - "question_id": "gpt4_a56e767c", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "88c8df0e_3", - "answer_cf9e3940_3", - "f56e6152_1", - "answer_cf9e3940_2", - "ultrachat_257906", - "ultrachat_104129", - "d75245ea", - "answer_cf9e3940_1", - "dac54d1e_1", - "71621be0" - ] - }, - { - "answer_session_ids": [ - "answer_a4204937_2", - "answer_a4204937_1" - ], - "ndcg_at_10": 0.8772, - "ndcg_at_5": 0.8772, - "question_id": "6cb6f249", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_a4204937_1", - "9b6db1a9_2", - "ultrachat_404221", - "answer_a4204937_2", - "5edfe253_1", - "f584ba36_2", - "67620dbc_2", - "ultrachat_374192", - "cbe70f21", - "sharegpt_IkYYvSd_79" - ] - }, - { - "answer_session_ids": [ - "answer_c65042d7_3", - "answer_c65042d7_2", - "answer_c65042d7_1" - ], - "ndcg_at_10": 0.9066, - "ndcg_at_5": 1.0, - "question_id": "46a3abf7", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c65042d7_3", - "answer_c65042d7_2", - "09f2c6c5", - "28eb86ab", - "a95bedca", - "ultrachat_406179", - "c1e598cd_1", - "4fe91719_2", - "answer_c65042d7_1", - "e224317f_2" - ] - }, - { - "answer_session_ids": [ - "answer_ef74281f_2", - "answer_ef74281f_1", - "answer_ef74281f_3" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "36b9f61e", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ef74281f_2", - "answer_ef74281f_1", - "answer_ef74281f_3", - "e05baf83_2", - "3159942d_2", - "fc6f9c59", - "5829ab38", - "3ceb6783_2", - "fd6f60f0_3", - "85cd56c7_1" - ] - }, - { - "answer_session_ids": [ - "answer_8d015d9d_3", - "answer_8d015d9d_5", - "answer_8d015d9d_2", - "answer_8d015d9d_4", - "answer_8d015d9d_1" - ], - "ndcg_at_10": 0.9609, - "ndcg_at_5": 0.9829, - "question_id": "28dc39ac", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_8d015d9d_2", - "answer_8d015d9d_3", - "answer_8d015d9d_1", - "a7cd1c0f_1", - "answer_8d015d9d_4", - "bfa499f7", - "ultrachat_452868", - "answer_8d015d9d_5", - "5854eebc_1", - "95578ea8_2" - ] - }, - { - "answer_session_ids": [ - "answer_e7b0637e_2", - "answer_e7b0637e_1", - "answer_e7b0637e_3" - ], - "ndcg_at_10": 0.5574, - "ndcg_at_5": 0.5438, - "question_id": "gpt4_2f8be40d", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "ultrachat_217382", - "1d6e01e8", - "answer_e7b0637e_2", - "ultrachat_304474", - "answer_e7b0637e_1", - "sharegpt_L64rHOA_0", - "81b971b8_2", - "cb1f962f_4", - "answer_e7b0637e_3", - "dfe646a7_2" - ] - }, - { - "answer_session_ids": [ - "answer_fa526fc0_4", - "answer_fa526fc0_3", - "answer_fa526fc0_1", - "answer_fa526fc0_2" - ], - "ndcg_at_10": 0.8118, - "ndcg_at_5": 0.8772, - "question_id": "2e6d26dc", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_fa526fc0_4", - "sharegpt_vXNQZ2I_0", - "ultrachat_541982", - "answer_fa526fc0_3", - "7a3cbde3", - "14a520c8_2", - "answer_fa526fc0_1", - "answer_fa526fc0_2", - "sharegpt_IigLRfw_0", - "sharegpt_Wt2YDZs_35" - ] - }, - { - "answer_session_ids": [ - "answer_8858d9dc_3", - "answer_8858d9dc_1", - "answer_8858d9dc_4", - "answer_8858d9dc_2" - ], - "ndcg_at_10": 0.6509, - "ndcg_at_5": 0.6509, - "question_id": "gpt4_15e38248", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "sharegpt_mm9tc5g_0", - "answer_8858d9dc_2", - "1e7f3172_5", - "answer_8858d9dc_4", - "530742d1", - "d4ab49f1", - "ec616e7e_5", - "sharegpt_01VDd0u_0", - "15acfb99", - "8c9bc932_2" - ] - }, - { - "answer_session_ids": [ - "answer_733e443a_3", - "answer_733e443a_4", - "answer_733e443a_2", - "answer_733e443a_1" - ], - "ndcg_at_10": 0.4306, - "ndcg_at_5": 0.3869, - "question_id": "88432d0a", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "f777f641", - "1c8832b4_2", - "c9d35c00_2", - "d1f6080f_1", - "answer_733e443a_2", - "ultrachat_122782", - "ultrachat_351103", - "answer_733e443a_3", - "sharegpt_HpkiAOb_9", - "sharegpt_tESxx1y_0" - ] - }, - { - "answer_session_ids": [ - "answer_990c8992_2", - "answer_990c8992_1", - "answer_990c8992_3" - ], - "ndcg_at_10": 0.8278, - "ndcg_at_5": 0.8772, - "question_id": "80ec1f4f", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_990c8992_2", - "f6ce25ef", - "6884f11d", - "answer_990c8992_1", - "b909542d_1", - "d22c92b2", - "answer_990c8992_3", - "ultrachat_318441", - "ultrachat_497101", - "sharegpt_1kE6haW_52" - ] - }, - { - "answer_session_ids": [ - "answer_5a0d28f8_4", - "answer_5a0d28f8_2", - "answer_5a0d28f8_3", - "answer_5a0d28f8_1" - ], - "ndcg_at_10": 0.9439, - "ndcg_at_5": 0.9675, - "question_id": "d23cf73b", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_5a0d28f8_4", - "answer_5a0d28f8_2", - "66369fde_2", - "answer_5a0d28f8_3", - "b124e4a1_1", - "answer_5a0d28f8_1", - "ultrachat_224488", - "ultrachat_283390", - "ultrachat_456819", - "ultrachat_375734" - ] - }, - { - "answer_session_ids": [ - "answer_a679a86a_5", - "answer_a679a86a_4", - "answer_a679a86a_2", - "answer_a679a86a_3", - "answer_a679a86a_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_7fce9456", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_a679a86a_1", - "answer_a679a86a_5", - "answer_a679a86a_4", - "answer_a679a86a_3", - "541ecc45_3", - "1da409cd_1", - "sharegpt_OjaM9hX_225", - "191e1832", - "67154fb2_1", - "288e7d30" - ] - }, - { - "answer_session_ids": [ - "answer_c008e5df_1", - "answer_c008e5df_2", - "answer_c008e5df_3" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "d682f1a2", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c008e5df_3", - "answer_c008e5df_1", - "sharegpt_E7VUcfj_0", - "7332a779_2", - "9de29219_1", - "d7f9742a_2", - "sharegpt_LFPdgVp_22", - "e30307d8_2", - "58335f60_2", - "ba0ae49c_2" - ] - }, - { - "answer_session_ids": [ - "answer_a21f3697_1", - "answer_a21f3697_2", - "answer_a21f3697_3" - ], - "ndcg_at_10": 0.8711, - "ndcg_at_5": 0.9197, - "question_id": "7024f17c", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_a21f3697_1", - "sharegpt_rshCiS5_1", - "answer_a21f3697_2", - "4f838497_3", - "8c652eb0_1", - "answer_a21f3697_3", - "3c12ff0c", - "f8476198", - "d7281662_2", - "445e06ce_2" - ] - }, - { - "answer_session_ids": [ - "answer_203bf3fa_1", - "answer_203bf3fa_3", - "answer_203bf3fa_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_5501fe77", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_203bf3fa_1", - "answer_203bf3fa_3", - "bc5827b5", - "ultrachat_163385", - "021c0d09_2", - "7a2c618c_1", - "6dfb33f1_2", - "sharegpt_x7apNZ0_0", - "631e4016", - "13b79e39_3" - ] - }, - { - "answer_session_ids": [ - "answer_6a3b5c13_3", - "answer_6a3b5c13_1", - "answer_6a3b5c13_2", - "answer_6a3b5c13_4" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_2ba83207", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_6a3b5c13_1", - "answer_6a3b5c13_2", - "answer_6a3b5c13_4", - "answer_6a3b5c13_3", - "ultrachat_568233", - "cc3c5fa9_2", - "ultrachat_385963", - "ultrachat_224880", - "ea336da0", - "bc6de190_1" - ] - }, - { - "answer_session_ids": [ - "answer_eaa8e3ef_1", - "answer_eaa8e3ef_2" - ], - "ndcg_at_10": 0.8066, - "ndcg_at_5": 1.0, - "question_id": "2318644b", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_eaa8e3ef_2", - "920f808b", - "c4ea8219_1", - "e491f2d4_2", - "7d02ef5c_5", - "6ec23dc5_1", - "fc733b20_2", - "answer_eaa8e3ef_1", - "fcae5c39_2", - "7ff89315_3" - ] - }, - { - "answer_session_ids": [ - "answer_901a6763_2", - "answer_901a6763_4", - "answer_901a6763_1", - "answer_901a6763_3" - ], - "ndcg_at_10": 0.4914, - "ndcg_at_5": 0.4307, - "question_id": "2ce6a0f2", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "a1714d2c_2", - "c1634164_2", - "ultrachat_355740", - "answer_901a6763_4", - "23ddce03_1", - "600077b0_3", - "ultrachat_36844", - "answer_901a6763_2", - "answer_901a6763_1", - "3e90815f" - ] - }, - { - "answer_session_ids": [ - "answer_2504635e_3", - "answer_2504635e_2", - "answer_2504635e_1" - ], - "ndcg_at_10": 0.7929, - "ndcg_at_5": 1.0, - "question_id": "gpt4_d12ceb0e", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_2504635e_2", - "73f4798f", - "sharegpt_dxirwR4_25", - "2fe5510e_3", - "2f23dd1a", - "answer_2504635e_1", - "answer_2504635e_3", - "ultrachat_55863", - "d8e33f5c_abs_2", - "sharegpt_dIgdYPv_13" - ] - }, - { - "answer_session_ids": [ - "answer_39900a0a_3", - "answer_39900a0a_2", - "answer_39900a0a_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "00ca467f", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_39900a0a_3", - "answer_39900a0a_1", - "answer_39900a0a_2", - "e4cb6c56", - "ultrachat_145200", - "ultrachat_239705", - "07942c06", - "4951c0b9_1", - "36828c66_1", - "sharegpt_3cVjZ2b_16" - ] - }, - { - "answer_session_ids": [ - "answer_05d808e6_1", - "answer_05d808e6_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "b3c15d39", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_05d808e6_1", - "answer_05d808e6_2", - "5d4ff8db_1", - "a16a4158", - "sharegpt_H4fpIsr_9", - "sharegpt_inuIr9J_119", - "8d25b813_1", - "d298714d_1", - "sharegpt_V2j1zkI_0", - "sharegpt_5n9NBC8_0" - ] - }, - { - "answer_session_ids": [ - "answer_02b63d04_1", - "answer_02b63d04_5", - "answer_02b63d04_2", - "answer_02b63d04_3", - "answer_02b63d04_4" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_31ff4165", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_02b63d04_1", - "answer_02b63d04_5", - "answer_02b63d04_3", - "35201d43", - "sharegpt_tKmxch5_0", - "e2393b63_2", - "c625cff4_3", - "2d0b80f1", - "6aa87dc3", - "ultrachat_555273" - ] - }, - { - "answer_session_ids": [ - "answer_3e5fea0e_1", - "answer_3e5fea0e_2" - ], - "ndcg_at_10": 0.6053, - "ndcg_at_5": 0.6309, - "question_id": "eeda8a6d", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "sharegpt_PTZ2ime_0", - "answer_3e5fea0e_1", - "ea67b29e_1", - "cf7aad73_3", - "f3745025_1", - "answer_3e5fea0e_2", - "91c71874_1", - "sharegpt_JsCcnH4_0", - "sharegpt_wrXNAYQ_0", - "sharegpt_khNFc2W_0" - ] - }, - { - "answer_session_ids": [ - "answer_8f6b938d_1", - "answer_8f6b938d_3", - "answer_8f6b938d_4", - "answer_8f6b938d_2" - ], - "ndcg_at_10": 0.7397, - "ndcg_at_5": 0.7328, - "question_id": "2788b940", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "544fe66c_2", - "answer_8f6b938d_4", - "answer_8f6b938d_2", - "answer_8f6b938d_1", - "ff6e824b_1", - "sharegpt_vgBnRfE_0", - "answer_8f6b938d_3", - "7033540c_2", - "3f863722", - "sharegpt_xFriTfz_35" - ] - }, - { - "answer_session_ids": [ - "answer_e0956e0a_1", - "answer_e0956e0a_2" - ], - "ndcg_at_10": 0.525, - "ndcg_at_5": 0.5, - "question_id": "60bf93ed", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "a200b713_2", - "388c5534_3", - "answer_e0956e0a_1", - "4fe27c66_4", - "0e297c49", - "answer_e0956e0a_2", - "6414f676_4", - "ultrachat_7648", - "e9590b7d", - "sharegpt_f40Yvlz_0" - ] - }, - { - "answer_session_ids": [ - "answer_fcff2dc4_2", - "answer_fcff2dc4_1", - "answer_fcff2dc4_3" - ], - "ndcg_at_10": 0.8194, - "ndcg_at_5": 0.8772, - "question_id": "9d25d4e0", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_fcff2dc4_3", - "464f3821", - "a4b774c4", - "answer_fcff2dc4_1", - "37beb8da", - "ca555919_3", - "46d9d476", - "answer_fcff2dc4_2", - "39911f94", - "41abc171_1" - ] - }, - { - "answer_session_ids": [ - "answer_1de862d6_1", - "answer_1de862d6_3", - "answer_1de862d6_2" - ], - "ndcg_at_10": 0.906, - "ndcg_at_5": 0.906, - "question_id": "129d1232", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_1de862d6_2", - "69a9211c_2", - "answer_1de862d6_1", - "answer_1de862d6_3", - "5e4bb245_2", - "1ab255ba_1", - "sharegpt_De0lanR_0", - "sharegpt_jBwwg7B_0", - "bf3aebdb_1", - "sharegpt_YfIEYo1_0" - ] - }, - { - "answer_session_ids": [ - "answer_e7fe8c8b_1", - "answer_e7fe8c8b_2", - "answer_e7fe8c8b_3" - ], - "ndcg_at_10": 0.8711, - "ndcg_at_5": 0.9197, - "question_id": "60472f9c", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e7fe8c8b_1", - "b1815b81", - "answer_e7fe8c8b_2", - "7b74d32d_3", - "sharegpt_kF8hHMt_45", - "answer_e7fe8c8b_3", - "ultrachat_334324", - "ultrachat_126217", - "ultrachat_470951", - "sharegpt_5BPO9KJ_0" - ] - }, - { - "answer_session_ids": [ - "answer_3826dc55_1", - "answer_3826dc55_3", - "answer_3826dc55_5", - "answer_3826dc55_2", - "answer_3826dc55_4" - ], - "ndcg_at_10": 0.7157, - "ndcg_at_5": 0.7123, - "question_id": "gpt4_194be4b3", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "ultrachat_58180", - "answer_3826dc55_3", - "answer_3826dc55_5", - "31403c51", - "answer_3826dc55_4", - "ultrachat_385467", - "ultrachat_474217", - "answer_3826dc55_1", - "sharegpt_KwbJJ66_12", - "sharegpt_VO7dTG3_0" - ] - }, - { - "answer_session_ids": [ - "answer_cc021f81_2", - "answer_cc021f81_3", - "answer_cc021f81_1" - ], - "ndcg_at_10": 0.906, - "ndcg_at_5": 0.906, - "question_id": "a9f6b44c", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_cc021f81_3", - "837f258b", - "answer_cc021f81_1", - "answer_cc021f81_2", - "d4912cd9", - "ca896bdc_1", - "sharegpt_AqgYTct_0", - "ec830058_1", - "ultrachat_279504", - "b7e36cc5_2" - ] - }, - { - "answer_session_ids": [ - "answer_5cdf9bd2_2", - "answer_5cdf9bd2_1", - "answer_5cdf9bd2_3", - "answer_5cdf9bd2_4" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "d851d5ba", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_5cdf9bd2_2", - "answer_5cdf9bd2_1", - "answer_5cdf9bd2_4", - "answer_5cdf9bd2_3", - "sharegpt_0gZVK1A_15", - "68ace657_2", - "sharegpt_wduYbsX_0", - "sharegpt_dxirwR4_25", - "sharegpt_x7apNZ0_0", - "d77d4ac9_1" - ] - }, - { - "answer_session_ids": [ - "answer_4cef8a3c_3", - "answer_4cef8a3c_1", - "answer_4cef8a3c_2" - ], - "ndcg_at_10": 0.9675, - "ndcg_at_5": 0.9675, - "question_id": "5a7937c8", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_4cef8a3c_3", - "answer_4cef8a3c_2", - "3b38be11_2", - "answer_4cef8a3c_1", - "ultrachat_288747", - "ultrachat_377263", - "b6bc5b09_2", - "a76e7e3c_4", - "b70ac29f_2", - "07417b21" - ] - }, - { - "answer_session_ids": [ - "answer_728deb4d_5", - "answer_728deb4d_2", - "answer_728deb4d_3", - "answer_728deb4d_1", - "answer_728deb4d_4" - ], - "ndcg_at_10": 0.9709, - "ndcg_at_5": 1.0, - "question_id": "gpt4_ab202e7f", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_728deb4d_4", - "answer_728deb4d_5", - "answer_728deb4d_3", - "answer_728deb4d_2", - "ultrachat_433876", - "sharegpt_q3qJzui_0", - "29bc69b3", - "cdcbdf13", - "answer_728deb4d_1", - "sharegpt_xD2q9ER_14" - ] - }, - { - "answer_session_ids": [ - "answer_6350aa4f_1", - "answer_6350aa4f_2", - "answer_6350aa4f_3", - "answer_6350aa4f_4" - ], - "ndcg_at_10": 0.9709, - "ndcg_at_5": 1.0, - "question_id": "gpt4_e05b82a6", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_6350aa4f_2", - "answer_6350aa4f_4", - "answer_6350aa4f_3", - "88e01130_1", - "b405d3d8_1", - "answer_6350aa4f_1", - "aed6b1b6_1", - "386f6df9", - "ultrachat_43272", - "ultrachat_256330" - ] - }, - { - "answer_session_ids": [ - "answer_826d51da_3", - "answer_826d51da_4", - "answer_826d51da_2", - "answer_826d51da_1" - ], - "ndcg_at_10": 0.7614, - "ndcg_at_5": 1.0, - "question_id": "gpt4_731e37d7", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_826d51da_1", - "07820595_1", - "5365840b_2", - "267a912c", - "c566edbb_3", - "sharegpt_Rwql31f_62", - "answer_826d51da_2", - "7e4aa7c2_1", - "baa5b468_3", - "answer_826d51da_3" - ] - }, - { - "answer_session_ids": [ - "answer_60e8941a_2", - "answer_60e8941a_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "edced276", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_60e8941a_2", - "answer_60e8941a_1", - "1da409cd_1", - "f584ba36_2", - "59a6150d_3", - "726fa34a_1", - "5726dc37_2", - "f5e28561_4", - "93f23301_1", - "e4bbcd5f_4" - ] - }, - { - "answer_session_ids": [ - "answer_e0585cb5_2", - "answer_e0585cb5_1" - ], - "ndcg_at_10": 0.5, - "ndcg_at_5": 0.5, - "question_id": "10d9b85a", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "c51583cd_3", - "cbd1fe79_2", - "answer_e0585cb5_2", - "8dd2fca2", - "84889496_1", - "aa6afba8", - "d576152e_1", - "answer_e0585cb5_1", - "faac3c60", - "5cbfaf3e_4" - ] - }, - { - "answer_session_ids": [ - "answer_b6018747_2", - "answer_b6018747_4", - "answer_b6018747_1", - "answer_b6018747_3" - ], - "ndcg_at_10": 0.7107, - "ndcg_at_5": 0.6934, - "question_id": "e3038f8c", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "sharegpt_hgGAUvu_0", - "answer_b6018747_2", - "answer_b6018747_4", - "85846900_2", - "a3d8e134_2", - "answer_b6018747_3", - "answer_b6018747_1", - "sharegpt_s72sGRc_0", - "4629287b", - "sharegpt_WxCMnMO_5" - ] - }, - { - "answer_session_ids": [ - "answer_23759615_2", - "answer_23759615_3", - "answer_23759615_1" - ], - "ndcg_at_10": 0.9066, - "ndcg_at_5": 1.0, - "question_id": "2b8f3739", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_23759615_3", - "answer_23759615_1", - "dd25eeb5_6", - "c075835c", - "67d6f18f_1", - "285525b2_1", - "ec8691f5", - "sharegpt_ZH6IAa2_11", - "answer_23759615_2", - "ce6cae1b_3" - ] - }, - { - "answer_session_ids": [ - "answer_2bd23659_3", - "answer_2bd23659_2", - "answer_2bd23659_4", - "answer_2bd23659_1" - ], - "ndcg_at_10": 0.906, - "ndcg_at_5": 0.906, - "question_id": "1a8a66a6", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_2bd23659_3", - "54aa4875", - "answer_2bd23659_1", - "answer_2bd23659_2", - "1e91a236_1", - "dc71023e", - "sharegpt_lsiJM0Y_0", - "d6792df8", - "ebb5d262", - "ultrachat_287097" - ] - }, - { - "answer_session_ids": [ - "answer_923c0221_1", - "answer_923c0221_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "c2ac3c61", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_923c0221_2", - "answer_923c0221_1", - "0c483a3c", - "8e4e215b", - "sharegpt_2BSHRQc_25", - "6009ba4e_5", - "9f803df5_1", - "ultrachat_412857", - "ultrachat_38171", - "0b64c6cb_1" - ] - }, - { - "answer_session_ids": [ - "answer_7726e7e9_1", - "answer_7726e7e9_3", - "answer_7726e7e9_2" - ], - "ndcg_at_10": 0.8385, - "ndcg_at_5": 0.8772, - "question_id": "bf659f65", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_7726e7e9_1", - "f2835879_2", - "9d480413_5", - "answer_7726e7e9_2", - "ultrachat_465222", - "answer_7726e7e9_3", - "258645ba_3", - "71619051_1", - "ultrachat_377765", - "sharegpt_SBJezEx_4" - ] - }, - { - "answer_session_ids": [ - "answer_35c5419d_3", - "answer_35c5419d_2", - "answer_35c5419d_1" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "gpt4_372c3eed", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_35c5419d_3", - "4d04b866", - "answer_35c5419d_1", - "ultrachat_400797", - "ultrachat_290513", - "ultrachat_424492", - "0d396995_1", - "bc1f1051_5", - "c95f88fd_1", - "sharegpt_d1pTYei_39" - ] - }, - { - "answer_session_ids": [ - "answer_669318cf_2", - "answer_669318cf_1", - "answer_669318cf_3" - ], - "ndcg_at_10": 0.9469, - "ndcg_at_5": 0.9469, - "question_id": "gpt4_2f91af09", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_669318cf_2", - "answer_669318cf_1", - "22b3af37_1", - "ultrachat_42946", - "answer_669318cf_3", - "51c82c8e_2", - "d75245ea", - "541ecc45_1", - "fd6f60f0_2", - "e7e416f5_2" - ] - }, - { - "answer_session_ids": [ - "answer_da3c1266_3", - "answer_da3c1266_4", - "answer_da3c1266_1", - "answer_da3c1266_5", - "answer_da3c1266_2" - ], - "ndcg_at_10": 0.9829, - "ndcg_at_5": 0.9829, - "question_id": "81507db6", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_da3c1266_2", - "answer_da3c1266_4", - "answer_da3c1266_5", - "5c2f41a0_1", - "answer_da3c1266_1", - "9071ba70_3", - "ultrachat_162129", - "88c09142_2", - "42fa05d3", - "058f17e5_1" - ] - }, - { - "answer_session_ids": [ - "answer_edb03329" - ], - "ndcg_at_10": 0.6309, - "ndcg_at_5": 0.6309, - "question_id": "8a2466db", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "6a5b5a78", - "answer_edb03329", - "6dcf5fa0_1", - "3392c0c7", - "6be54739_3", - "d8b3e1c8_2", - "3d0c9f89_1", - "76967d22", - "sharegpt_MKMWjX0_25", - "sharegpt_osTHjYi_0" - ] - }, - { - "answer_session_ids": [ - "answer_555dfb94" - ], - "ndcg_at_10": 0.3869, - "ndcg_at_5": 0.3869, - "question_id": "06878be2", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "a6074da9_1", - "ca929779_1", - "3e9fce53_1", - "465e7ef9", - "answer_555dfb94", - "8fcfbe43_2", - "sharegpt_GSC090N_6", - "ultrachat_448784", - "79d53fd0_2", - "sharegpt_InRLwN7_0" - ] - }, - { - "answer_session_ids": [ - "answer_d87a6ef8" - ], - "ndcg_at_10": 0.0, - "ndcg_at_5": 0.0, - "question_id": "75832dbd", - "question_type": "single-session-preference", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 0.0, - "r_at_5": 0.0, - "top_session_ids": [ - "d49b1a24_1", - "1ef63c66", - "c884b67a_1", - "sharegpt_xni9yYO_6", - "ultrachat_165893", - "sharegpt_FYqt26U_0", - "47ec1674_2", - "11e9d591_1", - "sharegpt_fK7pfNf_0", - "ultrachat_93325" - ] - }, - { - "answer_session_ids": [ - "answer_d586e9cd" - ], - "ndcg_at_10": 0.5, - "ndcg_at_5": 0.5, - "question_id": "0edc2aef", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "f20e72e4_1", - "6a747f2e", - "answer_d586e9cd", - "08ca1f31_2", - "sharegpt_AsgRx0A_0", - "53dc1394", - "3c3a9042_3", - "8464304d_2", - "c6c3a982_1", - "52939d20" - ] - }, - { - "answer_session_ids": [ - "answer_9b182436" - ], - "ndcg_at_10": 0.5, - "ndcg_at_5": 0.5, - "question_id": "35a27287", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "ultrachat_267380", - "465f0d1b_1", - "answer_9b182436", - "58ae7c4f_3", - "880ee13e", - "e4a1b565_2", - "203bf3fa_2", - "c9b1c309_4", - "ultrachat_422353", - "7128c070_2" - ] - }, - { - "answer_session_ids": [ - "answer_0250ae1c" - ], - "ndcg_at_10": 0.5, - "ndcg_at_5": 0.5, - "question_id": "32260d93", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "c4d370d3_2", - "04bef53b", - "answer_0250ae1c", - "573dc24b_1", - "3f787a78_4", - "ultrachat_26485", - "ultrachat_171173", - "sharegpt_H4jw5s7_39", - "sharegpt_8jxRz9U_0", - "sharegpt_cMHMPNy_29" - ] - }, - { - "answer_session_ids": [ - "answer_6dc4305e" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "195a1a1b", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_6dc4305e", - "ultrachat_396124", - "9f12daa1_1", - "1b71c896_2", - "d03b6a05_2", - "ultrachat_576932", - "9b90460f_3", - "29695e1c_1", - "87fff4b4_1", - "ultrachat_199480" - ] - }, - { - "answer_session_ids": [ - "answer_8549e5e0" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "afdc33df", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_8549e5e0", - "0b0bc8ae_2", - "22f9f163_3", - "b6558bfb", - "afa0c2e7_2", - "d03b6a05_3", - "43b53ac4_1", - "ultrachat_381452", - "ultrachat_512490", - "e58109ed_1" - ] - }, - { - "answer_session_ids": [ - "answer_2fc6aabb" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "caf03d32", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_2fc6aabb", - "ec806228", - "6b137bce_4", - "eb0d8dc1_6", - "4374f9a6_2", - "ultrachat_501938", - "sharegpt_3D7Qpuq_14", - "af631aa3_1", - "bd6d0687", - "147e93c4" - ] - }, - { - "answer_session_ids": [ - "answer_f7b22c66" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "54026fce", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_f7b22c66", - "ba4db050_1", - "fab585c2_1", - "670c49e6", - "ultrachat_166270", - "ef5d4aa7_1", - "1c95e152_1", - "sharegpt_jyHBcl1_0", - "sharegpt_DnGdKtp_0", - "sharegpt_8YTuqXO_0" - ] - }, - { - "answer_session_ids": [ - "answer_92d5f7cd" - ], - "ndcg_at_10": 0.0, - "ndcg_at_5": 0.0, - "question_id": "06f04340", - "question_type": "single-session-preference", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 0.0, - "r_at_5": 0.0, - "top_session_ids": [ - "6e6fbb6b", - "91223fd5_1", - "sharegpt_h4ZC1fl_203", - "66bfa1db", - "0844dea6", - "42924d15", - "8b156015_2", - "728deb4d_4", - "sharegpt_MkLNumZ_0", - "sharegpt_tb5H6IH_3" - ] - }, - { - "answer_session_ids": [ - "answer_f6502d0f" - ], - "ndcg_at_10": 0.3869, - "ndcg_at_5": 0.3869, - "question_id": "6b7dfb22", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "574df152_1", - "9ee69ca6_3", - "377f8fd9_1", - "56859d37", - "answer_f6502d0f", - "be581797", - "ultrachat_443160", - "fc69fd44_3", - "603a8580_2", - "ultrachat_336051" - ] - }, - { - "answer_session_ids": [ - "answer_719502eb" - ], - "ndcg_at_10": 0.6309, - "ndcg_at_5": 0.6309, - "question_id": "1a1907b4", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "9585e0c6_1", - "answer_719502eb", - "4f3b36d4_5", - "3dced3e9_1", - "e42e7876_5", - "4e3da326_1", - "ultrachat_183428", - "ultrachat_490921", - "097c93d8", - "sharegpt_buGcwO5_0" - ] - }, - { - "answer_session_ids": [ - "answer_b10dce5e" - ], - "ndcg_at_10": 0.3562, - "ndcg_at_5": 0.0, - "question_id": "09d032c9", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 0.0, - "top_session_ids": [ - "b21bd3e2", - "3fc2244f", - "af631aa3_2", - "d3971322_1", - "612e23f1", - "answer_b10dce5e", - "26d9aaaf", - "9d1999d7_1", - "e8bfacec_2", - "sharegpt_e9sAtcZ_63" - ] - }, - { - "answer_session_ids": [ - "answer_772472c8" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "38146c39", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_772472c8", - "8064b6ca", - "909d57ca_2", - "5263736d_1", - "87aaee77", - "381b80a3", - "5806662d_1", - "a86b30e4", - "sharegpt_yBVQLaX_15", - "ultrachat_233519" - ] - }, - { - "answer_session_ids": [ - "answer_7c0ade93" - ], - "ndcg_at_10": 0.6309, - "ndcg_at_5": 0.6309, - "question_id": "d24813b1", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "3124bb28_1", - "answer_7c0ade93", - "084802f9_3", - "61a46ff7_3", - "c2a34674_1", - "ultrachat_494027", - "446173bb_1", - "1bc08e44", - "128f4e4d_3", - "sharegpt_tntUB4J_0" - ] - }, - { - "answer_session_ids": [ - "answer_1bde8d3b" - ], - "ndcg_at_10": 0.3869, - "ndcg_at_5": 0.3869, - "question_id": "57f827a0", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "d0f42e3f", - "1bff1675", - "e85e0eaf_1", - "2ef55f49_1", - "answer_1bde8d3b", - "1fc5074c_2", - "5150a4e9_2", - "d05d6391_1", - "6dd592c3", - "963e896f_1" - ] - }, - { - "answer_session_ids": [ - "answer_5e613445" - ], - "ndcg_at_10": 0.3155, - "ndcg_at_5": 0.0, - "question_id": "95228167", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 0.0, - "top_session_ids": [ - "b916da64_2", - "3d2e66f2_2", - "385a60c8_3", - "084802f9_2", - "99e0fd9f_3", - "5d8c99d3_3", - "7e17ae56_3", - "answer_5e613445", - "e2691068", - "c34b6a1c_1" - ] - }, - { - "answer_session_ids": [ - "answer_f3164f2c" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "505af2f5", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_f3164f2c", - "7d52ca48_2", - "4f2d6be4_2", - "5e4bb245_1", - "af631aa3_1", - "0e193841_8", - "sharegpt_cjulkGS_15", - "sharegpt_kjeGJvK_11", - "ultrachat_208471", - "9f8fa5e4_1" - ] - }, - { - "answer_session_ids": [ - "answer_8ee04a2e" - ], - "ndcg_at_10": 0.6309, - "ndcg_at_5": 0.6309, - "question_id": "75f70248", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "2d74df23_4", - "answer_8ee04a2e", - "53dc1394", - "75671e3f_1", - "204862e4_2", - "ff67236f_3", - "6f689aee", - "ultrachat_27339", - "sharegpt_ijZ2Bps_0", - "ultrachat_309088" - ] - }, - { - "answer_session_ids": [ - "answer_b0fac439" - ], - "ndcg_at_10": 0.4307, - "ndcg_at_5": 0.4307, - "question_id": "d6233ab6", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "e419b7c3_4", - "94bc18df_3", - "87cfeb28_2", - "answer_b0fac439", - "911f859b_5", - "f916c63a_2", - "ultrachat_457634", - "sharegpt_xni9yYO_6", - "b45e05f6", - "ultrachat_329160" - ] - }, - { - "answer_session_ids": [ - "answer_4d3be2ab" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "1da05512", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_4d3be2ab", - "f2565b3b_2", - "sharegpt_eSTYwMj_14", - "ce919391_2", - "6b93018e_2", - "6dfa3dc6_2", - "ce3ad11f_2", - "5aa88956_2", - "ultrachat_159846", - "6de70f46_1" - ] - }, - { - "answer_session_ids": [ - "answer_a1e169b1" - ], - "ndcg_at_10": 0.3869, - "ndcg_at_5": 0.3869, - "question_id": "fca70973", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "fc005760", - "f504b269", - "5d5e80c5_1", - "99b061c5_1", - "answer_a1e169b1", - "2f2884ad_1", - "ultrachat_3370", - "312bf939", - "sharegpt_lSeiu11_0", - "sharegpt_fPvtTbw_0" - ] - }, - { - "answer_session_ids": [ - "answer_8414cc57" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "b6025781", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_8414cc57", - "f20c73f5_3", - "1a693999_4", - "3a1ea336_1", - "d6956a2e_1", - "1d6744b5_1", - "612c8368_2", - "6446f6e6", - "ebe60348_1", - "2ef53f61_1" - ] - }, - { - "answer_session_ids": [ - "answer_8f15ac24" - ], - "ndcg_at_10": 0.5, - "ndcg_at_5": 0.5, - "question_id": "a89d7624", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "99f2f5b1_1", - "02f70006_1", - "answer_8f15ac24", - "a479a7d6_4", - "fd778d8e", - "39a056ae", - "082fc07f_1", - "168776ed_2", - "bef3247f_1", - "sharegpt_G3RxeFJ_94" - ] - }, - { - "answer_session_ids": [ - "answer_30f63ddb" - ], - "ndcg_at_10": 0.4307, - "ndcg_at_5": 0.4307, - "question_id": "b0479f84", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "6af947bf_1", - "0cf86cbe_2", - "fbfda981", - "answer_30f63ddb", - "457b1adb_1", - "94cffbd2_2", - "9090b1f0_2", - "ddf41a24", - "d1bb5323_1", - "1cde7ee4_1" - ] - }, - { - "answer_session_ids": [ - "answer_e6b6353d" - ], - "ndcg_at_10": 0.3155, - "ndcg_at_5": 0.0, - "question_id": "1d4e3b97", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 0.0, - "top_session_ids": [ - "ecb24dd8_2", - "809cabca_1", - "c51583cd_3", - "03633a51", - "fb2368b8_3", - "1b320137_1", - "12b6f0f1_3", - "answer_e6b6353d", - "5cd6ab1b", - "ultrachat_8041" - ] - }, - { - "answer_session_ids": [ - "answer_d03098f9" - ], - "ndcg_at_10": 0.3562, - "ndcg_at_5": 0.0, - "question_id": "07b6f563", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 0.0, - "top_session_ids": [ - "dcc7f430_1", - "66ab6260", - "5e7b1c98_1", - "13b79e39_2", - "0ac1a6f9_2", - "answer_d03098f9", - "bbccfe79", - "6af947bf_1", - "55c267aa", - "39358a85_2" - ] - }, - { - "answer_session_ids": [ - "answer_8da8c7e0" - ], - "ndcg_at_10": 0.4307, - "ndcg_at_5": 0.4307, - "question_id": "1c0ddc50", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "2aa70c9c_1", - "5439cbf0_2", - "b33e89b5_1", - "answer_8da8c7e0", - "c7ca6dff", - "a0aa5035", - "ultrachat_339938", - "2566382f_2", - "6cd203f7_1", - "0a7d5bf6_2" - ] - }, - { - "answer_session_ids": [ - "answer_cebb7159" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "0a34ad58", - "question_type": "single-session-preference", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_cebb7159", - "fb048b5d_1", - "9ca2a195_1", - "f849dd04", - "121efb3b_2", - "4b495a04_1", - "f2b8e6a8_1", - "58ad78d6_2", - "ultrachat_122225", - "f7e682c3_5" - ] - }, - { - "answer_session_ids": [ - "answer_5237bb0b_2", - "answer_5237bb0b_1" - ], - "ndcg_at_10": 0.8503, - "ndcg_at_5": 0.8503, - "question_id": "d3ab962e", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_5237bb0b_2", - "ultrachat_564044", - "3329e5e8_1", - "sharegpt_6fTnCnw_49", - "answer_5237bb0b_1", - "aa235649_1", - "ultrachat_164834", - "ultrachat_155769", - "sharegpt_CDacKfJ_0", - "ultrachat_248741" - ] - }, - { - "answer_session_ids": [ - "answer_bf633415_2", - "answer_bf633415_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "2311e44b", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_bf633415_2", - "answer_bf633415_1", - "b5b8f8f9_3", - "5854eebc_2", - "ultrachat_360918", - "c824c711_1", - "84fedd9f_2", - "c6bed037_1", - "sharegpt_YAiYuAD_7", - "2714885f_1" - ] - }, - { - "answer_session_ids": [ - "answer_4fb01417_2", - "answer_4fb01417_1" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "cc06de0d", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_4fb01417_1", - "01551a39_2", - "answer_4fb01417_2", - "55033f6f", - "d5eab084_2", - "1dbe77da_1", - "ultrachat_104440", - "66ffbb8b_1", - "670c49e6", - "92d74e73" - ] - }, - { - "answer_session_ids": [ - "answer_c69ee1f9_2", - "answer_c69ee1f9_1" - ], - "ndcg_at_10": 0.8503, - "ndcg_at_5": 0.8503, - "question_id": "a11281a2", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c69ee1f9_2", - "ultrachat_55839", - "bb2180e9_3", - "c4c4d85a", - "answer_c69ee1f9_1", - "b8adda7f_1", - "28209b6a_5", - "d95e64a7_4", - "9126833b", - "60f10fd3" - ] - }, - { - "answer_session_ids": [ - "answer_50940cb7_2", - "answer_50940cb7_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "4f54b7c9", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_50940cb7_1", - "answer_50940cb7_2", - "bcfd4ab7", - "dde142b2_1", - "ultrachat_518898", - "8a834aee_1", - "b89f68c4_1", - "49a91ed8", - "663337ce_1", - "ultrachat_473718" - ] - }, - { - "answer_session_ids": [ - "answer_dcb18b9b_2", - "answer_dcb18b9b_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "85fa3a3f", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_dcb18b9b_2", - "answer_dcb18b9b_1", - "8f707521_1", - "71110dbd_1", - "0d66562c_1", - "c6c06e6e", - "93ff2f73", - "d31c3ec8_1", - "b7fd3b48", - "sharegpt_3WzI4bW_0" - ] - }, - { - "answer_session_ids": [ - "answer_353d3c6d_2", - "answer_353d3c6d_1" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "9aaed6a3", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_353d3c6d_2", - "c846422a_1", - "answer_353d3c6d_1", - "819dfad7_2", - "3f9f66f1_1", - "2e89bd03_1", - "11e942da_2", - "sharegpt_HpdNUlz_9", - "1bf761b8_2", - "73726c40_1" - ] - }, - { - "answer_session_ids": [ - "answer_d8454588_1", - "answer_d8454588_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "1f2b8d4f", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_d8454588_2", - "answer_d8454588_1", - "d3ee5958_2", - "c9b1c309_4", - "c8c64252", - "b5c147c7_2", - "9e96d5b9_1", - "0bd59f15_3", - "23759615_1", - "sharegpt_OHIxTQm_22" - ] - }, - { - "answer_session_ids": [ - "answer_4eb6d671_2", - "answer_4eb6d671_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e6041065", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_4eb6d671_2", - "answer_4eb6d671_1", - "d79173aa_1", - "b73a4d04_1", - "49f2e396_2", - "4a006b77", - "ec616e7e_4", - "2a86d0da_2", - "bdcee74e_3", - "b7dca8f1_1" - ] - }, - { - "answer_session_ids": [ - "answer_58820c75_1", - "answer_58820c75_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "51c32626", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_58820c75_1", - "answer_58820c75_2", - "c051e740_2", - "sharegpt_hzwc0HI_13", - "84d7650f_1", - "809cbce9_1", - "sharegpt_ErOTMZ3_59", - "932ea3bf_1", - "ultrachat_273746", - "sharegpt_jBafAKm_0" - ] - }, - { - "answer_session_ids": [ - "answer_85a77c48_2", - "answer_85a77c48_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "d905b33f", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_85a77c48_1", - "answer_85a77c48_2", - "8cfa91c3_2", - "2d364fb0", - "a9b7f49e_1", - "368276ab_3", - "dd4a0e15", - "6423d6a4_2", - "7613d5ec_2", - "sharegpt_ektieAO_13" - ] - }, - { - "answer_session_ids": [ - "answer_80323f3f_1", - "answer_80323f3f_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "7405e8b1", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_80323f3f_1", - "answer_80323f3f_2", - "f2f2a606_3", - "a6b00510_4", - "6c1c5bc3_2", - "c578da1f_1", - "6e983235_1", - "3a4012a5_1", - "8f8f2ced_2", - "643d285a_3" - ] - }, - { - "answer_session_ids": [ - "answer_e9bb9500_2", - "answer_e9bb9500_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "f35224e0", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e9bb9500_2", - "answer_e9bb9500_1", - "96da07f9_4", - "e184e4c3_1", - "9393001b_3", - "ultrachat_215582", - "4648f214_2", - "42234f98", - "ultrachat_558221", - "sharegpt_Pqhvz5n_0" - ] - }, - { - "answer_session_ids": [ - "answer_743f03a1_2", - "answer_743f03a1_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "6456829e", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_743f03a1_1", - "answer_743f03a1_2", - "c44d1533_2", - "ultrachat_490336", - "ultrachat_518760", - "a8c73552", - "898cfd46_2", - "ultrachat_555576", - "3304f57b_3", - "ultrachat_247641" - ] - }, - { - "answer_session_ids": [ - "answer_feb5f98a_1", - "answer_feb5f98a_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "a4996e51", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_feb5f98a_2", - "answer_feb5f98a_1", - "252715f8_3", - "e9fba4f8", - "c1dac02f", - "082b7e52_2", - "d1a61aa1_3", - "ultrachat_506025", - "sharegpt_KjF70Iy_0", - "eea3e4be_2" - ] - }, - { - "answer_session_ids": [ - "answer_c8cc60d6_2", - "answer_c8cc60d6_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "3c1045c8", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c8cc60d6_2", - "ultrachat_561782", - "ultrachat_20893", - "b52f03a1_1", - "62541047", - "sharegpt_oIB2JG9_0", - "4f3b36d4_3", - "ultrachat_299649", - "sharegpt_GYqnAhC_190", - "cf422c07" - ] - }, - { - "answer_session_ids": [ - "answer_e552e1f9_1", - "answer_e552e1f9_2" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "60036106", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e552e1f9_2", - "sharegpt_moARQa7_0", - "answer_e552e1f9_1", - "sharegpt_ckvrgF1_99", - "c575291e_2", - "e9cb7125", - "ultrachat_466668", - "23a7f0ec", - "763f028a", - "ultrachat_134557" - ] - }, - { - "answer_session_ids": [ - "answer_3be95d43_1", - "answer_3be95d43_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "681a1674", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_3be95d43_2", - "answer_3be95d43_1", - "ultrachat_64650", - "fdd25ea1", - "50e8f2b5_1", - "d81e853b", - "sharegpt_8Kzmn95_0", - "7d33fcb4_2", - "ultrachat_436833", - "9e411500_1" - ] - }, - { - "answer_session_ids": [ - "answer_6702277b_1", - "answer_6702277b_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e25c3b8d", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_6702277b_2", - "answer_6702277b_1", - "sharegpt_40qcQLS_17", - "sharegpt_WYQSZ3n_0", - "90fea364_1", - "sharegpt_Nk10TjE_29", - "824634d9_1", - "ff201786_1", - "4256f981_2", - "ultrachat_363088" - ] - }, - { - "answer_session_ids": [ - "answer_6efce493_1", - "answer_6efce493_2" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "4adc0475", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_6efce493_1", - "bb27df5e_2", - "answer_6efce493_2", - "4fc2cf8c_1", - "fc6f9c59", - "2880eb6c_3", - "0bde5dac", - "sharegpt_wcTU09Q_31", - "debc34d7_2", - "ee659010_2" - ] - }, - { - "answer_session_ids": [ - "answer_9ef115d4_1", - "answer_9ef115d4_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "4bc144e2", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_9ef115d4_1", - "answer_9ef115d4_2", - "4f23a396_2", - "d75245ea", - "ultrachat_380790", - "42bcee92_2", - "099c1b6c_3", - "60207b41_3", - "26d9aaaf", - "2e2ae792_1" - ] - }, - { - "answer_session_ids": [ - "answer_f7fd1029_2", - "answer_f7fd1029_1" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "ef66a6e5", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_f7fd1029_1", - "48941fec_4", - "answer_f7fd1029_2", - "74c2d3a1_1", - "3f05a03e_1", - "41e778e4_2", - "afb58af0", - "295645f5_1", - "67045503", - "5273d4b8" - ] - }, - { - "answer_session_ids": [ - "answer_688f9a3f_1", - "answer_688f9a3f_2" - ], - "ndcg_at_10": 0.6934, - "ndcg_at_5": 0.6934, - "question_id": "5025383b", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "22f9af5f", - "answer_688f9a3f_2", - "answer_688f9a3f_1", - "f40fa3ed", - "2d21f921", - "ultrachat_50563", - "d10f0307_1", - "ultrachat_150500", - "sharegpt_BmS3AX0_10", - "ultrachat_137351" - ] - }, - { - "answer_session_ids": [ - "answer_17dc2f5b_2", - "answer_17dc2f5b_1" - ], - "ndcg_at_10": 0.6309, - "ndcg_at_5": 0.6309, - "question_id": "a1cc6108", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "298a07de_5", - "answer_17dc2f5b_2", - "fce56dfb_4", - "7285299a_4", - "6dfc99bf_1", - "001cefa7_2", - "fb328ace_1" - ] - }, - { - "answer_session_ids": [ - "answer_66c23110_1", - "answer_66c23110_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "9ee3ecd6", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_66c23110_2", - "answer_66c23110_1", - "fd56c5bd_2", - "a9eda378", - "6d9d5941", - "ultrachat_282862", - "fa0fa74d_2", - "ultrachat_96128", - "sharegpt_6cz1Sq6_264", - "42b45f6e_1" - ] - }, - { - "answer_session_ids": [ - "answer_419d21d5_2", - "answer_419d21d5_1" - ], - "ndcg_at_10": 0.8316, - "ndcg_at_5": 1.0, - "question_id": "3fdac837", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_419d21d5_2", - "92b06116_1", - "a98ff421_7", - "ultrachat_526748", - "c85ef24e_1", - "answer_419d21d5_1", - "ac8a2cd0", - "ultrachat_293546", - "88c8df0e_3", - "22210456_1" - ] - }, - { - "answer_session_ids": [ - "answer_5404a208_1", - "answer_5404a208_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "91b15a6e", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_5404a208_1", - "answer_5404a208_2", - "ultrachat_15125", - "f9a304e9_2", - "6169ef55_1", - "ultrachat_217922", - "7cfeb89c_2", - "ultrachat_338803", - "ultrachat_326780", - "ultrachat_222016" - ] - }, - { - "answer_session_ids": [ - "answer_a37bdf22_2", - "answer_a37bdf22_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "27016adc", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_a37bdf22_2", - "answer_a37bdf22_1", - "9f554b46_2", - "fa858208_2", - "28ea20bf", - "ultrachat_495725", - "ff5e0db0_1", - "sharegpt_3b5ojYa_39", - "ultrachat_131737", - "ultrachat_284316" - ] - }, - { - "answer_session_ids": [ - "answer_c9dfeaea_1", - "answer_c9dfeaea_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "720133ac", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c9dfeaea_2", - "answer_c9dfeaea_1", - "4c128d97_2", - "e200d96c_3", - "ff2de758", - "ultrachat_479833", - "sharegpt_bZg2XwX_69", - "0cf86cbe_2", - "ultrachat_112896", - "ultrachat_512422" - ] - }, - { - "answer_session_ids": [ - "answer_33c251f0_2", - "answer_33c251f0_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "77eafa52", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_33c251f0_1", - "answer_33c251f0_2", - "bc1f1051_4", - "sharegpt_qEukY6n_14", - "sharegpt_FlvsHKg_4", - "sharegpt_C9GjDsF_0", - "sharegpt_bpxJpAU_0", - "ultrachat_309057", - "9c4358a4", - "7b13b4d6_2" - ] - }, - { - "answer_session_ids": [ - "answer_35e36341_1", - "answer_35e36341_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "8979f9ec", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_35e36341_1", - "answer_35e36341_2", - "91c71874_1", - "ultrachat_466891", - "bcc9382a_2", - "ultrachat_562471", - "4c967baa_1", - "a6b4c4c2_1", - "f18ebe36_7", - "1064ed80_2" - ] - }, - { - "answer_session_ids": [ - "answer_35c9798c_2", - "answer_35c9798c_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "0100672e", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_35c9798c_2", - "answer_35c9798c_1", - "a4b223a7_1", - "091aa510_2", - "326e86e0_3", - "8237de6a_2", - "ultrachat_372868", - "07313722_2", - "b167a4a8", - "f60c9de5" - ] - }, - { - "answer_session_ids": [ - "answer_ef84b994_1", - "answer_ef84b994_2" - ], - "ndcg_at_10": 0.8503, - "ndcg_at_5": 0.8503, - "question_id": "a96c20ee", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ef84b994_1", - "1b51fb71_1", - "dff89d9d_1", - "9ca3534f_1", - "answer_ef84b994_2", - "2aa70c9c_2", - "sharegpt_opQbgPL_7", - "32e292f2", - "sharegpt_lWLBUhQ_539", - "sharegpt_00qHEQ6_0" - ] - }, - { - "answer_session_ids": [ - "answer_6cb8f792_1", - "answer_6cb8f792_2" - ], - "ndcg_at_10": 0.525, - "ndcg_at_5": 0.5, - "question_id": "92a0aa75", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "ultrachat_178564", - "sharegpt_YwmoLVl_0", - "answer_6cb8f792_1", - "a5854a8d_2", - "2bdd78cb_1", - "answer_6cb8f792_2", - "bd00a24d_2", - "sharegpt_xkWMnwQ_0", - "sharegpt_9l7gjYP_17", - "sharegpt_hPax8pw_11" - ] - }, - { - "answer_session_ids": [ - "answer_1bb63ea5_2", - "answer_1bb63ea5_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "3fe836c9", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_1bb63ea5_1", - "answer_1bb63ea5_2", - "e41b78c7_3", - "0a42783a", - "c81897cd_2", - "sharegpt_2xK3jjf_0", - "8129a7a8", - "f0d7d364", - "ba184a34", - "ultrachat_258020" - ] - }, - { - "answer_session_ids": [ - "answer_4cb841a8_1", - "answer_4cb841a8_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "1c549ce4", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_4cb841a8_1", - "answer_4cb841a8_2", - "f1e4daa8_2", - "f999b05c_1", - "d743153b_2", - "ultrachat_441412", - "162ff451", - "sharegpt_miOKGt1_13", - "35f052f9_2", - "sharegpt_yLBm8Vr_23" - ] - }, - { - "answer_session_ids": [ - "answer_cc1ced21_2", - "answer_cc1ced21_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "6c49646a", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_cc1ced21_2", - "answer_cc1ced21_1", - "d4380d3f", - "8acb29f9_2", - "ultrachat_565061", - "fffa6e59", - "9f220c66_1", - "6a2f306a", - "sharegpt_1X61wv2_7", - "edf2a486" - ] - }, - { - "answer_session_ids": [ - "answer_e184e4c3_2", - "answer_e184e4c3_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "1192316e", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e184e4c3_2", - "answer_e184e4c3_1", - "574df152_3", - "c5db849c_1", - "6cd7234e_2", - "ee82ed2c_2", - "21ef2d05_1", - "sharegpt_pZ3I356_11", - "1ba79ea8_1", - "ultrachat_457634" - ] - }, - { - "answer_session_ids": [ - "answer_dc5e537d_1", - "answer_dc5e537d_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "0ea62687", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_dc5e537d_1", - "answer_dc5e537d_2", - "fba28a35_3", - "26eedae1_1", - "0835368c", - "c5e9cbd8_1", - "d3ee5958_1", - "ultrachat_108112", - "sharegpt_HYI3HgX_0", - "sharegpt_Ktz3pBy_0" - ] - }, - { - "answer_session_ids": [ - "answer_3a5010af_2", - "answer_3a5010af_1" - ], - "ndcg_at_10": 0.8772, - "ndcg_at_5": 0.8772, - "question_id": "67e0d0f2", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_3a5010af_2", - "6868c94f", - "98cd882c_1", - "answer_3a5010af_1", - "ultrachat_217333", - "0cde5602_2", - "8c568d75_1", - "2252d3dd", - "sharegpt_tJkPZbz_2", - "sharegpt_GJpf1ou_0" - ] - }, - { - "answer_session_ids": [ - "answer_de64539a_1", - "answer_de64539a_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "bb7c3b45", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_de64539a_2", - "answer_de64539a_1", - "60ff3a8c", - "3891baea_4", - "a51dc9ca", - "64b7d9cc_2", - "ultrachat_305663", - "f3d43a12_3", - "4f2d6be4_1", - "752b10e6_1" - ] - }, - { - "answer_session_ids": [ - "answer_cbd08e3c_1", - "answer_cbd08e3c_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "ba358f49", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_cbd08e3c_1", - "2a2ea005_1", - "20b5caab_2", - "9cc709cb", - "8db0ab95_1", - "4ea6f550", - "385683f0_2", - "648d9cda", - "d964ed3b", - "4f838497_2" - ] - }, - { - "answer_session_ids": [ - "answer_872e8da2_2", - "answer_872e8da2_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "61f8c8f8", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_872e8da2_1", - "answer_872e8da2_2", - "a3107e2a_1", - "6e110a53_1", - "7b88c38b_2", - "2154c810_1", - "7af38385_2", - "e10846c4", - "sharegpt_iL51bzd_0", - "sharegpt_7jGo66i_99" - ] - }, - { - "answer_session_ids": [ - "answer_75eca223_2", - "answer_75eca223_1" - ], - "ndcg_at_10": 0.4575, - "ndcg_at_5": 0.4307, - "question_id": "60159905", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "21f8f481_3", - "d5f4d9fa_2", - "02eae87c_5", - "answer_75eca223_1", - "sharegpt_V7ePzVN_1", - "c351fa3b_1", - "c92b27e2_1", - "answer_75eca223_2", - "6ade84f0", - "c36f4196" - ] - }, - { - "answer_session_ids": [ - "answer_87e3a1cb_1", - "answer_87e3a1cb_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "ef9cf60a", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_87e3a1cb_2", - "answer_87e3a1cb_1", - "2357de36_2", - "7b88c38b_1", - "b330d6b1", - "sharegpt_FNyKOSt_0", - "12dd7681_4", - "7a34bc76_3", - "0babf5b6_2", - "de47d59b_1" - ] - }, - { - "answer_session_ids": [ - "answer_1881e7db_1", - "answer_1881e7db_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "73d42213", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_1881e7db_2", - "answer_1881e7db_1", - "sharegpt_ab6IEma_0", - "9614d065_1", - "sharegpt_G8j6c8J_87", - "sharegpt_ndynIbS_0", - "sharegpt_GKjb2be_0", - "ultrachat_275858", - "a13abca5_1", - "sharegpt_2Na4ASq_0" - ] - }, - { - "answer_session_ids": [ - "answer_92147866_1", - "answer_92147866_2" - ], - "ndcg_at_10": 0.8503, - "ndcg_at_5": 0.8503, - "question_id": "bc149d6b", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_92147866_1", - "7211c69e", - "86935301_2", - "451ecb23_3", - "answer_92147866_2", - "4f435d20_2", - "sharegpt_bmEgv3O_12", - "05be3ff0_3", - "eb5c65e2_1", - "db73b7e4_3" - ] - }, - { - "answer_session_ids": [ - "answer_80d6d664_2", - "answer_80d6d664_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "099778bb", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_80d6d664_2", - "answer_80d6d664_1", - "sharegpt_s1I3UkG_0", - "47103d08", - "0070840d_2", - "b1793bba", - "6bb8af95_1", - "33877350_2", - "95ce0e21_1", - "sharegpt_kshUqZh_6" - ] - }, - { - "answer_session_ids": [ - "answer_96c743d0_2", - "answer_96c743d0_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "09ba9854", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_96c743d0_1", - "answer_96c743d0_2", - "ultrachat_177260", - "32e6c308", - "7a2bdd15_1", - "ae32e467_2", - "8c63238e", - "864a563d_3", - "69a42467_2", - "8682e696_2" - ] - }, - { - "answer_session_ids": [ - "answer_23f3a657_2", - "answer_23f3a657_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "d6062bb9", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_23f3a657_1", - "answer_23f3a657_2", - "sharegpt_xuhzAgG_19", - "b35610f4_4", - "4f8caea3_1", - "5b6aedac", - "59c704ad_2", - "86298245_2", - "sharegpt_afCogMg_13", - "ultrachat_496826" - ] - }, - { - "answer_session_ids": [ - "answer_8de18468_2", - "answer_8de18468_1" - ], - "ndcg_at_10": 0.301, - "ndcg_at_5": 0.0, - "question_id": "157a136e", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 0.0, - "top_session_ids": [ - "ultrachat_255132", - "01049599_1", - "ultrachat_3770", - "sharegpt_qqo6qRj_0", - "a2e2cb72_5", - "sharegpt_6ZXBs1w_0", - "ultrachat_506827", - "298bf7a3_1", - "answer_8de18468_2", - "ultrachat_446084" - ] - }, - { - "answer_session_ids": [ - "answer_2e2085fa_2", - "answer_2e2085fa_1" - ], - "ndcg_at_10": 0.4307, - "ndcg_at_5": 0.4307, - "question_id": "c18a7dc8", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "9d5a389d", - "ultrachat_353377", - "5022812c", - "answer_2e2085fa_1", - "fc6c3985", - "f716720c_3", - "ultrachat_415226", - "37141183", - "ultrachat_463016", - "sharegpt_YPC6Oll_18" - ] - }, - { - "answer_session_ids": [ - "answer_16ece55f_2", - "answer_16ece55f_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "a3332713", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_16ece55f_2", - "answer_16ece55f_1", - "62be624f", - "67388dfc_2", - "3e59ee68_1", - "fe291ebc", - "117098d9_3", - "33994682_3", - "sharegpt_bYymtlW_0", - "sharegpt_8zrM3l4_0" - ] - }, - { - "answer_session_ids": [ - "answer_fa08bf49_1", - "answer_fa08bf49_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "55241a1f", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_fa08bf49_1", - "answer_fa08bf49_2", - "d117e2da", - "2aeb1268", - "76491ee4_1", - "9c9a22d8", - "60153a02_4", - "sharegpt_TzwvePR_0", - "46e4a8eb_2", - "1d105d8d" - ] - }, - { - "answer_session_ids": [ - "answer_47152166_2", - "answer_47152166_1" - ], - "ndcg_at_10": 0.6934, - "ndcg_at_5": 0.6934, - "question_id": "a08a253f", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "6e5130ed_2", - "answer_47152166_2", - "answer_47152166_1", - "798fa5f2", - "ultrachat_242484", - "33bc1102", - "c7fc2bda_1", - "ultrachat_329610", - "a79f1a04_1", - "ultrachat_403700" - ] - }, - { - "answer_session_ids": [ - "answer_cfcf5340_1", - "answer_cfcf5340_2" - ], - "ndcg_at_10": 0.8503, - "ndcg_at_5": 0.8503, - "question_id": "f0e564bc", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_cfcf5340_2", - "65ca99fb_2", - "798e4ba3_2", - "bc1f1051_1", - "answer_cfcf5340_1", - "sharegpt_W3YFpVK_7", - "sharegpt_3hCiKrk_0", - "06fe8cfb_1", - "ultrachat_197576", - "932a97c6" - ] - }, - { - "answer_session_ids": [ - "answer_254d8b09_1", - "answer_254d8b09_2" - ], - "ndcg_at_10": 0.8316, - "ndcg_at_5": 1.0, - "question_id": "078150f1", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_254d8b09_1", - "61b27d67_3", - "203a48bb_1", - "46e4a8eb_1", - "ultrachat_202549", - "answer_254d8b09_2", - "0e218647_2", - "57bf2079", - "ultrachat_276164", - "b27b0097_1" - ] - }, - { - "answer_session_ids": [ - "answer_e2278b24_1", - "answer_e2278b24_2" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "8cf4d046", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e2278b24_2", - "a30f52a4", - "answer_e2278b24_1", - "sharegpt_P1Okivz_9", - "sharegpt_6DlPq48_0", - "7ef465f3_1", - "sharegpt_WjpyexI_15", - "7a5c32b0_3", - "c8b1178f_1", - "c36f4196" - ] - }, - { - "answer_session_ids": [ - "answer_4934b2d7_2", - "answer_4934b2d7_1" - ], - "ndcg_at_10": 0.8316, - "ndcg_at_5": 1.0, - "question_id": "a346bb18", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_4934b2d7_2", - "771ba1d5", - "5726dc37_2", - "fd5448f0", - "84fb50bb_1", - "answer_4934b2d7_1", - "sharegpt_VeexfGM_0", - "sharegpt_cYNuFCK_50", - "sharegpt_ooEuxDg_0", - "e5713180_1" - ] - }, - { - "answer_session_ids": [ - "answer_6b9b2b1e_2", - "answer_6b9b2b1e_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "37f165cf", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_6b9b2b1e_2", - "answer_6b9b2b1e_1", - "20b1b65f_2", - "a41d46df", - "c85d1682_1", - "c41e97c9_2", - "sharegpt_2yaKUmC_0", - "395a2f92", - "c0f6571a_2", - "7e8e790a" - ] - }, - { - "answer_session_ids": [ - "answer_477ae455_1", - "answer_477ae455_2" - ], - "ndcg_at_10": 0.0, - "ndcg_at_5": 0.0, - "question_id": "8e91e7d9", - "question_type": "multi-session", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 0.0, - "r_at_5": 0.0, - "top_session_ids": [ - "sharegpt_PZrBCF2_0", - "ultrachat_147574", - "4e3da326_1", - "bc790dfb", - "2f980ae0_2", - "c3023a45_4", - "06170195_1", - "sharegpt_8lpX5R7_19", - "c2381e1a_2", - "sharegpt_Z35PJ8x_2" - ] - }, - { - "answer_session_ids": [ - "answer_f56e6152_2", - "answer_f56e6152_1" - ], - "ndcg_at_10": 0.4306, - "ndcg_at_5": 0.3869, - "question_id": "87f22b4a", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "02b99ec3_1", - "ultrachat_350147", - "28741edc_1", - "ultrachat_182225", - "answer_f56e6152_2", - "889076d6", - "c4c504d2_6", - "answer_f56e6152_1", - "a8ac3d1d_3", - "ea28bb75" - ] - }, - { - "answer_session_ids": [ - "answer_430d0a87_1", - "answer_430d0a87_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e56a43b9", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_430d0a87_1", - "answer_430d0a87_2", - "83e14506_2", - "0eef411a_2", - "38e81260_7", - "7596ba6a", - "ce21f0e4", - "sharegpt_XaM55h6_28", - "ultrachat_245457", - "ultrachat_12201" - ] - }, - { - "answer_session_ids": [ - "answer_fc81d1a2_1", - "answer_fc81d1a2_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "efc3f7c2", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_fc81d1a2_1", - "answer_fc81d1a2_2", - "ultrachat_38938", - "263f2395", - "ultrachat_166366", - "bbdc7b0a", - "ultrachat_102676", - "ultrachat_552063", - "48cb014c", - "fc005760" - ] - }, - { - "answer_session_ids": [ - "answer_2c637141_2", - "answer_2c637141_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "21d02d0d", - "question_type": "multi-session", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_2c637141_2", - "answer_2c637141_1", - "eb8195a6_1", - "ultrachat_71299", - "sharegpt_adeEi7y_7", - "06a39bb5_1", - "42d9467c_2", - "a15fa6eb", - "sharegpt_B9jlITg_0", - "414b8ddb" - ] - }, - { - "answer_session_ids": [ - "answer_d00ba6d0_1", - "answer_d00ba6d0_2" - ], - "ndcg_at_10": 0.8772, - "ndcg_at_5": 0.8772, - "question_id": "gpt4_59149c77", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_d00ba6d0_2", - "9e2c2a6c_2", - "9242fbe3_1", - "answer_d00ba6d0_1", - "6e672b84_1", - "sharegpt_XlNs8Lz_199", - "148f87a0_2", - "54a05e72_2", - "ultrachat_318178", - "ultrachat_280434" - ] - }, - { - "answer_session_ids": [ - "answer_3e9fce53_1", - "answer_3e9fce53_2", - "answer_3e9fce53_3" - ], - "ndcg_at_10": 0.9469, - "ndcg_at_5": 0.9469, - "question_id": "gpt4_f49edff3", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_3e9fce53_3", - "answer_3e9fce53_2", - "7d33fcb4_1", - "1a29bf1f", - "answer_3e9fce53_1", - "dc7c53fa_4", - "9f8fa5e4_2", - "acfdbdcb", - "4376945f", - "ultrachat_257206" - ] - }, - { - "answer_session_ids": [ - "answer_0b4a8adc_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "71017276", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_0b4a8adc_1", - "683c8358_1", - "58ec258f_2", - "sharegpt_tCLMKZY_0", - "10889cf0_1", - "4d6927c2_2", - "sharegpt_11WxPMZ_0", - "44e4ae7e", - "398f23e7", - "sharegpt_tmFunkX_20" - ] - }, - { - "answer_session_ids": [ - "answer_4bfcc250_4", - "answer_4bfcc250_3", - "answer_4bfcc250_2", - "answer_4bfcc250_1" - ], - "ndcg_at_10": 0.8529, - "ndcg_at_5": 0.8529, - "question_id": "b46e15ed", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_4bfcc250_3", - "aeab3296", - "fa34119a_1", - "answer_4bfcc250_2", - "answer_4bfcc250_1", - "4bf66c38_2", - "a410bab2_3", - "47311a4c_1", - "ultrachat_520419", - "f598d30f_1" - ] - }, - { - "answer_session_ids": [ - "answer_ff201786_1", - "answer_ff201786_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_fa19884c", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ff201786_2", - "answer_ff201786_1", - "a7b1b27c", - "f3745025_1", - "1b066639", - "f4f4b504_2", - "800cfd94", - "ultrachat_423544", - "cbf48918_1", - "bb2180e9_3" - ] - }, - { - "answer_session_ids": [ - "answer_f4ea84fb_3", - "answer_f4ea84fb_2", - "answer_f4ea84fb_1" - ], - "ndcg_at_10": 0.6462, - "ndcg_at_5": 0.6509, - "question_id": "0bc8ad92", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "0868982f_1", - "answer_f4ea84fb_3", - "f58c36b8_1", - "answer_f4ea84fb_2", - "ultrachat_349938", - "fccc9400", - "d3cc5bdc_1", - "answer_f4ea84fb_1", - "eca613fc", - "sharegpt_scMKXPh_9" - ] - }, - { - "answer_session_ids": [ - "answer_b51b6115_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "af082822", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_b51b6115_1", - "bc5827b5", - "facef6e0_1", - "b28990c8_7", - "d048ae4b_2", - "1dfef591_2", - "cae65795_2", - "cdf068b1_2", - "1fcb4134_2", - "54f0d6f9_1" - ] - }, - { - "answer_session_ids": [ - "answer_add9b012_2", - "answer_add9b012_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_4929293a", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_add9b012_2", - "answer_add9b012_1", - "623ea729_2", - "ultrachat_329640", - "sharegpt_oQacgfY_0", - "ultrachat_512282", - "b8bc1464_4", - "73353aee_1", - "c1e598cd_1", - "sharegpt_mWFmQxk_36" - ] - }, - { - "answer_session_ids": [ - "answer_a17423e7_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_b5700ca9", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_a17423e7_1", - "44c62f26", - "5053474c_2", - "a53c7542_1", - "33bdd89a_1", - "sharegpt_FqOC5e9_169", - "d743153b_1", - "ultrachat_366301", - "47382adf_1", - "ultrachat_383774" - ] - }, - { - "answer_session_ids": [ - "answer_dba89487_2", - "answer_dba89487_1" - ], - "ndcg_at_10": 0.8316, - "ndcg_at_5": 1.0, - "question_id": "9a707b81", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_dba89487_2", - "4abbeb2a_2", - "5a13d047_1", - "ultrachat_233519", - "6862e478_2", - "answer_dba89487_1", - "990f3ef9_6", - "3fcbb58c_4", - "e1d11615_1", - "59510ab6" - ] - }, - { - "answer_session_ids": [ - "answer_febde667_1", - "answer_febde667_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_1d4ab0c9", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_febde667_2", - "answer_febde667_1", - "ab855b34_2", - "3f1d1224_3", - "7257bd15_1", - "082b7e52_1", - "ultrachat_201007", - "2dbfff2f", - "ultrachat_310433", - "sharegpt_MIYBRJz_27" - ] - }, - { - "answer_session_ids": [ - "answer_c19bd2bf_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_e072b769", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c19bd2bf_1", - "5558a42e_1", - "4f7b5dc9_1", - "b9e32ff8_1", - "2fc9ce79_2", - "sharegpt_1VhiETr_37", - "66d3fdb7_2", - "ultrachat_371691", - "sharegpt_9Ln5Ood_0", - "sharegpt_VbIxRWJ_33" - ] - }, - { - "answer_session_ids": [ - "answer_b9e32ff8_1", - "answer_b9e32ff8_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "0db4c65d", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_b9e32ff8_2", - "answer_b9e32ff8_1", - "71dc2037_2", - "953d5a4b_1", - "667465d6_1", - "sharegpt_5NoXULS_0", - "e9e19917_1", - "b22a2540", - "d36d11b9_2", - "e6c3a50a" - ] - }, - { - "answer_session_ids": [ - "answer_661b711f_1", - "answer_661b711f_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_1d80365e", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_661b711f_1", - "answer_661b711f_2", - "f7a61595_2", - "ultrachat_293442", - "805e571c_1", - "554b054e_2", - "ultrachat_561552", - "ultrachat_271411", - "3be19aed_1", - "sharegpt_5uZ7IdY_0" - ] - }, - { - "answer_session_ids": [ - "answer_5d8c99d3_1", - "answer_5d8c99d3_2", - "answer_5d8c99d3_3" - ], - "ndcg_at_10": 0.4825, - "ndcg_at_5": 0.4307, - "question_id": "gpt4_7f6b06db", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "39c1bf6a", - "928890e4_3", - "ultrachat_31232", - "answer_5d8c99d3_2", - "631e4016", - "answer_5d8c99d3_3", - "d9ca4aca", - "4bf66c38_2", - "da42526b_1", - "sharegpt_3sKfamn_27" - ] - }, - { - "answer_session_ids": [ - "answer_c4df007f_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_6dc9b45b", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c4df007f_1", - "d8f48b0a", - "ultrachat_384387", - "9adc81d7_2", - "ultrachat_355155", - "ultrachat_35147", - "afa0c2e7_5", - "94ff80d3", - "sharegpt_jyynvvk_0", - "sharegpt_0SDNlJ3_63" - ] - }, - { - "answer_session_ids": [ - "answer_56521e65_1" - ], - "ndcg_at_10": 0.5, - "ndcg_at_5": 0.5, - "question_id": "gpt4_8279ba02", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "b909542d_1", - "85f19a30_1", - "answer_56521e65_1", - "b6018747_2", - "86c5d31d", - "e1c807f9", - "62b40a05_2", - "3a735bfa_1", - "ultrachat_78027", - "1a037873_2" - ] - }, - { - "answer_session_ids": [ - "answer_c862f65a_2", - "answer_c862f65a_3", - "answer_c862f65a_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_18c2b244", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c862f65a_3", - "answer_c862f65a_2", - "answer_c862f65a_1", - "dfbc46bd_1", - "7e2c6065", - "d9e1d5fd", - "aeb58d86_4", - "ultrachat_503659", - "84896d13", - "746ae854_2" - ] - }, - { - "answer_session_ids": [ - "answer_e9ad5914_1", - "answer_e9ad5914_2", - "answer_e9ad5914_3", - "answer_e9ad5914_4", - "answer_e9ad5914_5", - "answer_e9ad5914_6" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_a1b77f9c", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e9ad5914_3", - "answer_e9ad5914_4", - "answer_e9ad5914_6", - "answer_e9ad5914_1", - "answer_e9ad5914_5", - "answer_e9ad5914_2", - "199a5225_1", - "47ec1674_2", - "d52d478b", - "f4a4a28a" - ] - }, - { - "answer_session_ids": [ - "answer_447052a5_2", - "answer_447052a5_1" - ], - "ndcg_at_10": 0.8175, - "ndcg_at_5": 1.0, - "question_id": "gpt4_1916e0ea", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_447052a5_2", - "d3fe25ff", - "280352e9", - "4b72fe4c_1", - "fa1ce406_2", - "987b5d00", - "answer_447052a5_1", - "7a36e820_2", - "dffa3157", - "sharegpt_tPA8329_0" - ] - }, - { - "answer_session_ids": [ - "answer_4d5490f1_1", - "answer_4d5490f1_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_7a0daae1", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_4d5490f1_1", - "answer_4d5490f1_2", - "8c68df0d_1", - "014a5d36", - "819dfad7_2", - "2ff297a8_1", - "bf3aebdb_2", - "1c8832b4_1", - "sharegpt_ZWqMvoL_607", - "99883f38_3" - ] - }, - { - "answer_session_ids": [ - "answer_9b09d95b_1" - ], - "ndcg_at_10": 0.3333, - "ndcg_at_5": 0.0, - "question_id": "gpt4_468eb063", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 0.0, - "top_session_ids": [ - "e60a93ff_2", - "ultrachat_108278", - "sharegpt_Sf51OjE_8", - "d97db962_2", - "sharegpt_gfhUw40_17", - "sharegpt_1rIk8tS_0", - "answer_9b09d95b_1", - "ultrachat_91161", - "sharegpt_8M7HcM3_11", - "sharegpt_ipLglky_42" - ] - }, - { - "answer_session_ids": [ - "answer_7093d898_1", - "answer_7093d898_2", - "answer_7093d898_3", - "answer_7093d898_4", - "answer_7093d898_5", - "answer_7093d898_6" - ], - "ndcg_at_10": 0.6199, - "ndcg_at_5": 0.5706, - "question_id": "gpt4_7abb270c", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "5808284e", - "ultrachat_253954", - "answer_7093d898_2", - "answer_7093d898_1", - "39948bcf_4", - "answer_7093d898_6", - "087d2b0a_2", - "sharegpt_W6t3Ck0_0", - "answer_7093d898_5", - "ultrachat_41971" - ] - }, - { - "answer_session_ids": [ - "answer_16bd5ea5_1", - "answer_16bd5ea5_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_1e4a8aeb", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_16bd5ea5_1", - "answer_16bd5ea5_2", - "28621d6a_2", - "f7838a91_1", - "faad7d7a_1", - "bc5827b5", - "15564b61_2", - "fd18e0dd_1", - "ultrachat_413946", - "ultrachat_292933" - ] - }, - { - "answer_session_ids": [ - "answer_be07688f_1", - "answer_be07688f_2" - ], - "ndcg_at_10": 0.8503, - "ndcg_at_5": 0.8503, - "question_id": "gpt4_4fc4f797", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_be07688f_1", - "39566615_1", - "814d3b22", - "47c5482a", - "answer_be07688f_2", - "660cea1d", - "bdcee74e_3", - "7e02e59b_4", - "86298245_2", - "0c783e4a_1" - ] - }, - { - "answer_session_ids": [ - "answer_4bebc782_1", - "answer_4bebc782_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "4dfccbf7", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_4bebc782_1", - "answer_4bebc782_2", - "a5e69b52_1", - "8fa624b2_6", - "491ebc5d_2", - "36f8b380_1", - "ultrachat_327454", - "sharegpt_e1DTWDa_23", - "sharegpt_cXkL3cR_69", - "ultrachat_373047" - ] - }, - { - "answer_session_ids": [ - "answer_e831a29f_1", - "answer_e831a29f_2" - ], - "ndcg_at_10": 0.8503, - "ndcg_at_5": 0.8503, - "question_id": "gpt4_61e13b3c", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e831a29f_1", - "f3fcf4fd_2", - "a6f2320f_2", - "c0d099e6_2", - "answer_e831a29f_2", - "cc380119_3", - "8ca0085a_2", - "2b5c911e_3", - "8e10bf6b_1", - "b4923852" - ] - }, - { - "answer_session_ids": [ - "answer_e6c20e52_3", - "answer_e6c20e52_2", - "answer_e6c20e52_1" - ], - "ndcg_at_10": 0.5641, - "ndcg_at_5": 0.6309, - "question_id": "gpt4_45189cb4", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "5c16fe0b_1", - "answer_e6c20e52_3", - "ultrachat_307270", - "c9d35c00_2", - "dc3ee1d1_1", - "sharegpt_hgGAUvu_0", - "ebf5b3bc_2", - "6593cb8b_1", - "66ab6260", - "answer_e6c20e52_1" - ] - }, - { - "answer_session_ids": [ - "answer_c9d35c00_1", - "answer_c9d35c00_2" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "2ebe6c90", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c9d35c00_2", - "0f73eee7_2", - "answer_c9d35c00_1", - "sharegpt_jyMHY1J_31", - "8fa64936_1", - "9ccea2f0_2", - "c3c1f84a_1", - "4d1b9b0d_4", - "453d354e_1", - "ultrachat_68168" - ] - }, - { - "answer_session_ids": [ - "answer_8c64ce25_2", - "answer_8c64ce25_1", - "answer_8c64ce25_3" - ], - "ndcg_at_10": 0.3155, - "ndcg_at_5": 0.0, - "question_id": "gpt4_e061b84f", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 0.0, - "top_session_ids": [ - "0a6bf5e4_1", - "3275acf9", - "4cc9270f", - "bab3f409", - "73bc3176_2", - "539c2346_3", - "77974d02", - "answer_8c64ce25_3", - "ultrachat_129606", - "9345f7dc_4" - ] - }, - { - "answer_session_ids": [ - "answer_61d1be50_1", - "answer_61d1be50_2" - ], - "ndcg_at_10": 0.8772, - "ndcg_at_5": 0.8772, - "question_id": "370a8ff4", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_61d1be50_1", - "11c0b1c5", - "cfde748a", - "answer_61d1be50_2", - "sharegpt_UbMVpdp_93", - "2b4eaee0_1", - "sharegpt_gVwWKqg_0", - "sharegpt_JRQtEcd_115", - "31d5ad91_1", - "8c64ce25_1" - ] - }, - { - "answer_session_ids": [ - "answer_f999b05b_1", - "answer_f999b05b_4", - "answer_f999b05b_2", - "answer_f999b05b_5", - "answer_f999b05b_3" - ], - "ndcg_at_10": 0.8401, - "ndcg_at_5": 0.8855, - "question_id": "gpt4_d6585ce8", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_f999b05b_3", - "51c82c8e_2", - "answer_f999b05b_5", - "ultrachat_225491", - "answer_f999b05b_2", - "dc0802a9_1", - "c1e598cd_3", - "6ec9bed9", - "answer_f999b05b_4", - "answer_f999b05b_1" - ] - }, - { - "answer_session_ids": [ - "answer_f964cea3_1", - "answer_f964cea3_2" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "gpt4_4ef30696", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_f964cea3_2", - "80b3f093", - "answer_f964cea3_1", - "d9868305_2", - "sharegpt_ELYlA30_25", - "4b29957c_1", - "4067bede", - "1ccb08a0", - "750b437b_1", - "sharegpt_K8mXaIT_0" - ] - }, - { - "answer_session_ids": [ - "answer_e3aa84c4_2", - "answer_e3aa84c4_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_ec93e27f", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e3aa84c4_1", - "answer_e3aa84c4_2", - "5a994ed3_2", - "sharegpt_Sg0ctDe_14", - "6b63bdc5_2", - "5de5c1ed_1", - "85846900_2", - "ultrachat_357838", - "dd049be8", - "sharegpt_1ZQntAk_0" - ] - }, - { - "answer_session_ids": [ - "answer_88841f26_1", - "answer_88841f26_2" - ], - "ndcg_at_10": 0.5, - "ndcg_at_5": 0.5, - "question_id": "6e984301", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "b38766c1_2", - "535ce50e_2", - "answer_88841f26_1", - "b500d9be", - "55c23a23_2", - "ultrachat_402356", - "954c790c_2", - "answer_88841f26_2", - "2bc4ae7d", - "018426a5" - ] - }, - { - "answer_session_ids": [ - "answer_0dd54b7c_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "8077ef71", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_0dd54b7c_1", - "f7ffddbc", - "fd88019d", - "5365840b_1", - "ultrachat_468198", - "sharegpt_Wi7Op1u_39", - "sharegpt_11WxPMZ_0", - "d87a6ef8", - "a993bf57_2", - "sharegpt_hoQsVGz_51" - ] - }, - { - "answer_session_ids": [ - "answer_d8a1af6b_1", - "answer_d8a1af6b_2", - "answer_d8a1af6b_3", - "answer_d8a1af6b_4", - "answer_d8a1af6b_5" - ], - "ndcg_at_10": 0.6181, - "ndcg_at_5": 0.5013, - "question_id": "gpt4_f420262c", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "sharegpt_jFDrHZ3_0", - "sharegpt_B0riiep_0", - "116bae09", - "answer_d8a1af6b_4", - "answer_d8a1af6b_3", - "answer_d8a1af6b_5", - "answer_d8a1af6b_1", - "answer_d8a1af6b_2", - "ultrachat_446175", - "ultrachat_548517" - ] - }, - { - "answer_session_ids": [ - "answer_d97db962_1", - "answer_d97db962_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_8e165409", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_d97db962_2", - "answer_d97db962_1", - "sharegpt_JRQtEcd_115", - "4c9f6bf5_2", - "09599d45_1", - "ultrachat_572191", - "ultrachat_106695", - "d5b3cf54_4", - "f1fe6512_1", - "ultrachat_577133" - ] - }, - { - "answer_session_ids": [ - "answer_aed8cf17_1", - "answer_aed8cf17_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_74aed68e", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_aed8cf17_2", - "answer_aed8cf17_1", - "318048af_2", - "9523d17c", - "69a9211c_1", - "25b3e36c_2", - "470f29c0_1", - "50d66391_6", - "sharegpt_O3FeoHQ_153", - "sharegpt_DXFHJtN_0" - ] - }, - { - "answer_session_ids": [ - "answer_43ba3965_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "bcbe585f", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_43ba3965_1", - "a158b7e7", - "7fa8099b_1", - "a98ff421_4", - "44f94343_1", - "5dbe11b9", - "6bed7862", - "8166673a", - "896162a2", - "8953c8c8_2" - ] - }, - { - "answer_session_ids": [ - "answer_1e2369c9_1", - "answer_1e2369c9_2" - ], - "ndcg_at_10": 0.8772, - "ndcg_at_5": 0.8772, - "question_id": "gpt4_21adecb5", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_1e2369c9_1", - "ultrachat_555122", - "ultrachat_444790", - "answer_1e2369c9_2", - "e1f37e24", - "sharegpt_ipLglky_42", - "6c574737_2", - "128082f8_2", - "ultrachat_333888", - "19f2e2c5_2" - ] - }, - { - "answer_session_ids": [ - "answer_c18d480b_1" - ], - "ndcg_at_10": 0.4307, - "ndcg_at_5": 0.4307, - "question_id": "5e1b23de", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "1a5344e9_3", - "1380576d_1", - "ultrachat_153798", - "answer_c18d480b_1", - "809021de", - "c3f945f3_1", - "a13abca5_2", - "c28aed33", - "d8e33f5c_2", - "8278312e_1" - ] - }, - { - "answer_session_ids": [ - "answer_5850de18_2", - "answer_5850de18_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_98f46fc6", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_5850de18_2", - "answer_5850de18_1", - "sharegpt_rnL5VQO_0", - "6172cc72_3", - "sharegpt_hPTUZia_0", - "ultrachat_238480", - "8e3b479f_1", - "28897044", - "ultrachat_45511", - "32e292f2" - ] - }, - { - "answer_session_ids": [ - "answer_184c8f56_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_af6db32f", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_184c8f56_1", - "aee13015_6", - "ultrachat_552923", - "sharegpt_1jjEIai_173", - "e0b8c937_1", - "ea834a17_2", - "f26846e1_3", - "b7c70f28_3", - "8930493f_2", - "sharegpt_hDjMGr7_70" - ] - }, - { - "answer_session_ids": [ - "answer_0d4d0347_1", - "answer_0d4d0347_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "eac54adc", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_0d4d0347_2", - "answer_0d4d0347_1", - "b40f0a7a", - "ae051325_3", - "bf234051_2", - "32fc393b_2", - "ultrachat_387180", - "74133043", - "75a306dd_3", - "sharegpt_PgmAp0p_0" - ] - }, - { - "answer_session_ids": [ - "answer_ad8a76cb_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_7ddcf75f", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ad8a76cb_1", - "sharegpt_ZUxBndS_26", - "8c652eb0_2", - "sharegpt_mMdyu3t_0", - "79d122f0", - "7ba84ed3_2", - "9283c152", - "ultrachat_281183", - "07e6a3bd_1", - "sharegpt_5RciDTC_0" - ] - }, - { - "answer_session_ids": [ - "answer_f6d6e33f_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_a2d1d1f6", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_f6d6e33f_1", - "ultrachat_476519", - "f5cf8815_4", - "ultrachat_575969", - "894dd0c2_3", - "2c31d8be_3", - "sharegpt_gI1U1eK_4", - "90f7041a_2", - "a1bfb382_1", - "ultrachat_68856" - ] - }, - { - "answer_session_ids": [ - "answer_581ab834_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_85da3956", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_581ab834_1", - "fcc7fe77_2", - "ultrachat_373754", - "sharegpt_h1UXagU_1", - "14b000d9_1", - "sharegpt_i59HUGf_0", - "sharegpt_ewad5Sr_0", - "8da8c7e0", - "6c184bde_2", - "97f8a545_2" - ] - }, - { - "answer_session_ids": [ - "answer_550bb2d1_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_b0863698", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_550bb2d1_1", - "91e581ad_3", - "4a79d078_2", - "63bc9fa3", - "5b014f9b_1", - "554b054e_2", - "65f6f130_1", - "833750a1", - "bdcee74e_2", - "sharegpt_NITaMP6_0" - ] - }, - { - "answer_session_ids": [ - "answer_9793daa3_2", - "answer_9793daa3_1" - ], - "ndcg_at_10": 0.8316, - "ndcg_at_5": 1.0, - "question_id": "gpt4_68e94287", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_9793daa3_2", - "60876def_1", - "50f136f2_3", - "ultrachat_209526", - "74bcc8f4_1", - "answer_9793daa3_1", - "sharegpt_nh3eDLi_9", - "sharegpt_XzAEFL4_0", - "dadd6379_1", - "afc64a5c_1" - ] - }, - { - "answer_session_ids": [ - "answer_e28c1f0d_1", - "answer_e28c1f0d_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_e414231e", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e28c1f0d_1", - "answer_e28c1f0d_2", - "9f8fc173_3", - "d3065d85", - "1bab001f", - "1980bbfa_3", - "d4439279_2", - "a490e4b8_1", - "7e3fa056_2", - "ultrachat_524205" - ] - }, - { - "answer_session_ids": [ - "answer_cf021b36_1", - "answer_cf021b36_2", - "answer_cf021b36_3" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_7ca326fa", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_cf021b36_2", - "answer_cf021b36_3", - "answer_cf021b36_1", - "8d0ca228", - "b4025421", - "5429e768_1", - "sharegpt_7SCdbX2_0", - "1301fbd1", - "sharegpt_CYVtC0w_0", - "604de0a3_2" - ] - }, - { - "answer_session_ids": [ - "answer_e22d6aef_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_7bc6cf22", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e22d6aef_1", - "e6cc6157_1", - "09d021e7_1", - "ultrachat_215356", - "c82c18dd_1", - "ultrachat_330542", - "ultrachat_465755", - "20a1ca39", - "eca3b71c_2", - "3e43bf49_1" - ] - }, - { - "answer_session_ids": [ - "answer_c9d35c00_1", - "answer_c9d35c00_2" - ], - "ndcg_at_10": 0.3562, - "ndcg_at_5": 0.0, - "question_id": "2ebe6c92", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 0.0, - "top_session_ids": [ - "9d4312f6_3", - "fb303dd2_2", - "ca3a4e4f_1", - "54c5a89c_1", - "16756728_1", - "answer_c9d35c00_2", - "84048eba_5", - "sharegpt_9dKNTQo_0", - "b30a47be_2", - "ultrachat_575902" - ] - }, - { - "answer_session_ids": [ - "answer_8c64ce26_2", - "answer_8c64ce26_1", - "answer_8c64ce26_3" - ], - "ndcg_at_10": 0.3869, - "ndcg_at_5": 0.3869, - "question_id": "gpt4_e061b84g", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "ae15e8b6_1", - "9dac9e37_1", - "68705467_1", - "3d6a2b24_1", - "answer_8c64ce26_3", - "51221cac", - "ce584ba0", - "sharegpt_rB6IP4G_0", - "ultrachat_231231", - "ultrachat_391029" - ] - }, - { - "answer_session_ids": [ - "answer_0b4a8adc_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "71017277", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_0b4a8adc_1", - "85a1be56_2", - "976ca0d9", - "f8de4e92_4", - "5854eebc_2", - "e64f9553_1", - "40a1e681_1", - "ultrachat_557308", - "ultrachat_542251", - "e5b86922_1" - ] - }, - { - "answer_session_ids": [ - "answer_4bfcc251_4", - "answer_4bfcc251_3", - "answer_4bfcc251_2", - "answer_4bfcc251_1" - ], - "ndcg_at_10": 0.8667, - "ndcg_at_5": 0.8855, - "question_id": "b46e15ee", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_4bfcc251_1", - "918c4788_1", - "answer_4bfcc251_2", - "69664c72", - "answer_4bfcc251_3", - "c92239bc", - "answer_4bfcc251_4", - "495cf518_2", - "sharegpt_RhSlNyz_0", - "aa31c397_2" - ] - }, - { - "answer_session_ids": [ - "answer_f999b05c_1", - "answer_f999b05c_4", - "answer_f999b05c_2", - "answer_f999b05c_5", - "answer_f999b05c_3" - ], - "ndcg_at_10": 0.6154, - "ndcg_at_5": 0.5438, - "question_id": "gpt4_d6585ce9", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "a554ed79_4", - "58dd3f35", - "answer_f999b05c_3", - "87f5cb44", - "answer_f999b05c_2", - "answer_f999b05c_4", - "answer_f999b05c_5", - "a7aed4cc_2", - "17d91635_2", - "d8e33f5c_abs_1" - ] - }, - { - "answer_session_ids": [ - "answer_16bd5ea6_1", - "answer_16bd5ea6_2" - ], - "ndcg_at_10": 0.3333, - "ndcg_at_5": 0.0, - "question_id": "gpt4_1e4a8aec", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 0.0, - "top_session_ids": [ - "9f1f3bf4", - "b34a52aa_3", - "a8086137", - "4c8455cb_2", - "b1265fd7_1", - "sharegpt_9JZCfee_0", - "answer_16bd5ea6_2", - "e675c69a_1", - "1c662b7b_1", - "e41b78c7_2" - ] - }, - { - "answer_session_ids": [ - "answer_d8a1af6c_1", - "answer_d8a1af6c_2", - "answer_d8a1af6c_5", - "answer_d8a1af6c_4", - "answer_d8a1af6c_3" - ], - "ndcg_at_10": 0.967, - "ndcg_at_5": 0.9829, - "question_id": "gpt4_f420262d", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_d8a1af6c_5", - "answer_d8a1af6c_2", - "answer_d8a1af6c_1", - "27639dd8_3", - "answer_d8a1af6c_3", - "sharegpt_UUFCRmF_13", - "answer_d8a1af6c_4", - "09e6f061", - "e1023f28", - "00c7b769_1" - ] - }, - { - "answer_session_ids": [ - "answer_d00ba6d1_1", - "answer_d00ba6d1_2" - ], - "ndcg_at_10": 0.4486, - "ndcg_at_5": 0.4307, - "question_id": "gpt4_59149c78", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "sharegpt_GI6737T_2", - "765ce8a7_2", - "23754665", - "answer_d00ba6d1_2", - "a8ac3d1d_1", - "84762061_1", - "sharegpt_YENXhFv_0", - "ultrachat_131385", - "answer_d00ba6d1_1", - "b2341a22" - ] - }, - { - "answer_session_ids": [ - "answer_e28c1f0e_1", - "answer_e28c1f0e_2" - ], - "ndcg_at_10": 0.7904, - "ndcg_at_5": 1.0, - "question_id": "gpt4_e414231f", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e28c1f0e_1", - "ultrachat_150948", - "07a26dfa_5", - "deb5bb60", - "135f761b_1", - "78ef59d0_1", - "sharegpt_K8mXaIT_0", - "752c438c_3", - "fe82a033_2", - "answer_e28c1f0e_2" - ] - }, - { - "answer_session_ids": [ - "answer_add9b013_2", - "answer_add9b013_1" - ], - "ndcg_at_10": 0.0, - "ndcg_at_5": 0.0, - "question_id": "gpt4_4929293b", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 0.0, - "r_at_5": 0.0, - "top_session_ids": [ - "sharegpt_KFhIUCO_0", - "b4ed7223_3", - "cae65795_1", - "ultrachat_511609", - "bda611f6_3", - "ea8c38f8", - "sharegpt_MB6e5CL_23", - "sharegpt_15lfOiQ_39", - "78bab597_2", - "ultrachat_156176" - ] - }, - { - "answer_session_ids": [ - "answer_9b09d95b_1" - ], - "ndcg_at_10": 0.4307, - "ndcg_at_5": 0.4307, - "question_id": "gpt4_468eb064", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "a2f0054f", - "db1aefb6_3", - "4d4df0e0_1", - "answer_9b09d95b_1", - "1e5bd28d_2", - "sharegpt_0PLBHdX_15", - "ultrachat_109577", - "sharegpt_P54kbvt_0", - "ultrachat_337251", - "cc3c5fa9_1" - ] - }, - { - "answer_session_ids": [ - "answer_ff201787_1", - "answer_ff201787_2" - ], - "ndcg_at_10": 0.6509, - "ndcg_at_5": 0.6509, - "question_id": "gpt4_fa19884d", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "ccc87da9_1", - "answer_ff201787_1", - "5f9dd782", - "answer_ff201787_2", - "eb4d1c82_1", - "55da3cbf_2", - "f7a61595_3", - "f910dc31_4", - "ultrachat_525919", - "ultrachat_483034" - ] - }, - { - "answer_session_ids": [ - "answer_dba89488_2", - "answer_dba89488_1" - ], - "ndcg_at_10": 0.3333, - "ndcg_at_5": 0.0, - "question_id": "9a707b82", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 0.0, - "top_session_ids": [ - "8e78fa70_1", - "fab41c07", - "79d122f0", - "sharegpt_bUaSM2v_0", - "990f3ef9_2", - "f8e9d4fa", - "answer_dba89488_1", - "7a4d00b3_2", - "ultrachat_466884", - "09e97c6f_2" - ] - }, - { - "answer_session_ids": [ - "answer_0d4d0348_1", - "answer_0d4d0348_2" - ], - "ndcg_at_10": 0.301, - "ndcg_at_5": 0.0, - "question_id": "eac54add", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 0.0, - "top_session_ids": [ - "2ea6fda4_1", - "7bb01253", - "7966888b_2", - "577da76d_2", - "2946bf12", - "c3023a45_5", - "773aebbd_3", - "369695b4_2", - "answer_0d4d0348_2", - "28269633_2" - ] - }, - { - "answer_session_ids": [ - "answer_4bebc783_1", - "answer_4bebc783_2" - ], - "ndcg_at_10": 0.7977, - "ndcg_at_5": 1.0, - "question_id": "4dfccbf8", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_4bebc783_1", - "sharegpt_3D3oQC0_213", - "a63ad8e3_3", - "1c177942_4", - "b4f63a70_3", - "7db28e68_1", - "1bc87711_1", - "edd89480_1", - "answer_4bebc783_2", - "ultrachat_444490" - ] - }, - { - "answer_session_ids": [ - "answer_f4ea84fc_3", - "answer_f4ea84fc_2", - "answer_f4ea84fc_1" - ], - "ndcg_at_10": 0.9469, - "ndcg_at_5": 0.9469, - "question_id": "0bc8ad93", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_f4ea84fc_1", - "answer_f4ea84fc_2", - "97338ddd_1", - "2c185a2b_1", - "answer_f4ea84fc_3", - "ultrachat_260376", - "dd25eeb5_2", - "7285299a_2", - "d59d1a8d_1", - "0d8f8cc0_2" - ] - }, - { - "answer_session_ids": [ - "answer_88841f27_1", - "answer_88841f27_2" - ], - "ndcg_at_10": 0.3562, - "ndcg_at_5": 0.0, - "question_id": "6e984302", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 0.0, - "top_session_ids": [ - "7285299a_8", - "de1f4aec_1", - "3535dbf0_4", - "7cd7c296_1", - "c9dc9158_1", - "answer_88841f27_1", - "6efebca9", - "96c743d0_2", - "c03fc56c_1", - "sharegpt_4Wjx3pH_0" - ] - }, - { - "answer_session_ids": [ - "answer_56521e66_1" - ], - "ndcg_at_10": 0.0, - "ndcg_at_5": 0.0, - "question_id": "gpt4_8279ba03", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 0.0, - "r_at_5": 0.0, - "top_session_ids": [ - "b357fb8b_2", - "ultrachat_526748", - "2ef55f49_3", - "864a563d_5", - "49b78a55_1", - "518d26d3_4", - "bb107057_2", - "ba9f938b_1", - "570fe405", - "50d66391_4" - ] - }, - { - "answer_session_ids": [ - "answer_a17423e8_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_b5700ca0", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_a17423e8_1", - "534be93d_1", - "93395e5f_3", - "109e07e8_2", - "sharegpt_FpTfRvR_0", - "ultrachat_23298", - "b403d6f0_4", - "e789afdb_2", - "ultrachat_101752", - "ultrachat_160178" - ] - }, - { - "answer_session_ids": [ - "answer_9793daa4_2", - "answer_9793daa4_1" - ], - "ndcg_at_10": 0.6309, - "ndcg_at_5": 0.6309, - "question_id": "gpt4_68e94288", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "4c49e37f", - "answer_9793daa4_1", - "128f4e4d_2", - "9d2fdc71", - "c99dcd81", - "0a7c4f1c_1", - "767b1b89", - "cff08ec1_1", - "ultrachat_396489", - "d2606fe9" - ] - }, - { - "answer_session_ids": [ - "answer_4be1b6b4_1", - "answer_4be1b6b4_2", - "answer_4be1b6b4_3" - ], - "ndcg_at_10": 0.5391, - "ndcg_at_5": 0.5, - "question_id": "gpt4_2655b836", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "d8a1af6b_4", - "9e05d9a0", - "answer_4be1b6b4_2", - "ultrachat_268202", - "e3e66b50", - "0f2434e8", - "answer_4be1b6b4_3", - "answer_4be1b6b4_1", - "b876911c", - "ea8cc87b" - ] - }, - { - "answer_session_ids": [ - "answer_1c6b85ea_2", - "answer_1c6b85ea_1" - ], - "ndcg_at_10": 0.6934, - "ndcg_at_5": 0.6934, - "question_id": "gpt4_2487a7cb", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "4f5880c6_6", - "answer_1c6b85ea_1", - "answer_1c6b85ea_2", - "sharegpt_QN26oUg_0", - "67a8ef5f", - "f87fea58_4", - "sharegpt_CHaU4ax_10", - "sharegpt_XNcyasy_17", - "4c1982d8_2", - "sharegpt_EJC4YOh_0" - ] - }, - { - "answer_session_ids": [ - "answer_b535969f_1", - "answer_b535969f_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_76048e76", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_b535969f_1", - "answer_b535969f_2", - "6a78e959_2", - "33a39aa7_2", - "88e01130_1", - "33630c9e_2", - "sharegpt_okVqBzp_13", - "ultrachat_273271", - "sharegpt_inuIr9J_119", - "sharegpt_vx14orQ_11" - ] - }, - { - "answer_session_ids": [ - "answer_5328c3c2_2", - "answer_5328c3c2_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_2312f94c", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_5328c3c2_2", - "answer_5328c3c2_1", - "a764b677_2", - "869fe9ba_2", - "sharegpt_jE25Ibo_0", - "3bbf4800_1", - "a1c1739c_3", - "e831a29f_2", - "ultrachat_44021", - "sharegpt_XYtuANl_9" - ] - }, - { - "answer_session_ids": [ - "answer_e936197f_1", - "answer_e936197f_2" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "0bb5a684", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e936197f_1", - "1a14d6cb_1", - "answer_e936197f_2", - "3c8e7c0e_3", - "ultrachat_62919", - "565929a2_2", - "b10d0907_2", - "826d51da_3", - "d794dec9_2", - "sharegpt_d3UV0eu_0" - ] - }, - { - "answer_session_ids": [ - "answer_6ea1541e_2", - "answer_6ea1541e_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "08f4fc43", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_6ea1541e_1", - "answer_6ea1541e_2", - "f3bd00ec_2", - "ultrachat_292416", - "sharegpt_JJu53iW_0", - "50303fd0_1", - "9216c4d3_4", - "sharegpt_zqQa6sO_0", - "sharegpt_wXqtiUe_0", - "cdbd6862_1" - ] - }, - { - "answer_session_ids": [ - "answer_d39b7977_1", - "answer_d39b7977_2" - ], - "ndcg_at_10": 0.8772, - "ndcg_at_5": 0.8772, - "question_id": "2c63a862", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_d39b7977_2", - "93a5cfe6", - "33baaaab_1", - "answer_d39b7977_1", - "1854abc7_2", - "ad175dc4_2", - "0dde26d4_3", - "ultrachat_179062", - "ultrachat_149914", - "918c4788_2" - ] - }, - { - "answer_session_ids": [ - "answer_7a4a93f1_2", - "answer_7a4a93f1_1" - ], - "ndcg_at_10": 0.8772, - "ndcg_at_5": 0.8772, - "question_id": "gpt4_385a5000", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_7a4a93f1_2", - "8062f4ab_2", - "80aefc49_1", - "answer_7a4a93f1_1", - "29d27eaa_2", - "sharegpt_hcaZN94_507", - "28f74b09_1", - "ultrachat_480527", - "ultrachat_528011", - "ultrachat_286149" - ] - }, - { - "answer_session_ids": [ - "answer_1cc3cd0c_1", - "answer_1cc3cd0c_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "2a1811e2", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_1cc3cd0c_2", - "answer_1cc3cd0c_1", - "b0e040c1", - "901a6763_2", - "ecfd2047_1", - "e9fb10e7_1", - "ultrachat_187883", - "sharegpt_c9R1uSD_0", - "89e05490", - "f5a4d792_1" - ] - }, - { - "answer_session_ids": [ - "answer_b3763b6b_1", - "answer_b3763b6b_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "bbf86515", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_b3763b6b_1", - "answer_b3763b6b_2", - "df003c93_2", - "592cb8d2", - "75b98d2a_1", - "0c260a71_2", - "e9e19917_1", - "33d0e4b3_2", - "sharegpt_d1Rh99l_63", - "sharegpt_G8j6c8J_38" - ] - }, - { - "answer_session_ids": [ - "answer_099c1b6c_5", - "answer_099c1b6c_2", - "answer_099c1b6c_4", - "answer_099c1b6c_3", - "answer_099c1b6c_1" - ], - "ndcg_at_10": 0.9896, - "ndcg_at_5": 1.0, - "question_id": "gpt4_5dcc0aab", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_099c1b6c_3", - "answer_099c1b6c_1", - "answer_099c1b6c_5", - "answer_099c1b6c_2", - "353d3c6d_1", - "answer_099c1b6c_4", - "3a192d16_1", - "3891baea_3", - "b10c041b_1", - "b31f923a_2" - ] - }, - { - "answer_session_ids": [ - "answer_c4e5d969_2", - "answer_c4e5d969_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_0b2f1d21", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c4e5d969_2", - "answer_c4e5d969_1", - "1cafd864_3", - "f31a0422_1", - "sharegpt_rhWNt54_19", - "ultrachat_455430", - "ultrachat_49268", - "sharegpt_4pDZgCL_14", - "5260c678_2", - "ultrachat_1415" - ] - }, - { - "answer_session_ids": [ - "answer_932ea3bf_2", - "answer_932ea3bf_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "f0853d11", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_932ea3bf_2", - "answer_932ea3bf_1", - "bd8708e2_2", - "sharegpt_9upOSlE_27", - "10c0752c", - "b16a8656_2", - "6e436d54", - "ultrachat_325202", - "ultrachat_114660", - "4c967baa_2" - ] - }, - { - "answer_session_ids": [ - "answer_d50a8a33_1", - "answer_d50a8a33_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_6ed717ea", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_d50a8a33_2", - "answer_d50a8a33_1", - "637bc32a_2", - "15ed03f0_1", - "7e7ecab6", - "d4b6ec0b", - "sharegpt_HQMYIkt_0", - "sharegpt_PuRfIep_7", - "8e657ff7_1", - "e0585cb5_1" - ] - }, - { - "answer_session_ids": [ - "answer_b3070ec4_2", - "answer_b3070ec4_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_70e84552", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_b3070ec4_1", - "answer_b3070ec4_2", - "ultrachat_352927", - "2b9b6224_2", - "30491f6f_1", - "45b155ec_2", - "3f787a78_1", - "ultrachat_573894", - "6ebd9e18_1", - "d5527e62" - ] - }, - { - "answer_session_ids": [ - "answer_4ffa04a2_1", - "answer_4ffa04a2_6", - "answer_4ffa04a2_4", - "answer_4ffa04a2_3", - "answer_4ffa04a2_5", - "answer_4ffa04a2_2" - ], - "ndcg_at_10": 0.9797, - "ndcg_at_5": 1.0, - "question_id": "a3838d2b", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_4ffa04a2_1", - "answer_4ffa04a2_6", - "answer_4ffa04a2_4", - "answer_4ffa04a2_2", - "answer_4ffa04a2_5", - "06317e04", - "8cf0d2a9", - "10305ca9_2", - "638e7094_1", - "answer_4ffa04a2_3" - ] - }, - { - "answer_session_ids": [ - "answer_e5131a1b_2", - "answer_e5131a1b_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_93159ced", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e5131a1b_2", - "ultrachat_387032", - "460be54a", - "ultrachat_365033", - "sharegpt_A83arI9_383", - "sharegpt_9srz3L6_0", - "ultrachat_162445", - "ultrachat_475376", - "ultrachat_304047", - "ultrachat_244924" - ] - }, - { - "answer_session_ids": [ - "answer_3e11e0ae_1", - "answer_3e11e0ae_2" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "gpt4_2d58bcd6", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_3e11e0ae_1", - "0bd0a01c", - "answer_3e11e0ae_2", - "be32ad25_1", - "1bf8b212", - "sharegpt_4tLDGmp_0", - "sharegpt_Xl3kKYf_15", - "3c53154d_1", - "1b205d70_1", - "47e1f8dd_1" - ] - }, - { - "answer_session_ids": [ - "answer_30dfe889_1", - "answer_30dfe889_2" - ], - "ndcg_at_10": 0.6934, - "ndcg_at_5": 0.6934, - "question_id": "gpt4_65aabe59", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "6a7e3aec", - "answer_30dfe889_2", - "answer_30dfe889_1", - "77b8fcd9_1", - "10771408_4", - "31cde680", - "05a37787", - "ultrachat_190669", - "78bcce6a", - "ultrachat_169466" - ] - }, - { - "answer_session_ids": [ - "answer_ab603dd5_1", - "answer_ab603dd5_2" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "982b5123", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ab603dd5_1", - "07ba9acd_2", - "answer_ab603dd5_2", - "e36b0031", - "1bfd5a8b_3", - "359a2a0d", - "f89f184e_1", - "bf8ab267", - "91168bd2_1", - "a200b713_4" - ] - }, - { - "answer_session_ids": [ - "answer_5e3bb940_3", - "answer_5e3bb940_2", - "answer_5e3bb940_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "b9cfe692", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_5e3bb940_1", - "answer_5e3bb940_2", - "answer_5e3bb940_3", - "ed88aded_2", - "sharegpt_oXgiN7q_53", - "sharegpt_JRQtEcd_115", - "2714885f_3", - "da650f3b", - "91d847c1_6", - "ultrachat_265626" - ] - }, - { - "answer_session_ids": [ - "answer_0a00c163_1", - "answer_0a00c163_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_4edbafa2", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_0a00c163_2", - "answer_0a00c163_1", - "22be78fc_2", - "18c0af9c", - "4a006b77", - "725f95e1", - "sharegpt_TQNHiIB_12", - "sharegpt_jZOf9E5_21", - "sharegpt_6MPnc5F_9", - "7533d546_5" - ] - }, - { - "answer_session_ids": [ - "answer_70dc7d08_2", - "answer_70dc7d08_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "c8090214", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_70dc7d08_1", - "answer_70dc7d08_2", - "a4c2a6f7_2", - "sharegpt_xOzzaof_0", - "005ec7cf_1", - "d6c6dadf", - "162ad3bf_1", - "c21c9d97", - "sharegpt_BQ3jrV3_0", - "sharegpt_3cVjZ2b_8" - ] - }, - { - "answer_session_ids": [ - "answer_fb793c87_1", - "answer_fb793c87_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_483dd43c", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_fb793c87_1", - "answer_fb793c87_2", - "3fd56ebf", - "52161c9e_1", - "19c24c11", - "29695e1c_3", - "ultrachat_48790", - "sharegpt_Sg0ctDe_0", - "4243ce01_1", - "ultrachat_374858" - ] - }, - { - "answer_session_ids": [ - "answer_cf425855_1", - "answer_cf425855_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e4e14d04", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_cf425855_2", - "answer_cf425855_1", - "a92165a5", - "ffc627f2", - "5944b36a_1", - "ultrachat_542420", - "f8de4e92_2", - "1c1f5ccc_3", - "ultrachat_325291", - "ultrachat_200098" - ] - }, - { - "answer_session_ids": [ - "answer_cdba3d9f_1", - "answer_cdba3d9f_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "c9f37c46", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_cdba3d9f_1", - "answer_cdba3d9f_2", - "b38766c1_3", - "0cf86cbe_1", - "4ffa04a2_1", - "2ba1710e_1", - "02df1225_1", - "7c9b09ac", - "211bb1c2_1", - "9d4312f6_3" - ] - }, - { - "answer_session_ids": [ - "answer_9af4e346_1", - "answer_9af4e346_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_2c50253f", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_9af4e346_2", - "answer_9af4e346_1", - "ultrachat_120586", - "sharegpt_a6pZtrg_0", - "0138fdec", - "dd81b163_2", - "b6f35f3a_1", - "65bc93cf_2", - "sharegpt_ueZYbCA_15", - "ultrachat_267493" - ] - }, - { - "answer_session_ids": [ - "answer_5e3eeb12_2", - "answer_5e3eeb12_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "dcfa8644", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_5e3eeb12_2", - "answer_5e3eeb12_1", - "b2243528_1", - "0d129c99", - "ultrachat_431004", - "7de51ffe_1", - "sharegpt_NCuLVp2_0", - "b906870f_1", - "4eedac88_1", - "sharegpt_hUAh1Tu_0" - ] - }, - { - "answer_session_ids": [ - "answer_b9d9150e_2", - "answer_b9d9150e_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_b4a80587", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_b9d9150e_2", - "answer_b9d9150e_1", - "ultrachat_389307", - "3cb41ab8_1", - "sharegpt_4tqCoYJ_115", - "9071ba70_2", - "55bcfb77", - "sharegpt_IT52MRw_37", - "sharegpt_HxOhjmq_29", - "9b4f5c73" - ] - }, - { - "answer_session_ids": [ - "answer_8a42fedf_1", - "answer_8a42fedf_3", - "answer_8a42fedf_2" - ], - "ndcg_at_10": 0.9218, - "ndcg_at_5": 1.0, - "question_id": "gpt4_9a159967", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_8a42fedf_2", - "answer_8a42fedf_3", - "ultrachat_534927", - "253742b4_3", - "300cfc4a", - "4ea94f85", - "answer_8a42fedf_1", - "7169e342_2", - "sharegpt_MJrfKtS_0", - "75671e3f_1" - ] - }, - { - "answer_session_ids": [ - "answer_be73098b_2", - "answer_be73098b_1" - ], - "ndcg_at_10": 0.8316, - "ndcg_at_5": 1.0, - "question_id": "cc6d1ec1", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_be73098b_2", - "22db3cc3_4", - "9a297400_4", - "aabf201b", - "ultrachat_316268", - "answer_be73098b_1", - "e549b6b2_2", - "33baaaab_3", - "ultrachat_436945", - "sharegpt_iNlPmuC_9" - ] - }, - { - "answer_session_ids": [ - "answer_72d9aa58_1", - "answer_72d9aa58_2" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "gpt4_8c8961ae", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_72d9aa58_1", - "3b22d17b", - "answer_72d9aa58_2", - "1d14730c_1", - "1cde7ee4_2", - "02f9bedb_2", - "b77d0b58", - "37f0ce6b_1", - "ultrachat_265707", - "d7281662_2" - ] - }, - { - "answer_session_ids": [ - "answer_da704e79_2", - "answer_da704e79_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_d9af6064", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_da704e79_1", - "answer_da704e79_2", - "sharegpt_w9YUs4c_0", - "sharegpt_VbIxRWJ_33", - "0d9a6f01", - "98c83fe4", - "ultrachat_519452", - "sharegpt_brc2wJS_109", - "3e3b6d77_2", - "6cff6108" - ] - }, - { - "answer_session_ids": [ - "answer_6a78e959_2", - "answer_6a78e959_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_7de946e7", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_6a78e959_2", - "answer_6a78e959_1", - "caa00337_2", - "sharegpt_6Yfj9A3_5", - "sharegpt_mjN4DdI_0", - "6155addd_2", - "f1cf6f00_1", - "c4d370d3_1", - "sharegpt_BacsrkR_0", - "ultrachat_560313" - ] - }, - { - "answer_session_ids": [ - "answer_991d55e5_1", - "answer_991d55e5_2" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "d01c6aa8", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_991d55e5_2", - "2adf224b_2", - "answer_991d55e5_1", - "64460e9d_1", - "f2fa97f5_1", - "fc9a62c0_1", - "7d1f5395", - "ultrachat_333097", - "51fe163e_2", - "sharegpt_YLhXcw8_79" - ] - }, - { - "answer_session_ids": [ - "answer_54f0d6f9_1", - "answer_54f0d6f9_2" - ], - "ndcg_at_10": 0.8772, - "ndcg_at_5": 0.8772, - "question_id": "993da5e2", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_54f0d6f9_1", - "bcf96d28_3", - "b16a8656_2", - "answer_54f0d6f9_2", - "8953c8c8_3", - "a6c0f032_3", - "c69a28f0", - "ultrachat_266485", - "sharegpt_wpbTVG0_0", - "sharegpt_v11Tpg9_0" - ] - }, - { - "answer_session_ids": [ - "answer_016f6bd4_2", - "answer_016f6bd4_1" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "a3045048", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_016f6bd4_1", - "536a334d", - "answer_016f6bd4_2", - "aaf71ce2_2", - "d837b5fa", - "5cb58cc6", - "sharegpt_asoSnhq_0", - "d12d5a98_1", - "ultrachat_406386", - "sharegpt_LC9Pzkr_0" - ] - }, - { - "answer_session_ids": [ - "answer_8464304d_1", - "answer_8464304d_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_d31cdae3", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_8464304d_2", - "answer_8464304d_1", - "41743aae_2", - "ultrachat_74113", - "de695a4e", - "55033f6f", - "809cabca_2", - "ultrachat_324099", - "5ace29d7_1", - "ultrachat_169591" - ] - }, - { - "answer_session_ids": [ - "answer_aa930b56_1", - "answer_aa930b56_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_cd90e484", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_aa930b56_2", - "answer_aa930b56_1", - "9aad36bb_1", - "08c306e2", - "ultrachat_347068", - "4d452cdb_3", - "835e1b84_2", - "4bfcc251_1", - "6043dcf6_1", - "018eac29" - ] - }, - { - "answer_session_ids": [ - "answer_e60a93ff_1", - "answer_e60a93ff_2" - ], - "ndcg_at_10": 0.6509, - "ndcg_at_5": 0.6509, - "question_id": "gpt4_88806d6e", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "ultrachat_562852", - "answer_e60a93ff_1", - "sharegpt_bJh9LPd_0", - "answer_e60a93ff_2", - "20b1b65f_1", - "sharegpt_XYtuANl_11", - "ultrachat_258611", - "c11fb634_1", - "3c11dc79", - "24bfc009_5" - ] - }, - { - "answer_session_ids": [ - "answer_5fcca8bc_2", - "answer_5fcca8bc_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_4cd9eba1", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_5fcca8bc_2", - "answer_5fcca8bc_1", - "f54d04cc", - "456860e7_2", - "6b9b2b1e_1", - "d61f3bb5", - "8242fa3f_1", - "2ef53f61_4", - "0e193841_6", - "sharegpt_2nvkR60_0" - ] - }, - { - "answer_session_ids": [ - "answer_544fe66c_2", - "answer_544fe66c_1", - "answer_544fe66c_3" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_93f6379c", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_544fe66c_2", - "answer_544fe66c_3", - "1b9b2960_3", - "b8a688f6", - "sharegpt_cfESFSo_0", - "e06c6d99_5", - "c07b8034", - "ultrachat_443550", - "d3a34811", - "82e27408_1" - ] - }, - { - "answer_session_ids": [ - "answer_436d4309_2", - "answer_436d4309_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "b29f3365", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_436d4309_2", - "answer_436d4309_1", - "6b7605d1_1", - "6a2f9452_1", - "06ee3665", - "ultrachat_243663", - "95c9666f_5", - "sharegpt_K0i5Zon_0", - "ultrachat_58020", - "4ccc2061" - ] - }, - { - "answer_session_ids": [ - "answer_7a36e820_2", - "answer_7a36e820_3", - "answer_7a36e820_1" - ], - "ndcg_at_10": 0.6871, - "ndcg_at_5": 0.6934, - "question_id": "gpt4_2f56ae70", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "3aac691d_2", - "answer_7a36e820_2", - "answer_7a36e820_1", - "f8e8d445_4", - "647f5851", - "ultrachat_159439", - "answer_7a36e820_3", - "83cd701b_3", - "e61d966a_1", - "sharegpt_KzDwaf1_165" - ] - }, - { - "answer_session_ids": [ - "answer_aaf71ce2_3", - "answer_aaf71ce2_2", - "answer_aaf71ce2_1" - ], - "ndcg_at_10": 0.8772, - "ndcg_at_5": 0.8772, - "question_id": "6613b389", - "question_type": "temporal-reasoning", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_aaf71ce2_2", - "e3d8da67", - "5eef2132", - "answer_aaf71ce2_3", - "55bc2e1e", - "2780323d_3", - "f1cc1ddb_1", - "1a65880d", - "2366adbc", - "86c8f348_1" - ] - }, - { - "answer_session_ids": [ - "answer_5a78688d_1", - "answer_5a78688d_2" - ], - "ndcg_at_10": 0.8772, - "ndcg_at_5": 0.8772, - "question_id": "gpt4_78cf46a3", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_5a78688d_1", - "b89a3ec3", - "sharegpt_rx39ipj_0", - "answer_5a78688d_2", - "sharegpt_89M8Aiz_0", - "52b19cba_1", - "ultrachat_305042", - "e8e65a1f", - "a8b4bfb3_1", - "27099532_2" - ] - }, - { - "answer_session_ids": [ - "answer_a68db5db_2", - "answer_a68db5db_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_0a05b494", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_a68db5db_2", - "answer_a68db5db_1", - "0abe3e51", - "sharegpt_24ey7mz_0", - "ultrachat_135150", - "48b68664", - "30677a32", - "ultrachat_273213", - "18687689", - "ultrachat_105014" - ] - }, - { - "answer_session_ids": [ - "answer_faad7d7a_2", - "answer_faad7d7a_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_1a1dc16d", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_faad7d7a_2", - "answer_faad7d7a_1", - "87f1f950_2", - "c12d5181_1", - "14940349_2", - "ultrachat_428346", - "ultrachat_63236", - "ultrachat_92302", - "8bd3814a_2", - "661ee6d9_1" - ] - }, - { - "answer_session_ids": [ - "answer_11a8f823_2", - "answer_11a8f823_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_2f584639", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_11a8f823_2", - "answer_11a8f823_1", - "55ca8b20_3", - "ca555919_3", - "sharegpt_bq9G6bT_2", - "4b3a0ee6_1", - "c6eb17f0_1", - "ultrachat_101401", - "68131948", - "d441f4c5" - ] - }, - { - "answer_session_ids": [ - "answer_53582e7e_2", - "answer_53582e7e_1" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_213fd887", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_53582e7e_2", - "answer_53582e7e_1", - "9bdbfc62_2", - "10771408_1", - "ultrachat_165091", - "sharegpt_EXJAjmw_7", - "8aa17385", - "dc1f9713_1", - "368276ab_3", - "d10f0307_3" - ] - }, - { - "answer_session_ids": [ - "answer_b10f3828_1", - "answer_b10f3828_2" - ], - "ndcg_at_10": 0.6509, - "ndcg_at_5": 0.6509, - "question_id": "gpt4_5438fa52", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "ultrachat_69599", - "answer_b10f3828_2", - "a8e24e22", - "answer_b10f3828_1", - "290fe09a", - "sharegpt_LVUZee8_35", - "b4f07fef_2", - "ab8dce45", - "d68bab3c_1", - "528f481d_6" - ] - }, - { - "answer_session_ids": [ - "answer_d8e33f5c_1", - "answer_d8e33f5c_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "gpt4_c27434e8", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_d8e33f5c_2", - "answer_d8e33f5c_1", - "sharegpt_9XlbWvL_0", - "16933961", - "6e2cca63_3", - "4d578deb", - "fd56c5bd_4", - "5d1c7bf7", - "ultrachat_212645", - "sharegpt_DVswHqE_0" - ] - }, - { - "answer_session_ids": [ - "answer_65600ff6_2", - "answer_65600ff6_1" - ], - "ndcg_at_10": 0.6241, - "ndcg_at_5": 0.6241, - "question_id": "gpt4_fe651585", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "e30c4a7b_2", - "answer_65600ff6_1", - "ff6b8624", - "9f220c66_1", - "answer_65600ff6_2", - "ultrachat_576354", - "da828711", - "sharegpt_izuEpAx_20", - "sharegpt_mJ7MnQS_219", - "cee0b8d5" - ] - }, - { - "answer_session_ids": [ - "answer_124f5dc3_2", - "answer_124f5dc3_1" - ], - "ndcg_at_10": 0.6053, - "ndcg_at_5": 0.6309, - "question_id": "8c18457d", - "question_type": "temporal-reasoning", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "98cd882c_2", - "answer_124f5dc3_1", - "deb12028_5", - "3c8e7c0e_2", - "2dfc1443_1", - "answer_124f5dc3_2", - "b53f447a_1", - "dc0802a9_3", - "sharegpt_2wY0N2C_0", - "1a5369b9" - ] - }, - { - "answer_session_ids": [ - "answer_a25d4a91_1", - "answer_a25d4a91_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "6a1eabeb", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_a25d4a91_2", - "answer_a25d4a91_1", - "d03995c6", - "4aaf678f_3", - "59a6150d_1", - "6da0bba2_1", - "9f80607d", - "6387e969", - "sharegpt_ZWqMvoL_178", - "sharegpt_9ApEWXK_0" - ] - }, - { - "answer_session_ids": [ - "answer_3f9693b7_1", - "answer_3f9693b7_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "6aeb4375", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_3f9693b7_2", - "answer_3f9693b7_1", - "402e0082_5", - "sharegpt_MWlvrvZ_0", - "1bc87711_1", - "2ed7c45e_2", - "sharegpt_wbbH6cZ_0", - "93bed3d8_1", - "e47d38f4_2", - "b65a30db" - ] - }, - { - "answer_session_ids": [ - "answer_0b1a0942_1", - "answer_0b1a0942_2" - ], - "ndcg_at_10": 0.8772, - "ndcg_at_5": 0.8772, - "question_id": "830ce83f", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_0b1a0942_1", - "8d77be9a_2", - "47c5482a", - "answer_0b1a0942_2", - "dfbda5e9", - "020149a5_1", - "639cec58", - "8bc305e4_2", - "d1e7d11c_2", - "e19c8fd9_1" - ] - }, - { - "answer_session_ids": [ - "answer_3a6f1e82_1", - "answer_3a6f1e82_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "852ce960", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_3a6f1e82_1", - "answer_3a6f1e82_2", - "b0fd1357", - "aaf22693", - "d36d11b9_2", - "a110c4dc_1", - "a76e7e3c_1", - "sharegpt_FYhsZ0Q_0", - "010a28ab_3", - "sharegpt_uiECyCD_66" - ] - }, - { - "answer_session_ids": [ - "answer_6a4f8626_1", - "answer_6a4f8626_2" - ], - "ndcg_at_10": 0.8316, - "ndcg_at_5": 1.0, - "question_id": "945e3d21", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_6a4f8626_2", - "sharegpt_E7VUcfj_0", - "5fcca8bc_1", - "9418c255", - "ultrachat_518638", - "answer_6a4f8626_1", - "dc880372_3", - "sharegpt_zn81iL6_9", - "87252d80_2", - "9bbd38a6" - ] - }, - { - "answer_session_ids": [ - "answer_eecb10d9_1", - "answer_eecb10d9_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "d7c942c3", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_eecb10d9_2", - "answer_eecb10d9_1", - "sharegpt_ezPxUrC_0", - "sharegpt_LOF3smB_25", - "ddf0f116_3", - "sharegpt_vYQdLPe_2", - "sharegpt_OTILwRq_0", - "e5b72a5d_3", - "b2243528_1", - "d15c9567_2" - ] - }, - { - "answer_session_ids": [ - "answer_c44b9df4_1", - "answer_c44b9df4_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "71315a70", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c44b9df4_2", - "answer_c44b9df4_1", - "ultrachat_399278", - "c86a22a2_2", - "b5e94516_2", - "8cfa91c3_1", - "ultrachat_42515", - "sharegpt_ZUxBndS_26", - "ultrachat_119182", - "a08fbe88_1" - ] - }, - { - "answer_session_ids": [ - "answer_e1403127_1", - "answer_e1403127_2" - ], - "ndcg_at_10": 0.8772, - "ndcg_at_5": 0.8772, - "question_id": "89941a93", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e1403127_1", - "8e72316f", - "ultrachat_400703", - "answer_e1403127_2", - "ultrachat_349321", - "sharegpt_bn0XI2B_0", - "ultrachat_352196", - "ultrachat_214088", - "ultrachat_150565", - "ultrachat_151269" - ] - }, - { - "answer_session_ids": [ - "answer_73540165_1", - "answer_73540165_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "ce6d2d27", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_73540165_2", - "answer_73540165_1", - "7a4d00b3_2", - "d8d7eddf_2", - "48ad2a93", - "a46d10a2_1", - "9f6bec4f", - "cd92ac85_1", - "4da24fe5_2", - "sharegpt_jZOf9E5_61" - ] - }, - { - "answer_session_ids": [ - "answer_02e66dec_1", - "answer_02e66dec_2" - ], - "ndcg_at_10": 0.6934, - "ndcg_at_5": 0.6934, - "question_id": "9ea5eabc", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "418c27fa", - "answer_02e66dec_1", - "answer_02e66dec_2", - "62e6593e_1", - "ultrachat_243996", - "d1f30ac6_5", - "6df6fff4", - "8c8b180e", - "7c44bc2d_1", - "6e2cca63_1" - ] - }, - { - "answer_session_ids": [ - "answer_7e9ad7b4_1", - "answer_7e9ad7b4_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "07741c44", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_7e9ad7b4_2", - "answer_7e9ad7b4_1", - "sharegpt_UGg8d44_4", - "af4d4ecd_4", - "sharegpt_CEWG2XW_0", - "d1d6b55b_1", - "3b4c63ba_3", - "aa71db11_1", - "1e0c4b50", - "sharegpt_fH7KHW1_119" - ] - }, - { - "answer_session_ids": [ - "answer_0eb23770_1", - "answer_0eb23770_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "a1eacc2a", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_0eb23770_1", - "answer_0eb23770_2", - "1f035408", - "7daa121d_2", - "521255f7_2", - "fde0ca18", - "93f23301_1", - "80082060_4", - "1ccb1920", - "sharegpt_ePBzYD0_57" - ] - }, - { - "answer_session_ids": [ - "answer_e2f4f947_1", - "answer_e2f4f947_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "184da446", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e2f4f947_2", - "answer_e2f4f947_1", - "bf633415_2", - "82a89303", - "85a49fe0_2", - "341a737e", - "ultrachat_195877", - "sharegpt_w7bjHzU_103", - "30a3c930", - "61ad6074" - ] - }, - { - "answer_session_ids": [ - "answer_8748f791_1", - "answer_8748f791_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "031748ae", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_8748f791_1", - "answer_8748f791_2", - "ultrachat_406392", - "5ac7f9c5_3", - "e0f71050_2", - "39a056ae", - "ultrachat_556006", - "ultrachat_69185", - "ultrachat_219083", - "7a085b95_1" - ] - }, - { - "answer_session_ids": [ - "answer_766ab8da_1", - "answer_766ab8da_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "4d6b87c8", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_766ab8da_2", - "answer_766ab8da_1", - "5c85023a_1", - "cffde796", - "ultrachat_40253", - "60d4332d", - "sharegpt_NUyiNZE_0", - "9af4e346_1", - "sharegpt_mmYyki5_0", - "a0201607_3" - ] - }, - { - "answer_session_ids": [ - "answer_d6d2eba8_1", - "answer_d6d2eba8_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "0f05491a", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_d6d2eba8_1", - "answer_d6d2eba8_2", - "27aba903_2", - "0c260a71_3", - "9760f4ca", - "1b205d70_2", - "e2efa128", - "13905cf6", - "9260a72d", - "sharegpt_DM7idTU_0" - ] - }, - { - "answer_session_ids": [ - "answer_cdbe2250_1", - "answer_cdbe2250_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "08e075c7", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_cdbe2250_2", - "answer_cdbe2250_1", - "b1d9d555_1", - "f7344cf4", - "ae77c245", - "25b3e36c_1", - "77827bdf", - "ultrachat_359377", - "ultrachat_556776", - "ultrachat_269248" - ] - }, - { - "answer_session_ids": [ - "answer_b191df5b_1", - "answer_b191df5b_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "f9e8c073", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_b191df5b_1", - "answer_b191df5b_2", - "0a042aef_2", - "ultrachat_88473", - "ultrachat_219619", - "445e6a7a_1", - "sharegpt_bLaSZdD_0", - "22db3cc3_1", - "50b32eed_2", - "d5283595_1" - ] - }, - { - "answer_session_ids": [ - "answer_c7ddc051_1", - "answer_c7ddc051_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "41698283", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c7ddc051_2", - "answer_c7ddc051_1", - "631e4016", - "47a2bd5a_1", - "431ae25c", - "0e8ba03b_1", - "6554fb34_1", - "28e948f5", - "ultrachat_135408", - "sharegpt_7mlXHfH_41" - ] - }, - { - "answer_session_ids": [ - "answer_9282283d_1", - "answer_9282283d_2" - ], - "ndcg_at_10": 0.5912, - "ndcg_at_5": 0.6309, - "question_id": "2698e78f", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "d76030af_1", - "answer_9282283d_2", - "862218f3_2", - "ebd61d29_2", - "338281a4_1", - "d99bab55_2", - "answer_9282283d_1", - "89c1cdfa_2", - "1cf6c966_2", - "619e96bc" - ] - }, - { - "answer_session_ids": [ - "answer_67074b4b_1", - "answer_67074b4b_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "b6019101", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_67074b4b_1", - "answer_67074b4b_2", - "de43030f_2", - "ultrachat_72569", - "sharegpt_CvVLg2J_18", - "ultrachat_146343", - "9bd36386", - "sharegpt_aVExml0_235", - "ac089c83_4", - "59c6685a_2" - ] - }, - { - "answer_session_ids": [ - "answer_07664d43_1", - "answer_07664d43_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "45dc21b6", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_07664d43_1", - "answer_07664d43_2", - "2f57c190_1", - "5934b91f", - "sharegpt_mMdyu3t_0", - "eb34144a_2", - "0504a710_4", - "ultrachat_318979", - "sharegpt_NSacJOD_0", - "8acb29f9_1" - ] - }, - { - "answer_session_ids": [ - "answer_b0f3dfff_1", - "answer_b0f3dfff_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "5a4f22c0", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_b0f3dfff_2", - "answer_b0f3dfff_1", - "007e7d81_2", - "af778ece_1", - "dba97bb1_3", - "sharegpt_1ZrnYv8_23", - "57a6a404_2", - "sharegpt_cXkL3cR_77", - "sharegpt_IMtsj65_0", - "195f2929" - ] - }, - { - "answer_session_ids": [ - "answer_4dac77cb_1", - "answer_4dac77cb_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "6071bd76", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_4dac77cb_2", - "answer_4dac77cb_1", - "a58eeaf8", - "a8899aee_1", - "04616813_2", - "8d0b4363", - "9de53d68_1", - "ultrachat_551846", - "ultrachat_56513", - "34a230b3" - ] - }, - { - "answer_session_ids": [ - "answer_1a374afa_1", - "answer_1a374afa_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e493bb7c", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_1a374afa_2", - "answer_1a374afa_1", - "9161500f_2", - "7aaa23d7_2", - "sharegpt_jHGL67f_0", - "ultrachat_185447", - "ultrachat_151926", - "0aacd15f_1", - "ultrachat_415114", - "ultrachat_575969" - ] - }, - { - "answer_session_ids": [ - "answer_caf5b52e_1", - "answer_caf5b52e_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "618f13b2", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_caf5b52e_1", - "answer_caf5b52e_2", - "86b56135_2", - "748eb152_2", - "01a120b5", - "dda70510_2", - "a20bc58f", - "ultrachat_47868", - "ultrachat_270398", - "sharegpt_aF6djes_0" - ] - }, - { - "answer_session_ids": [ - "answer_d7de9a6a_1", - "answer_d7de9a6a_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "72e3ee87", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_d7de9a6a_2", - "answer_d7de9a6a_1", - "1c1f5ccc_4", - "ultrachat_523550", - "080a4000_2", - "ultrachat_195333", - "ultrachat_129606", - "sharegpt_KdbXhTW_9", - "ultrachat_233144", - "6f2c01fc_1" - ] - }, - { - "answer_session_ids": [ - "answer_d3bf812b_1", - "answer_d3bf812b_2" - ], - "ndcg_at_10": 0.4218, - "ndcg_at_5": 0.3869, - "question_id": "c4ea545c", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "sharegpt_JJu53iW_0", - "sharegpt_I0uNTjm_16", - "9f9b402a_5", - "ee7f5084_1", - "answer_d3bf812b_1", - "sharegpt_CPzL3ju_0", - "a1f43f8d_2", - "b2d284b1", - "answer_d3bf812b_2", - "ultrachat_369982" - ] - }, - { - "answer_session_ids": [ - "answer_a7b44747_1", - "answer_a7b44747_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "01493427", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_a7b44747_1", - "answer_a7b44747_2", - "55e6ff53", - "077f7bcb_3", - "780b6c7c_1", - "5a840928_1", - "29332a29_2", - "c2204106_3", - "88e73f02", - "sharegpt_kjeGJvK_13" - ] - }, - { - "answer_session_ids": [ - "answer_77f32504_1", - "answer_77f32504_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "6a27ffc2", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_77f32504_1", - "answer_77f32504_2", - "sharegpt_kkqlRlo_13", - "ab483540", - "08effb2b_2", - "sharegpt_KSCqghj_0", - "ultrachat_20656", - "sharegpt_5clkQjb_12", - "0651f7bd_2", - "9ded9e01_2" - ] - }, - { - "answer_session_ids": [ - "answer_52382508_1", - "answer_52382508_2" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "2133c1b5", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_52382508_1", - "ultrachat_354956", - "answer_52382508_2", - "ultrachat_502724", - "25b1afbc", - "8ca0085a_1", - "sharegpt_Jm8wWJN_0", - "cf021b36_1", - "09d14ddf_2", - "1126be1e_2" - ] - }, - { - "answer_session_ids": [ - "answer_fff743f5_1", - "answer_fff743f5_2" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "18bc8abd", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_fff743f5_2", - "ba6b67af_3", - "answer_fff743f5_1", - "ultrachat_379579", - "ultrachat_131871", - "sharegpt_afCogMg_34", - "a35679d5_2", - "sharegpt_vyHqfrX_0", - "sharegpt_hH4KVIN_10" - ] - }, - { - "answer_session_ids": [ - "answer_611b6e83_1", - "answer_611b6e83_2" - ], - "ndcg_at_10": 0.8772, - "ndcg_at_5": 0.8772, - "question_id": "db467c8c", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_611b6e83_2", - "6cc835bd", - "1a9f5723_1", - "answer_611b6e83_1", - "ultrachat_137520", - "16ebc8f8_1", - "ecfd2047_1", - "77cf4551_1", - "sharegpt_HBmf6q0_0", - "76967d22" - ] - }, - { - "answer_session_ids": [ - "answer_d08a934d_1", - "answer_d08a934d_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "7a87bd0c", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_d08a934d_1", - "answer_d08a934d_2", - "62369c56_1", - "b98914c6_2", - "2c543c87", - "381227c4_1", - "e250791e_3", - "55a59bc9_4", - "60f9246d", - "15519944_1" - ] - }, - { - "answer_session_ids": [ - "answer_f25c32f5_1", - "answer_f25c32f5_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e61a7584", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_f25c32f5_2", - "answer_f25c32f5_1", - "ultrachat_183954", - "82159da4_1", - "1fd16470_2", - "d02bcc9e_2", - "51add7da_1", - "ultrachat_367590", - "sharegpt_o0MporM_0", - "e1416816_2" - ] - }, - { - "answer_session_ids": [ - "answer_79c395a9_1", - "answer_79c395a9_2" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "1cea1afa", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_79c395a9_2", - "7af38385_1", - "answer_79c395a9_1", - "602ad002_1", - "52d46b4a_1", - "ultrachat_51727", - "ultrachat_67100", - "e9ad5914_5", - "ultrachat_255553", - "sharegpt_X1NXUxZ_0" - ] - }, - { - "answer_session_ids": [ - "answer_babbaccb_1", - "answer_babbaccb_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "ed4ddc30", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_babbaccb_2", - "answer_babbaccb_1", - "d2f4a309", - "8b726b64_1", - "ultrachat_238480", - "4ffa04a2_4", - "499652a0_3", - "ultrachat_342966", - "5d2dc022_1", - "a0c40934_1" - ] - }, - { - "answer_session_ids": [ - "answer_966cecbb_1", - "answer_966cecbb_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "8fb83627", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_966cecbb_1", - "answer_966cecbb_2", - "9315f268_1", - "c15319aa_1", - "02b81ad9", - "0638f6f6_1", - "ultrachat_172298", - "88e01130_1", - "be050ab3_2", - "ultrachat_307166" - ] - }, - { - "answer_session_ids": [ - "answer_8c0712af_1", - "answer_8c0712af_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "b01defab", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_8c0712af_2", - "answer_8c0712af_1", - "b1d9eb66_2", - "sharegpt_VbIxRWJ_33", - "1dcfb9a0_1", - "9aa07e25", - "fc69fd44_7", - "ultrachat_479632", - "sharegpt_fB2ri01_0", - "3891baea_2" - ] - }, - { - "answer_session_ids": [ - "answer_bcce0b73_1", - "answer_bcce0b73_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "22d2cb42", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_bcce0b73_1", - "answer_bcce0b73_2", - "74932466_1", - "060aeced", - "eec72a82_4", - "5a76fadb_2", - "ff898925", - "ultrachat_18212", - "sharegpt_hGQUdgE_10", - "ultrachat_227551" - ] - }, - { - "answer_session_ids": [ - "answer_f2f998c7_1", - "answer_f2f998c7_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "0e4e4c46", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_f2f998c7_1", - "answer_f2f998c7_2", - "b42cbaf0_1", - "60876def_2", - "0773943b", - "71d1e644", - "sharegpt_1YtwybC_15", - "ultrachat_467820", - "ultrachat_180729", - "debc34d7_1" - ] - }, - { - "answer_session_ids": [ - "answer_2cec623b_1", - "answer_2cec623b_2" - ], - "ndcg_at_10": 0.9197, - "ndcg_at_5": 0.9197, - "question_id": "4b24c848", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_2cec623b_2", - "d9e0c03e", - "answer_2cec623b_1", - "1815d1b7", - "sharegpt_iqJr8nh_0", - "d38e5b38", - "ultrachat_198571", - "16e094b3_1", - "d0c1453b_1", - "b0855671_2" - ] - }, - { - "answer_session_ids": [ - "answer_c9f5693c_1", - "answer_c9f5693c_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "7e974930", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c9f5693c_1", - "answer_c9f5693c_2", - "705b5399_3", - "a1e169b1", - "898ce7a5_1", - "bf139883", - "ultrachat_380252", - "07b2420e_1", - "8bd5b3c4_1", - "2cc24b7c_2" - ] - }, - { - "answer_session_ids": [ - "answer_8afdebac_1", - "answer_8afdebac_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "603deb26", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_8afdebac_2", - "answer_8afdebac_1", - "c77250d2", - "433ce12c", - "58e5adf2", - "f379d356_3", - "88611fff", - "7b0dea50_1", - "sharegpt_z6pSZ19_0", - "sharegpt_etklNfN_11" - ] - }, - { - "answer_session_ids": [ - "answer_b28f2c7a_1", - "answer_b28f2c7a_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "59524333", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_b28f2c7a_2", - "answer_b28f2c7a_1", - "ultrachat_458180", - "e4882e09_5", - "9b9ff448_1", - "4507b423", - "sharegpt_KhxXaow_0", - "sharegpt_5FHWj0a_0", - "ultrachat_493944", - "sharegpt_7gIKsqO_104" - ] - }, - { - "answer_session_ids": [ - "answer_8d63a897_1", - "answer_8d63a897_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "5831f84d", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_8d63a897_1", - "answer_8d63a897_2", - "6efce493_2", - "ef84b994_abs_2", - "41c90f4c_2", - "ff67236f_1", - "83dae08b", - "sharegpt_NHidbra_0", - "b818e2a1_1", - "sharegpt_3Rk1i3g_0" - ] - }, - { - "answer_session_ids": [ - "answer_8a791264_1", - "answer_8a791264_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "eace081b", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_8a791264_2", - "answer_8a791264_1", - "4dae77d3", - "413b57cb_3", - "9316aae3_1", - "4d04b866", - "ultrachat_455723", - "164bd5a4", - "29a18eab_3", - "sharegpt_HOO59Ue_137" - ] - }, - { - "answer_session_ids": [ - "answer_90de9b4d_1", - "answer_90de9b4d_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "affe2881", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_90de9b4d_1", - "answer_90de9b4d_2", - "fb5dd87f_7", - "ultrachat_443550", - "f334878c", - "6c4a5aee", - "1b71c896_2", - "sharegpt_eM7toF0_0", - "ultrachat_156501", - "ultrachat_541196" - ] - }, - { - "answer_session_ids": [ - "answer_dcd74827_1", - "answer_dcd74827_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "50635ada", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_dcd74827_1", - "answer_dcd74827_2", - "9f9b402a_5", - "2e53911b_1", - "42bcee92_2", - "add7cc1e", - "cb88f886_1", - "1de8082c", - "35bc3132_2", - "b6f35f3a_1" - ] - }, - { - "answer_session_ids": [ - "answer_ac0140ce_1", - "answer_ac0140ce_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e66b632c", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ac0140ce_2", - "answer_ac0140ce_1", - "84098c1b", - "259d58da_2", - "sharegpt_nspc8Nf_199", - "75621e39_1", - "4f51038b", - "805528d6_1", - "487fd03c", - "b3070ec4_1" - ] - }, - { - "answer_session_ids": [ - "answer_a22b654d_1", - "answer_a22b654d_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "0ddfec37", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_a22b654d_2", - "answer_a22b654d_1", - "a8a0db2b_1", - "147ab7e9_7", - "0a699029_2", - "sharegpt_UyBTuTv_0", - "ultrachat_139747", - "sharegpt_cXkL3cR_45", - "f2199726_2", - "60f9246d" - ] - }, - { - "answer_session_ids": [ - "answer_25df025b_1", - "answer_25df025b_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "f685340e", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_25df025b_1", - "answer_25df025b_2", - "a4815e8e_2", - "e862f726_1", - "5035d8d6", - "e7dcd5df", - "4a9eb139_2", - "142e3203_1", - "sharegpt_RuTDId0_29", - "992eb1f3_2" - ] - }, - { - "answer_session_ids": [ - "answer_a5b68517_1", - "answer_a5b68517_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "cc5ded98", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_a5b68517_1", - "answer_a5b68517_2", - "f14a4532_1", - "c63b2c8f", - "ultrachat_323437", - "bd8708e2_2", - "b13d5e4e_1", - "sharegpt_IEsU45Z_23", - "sharegpt_BJmgxmf_0", - "753b7c1f_1" - ] - }, - { - "answer_session_ids": [ - "answer_35d6c0be_1", - "answer_35d6c0be_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "dfde3500", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_35d6c0be_1", - "answer_35d6c0be_2", - "ad1c7ac6", - "86e218e3_2", - "fa68c9d6_2", - "5c18cb32_2", - "59c704ad_1", - "134127c2_2", - "ultrachat_368201", - "7de51ffe_1" - ] - }, - { - "answer_session_ids": [ - "answer_d6028d6e_1", - "answer_d6028d6e_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "69fee5aa", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_d6028d6e_1", - "answer_d6028d6e_2", - "a554ed79_2", - "a0b8b0ad_7", - "8464304d_2", - "ultrachat_152449", - "6685e6cd", - "42adb80d_2", - "2af27403_2", - "5d284d7d_2" - ] - }, - { - "answer_session_ids": [ - "answer_94650bfa_1", - "answer_94650bfa_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "7401057b", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_94650bfa_1", - "answer_94650bfa_2", - "c38a915a_1", - "30f20967_2", - "8de5b7cf_2", - "c3757c5b", - "1450ffad", - "sharegpt_Ah0tRO5_0", - "ultrachat_402967", - "ultrachat_385162" - ] - }, - { - "answer_session_ids": [ - "answer_ae3a122b_1", - "answer_ae3a122b_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "cf22b7bf", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ae3a122b_2", - "answer_ae3a122b_1", - "36d5bbde_1", - "457e26bf", - "8ab0292c_1", - "8064b6ca", - "22fe1c93", - "c8fcdf5a_1", - "e201572a", - "ultrachat_345143" - ] - }, - { - "answer_session_ids": [ - "answer_5126c02d_1", - "answer_5126c02d_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "a2f3aa27", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_5126c02d_1", - "answer_5126c02d_2", - "b65f51fb_5", - "57bf4122_1", - "2fa50779_2", - "a8028ce7_2", - "104055de", - "ultrachat_475462", - "a135a911", - "4cf2ea34" - ] - }, - { - "answer_session_ids": [ - "answer_0cdbca92_1", - "answer_0cdbca92_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "c7dc5443", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_0cdbca92_1", - "answer_0cdbca92_2", - "064e1984_1", - "sharegpt_6cz1Sq6_264", - "ultrachat_435372", - "debc34d7_2", - "ultrachat_308370", - "8a355b36_1", - "728fc45a", - "aefbe381" - ] - }, - { - "answer_session_ids": [ - "answer_da72b1b4_1", - "answer_da72b1b4_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "06db6396", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_da72b1b4_1", - "answer_da72b1b4_2", - "6ebdc1fe_2", - "sharegpt_2pwdGxJ_16", - "ultrachat_456942", - "5bd9f1e6_1", - "35038174_4", - "8235b87d", - "824634d9_2", - "sharegpt_C26CwPQ_0" - ] - }, - { - "answer_session_ids": [ - "answer_cd345582_1", - "answer_cd345582_2" - ], - "ndcg_at_10": 0.6053, - "ndcg_at_5": 0.6309, - "question_id": "3ba21379", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "49576fd6", - "answer_cd345582_1", - "67620dbc_4", - "d557e57a", - "sharegpt_YGmoq5F_7", - "answer_cd345582_2", - "ultrachat_224436", - "ultrachat_74850", - "d68838d0_2", - "f64c4793_1" - ] - }, - { - "answer_session_ids": [ - "answer_c6a0c6c2_1", - "answer_c6a0c6c2_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "9bbe84a2", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_c6a0c6c2_1", - "answer_c6a0c6c2_2", - "114c1fb7_6", - "ef69c258", - "sharegpt_hcaZN94_546", - "362aa011", - "acad845c_1", - "1fc31387", - "a0f8468c_2", - "sharegpt_hLkmbBL_0" - ] - }, - { - "answer_session_ids": [ - "answer_67be2c38_1", - "answer_67be2c38_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "10e09553", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_67be2c38_2", - "answer_67be2c38_1", - "7d1f5395", - "e5f785de", - "d3da4592_2", - "918c4788_1", - "ff67236f_2", - "ultrachat_196381", - "f4e2933b", - "sharegpt_YPM1YqM_0" - ] - }, - { - "answer_session_ids": [ - "answer_4a97ae40_1", - "answer_4a97ae40_2" - ], - "ndcg_at_10": 0.6934, - "ndcg_at_5": 0.6934, - "question_id": "dad224aa", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "6de8645d_1", - "answer_4a97ae40_2", - "answer_4a97ae40_1", - "c86a22a2_1", - "sharegpt_opQbgPL_13", - "56b5b760", - "dd345e24_2", - "ultrachat_106352", - "ultrachat_468289", - "5c690047_2" - ] - }, - { - "answer_session_ids": [ - "answer_f377cda7_1", - "answer_f377cda7_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "ba61f0b9", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_f377cda7_1", - "answer_f377cda7_2", - "sharegpt_iqJr8nh_0", - "17a1e5c1", - "c9213a03_1", - "85846900_6", - "495cf518_1", - "6b7605d1_2", - "sharegpt_s8Opwwu_0", - "ad578e11_3" - ] - }, - { - "answer_session_ids": [ - "answer_e3892371_1", - "answer_e3892371_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "42ec0761", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e3892371_2", - "answer_e3892371_1", - "27d0d3e7", - "4fb01417_2", - "sharegpt_vWO8y7i_35", - "f849d84c", - "493bd421_2", - "53acd215", - "b37a44e4_2", - "37c3726a_4" - ] - }, - { - "answer_session_ids": [ - "answer_1cb52d0a_1", - "answer_1cb52d0a_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "5c40ec5b", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_1cb52d0a_2", - "answer_1cb52d0a_1", - "ultrachat_333871", - "be050ab3_2", - "a200b713_3", - "9eb7bb35", - "5cc9e1ed_3", - "ultrachat_122152", - "ultrachat_91084", - "6446f6e6" - ] - }, - { - "answer_session_ids": [ - "answer_626e93c4_1", - "answer_626e93c4_2" - ], - "ndcg_at_10": 0.8503, - "ndcg_at_5": 0.8503, - "question_id": "c6853660", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_626e93c4_2", - "6ed1c85e", - "ultrachat_223464", - "33994682_3", - "answer_626e93c4_1", - "71dc2037_2", - "a4863bcb_2", - "ultrachat_544305", - "f8ab60d7", - "bed5fbab" - ] - }, - { - "answer_session_ids": [ - "answer_f762ad8d_1", - "answer_f762ad8d_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "26bdc477", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_f762ad8d_1", - "answer_f762ad8d_2", - "27ce9f13_2", - "sharegpt_EeXTDjc_23", - "ultrachat_438698", - "37b3c75d", - "551eae69_1", - "sharegpt_lWLBUhQ_517", - "sharegpt_pRBsKdE_5", - "sharegpt_bq9G6bT_2" - ] - }, - { - "answer_session_ids": [ - "answer_3bf5b73b_1", - "answer_3bf5b73b_2" - ], - "ndcg_at_10": 0.6309, - "ndcg_at_5": 0.6309, - "question_id": "0977f2af", - "question_type": "knowledge-update", - "r_all_at_10": 0.0, - "r_all_at_5": 0.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "b1d9eb66_4", - "answer_3bf5b73b_2", - "b24eddd9_1", - "f115a3db_1", - "ultrachat_554962", - "809cbce9_3", - "357e33b5", - "1692563c_1", - "9b083158_1", - "sharegpt_FqOC5e9_45" - ] - }, - { - "answer_session_ids": [ - "answer_e1403127_1", - "answer_e1403127_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "89941a94", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_e1403127_2", - "answer_e1403127_1", - "6e5130ed_1", - "de275215_3", - "74b0227e_2", - "6a7d7055_2", - "f334878c", - "2ec60f51_3", - "522dd987_1", - "sharegpt_CQ3HZO9_71" - ] - }, - { - "answer_session_ids": [ - "answer_7e9ad7b4_1", - "answer_7e9ad7b4_2" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "07741c45", - "question_type": "knowledge-update", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_7e9ad7b4_2", - "answer_7e9ad7b4_1", - "37d65d40_3", - "af4a2fd1", - "66c23110_2", - "ce21f0e4", - "ultrachat_162268", - "ultrachat_267176", - "e06c6d99_3", - "ultrachat_27772" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_5Lzox6N_0" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "7161e7e2", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_5Lzox6N_0", - "3c3fee41", - "5850de18_2", - "ultrachat_68050", - "sharegpt_yguSguz_0", - "6b17c99b_2", - "dba97bb1_4", - "5aed819f", - "sharegpt_PjFSCFK_9", - "sharegpt_6kDZTZ2_0" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_234453" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "c4f10528", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_234453", - "107d40f0", - "51c89181_1", - "29116b50_1", - "689fec3d_1", - "11248f50", - "6cf1848e_2", - "sharegpt_yMwEvl7_557", - "21ff9566_2", - "e12cfbcd" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_YkWn1Ne_0" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "89527b6b", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_YkWn1Ne_0", - "ultrachat_117600", - "48d385f0_1", - "644e1d00", - "51c262b6_2", - "4b637e89_1", - "ultrachat_84388", - "sharegpt_htSCqmh_171", - "9c722816", - "07e363d9_2" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_480665" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e9327a54", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_480665", - "ba944cdc", - "11decc61", - "c58fdf1e_2", - "c0d099e6_2", - "28e0ea23_2", - "7613d5ec_2", - "b65f51fb_5", - "sharegpt_NP5eyFN_5", - "sharegpt_Q7EQHIJ_0" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_448704" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "4c36ccef", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_448704", - "3dc8ba5f_2", - "07e6a3bd_1", - "5fc83435_2", - "1ba83815_4", - "89749c78_1", - "7b1d1f43_2", - "sharegpt_m8F4NxL_0", - "940629b2_1", - "sharegpt_1PRjwYK_11" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_IUWQYGQ_0" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "6ae235be", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_IUWQYGQ_0", - "0a1e90d1_2", - "99188307", - "809cabca_1", - "31cde680", - "8e608aad", - "39358a85_1", - "61a91bc9_2", - "cb692a80_3", - "sharegpt_KzDwaf1_211" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_370515" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "7e00a6cb", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_370515", - "4dd1ba8d_2", - "b7dca8f1_1", - "24f78a31_2", - "2fa50779_5", - "956ce238", - "39358a85_3", - "9998814e_2", - "ultrachat_188131", - "a9981dc6_2" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_hA7AkP3_0" - ], - "ndcg_at_10": 0.4307, - "ndcg_at_5": 0.4307, - "question_id": "1903aded", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "66369fde_1", - "184efb91_3", - "9b4c070c", - "answer_sharegpt_hA7AkP3_0", - "ultrachat_110523", - "sharegpt_64onasJ_0", - "201df030", - "8adb232f_1", - "f8246a66_1", - "sharegpt_emE20LI_0" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_cGdjmYo_0" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "ceb54acb", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_cGdjmYo_0", - "511bdbe3_2", - "197d99cc", - "7aa2a9af_2", - "de293134_1", - "f6fd00cf", - "0bbd7094_1", - "ultrachat_150417", - "63b72857", - "c60d5c8a_1" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_m2xJfjo_0" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "f523d9fe", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_m2xJfjo_0", - "193c23bd_1", - "22aa7cca_4", - "debc34d7_1", - "9ccea2f0_1", - "73f75ab4", - "a07f9d5c_1", - "ultrachat_391251", - "661b711f_1", - "fbfda981" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_113156" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "0e5e2d1a", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_113156", - "f84b4266", - "ultrachat_201585", - "c756c34e", - "5d79fd65_2", - "22f9f163_1", - "sharegpt_DcVIUDF_0", - "sharegpt_GPluPmJ_7", - "sharegpt_btv4SkQ_0", - "c4e5d969_2" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_187684" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "fea54f57", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_187684", - "34590cab_1", - "0aa99e2e_1", - "c8854b28", - "712e923c", - "ee659010_2", - "ultrachat_435757", - "ultrachat_129340", - "ultrachat_562162", - "a1e90def_1" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_374124" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "cc539528", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_374124", - "18807892_4", - "8bd3814a_1", - "15a8ffb7_2", - "faa194ea", - "20507d15_1", - "fe1faff9_1", - "sharegpt_0z277Yx_0", - "2a394047", - "sharegpt_1SSsoox_0" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_459954" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "dc439ea3", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_459954", - "1bfd5a8b_2", - "eb7366f2_1", - "6ef99651_1", - "5a81277c", - "ultrachat_186512", - "49d8a382", - "cb5637b3", - "sharegpt_pttddrt_5", - "ultrachat_201433" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_hn3IS1q_0" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "18dcd5a5", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_hn3IS1q_0", - "6ea1541e_1", - "ultrachat_425586", - "959e6cb0_2", - "33d16079_2", - "d9fb1588_1", - "c00077f5_2", - "sharegpt_d1tE544_5", - "sharegpt_MW0whoh_79", - "ultrachat_39471" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_275993" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "488d3006", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_275993", - "ce19df54_3", - "a68090b0_2", - "9a57d65b", - "82b82901", - "a394f6b5_1", - "d53b6bfe_1", - "sharegpt_d9F8HCC_0", - "1af2c0fb_1", - "7e4dab66_1" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_U4oCSfU_7" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "58470ed2", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_U4oCSfU_7", - "sharegpt_oPOTyid_141", - "ff49b5a5_3", - "4050ebff_6", - "a49a1438_2", - "2f2884ad_3", - "b6018747_4", - "ultrachat_458590", - "e45db473_2", - "sharegpt_e8sgAJX_0" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_HFMn2ZX_0" - ], - "ndcg_at_10": 0.6309, - "ndcg_at_5": 0.6309, - "question_id": "8cf51dda", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "daa09295", - "answer_sharegpt_HFMn2ZX_0", - "sharegpt_ftoCcOn_0", - "507514d9_1", - "27639dd8_2", - "ultrachat_365938", - "3358b5ca_2", - "9f0f59d1_1", - "sharegpt_jpiaPoJ_738", - "sharegpt_yoSUjfa_0" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_348449" - ], - "ndcg_at_10": 0.5, - "ndcg_at_5": 0.5, - "question_id": "1d4da289", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "09f1c793", - "ultrachat_213759", - "answer_ultrachat_348449", - "4cd929f8_1", - "517ad0f0", - "sharegpt_eVmxjQZ_0", - "bf85791d", - "a05c7d7d", - "ultrachat_322093", - "ultrachat_329521" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_467053" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "8464fc84", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_467053", - "21d86ebb_2", - "6ac43c5b_4", - "bdc804df_2", - "602b4074_3", - "1d5998ca_1", - "d43b63a6", - "7a0abbe2_2", - "98d39a6d", - "9582aa59_2" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_563222" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "8aef76bc", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_563222", - "146dfabe", - "5209e813_2", - "27d378b2_2", - "08e084f1_1", - "sharegpt_n3Hy63g_0", - "ultrachat_351310", - "ultrachat_368669", - "44b1595a", - "c7f6643e_2" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_417348" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "71a3fd6b", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_417348", - "8419b385_3", - "7f7e26d2_2", - "878271ae_1", - "36743359_3", - "sharegpt_1L5GXZJ_0", - "sharegpt_n3kgf14_17", - "ultrachat_168420", - "b7146673_1", - "sharegpt_asilfXk_321" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_2kpncbX_13" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "2bf43736", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_2kpncbX_13", - "16c3be68_1", - "8897a2e5", - "661ee6d9_1", - "2e89bd03_1", - "7e4cd916_2", - "d79173aa_3", - "445e6a7a_1", - "1a5fb2cd", - "a93750ef_2" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_334948" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "70b3e69b", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_334948", - "09e1a929_2", - "ultrachat_47234", - "sharegpt_oTNStu2_51", - "dd92723f_1", - "ultrachat_478639", - "1388b204_2", - "sharegpt_O8qJclh_0", - "sharegpt_s0wsteT_0", - "c2204106_2" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_6pWK9yx_0" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "8752c811", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_6pWK9yx_0", - "d0a41222_1", - "93f23301_2", - "06b5b860", - "9db30f1c_3", - "5c6a5b7c_1", - "3e48ed93", - "02b63d04_1", - "ultrachat_160178", - "9272a6a2_2" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_CaxTGYP_0" - ], - "ndcg_at_10": 0.6309, - "ndcg_at_5": 0.6309, - "question_id": "3249768e", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "ed1d68f1_1", - "answer_sharegpt_CaxTGYP_0", - "0868982f_1", - "71f2b8ba", - "ultrachat_77515", - "56cf2973", - "172b47cb_2", - "e0bd5f14_2", - "23450b93_3", - "sharegpt_M5zaN8c_0" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_115151" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "1b9b7252", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_115151", - "3829d412_2", - "d3cc5bdc_4", - "53ab4734", - "af4d4ecd_6", - "0e726047", - "330c05e7_1", - "b2e5a019", - "4f6e1b77_1", - "sharegpt_1AhFMpw_5" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_d6JJiqH_76" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "1568498a", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_d6JJiqH_76", - "sharegpt_kRkQrRx_0", - "38de4c2a_1", - "6ad46850_2", - "70fed904", - "4022d960_2", - "sharegpt_trljpZ5_27", - "sharegpt_69agvDU_27", - "983b1954", - "f8e8d445_2" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_H9PiM5G_0" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "6222b6eb", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_H9PiM5G_0", - "caf7480d_1", - "sharegpt_cyc2xbb_0", - "40a90d51", - "2fd5e208", - "544c1e7b", - "b35fadaa", - "facb94a8_3", - "2a500dce_1", - "5209e813_2" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_13075" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e8a79c70", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_13075", - "sharegpt_ha8kUEr_0", - "c0d099e6_2", - "sharegpt_vKSMO9J_69", - "742ea86b_1", - "89aed484_2", - "c2b2b2ea_1", - "87252d80_2", - "ultrachat_225061", - "sharegpt_tfIfm00_14" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_252214" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "d596882b", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_252214", - "dc9bf721_1", - "71888aff_2", - "456807b7_1", - "bef3abcd", - "0dc2efcc_2", - "ultrachat_299672", - "087d2b0a_1", - "5fb3b5ac", - "ultrachat_306390" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_5m7gg5F_0" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e3fc4d6e", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_5m7gg5F_0", - "4d261642", - "sharegpt_TVK4gjw_24", - "d31c3ec8_1", - "ultrachat_161238", - "ultrachat_325941", - "sharegpt_sXKNzPE_0", - "ultrachat_495326", - "ultrachat_443354", - "sharegpt_nAacldL_21" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_427265" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "51b23612", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_427265", - "255c753b_1", - "8672f398_2", - "409015ef_1", - "17f444f3", - "ultrachat_343", - "ultrachat_522170", - "ultrachat_555062", - "ultrachat_221034", - "848f9a52" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_94624" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "3e321797", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_94624", - "9252145e_2", - "11302335_4", - "57144028_2", - "00305dd2_1", - "193c23bd_1", - "385a60c8_1", - "sharegpt_OzK1xD9_0", - "ultrachat_557903", - "sharegpt_EZmhz8p_0" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_195444" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e982271f", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_195444", - "eb0d8dc1_4", - "8a7c65c1_1", - "f72c924e", - "db65997e", - "ultrachat_383774", - "ultrachat_247528", - "sharegpt_Xlpa70t_0", - "3e8f07c9_1", - "934419a6_1" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_NoDZzot_7" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "352ab8bd", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_NoDZzot_7", - "6593cb8b_6", - "337cbfc8_2", - "d600c646", - "55a59bc9_2", - "4a42c62b", - "ultrachat_194309", - "ultrachat_101751", - "27971ce6_5", - "sharegpt_SM8NnLU_13" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_39395" - ], - "ndcg_at_10": 0.3869, - "ndcg_at_5": 0.3869, - "question_id": "fca762bc", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "06a34d82_1", - "d0cdfddb_2", - "8fcc3543_3", - "2bc4ae7d", - "answer_ultrachat_39395", - "42d9467c_1", - "sharegpt_wrN9uUo_77", - "3b7a6ecd_1", - "136dfd61_2", - "sharegpt_ZgtoeGU_0" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_i0tMT9q_9" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "7a8d0b71", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_i0tMT9q_9", - "f07752be_1", - "f8169a6d", - "7a8f5003", - "d5eab084_2", - "c14e3f09_2", - "147ab7e9_5", - "f20e72e4_2", - "sharegpt_P0Qe2Gx_0", - "881d8e9b_1" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_269020" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "a40e080f", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_269020", - "4d32ba1b_1", - "85736a07_2", - "sharegpt_fyAkcGP_81", - "89c7a48a_2", - "4fd6129f_1", - "cd38b6ee", - "ed109bc3", - "sharegpt_oXgiN7q_327", - "c9489af0_1" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_289157" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "8b9d4367", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_289157", - "sharegpt_WZzri9J_0", - "7babf05b", - "031061e6_4", - "sharegpt_iHP3XDi_39", - "5534efcb_1", - "ultrachat_79945", - "36743359_1", - "007e7d81_3", - "sharegpt_MHiC4J8_0" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_4aJsGCH_0" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "5809eb10", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_4aJsGCH_0", - "fc3d6c80", - "sharegpt_GDFyxeT_0", - "9b820206", - "sharegpt_obCC1BP_103", - "d3069305_2", - "22ce6625_1", - "ultrachat_536416", - "ultrachat_275993", - "sharegpt_9M4WbfX_0" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_81riySf_0" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "41275add", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_81riySf_0", - "5b83c26e_3", - "33a39aa7_1", - "b3619c2c", - "a68db5db_1", - "4d84cbfa", - "ultrachat_36085", - "ultrachat_556585", - "sharegpt_dUoAhvI_71", - "sharegpt_fV5wNsl_0" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_qTi81nS_0" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "4388e9dd", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_qTi81nS_0", - "cfd21744_2", - "3197d998_7", - "1ebd753f_2", - "ultrachat_412706", - "1b8f0ba5_4", - "fc2d70fe_1", - "0826def6_1", - "cf9eda76", - "9ff53e00_1" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_i9adwQn_0" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "4baee567", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_i9adwQn_0", - "sharegpt_wduYbsX_0", - "sharegpt_NxG2nGm_91", - "385683f0_1", - "sharegpt_kt5ZCxz_10", - "ec830058_5", - "sharegpt_hRVKLa2_0", - "ultrachat_427430", - "d5415748", - "ultrachat_210474" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_hChsWOp_97" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "561fabcd", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_hChsWOp_97", - "864a563d_4", - "34b38398_2", - "sharegpt_QaiMtpK_0", - "aee13015_3", - "3be19aed_1", - "83fb74bf_1", - "d75f4fb5_1", - "33da50d0_5", - "ultrachat_194831" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_2BSXlAr_0" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "b759caee", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_2BSXlAr_0", - "sharegpt_6gBVFdg_1", - "b41aae7f_1", - "bab41bb6_5", - "d3575920", - "ultrachat_554750", - "f55fee5b_4", - "sharegpt_I9veIBa_1", - "ultrachat_281823", - "0bad887e_2" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_GYqnAhC_190" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "ac031881", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_GYqnAhC_190", - "eb95f93c", - "6608c1da", - "sharegpt_LgJ688S_0", - "e7989082_1", - "c986f83a", - "ultrachat_58138", - "2dbe975c", - "sharegpt_cIj2dFl_4", - "19c24c11" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_446979" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "28bcfaac", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_446979", - "8d410160", - "be05680b_2", - "17cb8d2f_5", - "135a62d7_2", - "a27a2811", - "044e200f_2", - "ultrachat_433665", - "989ad9e6_3", - "sharegpt_MO9Xxet_71" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_294807" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "16c90bf4", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_294807", - "8464304d_2", - "5053474c_2", - "b8770374_2", - "sharegpt_Ou9rQ6U_8", - "b0da5097_2", - "ad72ce15", - "d576152e_4", - "d4230511_1", - "32449a42" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_519486" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "c8f1aeed", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_519486", - "sharegpt_ebYKddM_87", - "aa7a2ec0_1", - "a53fdd02_2", - "af631aa3_3", - "48a72204_3", - "f7cd21e5", - "sharegpt_fH5kcER_0", - "12cd736c", - "c3023a45_5" - ] - }, - { - "answer_session_ids": [ - "answer_sharegpt_SS141vi_0" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "eaca4986", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_sharegpt_SS141vi_0", - "3477c37c_1", - "931c521e_1", - "ultrachat_311235", - "7bc2c7a3_1", - "1c1a2b7f_2", - "564761aa", - "24ca2090_2", - "sharegpt_NelPQLT_0", - "0e543e9e_2" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_456407" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "c7cf7dfd", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_456407", - "de5a59bf", - "95cec590_1", - "285a29e7_1", - "sharegpt_f28uI6i_31", - "780b6c7c_1", - "c1e681e7_2", - "e27fe923_5", - "1382d6fb_1", - "9db6d988_2" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_174360" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e48988bc", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_174360", - "6e110a53_1", - "ultrachat_494373", - "e388463b_1", - "3fd56ebf", - "ff6bccaa_3", - "476b1cbb", - "sharegpt_TzwvePR_21", - "ultrachat_172828", - "ultrachat_307163" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_440262" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "1de5cff2", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_440262", - "4c6775ed_1", - "848d721f_1", - "8fcfbe43_3", - "ultrachat_58031", - "64048e35", - "28031c43_2", - "c14dbe17", - "42d9467c_1", - "4cb33063" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_403752" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "65240037", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_403752", - "37141183", - "46f07bd0_2", - "f8dc1273", - "fce669f5_3", - "0984a772", - "7df4c735_1", - "3c3a9042_1", - "dba5a924", - "sharegpt_Caxpnd3_6" - ] - }, - { - "answer_session_ids": [ - "answer_ultrachat_399000" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "778164c6", - "question_type": "single-session-assistant", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_ultrachat_399000", - "sharegpt_ki9IVDq_6", - "ultrachat_346016", - "483ddd90", - "490bb46d_1", - "05b551b6", - "b605741a_1", - "ultrachat_144598", - "c15dadce_4", - "35201d43" - ] - } - ], - "runtime_seconds": 251.044, - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_old_brainctl_brain.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_old_brainctl_brain.json deleted file mode 100644 index c02f524..0000000 --- a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_old_brainctl_brain.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\longmemeval_old_brainctl_brain.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "measured": false, - "metadata": {}, - "metrics": { - "ndcg_at_10": 0.9253, - "ndcg_at_5": 0.9204, - "r_at_10": 0.9894, - "r_at_5": 0.9681 - }, - "mode": "brain", - "notes": [ - "top_k=10" - ], - "primary_metric": "r_at_5", - "primary_metric_value": 0.9681, - "reference_kind": "historical", - "runtime_seconds": 85.439, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_old_brainctl_cmd.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_old_brainctl_cmd.json deleted file mode 100644 index d110c50..0000000 --- a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/longmemeval_old_brainctl_cmd.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\longmemeval_old_brainctl_cmd.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "measured": false, - "metadata": {}, - "metrics": { - "ndcg_at_10": 0.9247, - "ndcg_at_5": 0.9206, - "r_at_10": 0.9894, - "r_at_5": 0.9702 - }, - "mode": "cmd", - "notes": [ - "top_k=10" - ], - "primary_metric": "r_at_5", - "primary_metric_value": 0.9702, - "reference_kind": "historical", - "runtime_seconds": 130.863, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/membench_mempalace_raw_turn.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/membench_mempalace_raw_turn.json deleted file mode 100644 index 7654261..0000000 --- a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/membench_mempalace_raw_turn.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\membench_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\membench_mempalace_raw_turn.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" - }, - "benchmark": "membench", - "caveats": [ - "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 200, - "measured": false, - "metadata": {}, - "metrics": { - "hit_at_5": 0.885, - "simple_hit_at_5": 0.885, - "top_k": 5 - }, - "mode": "raw_turn", - "notes": [ - "FirstAgent slice only", - "turn-level retrieval", - "topic=all" - ], - "primary_metric": "hit_at_5", - "primary_metric_value": 0.885, - "reference_kind": "historical", - "runtime_seconds": 804.35, - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "partial", - "system_name": "mempalace" -} \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/membench_new_brainctl_cmd_turn.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/membench_new_brainctl_cmd_turn.json deleted file mode 100644 index ff93a9f..0000000 --- a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/membench_new_brainctl_cmd_turn.json +++ /dev/null @@ -1,3369 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\membench_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\membench_new_brainctl_cmd_turn.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" - }, - "benchmark": "membench", - "caveats": [ - "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 200, - "measured": true, - "metadata": {}, - "metrics": { - "hit_at_5": 0.895, - "simple_hit_at_5": 0.895, - "top_k": 5 - }, - "mode": "cmd_turn", - "notes": [ - "FirstAgent slice only", - "turn-level retrieval", - "topic=all" - ], - "primary_metric": "hit_at_5", - "primary_metric_value": 0.895, - "reference_kind": "measured", - "rows": [ - { - "category": "simple", - "hit_at_k": false, - "question": "What is the name of my niece's company?", - "retrieved_ids": [], - "target_ids": [ - "119" - ], - "tid": 0, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the email address of my cousin, who is male?", - "retrieved_ids": [ - "simple_roles_1|sid=59|g=59|s=2|t=17", - "simple_roles_1|sid=143|g=143|s=6|t=18", - "simple_roles_1|sid=101|g=101|s=4|t=18", - "simple_roles_1|sid=33|g=33|s=1|t=12", - "simple_roles_1|sid=42|g=42|s=2|t=0" - ], - "target_ids": [ - "59" - ], - "tid": 1, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the position of my brother?", - "retrieved_ids": [ - "simple_roles_2|sid=85|g=85|s=4|t=1", - "simple_roles_2|sid=84|g=84|s=4|t=0", - "simple_roles_2|sid=28|g=28|s=1|t=7", - "simple_roles_2|sid=41|g=41|s=1|t=20", - "simple_roles_2|sid=123|g=123|s=5|t=18" - ], - "target_ids": [ - "84" - ], - "tid": 2, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the height of my aunt?", - "retrieved_ids": [ - "simple_roles_3|sid=13|g=13|s=0|t=13", - "simple_roles_3|sid=0|g=0|s=0|t=0", - "simple_roles_3|sid=125|g=125|s=6|t=1", - "simple_roles_3|sid=19|g=19|s=0|t=19", - "simple_roles_3|sid=98|g=98|s=4|t=16" - ], - "target_ids": [ - "13" - ], - "tid": 3, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the contact number for my subordinate?", - "retrieved_ids": [ - "simple_roles_4|sid=40|g=40|s=1|t=19", - "simple_roles_4|sid=135|g=135|s=6|t=9", - "simple_roles_4|sid=59|g=59|s=2|t=17", - "simple_roles_4|sid=98|g=98|s=4|t=14", - "simple_roles_4|sid=9|g=9|s=0|t=9" - ], - "target_ids": [ - "40" - ], - "tid": 4, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What hobby does my brother have?", - "retrieved_ids": [ - "simple_roles_5|sid=63|g=63|s=3|t=0", - "simple_roles_5|sid=81|g=81|s=3|t=18", - "simple_roles_5|sid=11|g=11|s=0|t=11", - "simple_roles_5|sid=77|g=77|s=3|t=14", - "simple_roles_5|sid=159|g=159|s=7|t=16" - ], - "target_ids": [ - "81" - ], - "tid": 5, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What hobby does my nephew have?", - "retrieved_ids": [ - "simple_roles_6|sid=103|g=103|s=5|t=0", - "simple_roles_6|sid=98|g=98|s=4|t=15", - "simple_roles_6|sid=52|g=52|s=2|t=10", - "simple_roles_6|sid=154|g=154|s=7|t=10", - "simple_roles_6|sid=105|g=105|s=5|t=2" - ], - "target_ids": [ - "105" - ], - "tid": 6, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the work location of my sister?", - "retrieved_ids": [ - "simple_roles_7|sid=144|g=144|s=7|t=0", - "simple_roles_7|sid=68|g=68|s=3|t=7", - "simple_roles_7|sid=145|g=145|s=7|t=1", - "simple_roles_7|sid=39|g=39|s=1|t=19", - "simple_roles_7|sid=127|g=127|s=6|t=3" - ], - "target_ids": [ - "144" - ], - "tid": 7, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the height of my male cousin?", - "retrieved_ids": [ - "simple_roles_8|sid=64|g=64|s=3|t=4", - "simple_roles_8|sid=82|g=82|s=4|t=1", - "simple_roles_8|sid=102|g=102|s=5|t=0", - "simple_roles_8|sid=60|g=60|s=3|t=0", - "simple_roles_8|sid=80|g=80|s=3|t=20" - ], - "target_ids": [ - "64" - ], - "tid": 8, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": false, - "question": "What is the educational background of that coworker?", - "retrieved_ids": [ - "simple_roles_9|sid=104|g=104|s=5|t=2", - "simple_roles_9|sid=48|g=48|s=2|t=7", - "simple_roles_9|sid=112|g=112|s=5|t=10", - "simple_roles_9|sid=138|g=138|s=6|t=16", - "simple_roles_9|sid=50|g=50|s=2|t=9" - ], - "target_ids": [ - "113" - ], - "tid": 9, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What does the coworker do for a living?", - "retrieved_ids": [ - "simple_roles_10|sid=123|g=123|s=6|t=0", - "simple_roles_10|sid=124|g=124|s=6|t=1", - "simple_roles_10|sid=140|g=140|s=6|t=17", - "simple_roles_10|sid=86|g=86|s=4|t=4", - "simple_roles_10|sid=134|g=134|s=6|t=11" - ], - "target_ids": [ - "124" - ], - "tid": 10, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": false, - "question": "What is the location of my father's workplace?", - "retrieved_ids": [ - "simple_roles_11|sid=81|g=81|s=4|t=0", - "simple_roles_11|sid=157|g=157|s=7|t=14", - "simple_roles_11|sid=94|g=94|s=4|t=13", - "simple_roles_11|sid=136|g=136|s=6|t=13", - "simple_roles_11|sid=153|g=153|s=7|t=10" - ], - "target_ids": [ - "92" - ], - "tid": 11, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the work location of the subordinate?", - "retrieved_ids": [ - "simple_roles_12|sid=32|g=32|s=1|t=11", - "simple_roles_12|sid=21|g=21|s=1|t=0", - "simple_roles_12|sid=46|g=46|s=2|t=4", - "simple_roles_12|sid=131|g=131|s=6|t=7", - "simple_roles_12|sid=47|g=47|s=2|t=5" - ], - "target_ids": [ - "32" - ], - "tid": 12, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What hobby does the coworker have?", - "retrieved_ids": [ - "simple_roles_13|sid=109|g=109|s=5|t=6", - "simple_roles_13|sid=158|g=158|s=7|t=15", - "simple_roles_13|sid=70|g=70|s=3|t=8", - "simple_roles_13|sid=148|g=148|s=7|t=5", - "simple_roles_13|sid=143|g=143|s=7|t=0" - ], - "target_ids": [ - "148" - ], - "tid": 13, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the position of the boss?", - "retrieved_ids": [ - "simple_roles_14|sid=42|g=42|s=2|t=0", - "simple_roles_14|sid=61|g=61|s=2|t=19", - "simple_roles_14|sid=56|g=56|s=2|t=14", - "simple_roles_14|sid=62|g=62|s=2|t=20", - "simple_roles_14|sid=11|g=11|s=0|t=11" - ], - "target_ids": [ - "56" - ], - "tid": 14, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the email address of the subordinate?", - "retrieved_ids": [ - "simple_roles_15|sid=13|g=13|s=0|t=13", - "simple_roles_15|sid=96|g=96|s=4|t=13", - "simple_roles_15|sid=35|g=35|s=1|t=13", - "simple_roles_15|sid=160|g=160|s=7|t=16", - "simple_roles_15|sid=73|g=73|s=3|t=11" - ], - "target_ids": [ - "13" - ], - "tid": 15, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the hometown of my aunt?", - "retrieved_ids": [ - "simple_roles_16|sid=123|g=123|s=6|t=0", - "simple_roles_16|sid=143|g=143|s=6|t=20", - "simple_roles_16|sid=85|g=85|s=4|t=3", - "simple_roles_16|sid=142|g=142|s=6|t=19", - "simple_roles_16|sid=32|g=32|s=1|t=11" - ], - "target_ids": [ - "143" - ], - "tid": 16, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the hometown of my sister?", - "retrieved_ids": [ - "simple_roles_17|sid=20|g=20|s=1|t=0", - "simple_roles_17|sid=24|g=24|s=1|t=4", - "simple_roles_17|sid=32|g=32|s=1|t=12", - "simple_roles_17|sid=5|g=5|s=0|t=5", - "simple_roles_17|sid=128|g=128|s=6|t=5" - ], - "target_ids": [ - "24" - ], - "tid": 17, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the education background of that coworker?", - "retrieved_ids": [ - "simple_roles_18|sid=145|g=145|s=7|t=1", - "simple_roles_18|sid=21|g=21|s=1|t=0", - "simple_roles_18|sid=24|g=24|s=1|t=3", - "simple_roles_18|sid=144|g=144|s=7|t=0", - "simple_roles_18|sid=165|g=165|s=7|t=21" - ], - "target_ids": [ - "144" - ], - "tid": 18, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the occupation of that subordinate?", - "retrieved_ids": [ - "simple_roles_19|sid=97|g=97|s=4|t=14" - ], - "target_ids": [ - "97" - ], - "tid": 19, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What hobby does my nephew have?", - "retrieved_ids": [ - "simple_roles_20|sid=0|g=0|s=0|t=0", - "simple_roles_20|sid=6|g=6|s=0|t=6", - "simple_roles_20|sid=115|g=115|s=5|t=9", - "simple_roles_20|sid=34|g=34|s=1|t=13", - "simple_roles_20|sid=72|g=72|s=3|t=8" - ], - "target_ids": [ - "6" - ], - "tid": 20, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What hobby does my coworker have?", - "retrieved_ids": [ - "simple_roles_21|sid=61|g=61|s=3|t=0", - "simple_roles_21|sid=72|g=72|s=3|t=11", - "simple_roles_21|sid=105|g=105|s=5|t=3", - "simple_roles_21|sid=70|g=70|s=3|t=9", - "simple_roles_21|sid=139|g=139|s=6|t=17" - ], - "target_ids": [ - "70" - ], - "tid": 21, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": false, - "question": "What is the position of my mother?", - "retrieved_ids": [ - "simple_roles_22|sid=21|g=21|s=1|t=0", - "simple_roles_22|sid=37|g=37|s=1|t=16", - "simple_roles_22|sid=36|g=36|s=1|t=15", - "simple_roles_22|sid=158|g=158|s=7|t=16", - "simple_roles_22|sid=115|g=115|s=5|t=13" - ], - "target_ids": [ - "38" - ], - "tid": 22, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the name of my sister?", - "retrieved_ids": [ - "simple_roles_23|sid=19|g=19|s=0|t=19", - "simple_roles_23|sid=0|g=0|s=0|t=0", - "simple_roles_23|sid=13|g=13|s=0|t=13", - "simple_roles_23|sid=40|g=40|s=2|t=0", - "simple_roles_23|sid=38|g=38|s=1|t=17" - ], - "target_ids": [ - "19" - ], - "tid": 23, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the name of my subordinate's company?", - "retrieved_ids": [ - "simple_roles_24|sid=61|g=61|s=2|t=19", - "simple_roles_24|sid=9|g=9|s=0|t=9", - "simple_roles_24|sid=42|g=42|s=2|t=0", - "simple_roles_24|sid=41|g=41|s=1|t=20", - "simple_roles_24|sid=114|g=114|s=5|t=10" - ], - "target_ids": [ - "61" - ], - "tid": 24, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the age of the subordinate?", - "retrieved_ids": [ - "simple_roles_25|sid=75|g=75|s=3|t=16" - ], - "target_ids": [ - "75" - ], - "tid": 25, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What hobby does the subordinate enjoy?", - "retrieved_ids": [ - "simple_roles_26|sid=135|g=135|s=6|t=12", - "simple_roles_26|sid=88|g=88|s=4|t=5", - "simple_roles_26|sid=32|g=32|s=1|t=11", - "simple_roles_26|sid=29|g=29|s=1|t=8", - "simple_roles_26|sid=11|g=11|s=0|t=11" - ], - "target_ids": [ - "135" - ], - "tid": 26, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": false, - "question": "What level of education has my mother completed?", - "retrieved_ids": [ - "simple_roles_27|sid=27|g=27|s=1|t=7", - "simple_roles_27|sid=6|g=6|s=0|t=6", - "simple_roles_27|sid=129|g=129|s=6|t=7", - "simple_roles_27|sid=139|g=139|s=6|t=17", - "simple_roles_27|sid=44|g=44|s=2|t=3" - ], - "target_ids": [ - "118" - ], - "tid": 27, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "When is my subordinate's birthday?", - "retrieved_ids": [ - "simple_roles_28|sid=15|g=15|s=0|t=15", - "simple_roles_28|sid=109|g=109|s=5|t=6", - "simple_roles_28|sid=44|g=44|s=2|t=2", - "simple_roles_28|sid=127|g=127|s=6|t=3", - "simple_roles_28|sid=0|g=0|s=0|t=0" - ], - "target_ids": [ - "15" - ], - "tid": 28, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the position of my sister?", - "retrieved_ids": [ - "simple_roles_29|sid=163|g=163|s=7|t=18", - "simple_roles_29|sid=93|g=93|s=4|t=10", - "simple_roles_29|sid=145|g=145|s=7|t=0", - "simple_roles_29|sid=17|g=17|s=0|t=17", - "simple_roles_29|sid=100|g=100|s=4|t=17" - ], - "target_ids": [ - "163" - ], - "tid": 29, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": false, - "question": "What is the hometown of my female cousin?", - "retrieved_ids": [], - "target_ids": [ - "24" - ], - "tid": 30, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the height of my uncle?", - "retrieved_ids": [ - "simple_roles_31|sid=67|g=67|s=3|t=7", - "simple_roles_31|sid=141|g=141|s=7|t=1", - "simple_roles_31|sid=121|g=121|s=6|t=1", - "simple_roles_31|sid=21|g=21|s=1|t=1", - "simple_roles_31|sid=60|g=60|s=3|t=0" - ], - "target_ids": [ - "67" - ], - "tid": 31, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the contact number for my male cousin?", - "retrieved_ids": [ - "simple_roles_32|sid=141|g=141|s=6|t=19", - "simple_roles_32|sid=109|g=109|s=5|t=8", - "simple_roles_32|sid=52|g=52|s=2|t=11", - "simple_roles_32|sid=74|g=74|s=3|t=13", - "simple_roles_32|sid=91|g=91|s=4|t=10" - ], - "target_ids": [ - "141" - ], - "tid": 32, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "How old is the coworker?", - "retrieved_ids": [ - "simple_roles_33|sid=151|g=151|s=7|t=7", - "simple_roles_33|sid=62|g=62|s=3|t=0", - "simple_roles_33|sid=162|g=162|s=7|t=18", - "simple_roles_33|sid=161|g=161|s=7|t=17", - "simple_roles_33|sid=144|g=144|s=7|t=0" - ], - "target_ids": [ - "151" - ], - "tid": 33, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the email address of that coworker?", - "retrieved_ids": [ - "simple_roles_34|sid=26|g=26|s=1|t=6", - "simple_roles_34|sid=139|g=139|s=6|t=15", - "simple_roles_34|sid=113|g=113|s=5|t=10", - "simple_roles_34|sid=96|g=96|s=4|t=13", - "simple_roles_34|sid=72|g=72|s=3|t=10" - ], - "target_ids": [ - "26" - ], - "tid": 34, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "Could anyone provide the contact number for my subordinate?", - "retrieved_ids": [ - "simple_roles_35|sid=16|g=16|s=0|t=16", - "simple_roles_35|sid=109|g=109|s=5|t=6", - "simple_roles_35|sid=77|g=77|s=3|t=17", - "simple_roles_35|sid=148|g=148|s=7|t=3", - "simple_roles_35|sid=55|g=55|s=2|t=15" - ], - "target_ids": [ - "109" - ], - "tid": 35, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the education background of the coworker?", - "retrieved_ids": [ - "simple_roles_36|sid=104|g=104|s=5|t=0", - "simple_roles_36|sid=61|g=61|s=2|t=19", - "simple_roles_36|sid=122|g=122|s=5|t=18", - "simple_roles_36|sid=112|g=112|s=5|t=8", - "simple_roles_36|sid=127|g=127|s=6|t=3" - ], - "target_ids": [ - "112" - ], - "tid": 36, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "When is my sister's birthday?", - "retrieved_ids": [ - "simple_roles_37|sid=139|g=139|s=6|t=18", - "simple_roles_37|sid=44|g=44|s=2|t=3", - "simple_roles_37|sid=118|g=118|s=5|t=17", - "simple_roles_37|sid=121|g=121|s=6|t=0", - "simple_roles_37|sid=20|g=20|s=1|t=0" - ], - "target_ids": [ - "139" - ], - "tid": 37, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What hobby does my nephew have?", - "retrieved_ids": [ - "simple_roles_38|sid=42|g=42|s=2|t=0", - "simple_roles_38|sid=48|g=48|s=2|t=6", - "simple_roles_38|sid=14|g=14|s=0|t=14", - "simple_roles_38|sid=10|g=10|s=0|t=10", - "simple_roles_38|sid=72|g=72|s=3|t=9" - ], - "target_ids": [ - "48" - ], - "tid": 38, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the height of my uncle?", - "retrieved_ids": [ - "simple_roles_39|sid=118|g=118|s=5|t=16", - "simple_roles_39|sid=102|g=102|s=5|t=0", - "simple_roles_39|sid=108|g=108|s=5|t=6", - "simple_roles_39|sid=82|g=82|s=4|t=1", - "simple_roles_39|sid=70|g=70|s=3|t=10" - ], - "target_ids": [ - "118" - ], - "tid": 39, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "When is my father's birthday?", - "retrieved_ids": [ - "simple_roles_40|sid=41|g=41|s=2|t=0", - "simple_roles_40|sid=84|g=84|s=4|t=3", - "simple_roles_40|sid=50|g=50|s=2|t=9", - "simple_roles_40|sid=59|g=59|s=2|t=18", - "simple_roles_40|sid=3|g=3|s=0|t=3" - ], - "target_ids": [ - "50" - ], - "tid": 40, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the hometown of my father?", - "retrieved_ids": [ - "simple_roles_41|sid=129|g=129|s=6|t=5", - "simple_roles_41|sid=46|g=46|s=2|t=4", - "simple_roles_41|sid=99|g=99|s=4|t=16", - "simple_roles_41|sid=21|g=21|s=1|t=0", - "simple_roles_41|sid=56|g=56|s=2|t=14" - ], - "target_ids": [ - "99" - ], - "tid": 41, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": false, - "question": "What position does the coworker hold?", - "retrieved_ids": [], - "target_ids": [ - "38" - ], - "tid": 42, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What position does that coworker hold?", - "retrieved_ids": [ - "simple_roles_43|sid=105|g=105|s=5|t=4", - "simple_roles_43|sid=110|g=110|s=5|t=9", - "simple_roles_43|sid=22|g=22|s=1|t=2", - "simple_roles_43|sid=34|g=34|s=1|t=14", - "simple_roles_43|sid=14|g=14|s=0|t=14" - ], - "target_ids": [ - "34" - ], - "tid": 43, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is my male cousin's email address?", - "retrieved_ids": [ - "simple_roles_44|sid=100|g=100|s=4|t=17", - "simple_roles_44|sid=51|g=51|s=2|t=10", - "simple_roles_44|sid=138|g=138|s=6|t=14", - "simple_roles_44|sid=156|g=156|s=7|t=12", - "simple_roles_44|sid=29|g=29|s=1|t=9" - ], - "target_ids": [ - "100" - ], - "tid": 44, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the highest level of education attained by my aunt?", - "retrieved_ids": [ - "simple_roles_45|sid=90|g=90|s=4|t=9", - "simple_roles_45|sid=42|g=42|s=2|t=0", - "simple_roles_45|sid=130|g=130|s=6|t=9", - "simple_roles_45|sid=51|g=51|s=2|t=9", - "simple_roles_45|sid=60|g=60|s=2|t=18" - ], - "target_ids": [ - "51" - ], - "tid": 45, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What hobby does my boss have?", - "retrieved_ids": [ - "simple_roles_46|sid=89|g=89|s=4|t=6", - "simple_roles_46|sid=9|g=9|s=0|t=9", - "simple_roles_46|sid=83|g=83|s=4|t=0", - "simple_roles_46|sid=112|g=112|s=5|t=8", - "simple_roles_46|sid=103|g=103|s=4|t=20" - ], - "target_ids": [ - "89" - ], - "tid": 46, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the age of my father?", - "retrieved_ids": [ - "simple_roles_47|sid=41|g=41|s=2|t=0", - "simple_roles_47|sid=101|g=101|s=5|t=0", - "simple_roles_47|sid=1|g=1|s=0|t=1", - "simple_roles_47|sid=108|g=108|s=5|t=7", - "simple_roles_47|sid=61|g=61|s=3|t=0" - ], - "target_ids": [ - "108" - ], - "tid": 47, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the hometown of my father?", - "retrieved_ids": [ - "simple_roles_48|sid=101|g=101|s=5|t=0", - "simple_roles_48|sid=64|g=64|s=3|t=3", - "simple_roles_48|sid=109|g=109|s=5|t=8", - "simple_roles_48|sid=54|g=54|s=2|t=12", - "simple_roles_48|sid=27|g=27|s=1|t=6" - ], - "target_ids": [ - "109" - ], - "tid": 48, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the email address of my father?", - "retrieved_ids": [ - "simple_roles_49|sid=6|g=6|s=0|t=6", - "simple_roles_49|sid=117|g=117|s=5|t=17", - "simple_roles_49|sid=91|g=91|s=4|t=12", - "simple_roles_49|sid=52|g=52|s=2|t=12", - "simple_roles_49|sid=31|g=31|s=1|t=11" - ], - "target_ids": [ - "52" - ], - "tid": 49, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": false, - "question": "What is the hobby of my mother?", - "retrieved_ids": [ - "simple_roles_50|sid=0|g=0|s=0|t=0", - "simple_roles_50|sid=71|g=71|s=3|t=8", - "simple_roles_50|sid=112|g=112|s=5|t=8", - "simple_roles_50|sid=30|g=30|s=1|t=9", - "simple_roles_50|sid=153|g=153|s=7|t=8" - ], - "target_ids": [ - "18" - ], - "tid": 50, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "When is my boss's birthday?", - "retrieved_ids": [ - "simple_roles_51|sid=79|g=79|s=3|t=18", - "simple_roles_51|sid=22|g=22|s=1|t=1", - "simple_roles_51|sid=49|g=49|s=2|t=8", - "simple_roles_51|sid=61|g=61|s=3|t=0", - "simple_roles_51|sid=81|g=81|s=3|t=20" - ], - "target_ids": [ - "79" - ], - "tid": 51, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the occupation of that subordinate?", - "retrieved_ids": [ - "simple_roles_52|sid=79|g=79|s=4|t=0", - "simple_roles_52|sid=7|g=7|s=0|t=7", - "simple_roles_52|sid=80|g=80|s=4|t=1" - ], - "target_ids": [ - "80" - ], - "tid": 52, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the level of education that my mother has completed?", - "retrieved_ids": [ - "simple_roles_53|sid=0|g=0|s=0|t=0", - "simple_roles_53|sid=93|g=93|s=4|t=11", - "simple_roles_53|sid=12|g=12|s=0|t=12", - "simple_roles_53|sid=111|g=111|s=5|t=8", - "simple_roles_53|sid=74|g=74|s=3|t=13" - ], - "target_ids": [ - "12" - ], - "tid": 53, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the work location of the coworker?", - "retrieved_ids": [ - "simple_roles_54|sid=117|g=117|s=5|t=16", - "simple_roles_54|sid=112|g=112|s=5|t=11", - "simple_roles_54|sid=46|g=46|s=2|t=6", - "simple_roles_54|sid=113|g=113|s=5|t=12", - "simple_roles_54|sid=118|g=118|s=5|t=17" - ], - "target_ids": [ - "118" - ], - "tid": 54, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "How old is my brother?", - "retrieved_ids": [ - "simple_roles_55|sid=70|g=70|s=3|t=7", - "simple_roles_55|sid=42|g=42|s=2|t=0", - "simple_roles_55|sid=1|g=1|s=0|t=1", - "simple_roles_55|sid=123|g=123|s=6|t=0", - "simple_roles_55|sid=63|g=63|s=3|t=0" - ], - "target_ids": [ - "70" - ], - "tid": 55, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What does my boss do for a living?", - "retrieved_ids": [ - "simple_roles_56|sid=59|g=59|s=2|t=18", - "simple_roles_56|sid=41|g=41|s=2|t=0", - "simple_roles_56|sid=61|g=61|s=2|t=20", - "simple_roles_56|sid=60|g=60|s=2|t=19", - "simple_roles_56|sid=162|g=162|s=7|t=18" - ], - "target_ids": [ - "59" - ], - "tid": 56, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the birthday of my subordinate?", - "retrieved_ids": [ - "simple_roles_57|sid=93|g=93|s=4|t=11", - "simple_roles_57|sid=145|g=145|s=7|t=2", - "simple_roles_57|sid=125|g=125|s=6|t=2", - "simple_roles_57|sid=23|g=23|s=1|t=2", - "simple_roles_57|sid=82|g=82|s=4|t=0" - ], - "target_ids": [ - "93" - ], - "tid": 57, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the hometown of my female cousin?", - "retrieved_ids": [ - "simple_roles_58|sid=124|g=124|s=6|t=0", - "simple_roles_58|sid=68|g=68|s=3|t=5", - "simple_roles_58|sid=116|g=116|s=5|t=13", - "simple_roles_58|sid=142|g=142|s=6|t=18", - "simple_roles_58|sid=139|g=139|s=6|t=15" - ], - "target_ids": [ - "139" - ], - "tid": 58, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What position does my aunt hold?", - "retrieved_ids": [ - "simple_roles_59|sid=124|g=124|s=6|t=0", - "simple_roles_59|sid=131|g=131|s=6|t=7", - "simple_roles_59|sid=22|g=22|s=1|t=2", - "simple_roles_59|sid=143|g=143|s=6|t=19", - "simple_roles_59|sid=47|g=47|s=2|t=6" - ], - "target_ids": [ - "131" - ], - "tid": 59, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the date of my niece's birthday?", - "retrieved_ids": [ - "simple_roles_60|sid=59|g=59|s=2|t=18", - "simple_roles_60|sid=64|g=64|s=3|t=3", - "simple_roles_60|sid=151|g=151|s=7|t=8", - "simple_roles_60|sid=60|g=60|s=2|t=19", - "simple_roles_60|sid=41|g=41|s=2|t=0" - ], - "target_ids": [ - "59" - ], - "tid": 60, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": false, - "question": "What hobby does my father enjoy?", - "retrieved_ids": [ - "simple_roles_61|sid=54|g=54|s=2|t=12", - "simple_roles_61|sid=157|g=157|s=7|t=12", - "simple_roles_61|sid=111|g=111|s=5|t=7", - "simple_roles_61|sid=0|g=0|s=0|t=0", - "simple_roles_61|sid=28|g=28|s=1|t=7" - ], - "target_ids": [ - "7" - ], - "tid": 61, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the contact number for my brother?", - "retrieved_ids": [ - "simple_roles_62|sid=61|g=61|s=2|t=19", - "simple_roles_62|sid=139|g=139|s=6|t=15", - "simple_roles_62|sid=84|g=84|s=4|t=2", - "simple_roles_62|sid=118|g=118|s=5|t=15", - "simple_roles_62|sid=162|g=162|s=7|t=18" - ], - "target_ids": [ - "61" - ], - "tid": 62, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "How old is the coworker?", - "retrieved_ids": [ - "simple_roles_63|sid=64|g=64|s=3|t=3", - "simple_roles_63|sid=103|g=103|s=5|t=0", - "simple_roles_63|sid=145|g=145|s=7|t=1", - "simple_roles_63|sid=61|g=61|s=3|t=0", - "simple_roles_63|sid=41|g=41|s=2|t=0" - ], - "target_ids": [ - "64" - ], - "tid": 63, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the height of my niece?", - "retrieved_ids": [ - "simple_roles_64|sid=115|g=115|s=5|t=14", - "simple_roles_64|sid=123|g=123|s=6|t=1", - "simple_roles_64|sid=101|g=101|s=5|t=0", - "simple_roles_64|sid=1|g=1|s=0|t=1", - "simple_roles_64|sid=146|g=146|s=7|t=4" - ], - "target_ids": [ - "115" - ], - "tid": 64, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "When is my subordinate's birthday?", - "retrieved_ids": [ - "simple_roles_65|sid=13|g=13|s=0|t=13", - "simple_roles_65|sid=102|g=102|s=5|t=2", - "simple_roles_65|sid=23|g=23|s=1|t=3", - "simple_roles_65|sid=64|g=64|s=3|t=3", - "simple_roles_65|sid=133|g=133|s=6|t=12" - ], - "target_ids": [ - "13" - ], - "tid": 65, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the name of my female cousin?", - "retrieved_ids": [ - "simple_roles_66|sid=65|g=65|s=3|t=4", - "simple_roles_66|sid=144|g=144|s=7|t=0", - "simple_roles_66|sid=155|g=155|s=7|t=11", - "simple_roles_66|sid=163|g=163|s=7|t=19", - "simple_roles_66|sid=99|g=99|s=4|t=18" - ], - "target_ids": [ - "65" - ], - "tid": 66, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What does my brother do for a living?", - "retrieved_ids": [ - "simple_roles_67|sid=61|g=61|s=3|t=0", - "simple_roles_67|sid=67|g=67|s=3|t=6", - "simple_roles_67|sid=72|g=72|s=3|t=11", - "simple_roles_67|sid=47|g=47|s=2|t=6", - "simple_roles_67|sid=98|g=98|s=4|t=16" - ], - "target_ids": [ - "67" - ], - "tid": 67, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the location where my sister works?", - "retrieved_ids": [ - "simple_roles_68|sid=73|g=73|s=3|t=13", - "simple_roles_68|sid=69|g=69|s=3|t=9", - "simple_roles_68|sid=149|g=149|s=7|t=8", - "simple_roles_68|sid=60|g=60|s=3|t=0", - "simple_roles_68|sid=66|g=66|s=3|t=6" - ], - "target_ids": [ - "69" - ], - "tid": 68, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the occupation of that coworker?", - "retrieved_ids": [ - "simple_roles_69|sid=0|g=0|s=0|t=0", - "simple_roles_69|sid=8|g=8|s=0|t=8", - "simple_roles_69|sid=15|g=15|s=0|t=15", - "simple_roles_69|sid=58|g=58|s=2|t=18" - ], - "target_ids": [ - "8" - ], - "tid": 69, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the position of my niece?", - "retrieved_ids": [ - "simple_roles_70|sid=115|g=115|s=5|t=16", - "simple_roles_70|sid=99|g=99|s=5|t=0", - "simple_roles_70|sid=117|g=117|s=5|t=18", - "simple_roles_70|sid=138|g=138|s=6|t=19", - "simple_roles_70|sid=19|g=19|s=0|t=19" - ], - "target_ids": [ - "115" - ], - "tid": 70, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the date of my coworker's birthday?", - "retrieved_ids": [ - "simple_roles_71|sid=159|g=159|s=7|t=17", - "simple_roles_71|sid=142|g=142|s=7|t=0", - "simple_roles_71|sid=151|g=151|s=7|t=9", - "simple_roles_71|sid=155|g=155|s=7|t=13", - "simple_roles_71|sid=81|g=81|s=4|t=0" - ], - "target_ids": [ - "159" - ], - "tid": 71, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the date of my brother's birthday?", - "retrieved_ids": [ - "simple_roles_72|sid=28|g=28|s=1|t=7", - "simple_roles_72|sid=21|g=21|s=1|t=0", - "simple_roles_72|sid=64|g=64|s=3|t=3", - "simple_roles_72|sid=44|g=44|s=2|t=3", - "simple_roles_72|sid=40|g=40|s=1|t=19" - ], - "target_ids": [ - "28" - ], - "tid": 72, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the name of the company where the coworker works?", - "retrieved_ids": [ - "simple_roles_73|sid=67|g=67|s=3|t=7", - "simple_roles_73|sid=153|g=153|s=7|t=11", - "simple_roles_73|sid=126|g=126|s=6|t=4", - "simple_roles_73|sid=77|g=77|s=3|t=17", - "simple_roles_73|sid=9|g=9|s=0|t=9" - ], - "target_ids": [ - "67" - ], - "tid": 73, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is my male cousin's email address?", - "retrieved_ids": [ - "simple_roles_74|sid=12|g=12|s=0|t=12", - "simple_roles_74|sid=57|g=57|s=2|t=15", - "simple_roles_74|sid=77|g=77|s=3|t=14", - "simple_roles_74|sid=146|g=146|s=6|t=20", - "simple_roles_74|sid=0|g=0|s=0|t=0" - ], - "target_ids": [ - "12" - ], - "tid": 74, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What position does my uncle hold?", - "retrieved_ids": [ - "simple_roles_75|sid=83|g=83|s=4|t=0", - "simple_roles_75|sid=41|g=41|s=2|t=0", - "simple_roles_75|sid=61|g=61|s=2|t=20", - "simple_roles_75|sid=52|g=52|s=2|t=11", - "simple_roles_75|sid=57|g=57|s=2|t=16" - ], - "target_ids": [ - "52" - ], - "tid": 75, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What hobby does my niece enjoy?", - "retrieved_ids": [ - "simple_roles_76|sid=112|g=112|s=5|t=7", - "simple_roles_76|sid=134|g=134|s=6|t=9", - "simple_roles_76|sid=105|g=105|s=5|t=0", - "simple_roles_76|sid=50|g=50|s=2|t=8", - "simple_roles_76|sid=124|g=124|s=5|t=19" - ], - "target_ids": [ - "112" - ], - "tid": 76, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": false, - "question": "What is the occupation of my mother?", - "retrieved_ids": [ - "simple_roles_77|sid=21|g=21|s=1|t=0", - "simple_roles_77|sid=147|g=147|s=7|t=4", - "simple_roles_77|sid=133|g=133|s=6|t=10", - "simple_roles_77|sid=161|g=161|s=7|t=18", - "simple_roles_77|sid=89|g=89|s=4|t=7" - ], - "target_ids": [ - "40" - ], - "tid": 77, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "How old is my sister?", - "retrieved_ids": [ - "simple_roles_78|sid=47|g=47|s=2|t=7", - "simple_roles_78|sid=80|g=80|s=4|t=0", - "simple_roles_78|sid=101|g=101|s=5|t=0", - "simple_roles_78|sid=0|g=0|s=0|t=0", - "simple_roles_78|sid=40|g=40|s=2|t=0" - ], - "target_ids": [ - "47" - ], - "tid": 78, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the name of the boss?", - "retrieved_ids": [ - "simple_roles_79|sid=119|g=119|s=5|t=19", - "simple_roles_79|sid=111|g=111|s=5|t=11", - "simple_roles_79|sid=125|g=125|s=6|t=4", - "simple_roles_79|sid=100|g=100|s=5|t=0" - ], - "target_ids": [ - "119" - ], - "tid": 79, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the work location of my niece?", - "retrieved_ids": [ - "simple_roles_80|sid=49|g=49|s=2|t=8", - "simple_roles_80|sid=41|g=41|s=2|t=0", - "simple_roles_80|sid=110|g=110|s=5|t=8", - "simple_roles_80|sid=58|g=58|s=2|t=17", - "simple_roles_80|sid=122|g=122|s=5|t=20" - ], - "target_ids": [ - "49" - ], - "tid": 80, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the date of my niece's birthday?", - "retrieved_ids": [ - "simple_roles_81|sid=101|g=101|s=4|t=19", - "simple_roles_81|sid=144|g=144|s=7|t=1", - "simple_roles_81|sid=30|g=30|s=1|t=9", - "simple_roles_81|sid=134|g=134|s=6|t=12", - "simple_roles_81|sid=82|g=82|s=4|t=0" - ], - "target_ids": [ - "101" - ], - "tid": 81, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "Where does my aunt work?", - "retrieved_ids": [ - "simple_roles_82|sid=2|g=2|s=0|t=2", - "simple_roles_82|sid=138|g=138|s=6|t=12", - "simple_roles_82|sid=0|g=0|s=0|t=0", - "simple_roles_82|sid=70|g=70|s=3|t=6", - "simple_roles_82|sid=19|g=19|s=0|t=19" - ], - "target_ids": [ - "2" - ], - "tid": 82, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the hometown of my niece?", - "retrieved_ids": [ - "simple_roles_83|sid=44|g=44|s=2|t=2", - "simple_roles_83|sid=107|g=107|s=5|t=3", - "simple_roles_83|sid=42|g=42|s=2|t=0", - "simple_roles_83|sid=89|g=89|s=4|t=6", - "simple_roles_83|sid=121|g=121|s=5|t=17" - ], - "target_ids": [ - "44" - ], - "tid": 83, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the contact number for my subordinate?", - "retrieved_ids": [ - "simple_roles_84|sid=98|g=98|s=4|t=17", - "simple_roles_84|sid=37|g=37|s=1|t=16", - "simple_roles_84|sid=12|g=12|s=0|t=12", - "simple_roles_84|sid=139|g=139|s=6|t=17", - "simple_roles_84|sid=73|g=73|s=3|t=12" - ], - "target_ids": [ - "98" - ], - "tid": 84, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the date of my sister's birthday?", - "retrieved_ids": [ - "simple_roles_85|sid=59|g=59|s=2|t=18", - "simple_roles_85|sid=41|g=41|s=2|t=0", - "simple_roles_85|sid=139|g=139|s=6|t=16", - "simple_roles_85|sid=156|g=156|s=7|t=13", - "simple_roles_85|sid=52|g=52|s=2|t=11" - ], - "target_ids": [ - "59" - ], - "tid": 85, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the hometown of my aunt?", - "retrieved_ids": [ - "simple_roles_86|sid=60|g=60|s=3|t=0", - "simple_roles_86|sid=30|g=30|s=1|t=12", - "simple_roles_86|sid=75|g=75|s=3|t=15", - "simple_roles_86|sid=72|g=72|s=3|t=12", - "simple_roles_86|sid=64|g=64|s=3|t=4" - ], - "target_ids": [ - "75" - ], - "tid": 86, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": false, - "question": "What does my female cousin do for a living?", - "retrieved_ids": [], - "target_ids": [ - "58" - ], - "tid": 87, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the name of my mother's company?", - "retrieved_ids": [ - "simple_roles_88|sid=164|g=164|s=7|t=18", - "simple_roles_88|sid=23|g=23|s=1|t=3", - "simple_roles_88|sid=146|g=146|s=7|t=0", - "simple_roles_88|sid=108|g=108|s=5|t=5", - "simple_roles_88|sid=124|g=124|s=5|t=21" - ], - "target_ids": [ - "164" - ], - "tid": 88, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the name of my subordinate?", - "retrieved_ids": [ - "simple_roles_89|sid=143|g=143|s=7|t=0", - "simple_roles_89|sid=149|g=149|s=7|t=6", - "simple_roles_89|sid=15|g=15|s=0|t=15", - "simple_roles_89|sid=144|g=144|s=7|t=1", - "simple_roles_89|sid=67|g=67|s=3|t=6" - ], - "target_ids": [ - "149" - ], - "tid": 89, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "How old is my female cousin?", - "retrieved_ids": [ - "simple_roles_90|sid=16|g=16|s=0|t=16", - "simple_roles_90|sid=40|g=40|s=2|t=0", - "simple_roles_90|sid=20|g=20|s=1|t=0", - "simple_roles_90|sid=44|g=44|s=2|t=4", - "simple_roles_90|sid=81|g=81|s=4|t=0" - ], - "target_ids": [ - "16" - ], - "tid": 90, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the name of my father?", - "retrieved_ids": [ - "simple_roles_91|sid=26|g=26|s=1|t=5", - "simple_roles_91|sid=117|g=117|s=5|t=13", - "simple_roles_91|sid=15|g=15|s=0|t=15", - "simple_roles_91|sid=148|g=148|s=7|t=4", - "simple_roles_91|sid=66|g=66|s=3|t=3" - ], - "target_ids": [ - "26" - ], - "tid": 91, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "When is the boss's birthday?", - "retrieved_ids": [ - "simple_roles_92|sid=69|g=69|s=3|t=8", - "simple_roles_92|sid=3|g=3|s=0|t=3", - "simple_roles_92|sid=124|g=124|s=6|t=3", - "simple_roles_92|sid=80|g=80|s=3|t=19", - "simple_roles_92|sid=61|g=61|s=3|t=0" - ], - "target_ids": [ - "69" - ], - "tid": 92, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the occupation of that subordinate?", - "retrieved_ids": [ - "simple_roles_93|sid=81|g=81|s=4|t=0", - "simple_roles_93|sid=98|g=98|s=4|t=17" - ], - "target_ids": [ - "98" - ], - "tid": 93, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the email address of my niece?", - "retrieved_ids": [ - "simple_roles_94|sid=138|g=138|s=6|t=15", - "simple_roles_94|sid=82|g=82|s=3|t=20", - "simple_roles_94|sid=155|g=155|s=7|t=12", - "simple_roles_94|sid=123|g=123|s=6|t=0", - "simple_roles_94|sid=58|g=58|s=2|t=17" - ], - "target_ids": [ - "138" - ], - "tid": 94, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What does my sister do for a living?", - "retrieved_ids": [ - "simple_roles_95|sid=148|g=148|s=7|t=4", - "simple_roles_95|sid=45|g=45|s=2|t=4", - "simple_roles_95|sid=4|g=4|s=0|t=4", - "simple_roles_95|sid=144|g=144|s=7|t=0", - "simple_roles_95|sid=89|g=89|s=4|t=6" - ], - "target_ids": [ - "148" - ], - "tid": 95, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the name of that uncle?", - "retrieved_ids": [ - "simple_roles_96|sid=152|g=152|s=7|t=11", - "simple_roles_96|sid=141|g=141|s=7|t=0" - ], - "target_ids": [ - "152" - ], - "tid": 96, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": false, - "question": "What does my father do for a living?", - "retrieved_ids": [ - "simple_roles_97|sid=102|g=102|s=5|t=0", - "simple_roles_97|sid=87|g=87|s=4|t=5", - "simple_roles_97|sid=48|g=48|s=2|t=7", - "simple_roles_97|sid=66|g=66|s=3|t=4", - "simple_roles_97|sid=74|g=74|s=3|t=12" - ], - "target_ids": [ - "114" - ], - "tid": 97, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the position of the boss?", - "retrieved_ids": [ - "simple_roles_98|sid=70|g=70|s=3|t=8", - "simple_roles_98|sid=79|g=79|s=3|t=17", - "simple_roles_98|sid=62|g=62|s=3|t=0", - "simple_roles_98|sid=89|g=89|s=4|t=6", - "simple_roles_98|sid=81|g=81|s=3|t=19" - ], - "target_ids": [ - "70" - ], - "tid": 98, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": false, - "question": "What is the occupation of my mother?", - "retrieved_ids": [ - "simple_roles_99|sid=21|g=21|s=1|t=0", - "simple_roles_99|sid=7|g=7|s=0|t=7", - "simple_roles_99|sid=52|g=52|s=2|t=10", - "simple_roles_99|sid=84|g=84|s=4|t=1", - "simple_roles_99|sid=60|g=60|s=2|t=18" - ], - "target_ids": [ - "33" - ], - "tid": 99, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the name of the company where the boss works?", - "retrieved_ids": [ - "simple_roles_100|sid=57|g=57|s=2|t=18", - "simple_roles_100|sid=12|g=12|s=0|t=12", - "simple_roles_100|sid=39|g=39|s=2|t=0", - "simple_roles_100|sid=56|g=56|s=2|t=17", - "simple_roles_100|sid=54|g=54|s=2|t=15" - ], - "target_ids": [ - "57" - ], - "tid": 100, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the location where Father works?", - "retrieved_ids": [ - "simple_roles_101|sid=59|g=59|s=3|t=0", - "simple_roles_101|sid=13|g=13|s=0|t=13", - "simple_roles_101|sid=109|g=109|s=5|t=10", - "simple_roles_101|sid=69|g=69|s=3|t=10", - "simple_roles_101|sid=76|g=76|s=3|t=17" - ], - "target_ids": [ - "69" - ], - "tid": 101, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What hobby does my niece enjoy?", - "retrieved_ids": [ - "simple_roles_102|sid=104|g=104|s=5|t=1", - "simple_roles_102|sid=91|g=91|s=4|t=9", - "simple_roles_102|sid=131|g=131|s=6|t=7", - "simple_roles_102|sid=96|g=96|s=4|t=14", - "simple_roles_102|sid=103|g=103|s=5|t=0" - ], - "target_ids": [ - "104" - ], - "tid": 102, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the work location of my mother?", - "retrieved_ids": [ - "simple_roles_103|sid=44|g=44|s=2|t=1", - "simple_roles_103|sid=43|g=43|s=2|t=0", - "simple_roles_103|sid=104|g=104|s=4|t=20", - "simple_roles_103|sid=69|g=69|s=3|t=5", - "simple_roles_103|sid=116|g=116|s=5|t=11" - ], - "target_ids": [ - "43" - ], - "tid": 103, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What does my nephew do for a living?", - "retrieved_ids": [ - "simple_roles_104|sid=83|g=83|s=4|t=0", - "simple_roles_104|sid=90|g=90|s=4|t=7", - "simple_roles_104|sid=89|g=89|s=4|t=6", - "simple_roles_104|sid=129|g=129|s=6|t=5", - "simple_roles_104|sid=27|g=27|s=1|t=6" - ], - "target_ids": [ - "89" - ], - "tid": 104, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the position of my brother?", - "retrieved_ids": [ - "simple_roles_105|sid=0|g=0|s=0|t=0", - "simple_roles_105|sid=17|g=17|s=0|t=17", - "simple_roles_105|sid=137|g=137|s=6|t=17", - "simple_roles_105|sid=78|g=78|s=3|t=18", - "simple_roles_105|sid=118|g=118|s=5|t=18" - ], - "target_ids": [ - "17" - ], - "tid": 105, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the hometown of that coworker?", - "retrieved_ids": [ - "simple_roles_106|sid=21|g=21|s=1|t=0", - "simple_roles_106|sid=63|g=63|s=3|t=5", - "simple_roles_106|sid=37|g=37|s=1|t=16", - "simple_roles_106|sid=31|g=31|s=1|t=10", - "simple_roles_106|sid=30|g=30|s=1|t=9" - ], - "target_ids": [ - "30" - ], - "tid": 106, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What hobby does the boss have?", - "retrieved_ids": [ - "simple_roles_107|sid=13|g=13|s=0|t=13", - "simple_roles_107|sid=0|g=0|s=0|t=0", - "simple_roles_107|sid=31|g=31|s=1|t=11", - "simple_roles_107|sid=93|g=93|s=4|t=11", - "simple_roles_107|sid=18|g=18|s=0|t=18" - ], - "target_ids": [ - "13" - ], - "tid": 107, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is my male cousin's email address?", - "retrieved_ids": [ - "simple_roles_108|sid=122|g=122|s=6|t=0", - "simple_roles_108|sid=35|g=35|s=1|t=15", - "simple_roles_108|sid=112|g=112|s=5|t=11", - "simple_roles_108|sid=154|g=154|s=7|t=12", - "simple_roles_108|sid=123|g=123|s=6|t=1" - ], - "target_ids": [ - "122" - ], - "tid": 108, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the work location of my sister?", - "retrieved_ids": [ - "simple_roles_109|sid=76|g=76|s=3|t=14", - "simple_roles_109|sid=62|g=62|s=3|t=0", - "simple_roles_109|sid=67|g=67|s=3|t=5", - "simple_roles_109|sid=143|g=143|s=7|t=0", - "simple_roles_109|sid=15|g=15|s=0|t=15" - ], - "target_ids": [ - "76" - ], - "tid": 109, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the position of my nephew?", - "retrieved_ids": [ - "simple_roles_110|sid=82|g=82|s=4|t=0", - "simple_roles_110|sid=83|g=83|s=4|t=1", - "simple_roles_110|sid=46|g=46|s=2|t=5", - "simple_roles_110|sid=75|g=75|s=3|t=14", - "simple_roles_110|sid=119|g=119|s=5|t=17" - ], - "target_ids": [ - "83" - ], - "tid": 110, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": false, - "question": "What hobby does my father have?", - "retrieved_ids": [ - "simple_roles_111|sid=79|g=79|s=4|t=0", - "simple_roles_111|sid=111|g=111|s=5|t=11", - "simple_roles_111|sid=26|g=26|s=1|t=7", - "simple_roles_111|sid=68|g=68|s=3|t=9", - "simple_roles_111|sid=9|g=9|s=0|t=9" - ], - "target_ids": [ - "90" - ], - "tid": 111, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the contact number for my father?", - "retrieved_ids": [ - "simple_roles_112|sid=80|g=80|s=3|t=18", - "simple_roles_112|sid=40|g=40|s=1|t=19", - "simple_roles_112|sid=157|g=157|s=7|t=13", - "simple_roles_112|sid=121|g=121|s=5|t=18", - "simple_roles_112|sid=62|g=62|s=3|t=0" - ], - "target_ids": [ - "80" - ], - "tid": 112, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the name of the coworker?", - "retrieved_ids": [ - "simple_roles_113|sid=39|g=39|s=1|t=20", - "simple_roles_113|sid=131|g=131|s=6|t=8", - "simple_roles_113|sid=19|g=19|s=1|t=0", - "simple_roles_113|sid=28|g=28|s=1|t=9", - "simple_roles_113|sid=132|g=132|s=6|t=9" - ], - "target_ids": [ - "39" - ], - "tid": 113, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": false, - "question": "What is the name of Uncle's company?", - "retrieved_ids": [], - "target_ids": [ - "42" - ], - "tid": 114, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the height of the boss?", - "retrieved_ids": [ - "simple_roles_115|sid=36|g=36|s=1|t=15", - "simple_roles_115|sid=21|g=21|s=1|t=0", - "simple_roles_115|sid=31|g=31|s=1|t=10", - "simple_roles_115|sid=125|g=125|s=6|t=1", - "simple_roles_115|sid=84|g=84|s=4|t=1" - ], - "target_ids": [ - "36" - ], - "tid": 115, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the name of my uncle?", - "retrieved_ids": [ - "simple_roles_116|sid=30|g=30|s=1|t=10", - "simple_roles_116|sid=20|g=20|s=1|t=0", - "simple_roles_116|sid=54|g=54|s=2|t=12", - "simple_roles_116|sid=22|g=22|s=1|t=2", - "simple_roles_116|sid=9|g=9|s=0|t=9" - ], - "target_ids": [ - "30" - ], - "tid": 116, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": false, - "question": "What is the location of my father's workplace?", - "retrieved_ids": [ - "simple_roles_117|sid=144|g=144|s=7|t=0", - "simple_roles_117|sid=72|g=72|s=3|t=10", - "simple_roles_117|sid=163|g=163|s=7|t=19", - "simple_roles_117|sid=164|g=164|s=7|t=20", - "simple_roles_117|sid=47|g=47|s=2|t=6" - ], - "target_ids": [ - "162" - ], - "tid": 117, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "How old is my mother?", - "retrieved_ids": [ - "simple_roles_118|sid=20|g=20|s=1|t=0", - "simple_roles_118|sid=102|g=102|s=5|t=1", - "simple_roles_118|sid=0|g=0|s=0|t=0", - "simple_roles_118|sid=40|g=40|s=2|t=0", - "simple_roles_118|sid=122|g=122|s=6|t=0" - ], - "target_ids": [ - "102" - ], - "tid": 118, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": false, - "question": "What does my father do for a living?", - "retrieved_ids": [ - "simple_roles_119|sid=0|g=0|s=0|t=0", - "simple_roles_119|sid=25|g=25|s=1|t=5", - "simple_roles_119|sid=94|g=94|s=4|t=12", - "simple_roles_119|sid=87|g=87|s=4|t=5", - "simple_roles_119|sid=56|g=56|s=2|t=15" - ], - "target_ids": [ - "13" - ], - "tid": 119, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the name of that coworker?", - "retrieved_ids": [ - "simple_roles_120|sid=115|g=115|s=5|t=11", - "simple_roles_120|sid=104|g=104|s=5|t=0", - "simple_roles_120|sid=152|g=152|s=7|t=7", - "simple_roles_120|sid=116|g=116|s=5|t=12" - ], - "target_ids": [ - "115" - ], - "tid": 120, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the work location of my niece?", - "retrieved_ids": [ - "simple_roles_121|sid=5|g=5|s=0|t=5", - "simple_roles_121|sid=0|g=0|s=0|t=0", - "simple_roles_121|sid=140|g=140|s=6|t=15", - "simple_roles_121|sid=30|g=30|s=1|t=10", - "simple_roles_121|sid=125|g=125|s=6|t=0" - ], - "target_ids": [ - "5" - ], - "tid": 121, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "How old is my nephew?", - "retrieved_ids": [ - "simple_roles_122|sid=95|g=95|s=4|t=13", - "simple_roles_122|sid=82|g=82|s=4|t=0", - "simple_roles_122|sid=65|g=65|s=3|t=4", - "simple_roles_122|sid=101|g=101|s=4|t=19", - "simple_roles_122|sid=41|g=41|s=2|t=0" - ], - "target_ids": [ - "95" - ], - "tid": 122, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the work location of the boss?", - "retrieved_ids": [ - "simple_roles_123|sid=130|g=130|s=6|t=7", - "simple_roles_123|sid=63|g=63|s=3|t=1", - "simple_roles_123|sid=62|g=62|s=3|t=0", - "simple_roles_123|sid=142|g=142|s=6|t=19", - "simple_roles_123|sid=78|g=78|s=3|t=16" - ], - "target_ids": [ - "63" - ], - "tid": 123, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the name of the niece?", - "retrieved_ids": [ - "simple_roles_124|sid=123|g=123|s=6|t=0", - "simple_roles_124|sid=140|g=140|s=6|t=17", - "simple_roles_124|sid=84|g=84|s=4|t=2", - "simple_roles_124|sid=6|g=6|s=0|t=6" - ], - "target_ids": [ - "140" - ], - "tid": 124, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the height of my father?", - "retrieved_ids": [ - "simple_roles_125|sid=102|g=102|s=5|t=0", - "simple_roles_125|sid=63|g=63|s=3|t=2", - "simple_roles_125|sid=109|g=109|s=5|t=7", - "simple_roles_125|sid=22|g=22|s=1|t=1", - "simple_roles_125|sid=42|g=42|s=2|t=1" - ], - "target_ids": [ - "109" - ], - "tid": 125, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the name of the company where the boss works?", - "retrieved_ids": [ - "simple_roles_126|sid=1|g=1|s=0|t=1", - "simple_roles_126|sid=6|g=6|s=0|t=6", - "simple_roles_126|sid=148|g=148|s=7|t=7", - "simple_roles_126|sid=129|g=129|s=6|t=8", - "simple_roles_126|sid=18|g=18|s=0|t=18" - ], - "target_ids": [ - "1" - ], - "tid": 126, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the educational background of the boss?", - "retrieved_ids": [ - "simple_roles_127|sid=46|g=46|s=2|t=4", - "simple_roles_127|sid=56|g=56|s=2|t=14", - "simple_roles_127|sid=95|g=95|s=4|t=12", - "simple_roles_127|sid=92|g=92|s=4|t=9", - "simple_roles_127|sid=26|g=26|s=1|t=5" - ], - "target_ids": [ - "95" - ], - "tid": 127, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the contact number of my male cousin?", - "retrieved_ids": [ - "simple_roles_128|sid=120|g=120|s=5|t=16", - "simple_roles_128|sid=158|g=158|s=7|t=12", - "simple_roles_128|sid=13|g=13|s=0|t=13", - "simple_roles_128|sid=92|g=92|s=4|t=9", - "simple_roles_128|sid=90|g=90|s=4|t=7" - ], - "target_ids": [ - "120" - ], - "tid": 128, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the level of education that my sister has achieved?", - "retrieved_ids": [ - "simple_roles_129|sid=130|g=130|s=6|t=8", - "simple_roles_129|sid=132|g=132|s=6|t=10", - "simple_roles_129|sid=122|g=122|s=6|t=0", - "simple_roles_129|sid=114|g=114|s=5|t=12", - "simple_roles_129|sid=64|g=64|s=3|t=3" - ], - "target_ids": [ - "130" - ], - "tid": 129, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the position of my subordinate?", - "retrieved_ids": [ - "simple_roles_130|sid=0|g=0|s=0|t=0", - "simple_roles_130|sid=10|g=10|s=0|t=10", - "simple_roles_130|sid=57|g=57|s=2|t=18", - "simple_roles_130|sid=150|g=150|s=7|t=11", - "simple_roles_130|sid=42|g=42|s=2|t=3" - ], - "target_ids": [ - "10" - ], - "tid": 130, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the work location of the boss?", - "retrieved_ids": [ - "simple_roles_131|sid=76|g=76|s=3|t=16", - "simple_roles_131|sid=3|g=3|s=0|t=3", - "simple_roles_131|sid=104|g=104|s=5|t=4", - "simple_roles_131|sid=60|g=60|s=3|t=0", - "simple_roles_131|sid=69|g=69|s=3|t=9" - ], - "target_ids": [ - "76" - ], - "tid": 131, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the email address of the boss?", - "retrieved_ids": [ - "simple_roles_132|sid=10|g=10|s=0|t=10", - "simple_roles_132|sid=74|g=74|s=3|t=13", - "simple_roles_132|sid=92|g=92|s=4|t=11", - "simple_roles_132|sid=155|g=155|s=7|t=13", - "simple_roles_132|sid=117|g=117|s=5|t=15" - ], - "target_ids": [ - "10" - ], - "tid": 132, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "When is my coworker's birthday?", - "retrieved_ids": [ - "simple_roles_133|sid=160|g=160|s=7|t=16", - "simple_roles_133|sid=25|g=25|s=1|t=4", - "simple_roles_133|sid=144|g=144|s=7|t=0", - "simple_roles_133|sid=112|g=112|s=5|t=10", - "simple_roles_133|sid=152|g=152|s=7|t=8" - ], - "target_ids": [ - "160" - ], - "tid": 133, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What kind of education does my female cousin have?", - "retrieved_ids": [ - "simple_roles_134|sid=104|g=104|s=5|t=0", - "simple_roles_134|sid=160|g=160|s=7|t=16", - "simple_roles_134|sid=8|g=8|s=0|t=8", - "simple_roles_134|sid=112|g=112|s=5|t=8", - "simple_roles_134|sid=20|g=20|s=0|t=20" - ], - "target_ids": [ - "112" - ], - "tid": 134, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "When is my sister's birthday?", - "retrieved_ids": [ - "simple_roles_135|sid=42|g=42|s=2|t=2", - "simple_roles_135|sid=63|g=63|s=3|t=3", - "simple_roles_135|sid=87|g=87|s=4|t=7", - "simple_roles_135|sid=58|g=58|s=2|t=18", - "simple_roles_135|sid=40|g=40|s=2|t=0" - ], - "target_ids": [ - "42" - ], - "tid": 135, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What does my brother do for a living?", - "retrieved_ids": [ - "simple_roles_136|sid=93|g=93|s=4|t=11", - "simple_roles_136|sid=38|g=38|s=1|t=17", - "simple_roles_136|sid=102|g=102|s=5|t=0", - "simple_roles_136|sid=108|g=108|s=5|t=6", - "simple_roles_136|sid=83|g=83|s=4|t=1" - ], - "target_ids": [ - "108" - ], - "tid": 136, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the educational background of my sister?", - "retrieved_ids": [ - "simple_roles_137|sid=70|g=70|s=3|t=11", - "simple_roles_137|sid=20|g=20|s=1|t=0", - "simple_roles_137|sid=31|g=31|s=1|t=11", - "simple_roles_137|sid=112|g=112|s=5|t=11", - "simple_roles_137|sid=128|g=128|s=6|t=7" - ], - "target_ids": [ - "31" - ], - "tid": 137, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the email address of my aunt?", - "retrieved_ids": [ - "simple_roles_138|sid=104|g=104|s=5|t=2", - "simple_roles_138|sid=159|g=159|s=7|t=17", - "simple_roles_138|sid=39|g=39|s=1|t=19", - "simple_roles_138|sid=54|g=54|s=2|t=14", - "simple_roles_138|sid=77|g=77|s=3|t=16" - ], - "target_ids": [ - "77" - ], - "tid": 138, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "Could the contact number for the boss be provided?", - "retrieved_ids": [ - "simple_roles_139|sid=16|g=16|s=0|t=16", - "simple_roles_139|sid=100|g=100|s=4|t=17", - "simple_roles_139|sid=76|g=76|s=3|t=13", - "simple_roles_139|sid=139|g=139|s=6|t=15", - "simple_roles_139|sid=145|g=145|s=7|t=1" - ], - "target_ids": [ - "145" - ], - "tid": 139, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the age of the nephew?", - "retrieved_ids": [ - "simple_roles_140|sid=81|g=81|s=4|t=0", - "simple_roles_140|sid=91|g=91|s=4|t=10", - "simple_roles_140|sid=107|g=107|s=5|t=5", - "simple_roles_140|sid=0|g=0|s=0|t=0" - ], - "target_ids": [ - "91" - ], - "tid": 140, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the position of my aunt?", - "retrieved_ids": [ - "simple_roles_141|sid=7|g=7|s=0|t=7", - "simple_roles_141|sid=0|g=0|s=0|t=0", - "simple_roles_141|sid=150|g=150|s=7|t=3", - "simple_roles_141|sid=56|g=56|s=2|t=15", - "simple_roles_141|sid=23|g=23|s=1|t=3" - ], - "target_ids": [ - "7" - ], - "tid": 141, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the height of my uncle?", - "retrieved_ids": [ - "simple_roles_142|sid=128|g=128|s=6|t=5", - "simple_roles_142|sid=123|g=123|s=6|t=0", - "simple_roles_142|sid=76|g=76|s=3|t=14", - "simple_roles_142|sid=31|g=31|s=1|t=10", - "simple_roles_142|sid=75|g=75|s=3|t=13" - ], - "target_ids": [ - "128" - ], - "tid": 142, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the name of my nephew's company?", - "retrieved_ids": [ - "simple_roles_143|sid=111|g=111|s=5|t=9", - "simple_roles_143|sid=102|g=102|s=5|t=0", - "simple_roles_143|sid=8|g=8|s=0|t=8", - "simple_roles_143|sid=36|g=36|s=1|t=15", - "simple_roles_143|sid=0|g=0|s=0|t=0" - ], - "target_ids": [ - "111" - ], - "tid": 143, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the contact number for the boss?", - "retrieved_ids": [ - "simple_roles_144|sid=89|g=89|s=4|t=7", - "simple_roles_144|sid=30|g=30|s=1|t=10", - "simple_roles_144|sid=54|g=54|s=2|t=13", - "simple_roles_144|sid=113|g=113|s=5|t=10", - "simple_roles_144|sid=74|g=74|s=3|t=13" - ], - "target_ids": [ - "89" - ], - "tid": 144, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the email address of the subordinate?", - "retrieved_ids": [ - "simple_roles_145|sid=74|g=74|s=3|t=13", - "simple_roles_145|sid=118|g=118|s=5|t=17", - "simple_roles_145|sid=161|g=161|s=7|t=18", - "simple_roles_145|sid=31|g=31|s=1|t=11", - "simple_roles_145|sid=55|g=55|s=2|t=14" - ], - "target_ids": [ - "118" - ], - "tid": 145, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the educational background of my subordinate?", - "retrieved_ids": [ - "simple_roles_146|sid=61|g=61|s=3|t=0", - "simple_roles_146|sid=43|g=43|s=2|t=3", - "simple_roles_146|sid=50|g=50|s=2|t=10", - "simple_roles_146|sid=80|g=80|s=3|t=19", - "simple_roles_146|sid=4|g=4|s=0|t=4" - ], - "target_ids": [ - "80" - ], - "tid": 146, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is my mother's hobby?", - "retrieved_ids": [ - "simple_roles_147|sid=14|g=14|s=0|t=14", - "simple_roles_147|sid=130|g=130|s=6|t=10", - "simple_roles_147|sid=107|g=107|s=5|t=7", - "simple_roles_147|sid=50|g=50|s=2|t=10", - "simple_roles_147|sid=68|g=68|s=3|t=8" - ], - "target_ids": [ - "14" - ], - "tid": 147, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the work location of the subordinate?", - "retrieved_ids": [ - "simple_roles_148|sid=157|g=157|s=7|t=11", - "simple_roles_148|sid=121|g=121|s=5|t=17", - "simple_roles_148|sid=166|g=166|s=7|t=20", - "simple_roles_148|sid=45|g=45|s=2|t=4", - "simple_roles_148|sid=16|g=16|s=0|t=16" - ], - "target_ids": [ - "157" - ], - "tid": 148, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the name of the company where the boss works?", - "retrieved_ids": [ - "simple_roles_149|sid=131|g=131|s=6|t=8", - "simple_roles_149|sid=109|g=109|s=5|t=7", - "simple_roles_149|sid=152|g=152|s=7|t=9", - "simple_roles_149|sid=163|g=163|s=7|t=20", - "simple_roles_149|sid=68|g=68|s=3|t=6" - ], - "target_ids": [ - "152" - ], - "tid": 149, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the hometown of my uncle?", - "retrieved_ids": [ - "simple_roles_150|sid=38|g=38|s=2|t=0", - "simple_roles_150|sid=142|g=142|s=7|t=3", - "simple_roles_150|sid=102|g=102|s=5|t=4", - "simple_roles_150|sid=57|g=57|s=2|t=19", - "simple_roles_150|sid=50|g=50|s=2|t=12" - ], - "target_ids": [ - "50" - ], - "tid": 150, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What hobby does my father enjoy?", - "retrieved_ids": [ - "simple_roles_151|sid=23|g=23|s=1|t=3", - "simple_roles_151|sid=7|g=7|s=0|t=7", - "simple_roles_151|sid=61|g=61|s=3|t=0", - "simple_roles_151|sid=156|g=156|s=7|t=12", - "simple_roles_151|sid=66|g=66|s=3|t=5" - ], - "target_ids": [ - "66" - ], - "tid": 151, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What hobby does my uncle enjoy?", - "retrieved_ids": [ - "simple_roles_152|sid=139|g=139|s=6|t=18", - "simple_roles_152|sid=33|g=33|s=1|t=13", - "simple_roles_152|sid=9|g=9|s=0|t=9", - "simple_roles_152|sid=121|g=121|s=6|t=0", - "simple_roles_152|sid=141|g=141|s=6|t=20" - ], - "target_ids": [ - "139" - ], - "tid": 152, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": false, - "question": "What is the name of my cousin's company?", - "retrieved_ids": [], - "target_ids": [ - "112" - ], - "tid": 153, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the name of my uncle?", - "retrieved_ids": [ - "simple_roles_154|sid=115|g=115|s=5|t=12", - "simple_roles_154|sid=103|g=103|s=5|t=0", - "simple_roles_154|sid=120|g=120|s=5|t=17", - "simple_roles_154|sid=97|g=97|s=4|t=14", - "simple_roles_154|sid=68|g=68|s=3|t=5" - ], - "target_ids": [ - "115" - ], - "tid": 154, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": false, - "question": "What is the name of my mother's hometown?", - "retrieved_ids": [], - "target_ids": [ - "84" - ], - "tid": 155, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the name of my subordinate's company?", - "retrieved_ids": [ - "simple_roles_156|sid=10|g=10|s=0|t=10", - "simple_roles_156|sid=72|g=72|s=3|t=12", - "simple_roles_156|sid=0|g=0|s=0|t=0", - "simple_roles_156|sid=142|g=142|s=7|t=1", - "simple_roles_156|sid=37|g=37|s=1|t=17" - ], - "target_ids": [ - "10" - ], - "tid": 156, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the work location of the boss?", - "retrieved_ids": [ - "simple_roles_157|sid=156|g=156|s=7|t=13", - "simple_roles_157|sid=27|g=27|s=1|t=6", - "simple_roles_157|sid=89|g=89|s=4|t=7", - "simple_roles_157|sid=74|g=74|s=3|t=12", - "simple_roles_157|sid=143|g=143|s=7|t=0" - ], - "target_ids": [ - "156" - ], - "tid": 157, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": false, - "question": "What is the position of my mother?", - "retrieved_ids": [ - "simple_roles_158|sid=123|g=123|s=6|t=0", - "simple_roles_158|sid=93|g=93|s=4|t=12", - "simple_roles_158|sid=141|g=141|s=6|t=18", - "simple_roles_158|sid=160|g=160|s=7|t=16", - "simple_roles_158|sid=79|g=79|s=3|t=19" - ], - "target_ids": [ - "124" - ], - "tid": 158, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "When is my niece's birthday?", - "retrieved_ids": [ - "simple_roles_159|sid=73|g=73|s=3|t=11", - "simple_roles_159|sid=1|g=1|s=0|t=1", - "simple_roles_159|sid=100|g=100|s=4|t=18", - "simple_roles_159|sid=128|g=128|s=6|t=4", - "simple_roles_159|sid=62|g=62|s=3|t=0" - ], - "target_ids": [ - "73" - ], - "tid": 159, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What hobby does my father have?", - "retrieved_ids": [ - "simple_roles_160|sid=123|g=123|s=6|t=0", - "simple_roles_160|sid=45|g=45|s=2|t=3", - "simple_roles_160|sid=126|g=126|s=6|t=3", - "simple_roles_160|sid=153|g=153|s=7|t=8", - "simple_roles_160|sid=32|g=32|s=1|t=11" - ], - "target_ids": [ - "126" - ], - "tid": 160, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the email address of my mother?", - "retrieved_ids": [ - "simple_roles_161|sid=29|g=29|s=1|t=9", - "simple_roles_161|sid=54|g=54|s=2|t=13", - "simple_roles_161|sid=118|g=118|s=5|t=17", - "simple_roles_161|sid=98|g=98|s=4|t=16", - "simple_roles_161|sid=20|g=20|s=1|t=0" - ], - "target_ids": [ - "29" - ], - "tid": 161, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the age of my aunt?", - "retrieved_ids": [ - "simple_roles_162|sid=41|g=41|s=2|t=0", - "simple_roles_162|sid=42|g=42|s=2|t=1", - "simple_roles_162|sid=0|g=0|s=0|t=0", - "simple_roles_162|sid=121|g=121|s=5|t=18", - "simple_roles_162|sid=40|g=40|s=1|t=20" - ], - "target_ids": [ - "42" - ], - "tid": 162, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the work location of my aunt?", - "retrieved_ids": [ - "simple_roles_163|sid=3|g=3|s=0|t=3", - "simple_roles_163|sid=0|g=0|s=0|t=0", - "simple_roles_163|sid=123|g=123|s=6|t=2", - "simple_roles_163|sid=75|g=75|s=3|t=14", - "simple_roles_163|sid=82|g=82|s=4|t=1" - ], - "target_ids": [ - "3" - ], - "tid": 163, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the height of the boss?", - "retrieved_ids": [ - "simple_roles_164|sid=37|g=37|s=1|t=18", - "simple_roles_164|sid=19|g=19|s=1|t=0", - "simple_roles_164|sid=1|g=1|s=0|t=1", - "simple_roles_164|sid=51|g=51|s=2|t=13", - "simple_roles_164|sid=105|g=105|s=5|t=7" - ], - "target_ids": [ - "37" - ], - "tid": 164, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the educational background of the subordinate?", - "retrieved_ids": [ - "simple_roles_165|sid=107|g=107|s=5|t=3", - "simple_roles_165|sid=104|g=104|s=5|t=0", - "simple_roles_165|sid=114|g=114|s=5|t=10", - "simple_roles_165|sid=126|g=126|s=6|t=1", - "simple_roles_165|sid=15|g=15|s=0|t=15" - ], - "target_ids": [ - "114" - ], - "tid": 165, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the work location of the coworker?", - "retrieved_ids": [ - "simple_roles_166|sid=109|g=109|s=5|t=6", - "simple_roles_166|sid=122|g=122|s=5|t=19", - "simple_roles_166|sid=103|g=103|s=5|t=0", - "simple_roles_166|sid=87|g=87|s=4|t=5", - "simple_roles_166|sid=121|g=121|s=5|t=18" - ], - "target_ids": [ - "109" - ], - "tid": 166, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the occupation of my subordinate?", - "retrieved_ids": [ - "simple_roles_167|sid=49|g=49|s=2|t=7", - "simple_roles_167|sid=75|g=75|s=3|t=12", - "simple_roles_167|sid=136|g=136|s=6|t=12", - "simple_roles_167|sid=16|g=16|s=0|t=16", - "simple_roles_167|sid=32|g=32|s=1|t=11" - ], - "target_ids": [ - "49" - ], - "tid": 167, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the name of my brother's company?", - "retrieved_ids": [ - "simple_roles_168|sid=110|g=110|s=5|t=8", - "simple_roles_168|sid=86|g=86|s=4|t=5", - "simple_roles_168|sid=99|g=99|s=4|t=18", - "simple_roles_168|sid=22|g=22|s=1|t=2", - "simple_roles_168|sid=81|g=81|s=4|t=0" - ], - "target_ids": [ - "86" - ], - "tid": 168, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the date of my brother's birthday?", - "retrieved_ids": [ - "simple_roles_169|sid=61|g=61|s=3|t=1", - "simple_roles_169|sid=103|g=103|s=5|t=1", - "simple_roles_169|sid=81|g=81|s=4|t=0", - "simple_roles_169|sid=60|g=60|s=3|t=0", - "simple_roles_169|sid=20|g=20|s=1|t=2" - ], - "target_ids": [ - "61" - ], - "tid": 169, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the name of my brother?", - "retrieved_ids": [ - "simple_roles_170|sid=42|g=42|s=2|t=1", - "simple_roles_170|sid=41|g=41|s=2|t=0", - "simple_roles_170|sid=24|g=24|s=1|t=4", - "simple_roles_170|sid=62|g=62|s=3|t=0", - "simple_roles_170|sid=123|g=123|s=5|t=19" - ], - "target_ids": [ - "42" - ], - "tid": 170, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the hobby of that uncle?", - "retrieved_ids": [ - "simple_roles_171|sid=145|g=145|s=7|t=1", - "simple_roles_171|sid=111|g=111|s=5|t=9", - "simple_roles_171|sid=162|g=162|s=7|t=18", - "simple_roles_171|sid=159|g=159|s=7|t=15", - "simple_roles_171|sid=144|g=144|s=7|t=0" - ], - "target_ids": [ - "145" - ], - "tid": 171, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the email address of my father?", - "retrieved_ids": [ - "simple_roles_172|sid=128|g=128|s=6|t=6", - "simple_roles_172|sid=10|g=10|s=0|t=10", - "simple_roles_172|sid=75|g=75|s=3|t=14", - "simple_roles_172|sid=45|g=45|s=2|t=5", - "simple_roles_172|sid=122|g=122|s=6|t=0" - ], - "target_ids": [ - "128" - ], - "tid": 172, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the name of the hometown of that aunt?", - "retrieved_ids": [ - "simple_roles_173|sid=41|g=41|s=2|t=0", - "simple_roles_173|sid=88|g=88|s=4|t=6", - "simple_roles_173|sid=58|g=58|s=2|t=17", - "simple_roles_173|sid=45|g=45|s=2|t=4" - ], - "target_ids": [ - "58" - ], - "tid": 173, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the hometown of the boss?", - "retrieved_ids": [ - "simple_roles_174|sid=21|g=21|s=0|t=21", - "simple_roles_174|sid=0|g=0|s=0|t=0", - "simple_roles_174|sid=126|g=126|s=6|t=3" - ], - "target_ids": [ - "21" - ], - "tid": 174, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": false, - "question": "What is the hometown of my father?", - "retrieved_ids": [ - "simple_roles_175|sid=80|g=80|s=4|t=0", - "simple_roles_175|sid=99|g=99|s=4|t=19", - "simple_roles_175|sid=24|g=24|s=1|t=4", - "simple_roles_175|sid=19|g=19|s=0|t=19", - "simple_roles_175|sid=141|g=141|s=7|t=0" - ], - "target_ids": [ - "88" - ], - "tid": 175, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the name of the company where the subordinate works?", - "retrieved_ids": [ - "simple_roles_176|sid=88|g=88|s=4|t=7", - "simple_roles_176|sid=75|g=75|s=3|t=14", - "simple_roles_176|sid=151|g=151|s=7|t=9", - "simple_roles_176|sid=130|g=130|s=6|t=7", - "simple_roles_176|sid=157|g=157|s=7|t=15" - ], - "target_ids": [ - "75" - ], - "tid": 176, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What hobbies does my female cousin have?", - "retrieved_ids": [ - "simple_roles_177|sid=158|g=158|s=7|t=12", - "simple_roles_177|sid=73|g=73|s=3|t=10", - "simple_roles_177|sid=50|g=50|s=2|t=7", - "simple_roles_177|sid=146|g=146|s=7|t=0", - "simple_roles_177|sid=164|g=164|s=7|t=18" - ], - "target_ids": [ - "158" - ], - "tid": 177, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is my female cousin's email address?", - "retrieved_ids": [ - "simple_roles_178|sid=124|g=124|s=6|t=2", - "simple_roles_178|sid=58|g=58|s=2|t=16", - "simple_roles_178|sid=91|g=91|s=4|t=11", - "simple_roles_178|sid=122|g=122|s=6|t=0", - "simple_roles_178|sid=141|g=141|s=6|t=19" - ], - "target_ids": [ - "124" - ], - "tid": 178, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What hobby does my male cousin have?", - "retrieved_ids": [ - "simple_roles_179|sid=135|g=135|s=6|t=10", - "simple_roles_179|sid=63|g=63|s=3|t=0", - "simple_roles_179|sid=69|g=69|s=3|t=6", - "simple_roles_179|sid=113|g=113|s=5|t=8", - "simple_roles_179|sid=50|g=50|s=2|t=8" - ], - "target_ids": [ - "69" - ], - "tid": 179, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the height of my sister?", - "retrieved_ids": [ - "simple_roles_180|sid=79|g=79|s=3|t=18", - "simple_roles_180|sid=21|g=21|s=1|t=1", - "simple_roles_180|sid=61|g=61|s=3|t=0", - "simple_roles_180|sid=83|g=83|s=4|t=1", - "simple_roles_180|sid=106|g=106|s=5|t=3" - ], - "target_ids": [ - "79" - ], - "tid": 180, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the occupation of the boss?", - "retrieved_ids": [ - "simple_roles_181|sid=60|g=60|s=3|t=0", - "simple_roles_181|sid=73|g=73|s=3|t=13", - "simple_roles_181|sid=76|g=76|s=3|t=16" - ], - "target_ids": [ - "73" - ], - "tid": 181, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the name of my subordinate's company?", - "retrieved_ids": [ - "simple_roles_182|sid=79|g=79|s=3|t=18", - "simple_roles_182|sid=151|g=151|s=7|t=7", - "simple_roles_182|sid=57|g=57|s=2|t=17", - "simple_roles_182|sid=46|g=46|s=2|t=6", - "simple_roles_182|sid=61|g=61|s=3|t=0" - ], - "target_ids": [ - "79" - ], - "tid": 182, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "How old is my aunt?", - "retrieved_ids": [ - "simple_roles_183|sid=7|g=7|s=0|t=7", - "simple_roles_183|sid=17|g=17|s=0|t=17", - "simple_roles_183|sid=38|g=38|s=1|t=18", - "simple_roles_183|sid=0|g=0|s=0|t=0", - "simple_roles_183|sid=100|g=100|s=5|t=0" - ], - "target_ids": [ - "7" - ], - "tid": 183, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the work location of the uncle?", - "retrieved_ids": [ - "simple_roles_184|sid=106|g=106|s=5|t=3", - "simple_roles_184|sid=103|g=103|s=5|t=0", - "simple_roles_184|sid=88|g=88|s=4|t=6", - "simple_roles_184|sid=121|g=121|s=5|t=18", - "simple_roles_184|sid=157|g=157|s=7|t=14" - ], - "target_ids": [ - "106" - ], - "tid": 184, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What hobby does my brother enjoy?", - "retrieved_ids": [ - "simple_roles_185|sid=71|g=71|s=3|t=9", - "simple_roles_185|sid=20|g=20|s=1|t=0", - "simple_roles_185|sid=36|g=36|s=1|t=16", - "simple_roles_185|sid=159|g=159|s=7|t=15", - "simple_roles_185|sid=7|g=7|s=0|t=7" - ], - "target_ids": [ - "36" - ], - "tid": 185, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is my female cousin's email address?", - "retrieved_ids": [ - "simple_roles_186|sid=42|g=42|s=2|t=1", - "simple_roles_186|sid=0|g=0|s=0|t=0", - "simple_roles_186|sid=116|g=116|s=5|t=12", - "simple_roles_186|sid=139|g=139|s=6|t=16", - "simple_roles_186|sid=99|g=99|s=4|t=16" - ], - "target_ids": [ - "42" - ], - "tid": 186, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "How old is my female cousin?", - "retrieved_ids": [ - "simple_roles_187|sid=25|g=25|s=1|t=5", - "simple_roles_187|sid=0|g=0|s=0|t=0", - "simple_roles_187|sid=20|g=20|s=1|t=0", - "simple_roles_187|sid=104|g=104|s=5|t=0", - "simple_roles_187|sid=124|g=124|s=6|t=0" - ], - "target_ids": [ - "25" - ], - "tid": 187, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "When is my nephew's birthday?", - "retrieved_ids": [ - "simple_roles_188|sid=82|g=82|s=4|t=1", - "simple_roles_188|sid=41|g=41|s=1|t=20", - "simple_roles_188|sid=142|g=142|s=6|t=19", - "simple_roles_188|sid=81|g=81|s=4|t=0", - "simple_roles_188|sid=102|g=102|s=5|t=0" - ], - "target_ids": [ - "82" - ], - "tid": 188, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the contact number for my brother?", - "retrieved_ids": [ - "simple_roles_189|sid=44|g=44|s=2|t=4", - "simple_roles_189|sid=99|g=99|s=4|t=18", - "simple_roles_189|sid=10|g=10|s=0|t=10", - "simple_roles_189|sid=76|g=76|s=3|t=16", - "simple_roles_189|sid=114|g=114|s=5|t=12" - ], - "target_ids": [ - "44" - ], - "tid": 189, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the name of my uncle?", - "retrieved_ids": [ - "simple_roles_190|sid=0|g=0|s=0|t=0", - "simple_roles_190|sid=80|g=80|s=4|t=0", - "simple_roles_190|sid=88|g=88|s=4|t=8", - "simple_roles_190|sid=37|g=37|s=1|t=17", - "simple_roles_190|sid=6|g=6|s=0|t=6" - ], - "target_ids": [ - "88" - ], - "tid": 190, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the work location of my nephew?", - "retrieved_ids": [ - "simple_roles_191|sid=46|g=46|s=2|t=4", - "simple_roles_191|sid=89|g=89|s=4|t=6", - "simple_roles_191|sid=105|g=105|s=5|t=1", - "simple_roles_191|sid=42|g=42|s=2|t=0", - "simple_roles_191|sid=124|g=124|s=6|t=0" - ], - "target_ids": [ - "46" - ], - "tid": 191, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the hometown of my sister?", - "retrieved_ids": [ - "simple_roles_192|sid=42|g=42|s=2|t=0", - "simple_roles_192|sid=148|g=148|s=7|t=3", - "simple_roles_192|sid=60|g=60|s=2|t=18", - "simple_roles_192|sid=68|g=68|s=3|t=5", - "simple_roles_192|sid=51|g=51|s=2|t=9" - ], - "target_ids": [ - "51" - ], - "tid": 192, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What hobby does my niece have?", - "retrieved_ids": [ - "simple_roles_193|sid=21|g=21|s=1|t=0", - "simple_roles_193|sid=25|g=25|s=1|t=4", - "simple_roles_193|sid=135|g=135|s=6|t=12", - "simple_roles_193|sid=89|g=89|s=4|t=7", - "simple_roles_193|sid=133|g=133|s=6|t=10" - ], - "target_ids": [ - "25" - ], - "tid": 193, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the name of my nephew's company?", - "retrieved_ids": [ - "simple_roles_194|sid=121|g=121|s=6|t=0", - "simple_roles_194|sid=125|g=125|s=6|t=4", - "simple_roles_194|sid=7|g=7|s=0|t=7", - "simple_roles_194|sid=38|g=38|s=1|t=18", - "simple_roles_194|sid=108|g=108|s=5|t=7" - ], - "target_ids": [ - "125" - ], - "tid": 194, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "Where does my uncle work?", - "retrieved_ids": [ - "simple_roles_195|sid=93|g=93|s=4|t=9", - "simple_roles_195|sid=142|g=142|s=6|t=18", - "simple_roles_195|sid=161|g=161|s=7|t=17", - "simple_roles_195|sid=34|g=34|s=1|t=13", - "simple_roles_195|sid=84|g=84|s=4|t=0" - ], - "target_ids": [ - "93" - ], - "tid": 195, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the email address of the subordinate?", - "retrieved_ids": [ - "simple_roles_196|sid=31|g=31|s=1|t=10", - "simple_roles_196|sid=21|g=21|s=1|t=0", - "simple_roles_196|sid=53|g=53|s=2|t=11", - "simple_roles_196|sid=81|g=81|s=3|t=18", - "simple_roles_196|sid=18|g=18|s=0|t=18" - ], - "target_ids": [ - "31" - ], - "tid": 196, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the age of that coworker?", - "retrieved_ids": [ - "simple_roles_197|sid=49|g=49|s=2|t=9", - "simple_roles_197|sid=59|g=59|s=2|t=19", - "simple_roles_197|sid=17|g=17|s=0|t=17", - "simple_roles_197|sid=20|g=20|s=1|t=1", - "simple_roles_197|sid=101|g=101|s=5|t=0" - ], - "target_ids": [ - "49" - ], - "tid": 197, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What hobby does my brother have?", - "retrieved_ids": [ - "simple_roles_198|sid=122|g=122|s=6|t=0", - "simple_roles_198|sid=12|g=12|s=0|t=12", - "simple_roles_198|sid=49|g=49|s=2|t=9", - "simple_roles_198|sid=85|g=85|s=4|t=5", - "simple_roles_198|sid=127|g=127|s=6|t=5" - ], - "target_ids": [ - "127" - ], - "tid": 198, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What hobby does my niece have?", - "retrieved_ids": [ - "simple_roles_199|sid=123|g=123|s=6|t=0", - "simple_roles_199|sid=135|g=135|s=6|t=12", - "simple_roles_199|sid=8|g=8|s=0|t=8", - "simple_roles_199|sid=150|g=150|s=7|t=8", - "simple_roles_199|sid=131|g=131|s=6|t=8" - ], - "target_ids": [ - "131" - ], - "tid": 199, - "topic": "roles" - } - ], - "runtime_seconds": 140.08, - "series_name": "new_brainctl", - "source_path": null, - "status": "partial", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/membench_old_brainctl_cmd_turn.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/membench_old_brainctl_cmd_turn.json deleted file mode 100644 index bc94de0..0000000 --- a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/runs/membench_old_brainctl_cmd_turn.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\membench_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\membench_old_brainctl_cmd_turn.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" - }, - "benchmark": "membench", - "caveats": [ - "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 200, - "measured": false, - "metadata": {}, - "metrics": { - "hit_at_5": 0.93, - "simple_hit_at_5": 0.93, - "top_k": 5 - }, - "mode": "cmd_turn", - "notes": [ - "FirstAgent slice only", - "turn-level retrieval", - "topic=all" - ], - "primary_metric": "hit_at_5", - "primary_metric_value": 0.93, - "reference_kind": "historical", - "runtime_seconds": 140.592, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "partial", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/summary.csv b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/summary.csv deleted file mode 100644 index 2b6fc1e..0000000 --- a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/summary.csv +++ /dev/null @@ -1,15 +0,0 @@ -benchmark,series_name,system_name,mode,reference_kind,status,example_count,primary_metric,primary_metric_value,runtime_seconds,dataset_path,source_path,notes,caveats,abstention_evidence_recall,assistant_facts_evidence_recall,avg_recall,cat_1_recall,cat_2_recall,cat_3_recall,cat_4_recall,cat_5_recall,hit_at_5,implicit_connection_evidence_recall,ndcg_at_10,ndcg_at_5,perfect_rate,preference_evidence_recall,r_at_10,r_at_5,simple_hit_at_5,top_k,user_evidence_recall,zero_rate -longmemeval,old_brainctl,brainctl,brain,historical,full_same_machine,470,r_at_5,0.9681,85.439,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,top_k=10,,,,,,,,,,,,0.9253,0.9204,,,0.9894,0.9681,,,, -longmemeval,old_brainctl,brainctl,cmd,historical,full_same_machine,470,r_at_5,0.9702,130.863,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,top_k=10,,,,,,,,,,,,0.9247,0.9206,,,0.9894,0.9702,,,, -longmemeval,mempalace,mempalace,raw_session,historical,full_same_machine,470,r_at_5,0.966,695.36,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,top_k=10 | Runs MemPalace benchmark module raw session retrieval logic directly.,,,,,,,,,,,,0.8948,0.893,,,0.983,0.966,,,, -locomo,old_brainctl,brainctl,cmd_session,historical,full_same_machine,1986,avg_recall,0.9217,445.74,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,granularity=session | top_k=10,,,,0.9217,0.7614,0.92,0.6616,0.9738,0.9821,,,,,0.8817,,,,,10,,0.0438 -locomo,mempalace,mempalace,raw_session,historical,full_same_machine,1986,avg_recall,0.6028,2106.411,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,granularity=session | top_k=10,,,,0.6028,0.5899,0.6921,0.4604,0.5809,0.6188,,,,,0.5534,,,,,10,,0.3499 -membench,old_brainctl,brainctl,cmd_turn,historical,partial,200,hit_at_5,0.93,140.592,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,FirstAgent slice only | turn-level retrieval | topic=all,MemBench comparison is partial because ThirdAgent and noise-extended slices are not included.,,,,,,,,,0.93,,,,,,,,0.93,5,, -membench,mempalace,mempalace,raw_turn,historical,partial,200,hit_at_5,0.885,804.35,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,FirstAgent slice only | turn-level retrieval | topic=all,MemBench comparison is partial because ThirdAgent and noise-extended slices are not included.,,,,,,,,,0.885,,,,,,,,0.885,5,, -convomem,old_brainctl,brainctl,cmd,historical,blocked,0,avg_recall,,,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,limit_per_category=1 | top_k=10,Blocked while loading ConvoMem evidence data: ,,,,,,,,,,,,,,,,,,,, -convomem,mempalace,mempalace,raw,historical,blocked,0,avg_recall,,,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,limit_per_category=1 | top_k=10,Blocked while loading ConvoMem evidence data: ,,,,,,,,,,,,,,,,,,,, -longmemeval,new_brainctl,brainctl,brain,measured,full_same_machine,470,r_at_5,0.9574,403.852,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,,top_k=10 | Legacy 470-question session-level slice.,,,,,,,,,,,,0.872,0.8678,,,0.9894,0.9574,,,, -longmemeval,new_brainctl,brainctl,cmd,measured,full_same_machine,470,r_at_5,0.9596,251.044,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,,top_k=10 | Legacy 470-question session-level slice.,,,,,,,,,,,,0.8753,0.8715,,,0.9894,0.9596,,,, -locomo,new_brainctl,brainctl,cmd_session,measured,full_same_machine,1986,avg_recall,0.9145,157.701,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,,granularity=session | top_k=10,,,,0.9145,0.7533,0.9107,0.6607,0.9643,0.9798,,,,,0.8751,,,,,10,,0.0519 -membench,new_brainctl,brainctl,cmd_turn,measured,partial,200,hit_at_5,0.895,140.08,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,,FirstAgent slice only | turn-level retrieval | topic=all,MemBench comparison is partial because ThirdAgent and noise-extended slices are not included.,,,,,,,,,0.895,,,,,,,,0.895,5,, -convomem,new_brainctl,brainctl,cmd,measured,partial,5,avg_recall,0.6,1.514,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache,,categories=6 | limit_per_category=1 | top_k=10,"changing_evidence: discover failed: HTTP Error 404: Not Found | ConvoMem comparison is partial because it uses a bounded same-machine sample, not the full benchmark.",0.0,1.0,0.6,,,,,,,0.0,,,0.6,1.0,,,,10,1.0, diff --git a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/summary.json b/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/summary.json deleted file mode 100644 index 6fda86b..0000000 --- a/benchmarks/results/rlm_srlm_probe_refresh_20260423_174924/summary.json +++ /dev/null @@ -1,514 +0,0 @@ -{ - "generated_at_utc": "2026-04-23T18:05:25.239155+00:00", - "metadata": { - "argv": [ - "benchmarks\\compare_memory_engines.py", - "--label", - "rlm_srlm_probe_refresh", - "--membench-limit", - "200" - ], - "cwd": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl", - "datasets": { - "convomem_cache": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "locomo_data": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "longmemeval_data": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "membench_data": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent" - }, - "generated_at_utc": "2026-04-23T18:05:25.112789+00:00", - "git_commit": "9368f288279ae1d3395e37e89972038b74d66e3a", - "historical_summary_mode": "recovered", - "historical_summary_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "machine": "AMD64", - "platform": "Windows-11-10.0.26200-SP0", - "processor": "Intel64 Family 6 Model 189 Stepping 1, GenuineIntel", - "python_version": "3.14.4 (tags/v3.14.4:23116f9, Apr 7 2026, 14:10:54) [MSC v.1944 64 bit (AMD64)]" - }, - "notes": [ - "Historical old-BrainCTL and MemPalace series come from the recovered 2026-04-18 comparison bundle.", - "New BrainCTL series are rerun in the current checked-out repo using the legacy benchmark definitions.", - "MemBench remains intentionally partial because the legacy comparison only covered the FirstAgent slice.", - "ConvoMem remains a coverage/status benchmark here; it has no dedicated comparison chart in the legacy chart pack." - ], - "runs": [ - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\longmemeval_old_brainctl_brain.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "measured": false, - "metadata": {}, - "metrics": { - "ndcg_at_10": 0.9253, - "ndcg_at_5": 0.9204, - "r_at_10": 0.9894, - "r_at_5": 0.9681 - }, - "mode": "brain", - "notes": [ - "top_k=10" - ], - "primary_metric": "r_at_5", - "primary_metric_value": 0.9681, - "reference_kind": "historical", - "runtime_seconds": 85.439, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\longmemeval_old_brainctl_cmd.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "measured": false, - "metadata": {}, - "metrics": { - "ndcg_at_10": 0.9247, - "ndcg_at_5": 0.9206, - "r_at_10": 0.9894, - "r_at_5": 0.9702 - }, - "mode": "cmd", - "notes": [ - "top_k=10" - ], - "primary_metric": "r_at_5", - "primary_metric_value": 0.9702, - "reference_kind": "historical", - "runtime_seconds": 130.863, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\longmemeval_mempalace_raw_session.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "measured": false, - "metadata": {}, - "metrics": { - "ndcg_at_10": 0.8948, - "ndcg_at_5": 0.893, - "r_at_10": 0.983, - "r_at_5": 0.966 - }, - "mode": "raw_session", - "notes": [ - "top_k=10", - "Runs MemPalace benchmark module raw session retrieval logic directly." - ], - "primary_metric": "r_at_5", - "primary_metric_value": 0.966, - "reference_kind": "historical", - "runtime_seconds": 695.36, - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "mempalace" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\locomo_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\locomo_old_brainctl_cmd_session.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" - }, - "benchmark": "locomo", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "measured": false, - "metadata": {}, - "metrics": { - "avg_recall": 0.9217, - "cat_1_recall": 0.7614, - "cat_2_recall": 0.92, - "cat_3_recall": 0.6616, - "cat_4_recall": 0.9738, - "cat_5_recall": 0.9821, - "perfect_rate": 0.8817, - "top_k": 10, - "zero_rate": 0.0438 - }, - "mode": "cmd_session", - "notes": [ - "granularity=session", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": 0.9217, - "reference_kind": "historical", - "runtime_seconds": 445.74, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\locomo_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\locomo_mempalace_raw_session.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" - }, - "benchmark": "locomo", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "measured": false, - "metadata": {}, - "metrics": { - "avg_recall": 0.6028, - "cat_1_recall": 0.5899, - "cat_2_recall": 0.6921, - "cat_3_recall": 0.4604, - "cat_4_recall": 0.5809, - "cat_5_recall": 0.6188, - "perfect_rate": 0.5534, - "top_k": 10, - "zero_rate": 0.3499 - }, - "mode": "raw_session", - "notes": [ - "granularity=session", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": 0.6028, - "reference_kind": "historical", - "runtime_seconds": 2106.411, - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "mempalace" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\membench_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\membench_old_brainctl_cmd_turn.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" - }, - "benchmark": "membench", - "caveats": [ - "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 200, - "measured": false, - "metadata": {}, - "metrics": { - "hit_at_5": 0.93, - "simple_hit_at_5": 0.93, - "top_k": 5 - }, - "mode": "cmd_turn", - "notes": [ - "FirstAgent slice only", - "turn-level retrieval", - "topic=all" - ], - "primary_metric": "hit_at_5", - "primary_metric_value": 0.93, - "reference_kind": "historical", - "runtime_seconds": 140.592, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "partial", - "system_name": "brainctl" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\membench_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\membench_mempalace_raw_turn.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" - }, - "benchmark": "membench", - "caveats": [ - "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 200, - "measured": false, - "metadata": {}, - "metrics": { - "hit_at_5": 0.885, - "simple_hit_at_5": 0.885, - "top_k": 5 - }, - "mode": "raw_turn", - "notes": [ - "FirstAgent slice only", - "turn-level retrieval", - "topic=all" - ], - "primary_metric": "hit_at_5", - "primary_metric_value": 0.885, - "reference_kind": "historical", - "runtime_seconds": 804.35, - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "partial", - "system_name": "mempalace" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\convomem_old_brainctl_cmd.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" - }, - "benchmark": "convomem", - "caveats": [ - "Blocked while loading ConvoMem evidence data: " - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "example_count": 0, - "measured": false, - "metadata": {}, - "metrics": {}, - "mode": "cmd", - "notes": [ - "limit_per_category=1", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": null, - "reference_kind": "historical", - "runtime_seconds": null, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "blocked", - "system_name": "brainctl" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\convomem_mempalace_raw.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" - }, - "benchmark": "convomem", - "caveats": [ - "Blocked while loading ConvoMem evidence data: " - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "example_count": 0, - "measured": false, - "metadata": {}, - "metrics": {}, - "mode": "raw", - "notes": [ - "limit_per_category=1", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": null, - "reference_kind": "historical", - "runtime_seconds": null, - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "blocked", - "system_name": "mempalace" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\longmemeval_new_brainctl_brain.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "measured": true, - "metadata": {}, - "metrics": { - "ndcg_at_10": 0.872, - "ndcg_at_5": 0.8678, - "r_at_10": 0.9894, - "r_at_5": 0.9574 - }, - "mode": "brain", - "notes": [ - "top_k=10", - "Legacy 470-question session-level slice." - ], - "primary_metric": "r_at_5", - "primary_metric_value": 0.9574, - "reference_kind": "measured", - "runtime_seconds": 403.852, - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\longmemeval_new_brainctl_cmd.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "measured": true, - "metadata": {}, - "metrics": { - "ndcg_at_10": 0.8753, - "ndcg_at_5": 0.8715, - "r_at_10": 0.9894, - "r_at_5": 0.9596 - }, - "mode": "cmd", - "notes": [ - "top_k=10", - "Legacy 470-question session-level slice." - ], - "primary_metric": "r_at_5", - "primary_metric_value": 0.9596, - "reference_kind": "measured", - "runtime_seconds": 251.044, - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\locomo_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\locomo_new_brainctl_cmd_session.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" - }, - "benchmark": "locomo", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "measured": true, - "metadata": {}, - "metrics": { - "avg_recall": 0.9145, - "cat_1_recall": 0.7533, - "cat_2_recall": 0.9107, - "cat_3_recall": 0.6607, - "cat_4_recall": 0.9643, - "cat_5_recall": 0.9798, - "perfect_rate": 0.8751, - "top_k": 10, - "zero_rate": 0.0519 - }, - "mode": "cmd_session", - "notes": [ - "granularity=session", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": 0.9145, - "reference_kind": "measured", - "runtime_seconds": 157.701, - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\membench_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\membench_new_brainctl_cmd_turn.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" - }, - "benchmark": "membench", - "caveats": [ - "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 200, - "measured": true, - "metadata": {}, - "metrics": { - "hit_at_5": 0.895, - "simple_hit_at_5": 0.895, - "top_k": 5 - }, - "mode": "cmd_turn", - "notes": [ - "FirstAgent slice only", - "turn-level retrieval", - "topic=all" - ], - "primary_metric": "hit_at_5", - "primary_metric_value": 0.895, - "reference_kind": "measured", - "runtime_seconds": 140.08, - "series_name": "new_brainctl", - "source_path": null, - "status": "partial", - "system_name": "brainctl" - }, - { - "artifacts": { - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\runs\\convomem_new_brainctl_cmd.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\rlm_srlm_probe_refresh_20260423_174924\\charts\\coverage_status.png" - }, - "benchmark": "convomem", - "caveats": [ - "changing_evidence: discover failed: HTTP Error 404: Not Found", - "ConvoMem comparison is partial because it uses a bounded same-machine sample, not the full benchmark." - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "example_count": 5, - "measured": true, - "metadata": {}, - "metrics": { - "abstention_evidence_recall": 0.0, - "assistant_facts_evidence_recall": 1.0, - "avg_recall": 0.6, - "implicit_connection_evidence_recall": 0.0, - "perfect_rate": 0.6, - "preference_evidence_recall": 1.0, - "top_k": 10, - "user_evidence_recall": 1.0 - }, - "mode": "cmd", - "notes": [ - "categories=6", - "limit_per_category=1", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": 0.6, - "reference_kind": "measured", - "runtime_seconds": 1.514, - "series_name": "new_brainctl", - "source_path": null, - "status": "partial", - "system_name": "brainctl" - } - ] -} \ No newline at end of file diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/README.md b/benchmarks/results/smoke_legacy_compare_20260423_105447/README.md deleted file mode 100644 index 1a413ec..0000000 --- a/benchmarks/results/smoke_legacy_compare_20260423_105447/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# Legacy BrainCTL vs MemPalace comparison bundle - -- Current repo commit: `c130fbd47174a0932822db2ef7fe64e18a74d30d` -- Historical reference source: `C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json` -- Historical source mode: `recovered summary bundle` -- Command: `benchmarks\compare_memory_engines.py --label smoke_legacy_compare --longmemeval-limit 3 --locomo-limit 1 --membench-limit 3 --convomem-limit-per-category 1` - -## Datasets - -- LongMemEval: `C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json` -- LoCoMo: `C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json` -- MemBench FirstAgent: `C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent` -- ConvoMem cache: `C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache` - -## What is measured now - -- New BrainCTL reruns: LongMemEval `brain` and `cmd`, LoCoMo `cmd_session`, MemBench FirstAgent `cmd_turn`, and ConvoMem `cmd` coverage/status. -- Old BrainCTL and MemPalace are frozen historical reference series loaded from the recovered 2026-04-18 bundle. - -## Blocked or partial runs - -- convomem old_brainctl cmd: Blocked while loading ConvoMem evidence data: -- convomem mempalace raw: Blocked while loading ConvoMem evidence data: -- convomem new_brainctl cmd: Blocked while loading ConvoMem evidence data: HTTP Error 404: Not Found - -## Output files - -- `summary.json` and `summary.csv`: all series in one table. -- `comparison_table.json` and `comparison_table.csv`: long-form metric rows. -- `runs/*.json`: per-run payloads. -- `charts/*.png`: regenerated charts with old BrainCTL, new BrainCTL, and MemPalace together. diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/charts/aggregate_primary_metrics.png b/benchmarks/results/smoke_legacy_compare_20260423_105447/charts/aggregate_primary_metrics.png deleted file mode 100644 index 91ee6a4b5b60cfd85a4fc065daf6c30f817c8c73..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 72024 zcmdqJby!txyDy4~poofzq?D3McdImtbc!^S2`Jqt3ML5BNQWRynn{C*N(+K8r9rxT z5|cRh5Wn?(Yp=D=S$m)T$3ACV*ZYb%V~l4!&wc;u887ar+$JNXBqbsuB9oVsRU;xg zl0Zasi0;@?_)X`hudMKgu;Wc_$9s0>jxNTJ%!rhY9qn!G9Br%~&^enua9<> zol}sV&eGA*-a(j)3;l1e;Iw;W!BwN@^c_Bh#9mIvfr#j04D#Orc_+V6q60)k^0HFu zuJ332kA#Mf5%!i|&1a{UUp^cWa)yLNs_gW&BP2)8-BCX*`&LcCT`F{bUjNjo(DNyZ zZ|D2=O6M2$q6TN@oAK8Rnh8$d-|NIxOhiQ)>@@S?s2S+!=p^Aw{~XUB+(_Q?{reZ9 zQ<8`N{;2OyYWep^U&Vj<^%Ek>zrP+jbGYpv?~9=GMaBd)yI)v(Q`J3ng0d#zLJcLuA*N7HY=uxP!dTU*h&hO}F{2XIG-d?)>p zp{!^>c<|sTey%s)XV3eR(4&;Itm<-U$}!{f#`Mw4`0Hn3(2j8-1Gd#OViP6pXShxKU zG}c{UVV*_nvz=Fg??IWz3qJfU_fb6~#dC9cxt3g_euuCgJD>CJ`;?&58DXc@$-7g( z@{&jEgJpi#1=)33Y1n4#mo~R5$MBP_ZDH*O2W(>KW&G`*5u{j#JG!nNI(p(dj`ePm z?08S*%6E}`yTK|+0=hf@D?6^ync5*v)NOPor<%Xj$nqqW;C-5$`DC1aYjO@*Z2B^c zvPw=w+`X2_8w|gmM&P{?=UBd~U*Ob#8z<(WFi_#vQaZ~*bz+9`DK6W9XJm10E$nB; zy_j%roqW?9IipgGjWl~_8 zVNvNhxT9p0^*u$X+HLP26B&vToa2M;C#V%`{mGPbHrpmWd|>p)Z=|Tk4`8dSg4P~x zFV)e~&*v=cz>QOE2w`H94`Cd~Z_VU1wMy*G7KVsV&MXc3?D6HS^_4orpdzo`W47x2 z81`Q9;gvRT#c=lE;sN(2XUy8#b@S%PQ-gzh+bcn@DftvWYUS7t#5^4N#35T<=1qK>Vgz!8su+-&&}-Tf&37Nh z^#I8k#$(uON?yyin?i;at_K8c`|RBf7W&ICycK--pkVGpyf@`3nk`SAiM}c?4@sC~ zRN`?POC!WKzjAA4c}(k1eZXOLi>wd8nsRZ?zQAR8cy+3yZmPL zBG53X+-2dUTokui?%bNwh1Dg!rYF%n7J+kH{$tz5sjb!XnL(=SF3z)0jvVh9;0h-u zr#Z~17~Y%(r{eN_ViWETx^=iEMxd7O7GeEogpv1@<7-&WCj_Aubd11EV4{P;qc^M# zq!u|U>Vaqfvfeb_5`zvBnN%KRnd!nf2>Vy>Zn4qb4y2+Flxj6@-Cgg10&u7?f@{HX z5NeQ!XwJ^e5=?;;auJ-dwL5E0?u$cGE8?3&e&p}tCMwql99-k{ifx~5iNBy=QSpU| zTf0-cQvSX1tO1*jCdNQ3y4d2sT*kVDUt>ui?!@6I)@CGzpv^PR!v#Ee+ko6Jc- z5%X0`RfxJS=OgZcxA|0r!YM8O9vLw?p3IwmQlZ$cD@*4SjK@LexgN78=KWf=sj%*1 zuB)uCudO+`QfEnWShc4dfQe30A1~qaslsjLAg4i@sqM-{t3avh%{?Ud9B(7D{0CM>oO!@W(c+wp>f%&>;z=J&-l$wo_5-USOo}#--L_Vz4t;xliNb#N*Nv6OzP8o&!=HVjOdfHZ{&|fq z>3}1Cwi}I8H(J;#!{O8-I1Ow}GVdflAz_}=nOWO>xGJns7UZ-I_ddYmHGbh5o6^q- z<+^lSqEWT?fjpBs{+UM;-_cK&67UOyb_2LFWvIl5pf;Ih#~R9by%%#|%{4F~D~+u* zf>viMtxI-mt<0#(Q)s9$jBVU2ea11SIf_T5;=H@~jNdWpSKg=cT^0;i1XsmA&}Mrs zH?WMmNVo0G4${6j!$f*f)HT;FerjV)ETQ#H5v)1Xli1$fwe-5S-r4Sg7J^<~PLA@- zp+JEF*IN962>&+sxm8i(lT^oC33J8hx`$^AXHaF1Q|$+20#1_sDtVYZ=Q0U%@8EeZ zLz_5}nOZ)w5o&|v++EI-*L>r+&0fx}hgYSo?Hnf}>I*>ntE)264wUsCtlJEx*rsXD z??fd=JMYRDhi6Q_U3vj0sj23Tvh{@u3}tx8d|*N@-0!t~epW5vEG1cZmZXwjLc%PY z`^sp8$#EH(kCa?@lb&{28=BclO-$+R@-Ivbdsy`raOMq zA)*Q=(zT-;5^1mY@#*DvnD|cStc-EeeK`HRYBvl0USl=j%dS{z=UgDo#Aw4ibg$>$ z&U#JP1vbs*tW5pVM+xMlvrzcHpAS4*`5nJ-K`dNzn>dv=RJ(VQ_=HqvmQIKBHo6xp z8^5!_LZ-pzNv=?rOsg`X}BKLz;1+kf#Ty8I;(C z(Yx>)JLtLc1}T73TYzpNJ?4}Nu(pI|H_Ony{Cc#L^^;wgn5lPE>YX1olv@o`#muS+ z^W~>6cdFsIW*EKr zO}Fihr8J9N9#|ZKix-Omu=rr-(N|nm*S$(QkyVOegBxEeq-znykB% z0=txlhS9!|+XjJZl)URH zjDN{kR(71v?2QhKLxK1m`y;WgcUVvkvNCnTPScU`J3Bjm%8nLZxykPi9gQ8GgDRPS zzvkddfl9mOk@u$)X_d5F-!Lpx5whWCoX#<-rkSOjCdeQ1v}i5g^}ecGX!)Ez+aI?@ z_xlDd$dEk1QBe zw1ykeo~C@6XDMOVunlFhW0*fcURJFlNO^AIvRAbLs;k+vK4qkfr+?oVUjaXQRM;Tv`6(^et#jM5RDi268@P7 zfTDLw7{l5%bggjaLt^((jo%pb^M0Y&H4~DZotfhn45Hr%_W-{vUsyhX2P{>+DFH zjyFdw1M2JTYKh{hW5*G3bJw6;zombV6GIV>K0ew$4aJ1{#GB@o>;o9hhJe);;?$t- zA1moI)LtX>S<>!vA|l0^115pqTw*@Eed@`rkvC*MOB<$YrfIzCc_~%<0ZL&2<){w5im6a0&Fgw+DK-_p z(H}WAsEw@0cGccHrcL1-%)USD+uUb$Pp3b@COxko>AWMKd7&sD8bK;{Y>RITnszVt zMOLnUP_CgKVdlKrG1?Gltn#Ke$N5svePPY2v65v{c`w*JPH}QjU=s*(d9Xul?KdrE1{kEvju4hMqHsP~%yfKoKeQI9c;U__cetM8muUt=68LY~sk*C=WK%_z>{(BOR;dGgob# z2_9S`RakasNR4&CDQfKQ_dSoMb;Y07FP_QnoxihK(3iNH4F9wVKaE;^6(Skbuv9ueo+WA}U} z7u%{nQvRNHD_SVP>vraS38dD8KEV_>k0~DDUO#?;llNWt&8MAv0x2lku&fuve2c~L z_218FtVyw;3nzaYoE}t zI%Vti>%PQ~58n5aXyZ!!uylD(izFU7nizG@&h8{Ot*!gBS^C(L#TK=M__qP>3@5wH zUoxr>r|kJh;7GcZnRJiPXx(Xar)=AgT>Sa+iuLnf`DTiGPMI3RA3ObF)>5eAGPSrI zax|g=&65{Vc12fkPNuw>ZckNQ=AJz4Lz#8WO9gt7(T86DpQ-}vVnI$x^nJk)k1T=9BFni)Wg(2WH`-`^LooiKI1gbbyne8h#U;fHYyFFIMe2L}h4T9ZYr&gFN`n*!ZCS$s**K4FutMq_m+Bhex6!PjQS7d-*G zgr6Y)tY1MCfam6gbN<-S+^Xs9&U)bkiT+P}D0*->-0JhLfy}!t(`%XN!&#rWSea=( zFBy8w@lEv>=WMzG%~Y%Mbc+I71MTFNZGV~BlLkN=4bTBo;hV89)}GMFpa-k^3u%Ak z8Xc0UEKREP!(WdFm_QZ6PKFY9ChV zu^w%0i4I#F4xo*0YqKsGo{ZM!y_y3u1BSF`x;)GHzIuH)M+^x?QIbwPOYEr?q|tI! zZAe3BF-19@UKO4!>iB3<8NP)fqHYBet)xfc^u1SBla8v|b7 z4~5~68kvm~@jje{^#uWwwl)voQRX1Tj4=wglMoGhtsATq7%J|(y`7UOt_qF)#?DXR zSg31v5_}cw5<3?tr2&h`Sb8a!WiC?_5pAo^?)i`dT(Xp|wII7wM5*J)h7m2gz+;*PXPauRES0Nq{sLN*B9C;ULkTEkonSzYP4puVx)*B_u99mgz+FMV5 zS~Ia)UUcPH0v-JE=gqifW=n zmnYMvPx9f&UuQY9L#rH1I*6=w%60{fx!z1K6(_Zll7}M-X-ZK_M?M^t{4U*?p7lhN z(%qVTa#H%xPw$*Pg;Kz zlPSiWL%SFsNnaYN6WEDQ4hg*3D>CU;*#E@v5XQZ0LX8az=zqj<5Eg0-x#?={<()Q0 zudUT_ujG;D5T;i;X$L+UTFf!blJ=0wvfTT5ANy#mar}nmFAVLdrBkJU!S^05S>i3q zA2@~9II$Lj3~h{m4d@`jz%zfWS~{;Dc+575(uNr_Q1tb+e_N&|#{Jp^EWpQKk0D3? z36Mza71&JqPH9k1K0CWrLl|w1A6#wi!e7shwm%8~`h)l!BE%lMfLfNCOH|r8;n5TQ zHM-(P+3HcK{JyBuh^R}~97rYfAf_!72!+1TVQv6`4)G!`wxHONYQf8hl%$Wri`yz) zmp-dwty?YThSI#eiw1cUfV^5T-+wRSlD%1sH$92|EDf)A4VdIHK~Aktnx(XZgq(wt zeRs7()%?i~*J6+;Vj5OwH@CJLJr*h_ur~l2eJW^sf=J88A)u*HaB><|>09HnTQq9- zc2sXRM}`PFm`?$f$kH!Wd+_z;*G_ng?`a0fY8euE6LLtNO-3fppW}aBd4BOLTZlbL zElKvp=v2pt5Pq~SCrnD@@V)wQZCV4EWnf>mkCFZ}B4Nt5#Z7=y^E7ZX{=7n~ekJ0q z&>6|whcw4lo*lo?Cx(c#$dU70DT=#vn`#xQoPUo0Jt}yffrGZ|OJ83ix83d1QsPrL zZanz%7Lf-%_P^yR^?_A3x!Um<0lP-1i=&|EK5dme+qNiV{P`&|((=Jqju`LD@sbx$ zNmf4qfr}p6Jnt}%Z~D&tiQOFnRu7a0CUd2-i~Caj)2BpR=Nv}9k?C!`XV>8dz2wE0 zR~Le;mFYr;S4QfB#$Y}?BH+<0G#%@@(?2sxv-i)1p3c|%h@M?KFaiZ2?0sU4Lt&={ zZ1qi!AYss~P)(WO zZcG&|s)KSUU}DpY#gqpe{BsUNYtY(dFH?R)m1GTz?w^*Zk&O#Ek>u*|!~_c>ezWW3 zmv~&=|F`Ey;q^~1)$r{9^|Rl3szqHyA%-3(Ki^l%s0|-;nVpy@c;;g2%4Nz0VK|^{G z6chwERQ1t#^ED9EBkF=^MZ7!-__=rd=pH$w&pHHin<-q=ElghpmW=QNk$`NivgYYb z>fqg;{mXL|O852G>M5`Y`k8)L##4J?Q5T7tpDWq zZ@k8z4>v?`#aC>O$BgG$(`D@f8bwrpk~FxlY)p#bp)mKO>0B3wYPenI^}CBu5h)5` ztokNm_CvmoK#J8;~*pfMV21`NTPs6PMQMAI7 z2Z4egLuy^=3?PoX34{{Hs;a7d*QE#dm<-dDF7@Imm)Aj!Ru@C~8l`|O)ub!Pzj+!0 zsK^|tvZHMo_dZ}X9FWosMcU!VlS78L+5C#0$@rhRq*Q`>*2)U6PMg-;*%%301`cp1 zNWiS|Y)8Aaj3!}G$11WP-Y@q?i|xr=tp!3xI>bl7#9@|nLGHrIT8l7n4tV(CrA$hZ zcfJMDav|~~?pj4*i2Kl|$5{9X!E6qpQ2zn^(GwT%&+|=^Q;KIEJVGkByS-T}hIBVL zT#vFQ3_E4*eg*tjRFsnYp(Y#qHdj~8(r7Z5ipx^Y=EuM{-Ahz=%c={;bjhd+X$zRx zIamw(pW|PMyUf4O@IHC$*!8BOwR*tr&3tXB)0(<-bLq4N=dKum?0aXd1UuVyN}sKM zzW^>y)BF14@}%#eq{|a{B|Duct9ppj=#|YcwlBTsAkS%YVrO3 z!;*pS<=p&lzh_H&T7^XC2dlY2-r_aR8_>$sR0j0|HS-KJQzYh}sngjW!DSTQTWnXp z?YaM{uA9eTL=j_h9tz=zeMk+|n<)HcsP;GBO+QP69z4Da6AZqjwUtY41WZly=XFIi zs@hh+2-}+u9p@RKWYu)FM)R@tg|(g@9vNYPPY8q7QV-RswlD~vrYXF6W&hhmX<@yn zAV_j;4yr;5_&<+7D`EI|VN$iU_Ln)m1J)YgE*dMz1uFmVg#|8?Z?8!TRalO_ziup@ z@-6f6hF$f}dCTtY&6SqJPmw7!av}({fT*hP#kT#V=WRiTR21Lac5u`AGmQvqGZr+e zcVOxUC*Kb0*^OmZrlgQYJ;Q@AYS<&Ve?v~)P?DZZo%R&Jo%GI3CR(;T4{Sb*gME~I z*7a~c%9lxvjEqV>*0XK;N`9!{gp=++AL!2vD}&prozbd0KNYDj%&Kwh&q&X4Q(Uy| z#VWh6PSP{n4t%}nx)upV;N5A4TXInR$Kri%HmO;v4#!K&w#l-ib zZtaKLfK9)Hafz8LQkH|EY+2FD(!O!csH%cdDY7lAW9IJ~YL!01TcHLmc=XfbL(2$v zlGMz6&BxAZSW!Zf3iHDjwSMY6EHJ*#@09u|`*PEUGZdDnwSuE{!HEY}!3I<=S^*3^ z224&ZQThn@G*KkM4AL@7Ute8N?9MkEgE1S|1}~c5col4zQL|}agd|mbrZ^PVeps4E zD#^bEaC0zBy2Cx*zrVa_&$!3(;xvOlNzw- z15it|R)_A_#Gmn=VL<-?nd~js0gL_D=Sa_Ayrllm)WZLpctXxuzzGg3jBqhLCc+Nm zum6Dsh-CjYvGB#Y#rVKeoIly+KN#|c$t_Nr{gW{yIo#$OP#?wfp*&#!H$<}6bnQRQ zfUYjD5YB!L43Q9MH;NeDB2#dRp`9D%o4x>Ka#Jl$$toN@P|BC8)V!~_E*o+z6%w= ztP7)4aI4H^fzxMaEdu<~P8+Q~%OBGP((T5>9-t||1|yHYT;q@VX%?F~W0jgEI1Yh;3j?{;nE zRbXL~ww2Q|C>+XHZ7%WKL_!I@4W^j+!2wXgkA74G?=Ooa27uL7NGOmw&%*j{oa-(A zio*%CEdv%%fc0QD*HiQ-j<7ILc?eS+2-D7LrNHaRYB+~pb1Mb2()GusW+3q{kH^?4 zG)LZ09xGhaeLS-$zTXB1!bNT!J=eBqO*!5Z(5m2PjE+d82jTu5^*jzl;eg?SMlcX_ z7j;tQf~CE;mu|1D0L1w$rirdJrWI$S_E?+l90TT%Z`F}zPYe1ma@aF5(?8Yfpz96> zbE64xh4SDS=r%|72Epn)6_oincfQTm-*8Jt!tMeQoaQlff1OL*WiX%*6i`G0A-%mVr z0|x%tsRD8$k%SYMxDOwX2Mem;3a`rkN7D&k0U&uurz>?d2)iMLKC~Y4sln=MPjDGIJ^tc*UzWs!l!nS|ArQV` z0T3GuBi;mXCKaHX`LblJ()Fae_n=grv#4? zq#_P?xDA*_1fqI8RYUj*OqiIr)8Ih8`S}o+W5iWt-5CPVK@kZof!W-iVQ8%?7jqW$$R_@lw9N;vA}+#=c}*iGtPmswO2$-YW(3I>R`ql`P+Gwf#WuivF{^oU zB;SkROrdlrhg3pB!fkQrF-VYK`(^e&N0N7l?_3-7?hsY+O<8ZylZ4z?(+i<4s@3&Y z`&4^ENahTiCc94FgJ(cVZo-mcL+FV*Mz>R4AEt*;<2Q2GgJf|1HaJj|f!h6>_X>H`6qld%lX>F9e^OhWQRSs=F_bzQ}Obq_iXs<3njunCjNh zi$ZJgQ2Mmlp(+tIu@dVt5S(w;)CjY~m=!nZO+XijW`T2M(UY5HSg8w6Ejv_uKCzlL zKXOcH&}$&RkqO#|`XKS72whL|@4-#Ckv9U(H;Bn783716l>~WT zSBHkzF~9%}S1rd-j3*pOG(r%~pv#zx7~mm*6dljUj{DTJ`?vJ9j_`>kB)7x;M{fqGRu3h}Mz{GeA0$&Qi8yC@ zF4hp&bWJgO3qMYqNA01BybN5X+P^-nG4kF_?UoQNY%SBgGgAaNeHw7w@0l64R~NYW zHGYF2nTM`+oKgd@6_nu~cf-V=7YVy8A);ZXZP#@pKgS*O$9T=+kEMTCVQIQ=AQAdT zLSg33sUM}7*M&A2%n*=KrlD)F{f9Yz;Ls{eedeEqA1-LW zUJjKzm@-H6aBo2CM{r@&ShxNoal|HIEDN8#R`~_jX5$e1ITO||xBZZ$-bNNLRk3?E zY7%Yt$@jpxq2F!-e5myT0O0@+s{M7ictTzBwy*yi&C5g zD057EAixMx@Z%JJA9qzF{W^ZzKFfK&#lq|`F?2lS%}7h#R?=n%Dto(BzD^&rKDc3 z|0ocd@quVMuB#SJwT<0^m@(m~2d?;gKwy-%zWaVDe-XqlxSlDi$1_aOj#bh!C{U=SKqjzmq4BtK!>?Gw>q2G0$=99by*>Qwbs_-uYuCPq2ZR-f|L%Gy zV?Fr$xwm~jME6#X-&%lRAu|BEq9Yk)hP@^wx&Wz~E06X=xW{>O^Y2VsJ}r4$5{ZGD zqOAdQo-ED_kAULJ-Ab;X0HtQ!8$yXESuW`pS_WHHCx*=nXBb+tYGzr^BQ8e34A}fv z_y)i<^LN#sS$Z1gU)fWwow-udWobW4+p|4NNX zGzT0HTOPxM!5I^8(Hci>vjZ!sWz@Dy)_#%w4J99UXEBII{{`u;K*TBM3%k3!+z{oI zoN;{eiG+e#*1Cf7`5t`Y~@gOwK0z! z{xeFxgddE8hb4Aa-<5`O8VGBjo3V)ZGMD}YW7_~u5*wfO&+ARJqOSK3%44nreqJ7< zP*tB;5jqpgez~2oG4LFhq3BBAlmox_wrhv~XRsm*?{z~SEaN5UFl8K6WN+|_alBR` zg*M%42xkE#8!r8fG^&B57~RVZUg_IlhTFGXE8LqQ>DKW4Gy@~F@MY)^^&n`sltETd z5mx-EUIz52MVj|mM2>6usD8tq-(VrL4QFq%vd{yNvHh<>}#cINmPAA}uZbuP=fJC3hG<|Dw5%~e6yt-yS3s;j zZPk?(g`h?otqe6L0N1Z2k|NuSfjZC#HJ5Y+d++Fp7aJY3wAlwQ)$($LEth* z5^2~ufy90tuUO=Mt>^y{Pb!XqX4nqjo&we_Xz>ZIknarv9j|u_az0U#Lg_H&rM9=X zQ_Pyev1V}Xd9N?t_^i&3a;*eK72$e1A8H-}*fj+nkVfhb4heP#&f*HF1p>X3yJ8;r zLQHS2vER>ZgWhLa$c=%+UhJSH*Vx<+Wdw&Fv&Z^;7$V`?!$yQJkGoJYZya^WPu+mz0V4<%fl0Sd z?0-##=Z$`Ql}~epss-)mFH6-Fd4}v0oYP9uUBX}yr_=^ z-d9TY-%g||3w<9_z&Z{&6uZxE|DpNC(l)ne-Z9EQm5-(qZtty6$&^~yKN>w7klYA-`AZw@ppCWzk#XEhU&^f63Ul^ z9n;UUYqzOi;nFX;gBV-V_YLN)|7xDzGj0*=I#+~6w95Lvkya3XS+-MCJ zccy-4cwCjwYs-=68x)GNe;A^MK}3|2g^9)${!NtM|N2WS;4eu=ZfCqqYZIl|R5x z5s>^4EeSo*wx$rF<5k4aZghG7;<+Hdv8VvqpZ0hl@y%_rTGK1Rz1dMK#tal8b!K2Ye4znh$(B5KOaB7#D%S>*cY`PGS4-L&w zQ^Zr1rraM+dNhIO5OQ2pjauxGD_H1DMZx~A&|%7%$A^ehpgx8GQOeNHODe5G_zsln zsF?&DUpmr`0NybJlz59scpy5D0>XN| zclXDng8gvYqOk6ZUWh?c1OlJxK3oAs_{I##p-hmP1|gxV9^&BWCmn4N^1g}Gl+P20 z-2%7`J;QJFiw?}bz@M(S!DX$jQ+ivrcDXwoqg^KF?lea?=(X}}BRqr7ZF!6wJ7`m# z!9CQwka{x)U(;pQ_+BS%uk4OzljhsV8+!|{PJu7{_JEf}@Yj`z`pIFL(M+k@&wO3T z#wp0EHW$5uy07Cr*+#zvyS_j$jA31Z2K)<)ds_|7IBnVuz*9R!oFwZ+ptWMkbZWPu zC3hS~oLBzNuKmkhv?(0&#r?20`;Y7yKw`3}`|8!7Bg#?SH$lKbN&iRdbP`l&Vq+x$ioGpI!lCnmj*GASEM!Bm-3}g;4&6fg5<`#6C^LK6solVzcE8lSvhjIae-*F!8il@I6W4&6RJd*okD8)Xx@y{;+(4z;1QR?y{e2NI7olhhWoWwHGW^6>92j-`c*MJB*@*1b(8dLNJ>LUZ%=S2myTq%;SO#pFP5aI{cp1z zcvE=HkOllpED3R}5LCqkbnT-GkirG^7i`?LbxcU6e?QyPI7LWlJ(WEo=4x+*wb8Zs z{9B%9T&`4rRq^9bE1Thh>}uz^k0sF>9A2BtW=N=_sr4^$h=8(c_~Ad0%3#Qd@cb97 zQdqyrwT8;Us7$F|?4w_q$3!5AiTY<)-Ltp4BSqds+A{Xj;9uGq+O97w>Il1~{L0UoGnJD5YeyjJUO>y_W$BJ?I%nw?0dG+jFZiuM9UlItr zZ`e0;Z`)nkc9NLt7&)|Q_U&xLd!`uuGDj)=!j8c-za^uG3!&-j@rkdUpb>HX3wdM$ zwHws0zvYoaQZkTbEsjJ zXDOy%Mw=Ji^HW^}We-452h`*jB!xv=ND?KOGV^F#a-Ax;y_a#A5Q@5>22^TO{THi( zRD%G{)57Mx>mp4h4Lqq|P^n|98SnC~cC6i&H}Xzw;5|0IM+M!YkiEpQC+?fufBLb}^bi1PTps=v;!y29*<Ti-W_{!ONdLqNG_jY?9S9ixX?Eg8ycJHD?U46oTo*}${ zMtXlxiEM9|f7GJ*lc)UioB!U%cGq0k93B}X&|TqH;MNT0$fJK4AE)I1Wt03LN+|gy zl+!>rkVcFZbe7RG`yf!(0bVBh@@OlNG>7%;`zLei2m*uFhH4HWlq;0W$PfuLAyXS9 zV+Ekwrhx7rV>`XS^h9VAD3A=q07&QZVXR?bpveM4zuJpM7of}C*R20|U*=#MqymLm zhRs=m)B*p5PqtW}zsa-H-uha5l~)E&Zs-rVCl{V=43l~}`km1(l#?A3Q^jzPd3)48 z-X}*tyvArff-=mj1&`pXqjp+hQ5`G|kmh@W-UASXSRbM6I_>&?P()sWc!lT`;0iGS zPdrw*188j&Hj}=C9e^=2SwZ)5iVW*O2YQxCybApZaY)R4#55a2ueGejLGHA6u6k>l z%|Wyhs$#U~#&7@<7v(XpTmW>~SLVb94G3xC}j47EA)j8;1RS6bRo94$T!<;8Zq;b~GVD$^%sTo5JXxT<@IG zO_C*n{iYe0f@omx8}%|6G7X3z*4kyVuiHufr1QnAHTm@3Om{a35eh$$Im0W1d>(! zk4(J7Udyj%F`B2h^=XT;Zeg&U|Mq=W9}dcj`R+;{?fW~%b^OSt)lKWR>sCGam@lt4 ztGRCVRk+ENxwxrf?fMxh>Het9Wr1J<_k;12g$>uMw^Y-K)v06nUTZ z1LjT{lq~ZT$c|y!zkUzhG~z>BfKbDns{|N=#2=Dy=Ony6GeESf^(xxx68F^9O#j%~ z2A)ISvsMK*FTSf9ls!Ei3J?fdi2n2w{@wjGz}jboyd9UhkEOUq3nW_SWY@iX?hr?e zxY?7wGB+9BLMyHoy=sf&@fxsyXN(X$((GN3rW)?+OQd(o=KI}5X$T}(9Y2j>F4pc9 zDNBQ0?QdaEErU{!axDRdBnC_C-ZDoUAJ^edQ+Tj97%l|K7lLq+uWNr5qrML>B!7Y( zkNZK>qR=aA$@P2D{%n$UCJCFpQy~QD`o?#m@z=Ohvtg#4Qh+v}F}(RQ?E1Qd=lYv0 zogyWOpk4Tl*W=G343^CNbSW8N@jBWNEf&yMvK#+iXeUS~4b9H0jeGfk%#q8eit4wJ zLIfvzp6@;G{3YEQs?%L-kDLt(qxG#WzfHjPkggT=E8KylEU>Oyxu0R$*A^TQ)gvB0 zz?XCdHc!zJ*$oCOvtJjJYb zU2pT=?I;~7=p@(cbS=+jlsi=+{L66p>@^z+RPRebFHLY0lvOP&cR-#W=fD4nh$lE1 zD|lDeslv8jn1Ua*wh5H6MY%V@(q|I%zEA3E<mV9^6~N6oGoZI1-|nbi~DF%nE_@*1E|q|WOQ&7JM_wdzZ;hs zLOSD7i6KE|5WG+<2$9Iq7Zf7tN{v30=)JS1RqC@>)%)#k-tRr;!#}a-0yj1{Z#K9& z=8MOA=BiY>v{b_udQ-jiTVs35ktNofA#(Az^`bpt>(KZFh>lN7(6hhtlqaFytzAdw z{7E_b9qm0oCy!q1WBwb_Up#>jOwi}U5WW1xG*E9w+@6n}$t(Op!y@ajrnKrVp-gV+p=(v?x%p_t0i;ZHQ_<(mjitS{V64kn;^5l_^z#9lZ)sKM314 zDmkvhHl=8ixhow98Q2|{tDr#3!%oUnpxE4Q%cj@V)8Ah5!{3?zksJ^;%O%>Ct;-pY z-@enHwi3L3>AMUgzz=I079mbVbq&`FN z2?Sn_G|X+-3}(Y2wLcC+-f;I6bRTAR7VE}}z^==C&$2qJK|xRBy9L2;qej7Kel#mQ zYJ`u*AGLPQF_@sQ(J$dWuMcxAnWFR%uS%i-(79+(^V8 zt!Ghr{}!%C9!ao3!Y*ySz`tcHk{yBikU)=|TTLUtlTI`ZeRiBEE+Tt~3rCN=f(S!Q zf(JzV@{MbcuJplkh2)SN3zF6YdGqOv`+oiSRP^BRFP|W4ng-Gpz%8eKJNKL!h8)QF z1gIbK5I#3-+!%6a;oP=+B<9itDxRx|s%Jc3w2q!=O@-zjs^(zDG-5>b%Pd}P@?RW| zXZe3b%{BLS%~y)s9T3<|{Qu)<63qc#UrU)#_$}>i|VF+yo2dh%lYm2w9t_+G7G!58TJjaJf48VhK zLzpH7p8bG`coDB@?ODj;R>qoG5CZUG|5GK?D_DJzr3tYMZQ#fBkdqEamlPky*9 zXb7OzRV*S+O|gt?9@iwaEOS4YECv~I&VJ3e%e$#epTGsX-*th9n2EoBae*stv^nbD z*7^b$AaUod4J23xUPmy(4JNL@^^Mmi{@-Y)0l2~;?F!dC2!r(;_S3`K1fOA2h#W(5 z2qf{KpmG{{EG z2bDHxljt|J+O&`BNHht7Pxqa`G0jKH?pGQhG1s-aW*jgzPQU?D;NdKfcDLQ!;5!i~ zFW5bL)rl8UY1hdJ}iob(6z8tWNW{=wvCxqRWQ`{uZ|BuRR zbMXaAew&LmZ2Rdo--xfMgOf1lV?4;pU@Ls8N~|sn1Pko>WM^SF6m}k_ zg7GkArgwb@_;n6*NUk}Y8aC@!5Uah8Ct=XqqG4IYXE5?a7n>gWV6#r_%@Q4=;&@0^H_serdQpFYD zE(Oh0iU^_PcN8B%_7E79L>}CpF#t5zr3#rq_)614LoKcq{RRX}$iM~UTX_gs z&Rv0c@Z)SP?cyKtn14j0zJI@Prk~~eo>O7&i=DlQBXW`n zYh4|2g2tW!gp6G;_sGRk*#@=tk$0+0J~h&aZmK5|f_ru0ZRXxCTb{{LaclwV1I00a zXLQzF*4pV(PUG&nAEiklH1}vf#7)2recbb=L~&Z%>MO-lya=hQMv8Ltpq59)y4Z?K z{)eHp)4!4r2eToE_#Z!KCNd6sP~8e6RgczA+1_PpdiQCPY#BJ=|Y+Rmwc39W8^x;Zi#(;7( zdlaMGd-h-a$H!X=G=yh_32T}fm4?R&{}*-d0Tk7?wTmK(3W$k~Zc&Vg3W5<(vKSFC zAVH!5k=W#%F<=%I1r-URf`kTz?k0m`Km;X&TLA zj&3@vwdR~-d}-vI^-x=4pYkEqF*TTOaH;|{p?DXsynNPa-d$2CK(i-h8O8-q43rPH zUO((giLLNv)V>aVU#vgj);Y3ZUiMmA{qZ+3P3;+zlQ)*k?(!%Y3+~wD>@jPg;mo(H z@f01!iSsiJhU@2#9T`vw>8YO|(lgRv^*yh(z1`Su{x5c-+#Upw<&=7h{M*b-s86M?(h`r=6D+$vmBAScg6kum>vrP}$v&&K% z|5&#q_B28lzgFCU=Y8N7?Y~Vb+xe7@zI-+knnfv#+nanlP)b#&#IpG-QZJp~PAXdD z`N$7^T2Rk(KhTKCIWtlI!*+S$G}CXz?zKSS>EqG>kQT#Dyu;^dq`NGf>g4CE%l>N? zz(c=`1EW)Y(`KTUMA9KfQ2|_Xl`-$3-4TwU=FX877mVq3X=o@a5&egf?2)`zCb6y( zOCpMZzhLz$5!)~F$RRu(Y<_ii(JNZR!~dWrTObT#;NR3_2Kv&UXiCk3x&u#0BQ$I` zDVNw;GEsjhbByDigA8@y#Q!f|;H&(rYr@370S;9O8}yee62OA7r~p;%?-)9Yd|Jxu=ALU(sIsum7rz;CQ zZQlZJlCC)qMD9o3@ozkXB^B?${?%!@jZ}&{tym%!-IG~#uy`_Y z+rZ`gepZ#vxD15aA=tvRQKUTk!(iB?%0w>NOEKq6~2*r=xE zky;bI*mk5;sgmYNmvI?8Ioo|S!(HC?NxF*EUDQ$1Uy?Zc9*gQF7p89zx9EiC*!0m^ zD`1|)6IL>N@5MIQh#pQu7BWX~vc*&~MJ#crBdXMN*AsuR=FeD-B52$LAS8HP&RvyQ z8UC~d{jVi^E7WnbqzoXRLB~zaFXhr1Ef*j@TWo+q2Cu=y?R@b1?wNk?`B6t{WixeR z4ro9Xq}ZX#gtn}x=o)v(aivgcf8)MOwvpZG1qJ~Jc9`GT@UX%x1;U8*zlvB^FJoLr z>rIZ1W3dfYFZ_1#?avRqf*o#^pe53;Pf5^Z{qg6;$gwX!u-=xI*KFIK9?o&awKdo& zf0j+vGp%Y-QW4=fJ$K;$M%H)(D<=kb-T%-sbqTd8*CCtNqZJK`TDCEB385$b&Cm9>Hdr z>s0SyJZ;cpW83oN(!#0r$JPzu>oo7lruXmGJMTF+;O+Y(xGP#TV`uku)5@1|0NV~b zj#@eWAxIj&|2Lu($bhH->^~v8lZYpY(10`xu%T(d)3Xj#!+aVA^cTDi2%N}6PErdL zaRE31a7ZXUN1c~|oVe=W>NtGu$A&^6CW&SZgyoeI8%X2+IPcGo{!xx}<~$RXr1gL7 zN^^YZm1L$@Fjk!^`b1i~(6K7|-7@pgJFaSkoda~f4r-Jgd60b=T8`KO9u!2RvggFG z@h0tr=V%E(SqzrzT0P1^3?cdHT3B5>ldY!lziRA^*}p^M_IT?s^mWC6@lTH;oiO`S zIb@+V|fw^9c z_)&ApUS}ZA&~~W1(ONd5)U^2UxUZ^Fb-=BjdU}l7)3bbuNH5Wye@&S9O<*=x!fBxT zw-@pyd1cJoEKCt{$=M?UBmy!DiHfu2U~olP?tY*b{Tr7f!G`?E@XnwG>PS4l<{U#> z4ImvVL$Paqfu0h?e$D77)WeyCb@F4~7XjfBG>VcZH!)4SA{LT?Pvhedg7 z@^6+jE#8ObWHn+Q9{{sWLDP&aYUn`na^beTwd3iA0y4Lo<-3g3I|*{!=$8}c0SL5KtGk|&xj_loV_GW8qj3&@u(3mQUI z^#BslG|CE+$1s>R)^p=szo)0?BB{8tktQ_V^D82Im8@WfkY0!AL{5fh???I~sp*$l z_!nO02p+>1;J44Ky#c|;5j5|p8*jW0{qy%`6Lgb51z6M8@#*@*UJvq@{PEl@KWV+t zZ70s>2FOO%qb#?aS)YT)Xb8L7iVZ)b=zgrlk_3WN~S5L^! zmm2$ZBwY^bFqjLC{HCQAbwssDp4ER-Ew1^aT4X=1Ke2t9*k_AtB^F|m3a&2i_xHB7 ziaD`*{i4GIqsQ-Uu=1>=8wTZv_TI>!qy#+K?NH26?SJfWuBq?Ti+xcQ-}9rq&gFgV zUo3rO#j@Rw3HFmii5uN7Obv7AThGD94RwbQ9T-dghk9BP$=JyA)1 zg4D-92;m!%-{I@&ss9N*v{Yc@d1Ev;Wa_w0UoN_<;Q>wbVqGajaKDc^n7x(mZ@hb7 zMC*uVoZe6SdP1TA=vDP{K+UPqbGSv4K$hUQF8~x+z3H=C6Xl{W=oSB0-XIcFEMZlDeY*kr0x&%gK&Zp=3pL zhSQNU*{GUEMVYp&EO!{INiI0@-5;9cLi|^!;77edm>GgvpPCef7lSz4je?T3BGusM zcBW3X{NU6*widE?qukx=i4SivrBhl0H%P>lh zp4yThtmRox-z&v174|IgbWA0X+;ffQRNIM~Wt(&}!$Hg|0Rl8G7m&6+nK?v@B!-7s zAAV!~<4`UdfuT{0R^OE3IqsUAgD{gBn>1q|B0)%%=q{|%fBEkIK~ftaYnVUyTMi+n zUqu_dx?D1@G)Al7=65*lwj%(il4j~(c_c6c(X~H}dGf?xF|wu+|9)~jNgy5P*Wh3v zX-OF!G4YxAAi>rl(KS(MlEPTH&lb@x;$bBEm@`n6xCBWgJv2=%@Z1sm>IqX_x|zl> z;%HP~f-kpAq-qYDJtV2>iPX5!wr76fg5Dw9j4sSuwPR0v`Q)irasLyj8EC3K9`oNg4QcmkqAdaO z41Vb^;Tk0Mu9^ANP^@~&Uptc<7s+eotS zd;wu%7DC$6*FNnD1iPz<+=M|s7rWd$fO3LPzd<=x{3N<7mAv^pZ>y7>R^0#7V#|w54za65me*9szqb_b-*1$8#$My`iZDcNg#an6s+S zVDI_S_y!Tytr);lpJ8-~TTFYg?-1;hXF45<)5G`Q@J*%^GFI{nnX^$TyJ*qc8@86V zpQ7kLeQJlgCtr%PZBZqg)XtHoe(S2VDj#`fxIOIF9^Gc1b>3r)e`gMH@m4Sds}AH; zzJAs;GIg0&kJu=)jf3b6if+YLp$xljuw~>7`ams*j8DH#*_$EE6yU1J#GA&S1ML)B zwzZn;A^rP7bo|6w48w#&WvW9Ar=y5w?B7pq40btb_AK4_m&Nc0?#Y`eReU?`}Eyr3sMn~sjC$O^<98<9MLZ0LoW8HAN?c@tzgG47% zYc3UQ;G_*E>cHL{R=)7eabzg)Hkf@%KmUbachdNVZ;{lV2GaGh>l0*Tx$N42%}MA%o zJp5>2JFZJ2H7WWj^>h0tTK&l+*UOtvC2TEBfROp~;|F;vd48HKDP-zIl{|j{i*R0W zb&?S-x=%SoQPa6VwEfqhyA0iY^402ZBap|>8Y$lgKb^{4Dqj`&9B0$!sruJ@xvn*- zYD>%cb&f}!QG>+pR8?}i62yn=r5*tVaAptQ)e9^kLNeuu&`Z9Q5&p7@nAycTwUaHm zyuBmk1_c8<)$2eZMTIcpJsH_7bfk}#tSDzJD*KUfr1<|^M;f_`-Q(M{qd%f4p%X~7HLdb}BI~B9N5xUR-K20eapv!|_*QsnXk-OqLyjE@kumi& zZnL{D@8y}8!Pp*!RMdPJIQu%wH3{9glZm{dj-FprszYb#By|NX01+xL>D7tFCXCWuo<0;r z`P0ly+|qRoMmw!|fFbclm>r+FhI$_R`D;_9LIy>bSPt@orM2V`w4OKtijI++lm5V9 z`3oQSLm_e=X3+`+0#aGsIu8g?9uS}+xwTFa_nTPRy>f2aGT+BLHr1=M=~2J7PBNqX zCbcD=a|UM5^xELYH6bx;;Iy^tuUVX<2Jj-X#W!u5WWA}kC}_c-4V%0Yhjo9Zq!`j- zyiv|1MLjOfeNXz|9m}5jeo0kvV|YT$c)s`*Y5_Nop=oazThRkuc9Sv~A4(Qzb#hj@3QI*x)3BBO}5+qWHy6g7U* z?;VF|_h|?#aNU&!p9$Cfb8X6t(_j145)UtXwB06_b&>cP@uQHQ!7ha^u5*{@LX~)f zjB@~-Nt{O%;&TzceGJI#dZ0W0sIU*iA}3%#CzeLfk)IoQg)4wfV>$`3w{2$<<`io| zlDd;tdlQuuCrR884fPm+2MFYxAb?TsiX91*yaSM^r=QDxtom8wrC>$egb1`{_ny{! zZqgDvb>H(8&yQtOszvXEIG}i(uVY6b4o%hdiALXEohgnyy6QLX6G4}EIff%za*r(< z?u*nCGc!m9cx9^75Q%`sQ@xzXI*I*9Anx^qj3do`N6Q?d%A`D+yiF};XwQ5k?vX(2 z>U%o$Jko1gIwEtWdZr1*A*?KnvA2!T!Kr=nlb_fKJKzKS@3OSFXUut)LSv%viB6tP zr(R#Ts7~MG$f^8F;}SKKf^RbzCIQw%BQBd zHQC%IcRCK?exNAv2o~TdB=FC#WJEJajhua(<$IHKQH-wg_b9~@3tH!;*?OZ>+^QLjwpDT+t~A7>E6Rpyz`aOhf`nbjmT03 z<}6Eoxr%5FWr98CB6uULp9D^#3;Kks`{nlt4QXG_Z^8i=58qksQxfy6tO&mV$7|m` zF+@}Yrc=k3`5SK;Y`GW!B}C87sh3}^W1}2S$EHt&zCRO}?{wFMzKRUZPzsi!Zw7WR zu=i46;R3}K0Gn^76x`3XDSN#GP@6yUuk%#&UdH!Au}tQ7sSH_Y9Gr%U>()l3DQ3Yd zdl*sS!IsaM_REcDtXar+{_B%vO+Cb8$aE(1ioub$hdaK#%{m?C@nw2TvT6)qlY+vz z2j075NoazXQRXzLk;l}ZqiQ)4M8{wz#-X(s$^s8?OaT9SN93HlPrqsw(`DqZn5GlD zmtem`Fb5@jcks=v&kq*^UVEgl!1@86KS<{_ROlWk=@IR<78aK+ylW9CQ%1YUxqz~~ zgl?`yf_NY8J0bG2KbiQaQXQCwVgJ=1I5~CE*p4;_u%!~uD-v>DkS&Eoh&`tLFOM`h zwtf}B0eOvVNKg54kv_H~o{k#mPrg(Ff(3RwI=#NOK9TgI88V4&X5QyP9qL(wh)G zWC5R3O=V4X_Twj8f8!beU<+y#rJYXRrq(anWz*=D@j_tuWkuwV>GyHwPnuOobi`=8 zAl21BhfL0h!|GP#DS7<4l>`>TTT{Dj-}HA^u%8!3#K|hk7f=`mInE&qJ#WYP{NL^H zxvx(u_4iIc25kFU*{Z_#5GJEW_Rpd4CT1(=n(5bSC?kbDfecsdw*3T3Zt7WKE&6j9 zUG=P4qEMgepo|Bs$vy~AdmeUhGCjj3%J3iW(#$r3ei_`s^+O2W3AW@#h{Pr&+m3-A z-r`^}{bt@KD-=$_M2G~8zuO&CZ&^;m_H1~Q#R?nXhL6L95&VnlA&uhc-(f5GjynyK zGQluseu_=AZwbY|7TP7^=K1$mCR1r_Cnr2eVx)|7H_A^aR1U)mo$moi`(G}fAkM_L z(kOowqy>^KA4UCLa2wUa!2nx@u&QEUCsKltQDl{Sw+|kq>5p>t_6&RUDk?*IS)Lx1 zNN6Id5kBQ}X3aW;lYy8=BToI%@BMB~s4yrB;`~h8tjkc*j5iPk#w4Bx#CR=2IPQ>V z8dmxmahd7YSFrDxK`HXqzW9TL(qirAQ*hsj)GXV0^%LR1O+qf)Bkysy7P#coF&F2a z%NF?jwPnh%PQUyO#EjYbbn2DTZEN@yW_pVtP4wfXJP;1eic@#WS((j)SuRTAo%!9-H@rmO`#`@|`Er&VoJ$Pk)F62811j#YA)lbMVq9wc7@5QW6}EtvQ>;i?mB z3wm(<7EnI0=m>kt9)&~wfIvuqNiyWm!lxHG_lTb?N4peBLKsf)|?d>21o$*Mmem|9ip+wPbwBWzR%K z`3AMrj#{UBae4ETaX$3%>0K$_g!?u;+K13svWp_CEfAX*5#U+m(G*<)h+F=>CjA3; z&wTSP;BiT!qtj6+0c2!C-`b>#6g%S+Ut(tTqq9*-wjj2zzK+Thc)7D9D|4>%<+;C9 z`#%8Y#ejEx_}Pl*A~+i5Wr*HN{wRg~#1ng{6}s_(#jolRzT?iT7`l91^O6yWc2YQ= z=$AqFW{S%c)@-^yMWdFS#zD+TzD^)9|MWRuz|4gu^vTZ?K$r5hTIgqs46t<(Z}@Rn zOG^J!$_&~dO>=*H#Q>e}g67c!rBJP^@WpESn(dG9y@y8e0m1$b=iGi7S#TEuy8i$H zCJYE-IXZ@QYxlPmy;DDqnOuq}wT*NwFMnc|kDubcDIo3mu)-0ymW(73DlIQxdDtxs z_OJg1_>WSj+xWuIV=BuUBFRs*7HjooaeU0fBF``S7DOxfu-}vS>b9N^K{fbG0kWyt z);h9LOT_d{vKe7B`k?f^aM>LpyH9;JT`kACys(f|;%kq}kE`QsTip3-6p=4KoV(I{ zZ*zX8J=U1HpQn%(O4qE&PBazaJhtr;}qzJl_Q?fwvKI7M1 zeg!Mcsbu7HY0`tQ#8s59gzHEm&%Wskq)-*K7s!ttgF@YSWV08N<$hcY1XB1Ylhye2LCfutZMtg`ko>wJn)W-kY~6GUZb*Ts z=q}oRAK%QXUX8uR6r-@Gp_+?yX==Tw+qdqqQ7??|Wmb%%k+-OFJ;}Rz0<;AJUL5BV zrS8u)%QdU!oQ#nZ_bv}98xi-)!#3Q4|HALy>vs-MRC^+M7e_-8R2cw;{O1}bQ^RUa z(bn>b6;--ABF~7I4^08|5FvYG;^_i8sEG5o2eWP4&XQD**sLwErC==UJDMTeWI4n0 zt-LD@Cc7SEbI}}NRq{2Zbk(F7X6iRwXU4Bu5{1(%h|6JW`*W5$v5UB$XC1y(9 z4e}f}7;UW@yjd zbtqp%z>;#(3}SZD{X}JS3j-D-i;eI8k;jo#UB$&T>J{HygcUJ$VjlN9#Fu4Nr7GeA zNc+cKl*9e!NJ%@Y>v>ImPmq zr&syjTCsckF%A2DKP!g7svCMTgnSYsHNJtnkr?-+e^G=`Mp5<$Zj^~RbJ0%sk1SFaU&A}un^{TCNw z>_55O&w6-4Y%Wtc0vnOXwH)Qdp(fas4^76R|ALLHlN?&(-4f`T`4JZ zW581rr!Y?`|Ju#8FE?f<^KZxJE}N_jKSN^#4Sp{_$|M z*yuC55T~x1ZSlkCx~3y*v-O8pERxEN)C9aK4~%82*S8n{0&nsaZm9<9u(w0f{8X(O zW%*(jhXqGIJS*G%Uvcm}L9Z;DH9io;x$>9UMpXCdC<@7B0aFs9PSaO`3=_Pxi4-6^&A5TH*YzbP3SJQNMhgAwB&Q_Doo4d1)0mS!-s`oD!hap;ke zZzQCoTFPS_MNMl@g2|1=83TYCc61qg$aM(pwterOY@~YjCBDe!c(Xit`V*CGfI};Z*Xy!@#FE4+xs?TN$7UmYnF<+Nf&md znwu5n3ZZh`gwlAsKj)H7YVXy{m;>mX@1pjFK-Xn0G4KLT<&cW-{QKz)n&we}m*Rys zu#j6Yk_>dwy>Zdxj>Nu_k8tdAx<{0GeYwrxOP?T~)O2@uj$C3~S7802Mn#U96sruC zvpM$eA-i6m%{`dmn)hbL_;b*TLV^`mqHWK;5r`K-y~QXYgypit@#J9rgE66wh2NLx zTT+a(GEq&CAj;vMye(gnHem8;!CSa9zPKHo*@oS^y3TH7WuZsk-kGdgN`JK`E{)yQ zR@(zfML91<0jcDr*Vl+T#L}8L6-Nd76J!oA9Jczp@AAQwwz3QRi-jB~2FsSae0Hlp z;rzMl+|jqpbz3J6%&7ZqWv|cuZE1gd!eG@H)qQy90-OF7`uS{2W(=@qqkl2c5jNpo z!1J2E-Z)n(a|B`6+mwLz->>) z1vQHFGuV!VaU@|3rnvWnX_NXj0TD**skFlra*is8GYk$06+9GWNjqLE+*Z#l)D}7t zY$~*73{|ZWKzfpYL$b?NAd)f=3O=MhdX9+4-IHkqJjMWysLlxpWGpI*!XLB~F1O<3 zMH7Lpyp>Z?hTcinS)xy$maTldEatMWuh$vb5HpNqd9?3qh~_@W_EIIZCg+8Mxo?on z0387Vol)<9YI?UGv5LR`(sT3E{>^n5y>^ueRMOP4@u_U+Ua2l9tDeNUjbyE@T}bJX zKRr53ou}PG)#=Z+fAYvK(D$dzgP5{=w^=LX&r>Jn7dVtk2)f!G$kGmzG`253ka2lG zDpX3@*G0Rvj*YA^dq|B}qEMSS{qIZ+jzv4OA2qeH-q>GvIstF!Z{A8mVo zqHZ)K`-CTOVuMvaza+W3j}R5^f1}3L3^kx?I(*bwN{ft7Y}RIpu22m#$A2Se4VxAB={4=5fBNIPk0R_QlrzUcQTO(u8G%=U zvy{fJqiYIrlZZ-=1+r*UoXXD{5@na%_6+xFlx4M)oidv&61fnU{8m}k7+A`ej3Zn_ zhz$4^V6M?OXO;VD&rXbLB9mD$m_#xUvjbjnHkTe;<$XM&j8_PQh|Qdx1b+? zO__34v9Z;C5}iStovHJn26Iq$<0Mqx-HMJqmTY3E)w6Hf}Zwsla)1p-zlK717o8l&3;p-7Kjc3^DlT)X|#!BTAL zW@zDoroOwB1_U|)?O4!OXB7Hk^6?m%*Njg16JA5LKVf#G4qt{%^8uwhtM??|CL7Bz z7`zu~4{gz!lI#qO!1{XWD*)ZQs6js(`Ar+b*ktrAoSvv(tS8tD;ttLcax@kIGZh+` zFX{)gfU`toTo!?|>sw8BI2w%~OYRZI6cxrZ6MKcz(3or;^a{=T`eN!HDA1Z2N^|X6 zKCQu)5{UZeGh&^eLkD3CxdqM`zo^5V=%tANNB`YFppFUvDzb=jva%AIU4x}Ya9Sb; z#HV_I!-PZyo&%mp@?3xatRAs;W1}2(+c62LS}~xl9oItZcHS>n8i&tb6abFX;KJ!= zf~`o@yAFxiQI`O;C*EAKTkY(}o0~{M3KG0-RtednVOIQvuBAp+Jza67jPq)uL;z+Q zl-W$W;eQFNV6ccGcVrvE2%zs+58jvo|0;>CUJGH^eO#YPhJJ=ConWLZ$Uu$AzjJ1WVaoK?L$U+!H-9Ng*K| z5()Cwj`Vou^U@l2txX!yq`4|_XA75?G%0*DtB`-=Aiejkgu8Bcr4RK@%jPQ{`0)m2uhrjRZp&-;R7r_Z(7#J{u2_Pden6Wa4V? zzNve(TIUAqwtxHki|Bpt;E>3JwN*9d7M@6D3^5^R50U(z9yPgaLgqsN9_jphyTLZ( z@oYWQ3sH~Cv44qO_=Gld9-|)h`D#e$im~BUZ|AvA+*D*%s0$@tKdoF{L4>6r_h1wm zI(N`rKjxGa1vtD-V>0ztXZ)UIBQ!v2X^xbo2Gh5>fsZE)vRca^^JJ;zE-f z?9|5xTD~B4C1%Sr&pWSy%qx+YgDdp@{e`*P`e*uu5&zgmcTG}P%pj6g!cJha4;ERT zx*Bu$PIX2k>l{fQ$7CYpCceliY@bAY*1@86(?DbTbB-&%Zc;^IvmLRUEI2U~n>jh* zhVUd>Ud|P*P)7POKzj?8$X$7Vd}A&Zue@?v2H4Fts}O}D zSrNIYAewKN%p}hi0f&DctRwtjR%IPFwR0FbC6=Lvicd1*873$HGkS@=;IFC09xjds zhGgKPG*d>IL?kjo9WxDt{yk{%N~REW#FoPT9`LK8a269FbD+)v?jlMeVk`yy^4`(Y z!|LBt7DXA!d$3f92%6COczm}f0n<5**8T!zIV6w5FB%s^^g@PrekNNaL6R2_V-cY7 zTX^!_X)W_AS`;EllrqN$bnQIHHNAbCbjGob9VDZ{@^Ys4taP;*wDA@*{(Ui|d2M6> zO^gc{-3p_Yq2-#QfWOypQnD484xNXHkOH*Jnog&t>{|={rsc}cYjG7&F*xR!{@K~$ zUD>GA>3>EDsRRhsKd`EE|3Ag;2Ej{K1d#j^6cR1qbg)GVT;Zkv6ud+x0_cAHbHZIc zj|1)t@bUoY<)r2XQya7^e#%aNdV;-j-v*uXsJ#`4^e3dS{0=uYDL>psWFio4+n$R z*DAWxc+1l8^+yZ;8O(X}K}mz8tm@k`@brbRKp&i%D|EsDO02R78er z{aiHO_w$5lZ7Tg=0PpADnr4#}9^lrG3<`zOw-8>+1@?+_s8@bO;3?mLcHJrGpC{Nh z@2{BF`8)k(J0vY9_%cAal#D_|Id=XZ=}!uMEdfhbd=)%&M*-BerSNmkP@nE7y@-c| z@)dg`-?}X@4p*6ocuNXuPpF2<)!uxK<@sm!!yA#DXP8iU^)ODvZH>0@1th47RHikV4k!Li zW$F@3@GHMjytB~Ua;xn`wnC~TGN!fSH&{(zDD1+AuspxRnN8@YWq_-ZKy?z61Y5eR zd5JIpI$=eG*{#DS=nuN-GqzeISV7bv`{b2lUg08!&4OpX-wDY*NS|7Z3Onb8Ih=ik z)S!#q%Pb}Tlk&9c(vv+i+fXnEb!*RThe>M|*L{+(34rE4sQ|GHkPxbfVuo2+bPwp9 z$Hy>6;m3JhYG*Bphf{XY)gLT6xN5%=o2=pISkFJ~S{~paL z8xhV#R87yAH7LBWcds<{`K%gNkX0rpY#`9j>5&KpoIFlp0sI&za_0#KT|D~lpk|$A zlp^6%)T***J*rUoFn$~6MW}b?^UH})*8&N1PM)TR(U zEh})9RR%gd z>=I`@o0GLfZs?IiFX5d(rwa&RlU+PjvTw;^9z#~aQ{F*SqdsgCstb&G!unU}cu-lp zP_LW~Fl7_KFhk)J0kO9OjVJ1RL;|<$h65zAmvceQ{sq)~<7w&8zpGIx?K;DeJw@Ir~?r&X0JTXnBy0 z=ja)FQ{jIVkyc~?md*3S%j7`{--7CvhzfW+upqZ7Hm6&6#dpi8( zV8k74ceMMgPw5+`^8LZ7Sc;h{Tj_Q>9pTNE+V+5R>W$4Yv2MnnTFBQsP;!{ z`k1+0K%bvxxKZJuX;&-p27pT-o6H)`d2=+T1*I1K4&@UCOL(5q*+iu2Nmgo_a1OFk;O(*2T5<<3{hog@fri`&=0A*4EjTv#x@zh)d6l%VVok2SiYkX?l;&L@ki|wMjVkzsQzqkcI*Q%Jmol?AuxSyU#yXj2S zWEI4Y(k3PEZSi`G3|1nkIpbHieTeBzAu)D&sDgRaZnP`g6VQH-RZv!qU3F2 z+I6F7+>KxnPz}2ifHxV1XZ(Pp+{)F-Jcf6?J8xw5w74UVZ(!NC@S&JKXxMFXqI?b& zRX|&6ZbbSJ!SaQUB;gw13-1MnNm2HgH0r0f1M~$CF3!%^p~q09)X{>xfD?7mN<-#A z@VUOBsE8_;=2Q*arlD;vRQqL*c2O8KX5o6p9E(v;<7JXA5rCyN31kG;)gPt9zYnn> zwIj)b1vv2Kc?AFeKwBDb!hvmRVuTqVv3RG9hrQ3s#X1E|Xi25@BQu_ekCo~=n{4GT zb|Li6-PgI3-mF%YM~R2NRP|@m>SJA$-_dG=%FLAl2Sf$pO0bXX9Sn9ZEoyld04w?9 zpdILne=J#2IW$PXRoczed|yu)tEzNMW|zEA%?M5DilhT8pG6I6@5 za;GOt{=7%@(LZg_Sk^%2EWM9Clg&d~jtL^toAw=_d)Gr*TR35ZvJpIA`?!T4NT~m{ zUrDEfZ!H%_S>0G)x$Z@W@~6p>n(?mTFWSNWAD0=|#WNq9|Hzl=&EH|St3W2S)y>O_ zEf-^97k(_ASv#og_g6b-{Gc8{qvv(gEvw#K(Ot9WuV{Z>&<&J7E7 zZsJ@jNCtp^E?`?2e4!QRVei|ie;1tDN@bVI4E1?#jc$dg%2c21Dk5;ipdA`6)?@Ry z*XX&lQ5DY2i(d5$OO^#!4uYMED^DDiHA2%&2V0Rx=BJ=ery(DwOmq}#qiXIg>8DZS zCg4cbK~vO$zjMm_a2`8Ioan7MsLZ}+SHO;4(V5DO^XktWm`KezFn^U9cjjaBtg(a= zvBNfZjlKkY5lb86iZOV0ySDa)+s)RoRO6(K@gGda>ADr8My*GbAN8yiao>O3ye9Sq zbdzskOB{(LnrH-~Zykj!hyvkRr|~VzjS&A~5Mv2i=Vs3VO#VIx2}11X7#V}741M2s z>;RZpYKoGV4mIJvpEIZq>X?9Ad)pE?PiOBKhz|-Xg8+=S6T!$&2T02vOIocobQ#ZB zw)M2P>~M=eMDmh@y&FQ^&*F;2kO8iiX)KAs(fr5a%&d|0sR#Pb`x*B3*r8tmJk%la zR88M!R0F=OwJ%W}KsZ72b=`4Y_0_576=I`UffT^i$JtG+@VO=684&@9S4P#Xz}IV- zkYzI1ivX%sdkoeZ14^VVw@NR9w8s!(Y&K`2SBfA$_kO?X!>w;G*Af)qOl)}!C9O5Y zO>bk!$t>1*+2frrS&D7x!#%oX?ext^xa(}~ulF*pj~|sdz?R|u^K9B+h&1bs&v-zV z!k!qo$hgZZ;J zd|an?qqBadh0V=3sk)Jlj@=D2n3MV3H**Y79X~Ug_^}4HVVuevmOdvXJTN zW#u%9?YMRzs{Q0!D}*G4$J(LAiCbJ%;s!ua_!kO{{n`WQ-xx>(n3W-^vANzRN1$`? znF9tBNp?^4Js#0Vs&dukNys~H{&JTs#~?j7P{X*9G@oNSeC<`UHL{uKrnk5W=}AG= zR!lLj+zcdI=~uD02l6AuUgi65@ZP*MkIE*FIg2G&5wsL&Zk>j z{bx*=`d?MEPrT>^tg3gKe}&wQ@}~idul0{wxz*BzL!S+5YdLiBXha`O=?YG&e+BmS zj1w)bS|=$(WTBig-jmWT@~*b`V@9v$>EdG?Z#%`KHE%7FQ$`E~9J8%~eykn97BWy# zi0kq{YBwdhW@a4xIum~voUew#x5ksrBM%0%M}DKvY|pR zwtOjf=daRd+NDBGO*gh2DYY}?K3-;)Jtyw9tO3pBxD&N5=dHV!p+^752(T3aCoAqx zL>BZZ7x2)ucgWM%9h{$&Ha3=WYAmt#7Jtr&bEk|8^;lyb3!#bC<)T5g9%hD5PwVr2 z;IaqaE6`=G{_@yceUT?xH|A+dJ^&(qer$5Nhbr71IlU z9=l2Zr?NXv{dOE*F`|o+Vy-$6coHDn&3?~S=W+&;;}@6j^LSf2Q=4n4*b1gpbE>>t zQf*zfrD;tmV=TSW)!xkNdt7D>e;L(md{pk7@)FUjug7k)Cf%1%t*lOe;A*+r-&69D z$MTW<0L!jIOV8$VM$c!FxEBY}{MMTI=82`{)8n1tIMknPI}N0ZfetWPF0Va}npp%q z&WWnLxXN?guGP2kMIcIWs`6rR4%eExwbajUo_dPrCax@O7l>(yhTbD?0~3T-C;Mie z96wdPw~MSZIg(xuN%~!RY%uxZjFnDcXNE;x;Hht)zvo$$UAM9rb2F@Yp&1<1(Vz4& zmFs-{0$ocp>l-hQu1b{Jb+X)|l0B)65suz@0E3j1Lkd$#?saDm*wfT&E6SSosXL5) zSb8e?m3CPx7$V1|^gYMD+Q|8#p>{<_ZPIe;1Fk)OZ&qREh2yeDWplIwlV27vp~m{! zT)rlycf(HPM}$7FzKe=Ci3cGhxB7o2cOJdE_QUnGJXw9$Qupaqi@9m>M7LLJA&iS&_i$hDPf=9BustUL#|3{i(g*L`ip%DlvMgT zPDKfCyp$-bTP`k;dj-#yVC5Cvj1hTW_Grp}3wWbOL(^{`X)@_~dmyQ<({tk2l`eKa zjhNwrOV@VD2RVL?e-|CES+Rgd9Q)H`&8=p0A0Q1f2*oj#A@%T9-Pd$CEL#)g1-5ayuIq<$hxBF zzgJ3FoV>|7Sm)e)YhS$G=l2U(Y4mOHYCzkeY`#&$N*|tZQfmzns};2i*er3r)Y@(I zONC*d)qqEMr0T@3?q1NNrXUrpWy3B;6^|gY&o0TqHuuEkUSFQczPt00{wm2Xz=kWW15>X4{x z_nom=D=+~8K*{BQ#b|C$+&gE1&TA3nlfC6mvjYl(H-z4DHh<#KRZ|z@HNxq}yro5L zd+>o(tZY|_jnSev;ugcBdPy21-@(z`|3~L;Y@c+bN2P2{MubnN#|mT+bG&=pa6EoD zSl&10AQsVpFn{QDwT=IDA!->nKUV{wb~Q>qmuO?h=*dU;0pJ#3fSIh*+B4AIOL$GV zSCl`4Qq3n|^`B>GR^in{(a9S9)Mz^-vbIIv(HIZHM6C+`XTbSdjiiZ<4wfUQva~8+ z6!Fy$vw>rx=ody^o@qLDX3+;9+AK@!@<-i+!pC13?-(W^6im)e8& z>6LKm)Ei90tgL6Gn3RIy8wc{C($*>Z*}L$tjga$-QI`QgGilLB>C3;R*hMZ^i*GpW zf7f*Tj?3z=6dpzvY?L#*){Pv|lxhV3yoS5F_t$#fAjgRvDF$jHkLA<$r&^DBm3L(L zE}1V>Gklag;JKsyv6b9&mW^~CX;)Ray3@69om}fBg6BmDo*G zk2!)3cWb=7wBzlZRX2YnyMBruq^@*pkBUSHb++N()BiQw>PE2y!B@_|&WeD;k8Mn> zfUAR)jkt(y>6X)`5Vi*P`C;PM0<*s-9zR4i`qOJI4H4uF!}e+k1ZEf07R`NvNPK!v zy2T%cW%aUIg(%$^V0)_23N-5-@H1ss6#e5LNbVNQ6}*Icga(XW7f~>==nu-_NEE^7 z0XGZurUQ7GD>00+15I8ML`CaA2>sz?#NM-!SQwvbyyNw>_vb&|S<|_AGj1PXu&6s$ z>!!b81N*$Nf+xHE1q=S@5d?eLMlh7%$o0T~{LP@!+&|xmbY0WGx?1p`*-LBou<;1; ziiBV5nGJ;Qp2$_HoGN2`TKU{|n zr+Tv=+U{Mi1dunh_kIq;s=l*xGHFI<$D>f~JfVK7$~8sJ26VXRokgvqx^M?T2>;ci6EbPLtSb;M zl)DagqS~Qwr~9s4TH7kmY^a|m2o-Oz@h@WGxToQ~N~y*p2D56@BtNI3Bj#nY z8bUD#+`nybr(@qCW?fY9xI@#WvRM6sxQ(taJQD5iI+#@yf4seHS!?fkPl+#`7M?d) z-!jWAU#iuD$T$Pzl7k<0veC_L?5+XBx>0fIUy^2}pDqueH)kNQP8)nH1&_wHx^f_P zC-Gfb!2TJa;gCI1RwEVa@?)7fnv~cA7cAIVWF*989E7U91ILd;HJJ+L_Upq^(n#Cp z{mjHn_}W^{lW}jPhNjknci}URIcuwsYHNYB$`M8HoK7a3&pdx`eM7ik@Q=3>+~Ky+ z6OU(aM__vO<-9sbe`6s6DMwK0C?HEwT)?=(($0I8K*YHJJ%=Vy$x$J8dc zc4Urrjvi%ibE@}zXRuHe*BR zN(&VA(jN_>cYWm~_t3&gBCAX+wau%QT3WCDpq}d@0UAh&&P%7_h;ra79ZBzZxt2Ww z%(x0$aBWuv$V8v-@~|blp}NyA>qTQ_jx)(GK~Ce|X!{Rrm^mJv(bffzc0|T7rqmK| zhB4%1ZPgo}na?g2^uXUmCK6kqtv{|~wsE88ohOo32Fo#j;(oVwXV#>r*8U9B5}_cJ zEe)Ql)_VQ=FsXBTvkvVOmd<(7+%x0SzFw{}W32(?_8J<5?F$VyXtI`D-CiuGd{LLb z`j(`mef%OVKKl=!H4J$|*E@x{#|caAat{Ei@(ml;X83z0&SYQOXBAU73Iv32ZYW{V z;?^Iuc+ePqa;e6lg9}2xN5{k^EM9-smpQfb9O743SQj4kV4uBNqzw{7(lq(W-pbCw zl{{LuRHv~Ue3}K)+pj0Kz7FW6$XuA&2KVOdfZn&!RW4&VC(#4i0?K{Fl+Qr$@F@O9 zfAluwV(Uj_#VxD0e;qacGE|zWov^Vy$12(@PrHIT_`-2n`*CW!vB%1A&A&-;F^&=x zYnL(0Dn{C}tW>HGB9sKTmr&LK45`rTTX(wf$M zTUw8K33oBHB$ZD_0s^91&F^M??W&hTlZA#$E?u&PN9v6S4)DY@O(u_i)3dD-L*k{n zIuD94l;stUk>c^{Li3j+I;>qG9s}A=q3msTrdex-xtClYpJ1P?$ub_?@>(t=UZI9} zW#{O)Bvx$(RWEaAT?O$phO@W5cc0+OSPwaMoZ>Vd*0?2{-GqMVSb;X3`z6MY=)3>= z5LfngouABE!WWa+Qdl)6$|dz`wca-&S!XW ze3^KDtLmOf&ztuX)kpJNr=Uo0?xXK#6+SLhX-rgXMcJi`ZJ>6Xn9sPMiLa@_tOLs} znL|xkI$j8i)2p7PsumfNe8|&XWIo5o_>2La@Q|3T+LK~`iHXE&JY>y0Pah*$TOR#o zTG-#^5M$%R?TAri6Y&hY)&YuUQ}n%Uo%QtIP_CvLmmVQA4|oM$51;aA%s#6zYz*A@ zbMwL5D{NTX6u)RGi?qpEH?V0GPCb-My{~SPC(fTV1X3Pd?T)}KJJPe%QZ8TTVk|ZX zT0ekOx_agUnbUB33@u3FZ`_v=`rZGi+_hr@Mv(~-L$x4QMTdbPHhc)q@i=tf3oSs&2DY*xlRVr6F`nw z!ts1{xFS+}3XWTu{rbamG>l!p%BI8n$n>5hBoPibb2L1n+co=}G%g#TeR)AgqoAEB zjaqurB508(KCdD6!Zg;m*YAjfh_raGZ)O_?-#jMnzLg`XLf1n!nB$;H76i3Dh$~Aw|D9XZ|;|n@hjH zMqOKlTtc(f8#?z7VI-mju=;H)CmXMe7fVOBW#zhWZg$zhWm#6HxFU?oP#i(5^^0ot)~uk~KTs;-BgtLS_2s^2Ke2AHRXc zq~Kude11{bw%_V7-y5#y;}Cr9DP!(_sAv>C_jtd(b${Z5B#0;*b*zVsCaI224vaae zPPA>$e5;Kv$G>7#GrlFoXr<BI+hXGoULFW(8PgFFT06r+G4!WgwaJC-O-eK1 zW3>Q?WJB4DHun5el&^+vKOPx(yv|vf2;?kv;i;^P7@kxKz_$gc`)#5Z3(ykrC&x2V zC)-?|nCDJ_n5;3h%IMeplr)9$uP)xdkE0JeP+Tk%=(rzFSDh5jB}O!oF@e z@BSwLpjfhvhzV-Wjajn!alYD`HUXy^#DX_n7da~0B8(7Ari=N za$-DPcOcEIbxYPm(;$i1B)H)!gA70FUb(G5SteCoU-H2>t3RaH`W+7g11BU+BM;2m zc#{Mmv)tROlj6BE9l|k;VsB@QpKG#+1<9wF2|No)=sKV`YK>zjLZLUd ztf>8rz$%T3=vDNW;}^to_uIN|nf#brvSi8H@xj&6p5<-q9!nnUsZr(cRdq!U>g?K8 zUH`K8d)w|Q2UzI7~Xm00$7E8FCT`h)5n&%8hVB~~vm6tp4) zpM7ug*y!m~E)$niN3_?jRZZ&r~i=`CwK4L$EF{lvdk$1+pc8p>Bj7BqZi|lRjQ9k*%EJ={RP)RHV6cDz$FxulNtuH@VV@HG~Z^omqXF^q7%YsT(7v)blt%{gxxc&b$ zIoEi%&FG^f)t<+L9U5fu3Td#87}=zYa3w07{R|WhF?uNpvi+qej)&MGyp0S3r_`nI z&SfmWtLk6-bGixBySi4$I{;ut73A&bAe>A9w_JU5*$zFo`5OC)@FYOaW8`_=zJ@1s zQ%8(~z4I^UC*Q6cl!R6HMPePZs zT^eZ*aFG?qrFBXXP(ru!<8mweu4&%#?wTeYYc?WEO8XfR)PY?5S11&dl{WHi#C*cM z%ejvqLX=RxO)eHf_O!xOxOv1*mJ2=r17w|4cF-MqCCdb5Rr{z{4mhj{Xs}d`xlPq| znFZLrY54$?1*uxqjs>!V-(U?4eR)drtFjpVqpEIIMQ;|Z-mBVNBc{T3(1l}5 z+L-+4cS4u?E?8wLSz~>XyBXYr@lqKP`vJ_)Yq&6mSi#>Mj4?WpQ^^J9kfxFjW74>K z>H+lb-iJHWi}%rmKvQoh_Kq`DEsi+UX04R>9K%Cr*juc;r4q~p6$Q1==t}qQJ#Kh1 zShzy}4}0Go)b!f@8Bhd4u^>f}VntEFLKCH^*hN4I9fBymDosiN3s?{=C?G8qrGzFW z^sb1C5I}0^B2q$A2mu0Q&m-J--`~#e?9S|OXJ>Zy{&(+iA^Dc`obzetHVY8k*+y(R z`*iIa;{1U8{}S!9*KWfXu^m*$+ zn2>{-W53GPSa#{1K_1cp`nIYY2;{xFCJONV)A8pFpYcoRHyJU3P}CgP@q6D1 ztWIXBHk|(l4fj8KrVbuh!+_u+P~47r+yF0cC3y1rD5j?!l5-@2n$*=(~qVzKHP0a zhnd%5c#9M+)9|=SP=m2DcyVy~<@9wrck#ia{*+f>bzBC_OZV~6&+xY{4vM>sNRQ70 z$eO8CYBL!0yjXJif7uD4w>yI-)D~dYYy(y6OWWfhC>>9SO$Cd(B1q6vrk~+}zXeE+ zt_b<{@n=QdnfMfV>~M#L3G{(cA6nt^FH&&^I8#2a^L7B5@YOimK-DD>+?Oo(8&3Sw zr9$uwAh~NZru<7E%ea_K#@4z2>;uP{)Hhwxm8yoEsn>vZGal zOU=}kD%Y?{W!H}+<==42N53A(a6Xx>e;b8V>gU;y5nrOL6sXDT=Uji*Mzq4oWd;~? z4x~&{WLgb*+WcXWj~>k4OU)h*sXd>KHV8NMD7abL;YA9`^E2aCDcW8@=__(3Ey@lYzdGH#5vNOMqi|DrhY3*ns$aO484eAn`lE+@R_GCqGn3>zqvt#83 z>Ji^|V?>ja7OQ}S9+k9+e=e_N5pie|F8eJdZBREc7N*Gk`0j)#A6-*v2baX6kBPA2 zdVF(4;PIKNBE?$;aKhFVUQXJpv+}6U#{O&h?8eRi+6wFYTy$k49R~AY_5To>TmDoS zjAa&*AVtD|0UM{h!XzyKgzD1$+%3( zBe3Q#MI-henmkd7)P;`D77o5~l)K-CY&$S54=-RExJ7e;JCtulzAGy13o^P@ z#Tx<&^BRz91NjNYGutLPy^c7guSiz`)Q2u0><2-nrCs9Tg5&_JY9wc^(@A#j5w8!p z7rZXbiXl04eJsV%?T{iN6GpGk*Ab?PAjG6iA5y)?1XHzMcxm^A$F*crjdJG_12gT( zFb7b2E#^DM)>GTon_X&a4js(BX!gWlkUPx|KIh}5R9Ykm5cA0z1RR*VBtSEcU?s@H za4#kstw4N0QjRLsK(+56wHn(XIuRb%D%~aIfMXZFQw~GH^{q#4Iufp_KxfL)Xy(B;B?G}mz08FA~;2n(&WIW zc@Wq4vNl4M(T6cF6?k$84n79J+v_e+#Jj?nS5+VZ6Kx)xbQCD=UGo83_M84aaCOpV zKLH6H;$q#?xTJ8a60kn2g4=_+RXB2$~WH0-MZ znjSjyjok-$kHeAMpmx(RLt^)8-vj+K& zu_;PQfJ4KE`)&}-{}cw!wd`Wp95xw$wU7H^>H?E~;Yp&2ODU&JcjCJ?bG zO|~I{4fw#`r+8@h_JBWSA9)dGZ&u#XEWonp?h`#_J<$A&wrS8LGca9-O3tW=4p#ih@Hp z8e;08XRs9dI}N6}d^#Iz?>Ku9`gx~T16k(HwDDq)Is_0ii<~D0?4|T_MC<5xCh!~9 zR9Y~L&&{uYZ&)yu@2k(^`hG-)`Vlb0Z!QCH2kg_I+<`YNL?f@u)!^x5drt;NpXJuH z^)L96M<(B#U4{~G+sPL1IPITLvt&#&e)nZ3MLoW^2;D}J_IB%(707t%KBuLsG`9P! z4w5=ViDSie)(f@vkI0QK!p)ZjLyen3N#Kom?d>B+VNa-a>&{cnB^Digv=iFCL`RZG zn!;gWDXrPtZYA_q#apG-&|H4XSLY?u1Z8p&h5-C-s%C$JrZ5xwZQJh7_FStE9>*O` z%A)3(Exct*f7BgDK}oG^I+T4iU=uc2MBY)f{@EOmbQ@C1Wj@%W86eO0`wZu|+r5?y zmuh-@ioQXa9QKL>B?o0Jg(QuwxEjjPWS|Y5jR1ZBZax}bn?opS{!GBBo9ZmT!6n-@ z2C!))&4uJO&BvE%7tkXtK@f9T(iLV{7&K)ZlYNtY^HylvPbRaqBb(uKvd~M~J5}*` z?_nTj&PR4k^w^wQ2QWJJNOh~vu{(Hg*7NwK5#Fec09>kXu%*>2MZnf%IFtWY{oWHh ze9x+K**8BAtB$+ImIFa@McI-B*`&eSr*Kk-JAwr&B=(mY6n%;wn)0>z-cU3Vtet&D zu_i*Q?43HHf@Zj6{YkYIcP!TKAmotRg4fAyMNEFJsOl;$y*g07$UZaS);jm6+IAOc zP;a3!7H7r^Y7R3Jr7#$ zg>%tfe=dOcE06{othzzEw$JWZF*u*@RVP;&P7I!KpVBLd z5M&cBiuN~0?#ozza=C}KFPZ!@f z2pXxZbx=TxYHmDJG}ke5d2Mw2%bxVjXlE2)uUBkfzLBwN{8?-pJTHec8qv2gEK9&9 z>9%K?*Vt1_jqEz|>@r~%q+H4f%E=jSWD6-PakmjXKCi~;DwPo2@}D)|xd z&UIn-Ftt59alMd&bVUaA%=8Wzuh3#OR~W1eVCd<(`2Z}+d5DBCa_i!?dF=!Yh*1YO z-hYH@SbXP}4sRNtnDXLuFjq|V)2%!l`S8;Ys98khlCa0n4nZ(jp)*D8X3}$m>ohn{ z#-N|b1103bIl@X8U+2$h*{j?H@(%_8ziGfGrSEhD$zA$Aox4^L=QUa|80}OsG8gxF z$v;TIuFF%=76`E{DRZ^F`|L9yRSC5Hk(g~%zWs7ZoR}KHCat5}43btd*`|I`MqbyX z-6n05b@`;D4Pq_T5^n%O?cjiStrl|27iEAJBpYgL3v|~5<2n<+@-6RG0`^d328O<$ z;??|+9t5ySE)3XhYL)^Z4!0i9MgXgkC|WTbap?NJG$=4YrwlIpGoFS0cV1HNB6CM)PR8bF@|F?m+HqOfD= zgU`{54{1KX{B%W9u8|^B#;<&{h$HVxZ{bHTGN_A8^+O*N!Xaq~aY*AUb@=T$w*L{` zy!@m=PwSzw>Wjt_D;t?2xS~?#kP%-=gu!4CGySh0h4y!_3&02o#Dlk+h5jC1KEHO~ z__vPQ1PpzHMsRZemH)!QF!k?0Rd<|?3|De6c)7lw|7BTm_tD#>4~4J|`$j5s1tDnN zD`B1ojoC=1n};TxihFDlfB}cVTZ35YlDS_IGk^)4(;EaV8j*{LN)mX6hg)#BH4;>@ zud0CpMjhXKqz#7FW^>NsLvG<5oi)lg)>E}IZ31}Tr)0pU$@HSH`m&|%{-v|2QGkhl za^fECaX5|hsIyPj>H4}w%CdD!?L@S)*H8c?*Ea~%P!+vqmx}D8T8)g_(#0cdBul2R z7J&*h4XBR0L#c^H zWbjvhKOEu%=E=@^3d9EhKaz>sQAl$9;cWr+3z4SnQ0X^Mw=dt+-m=p`L582I1B$$3 zy zSP!lPmBLYs#__mLNLlpmwMe^E;RM)^1b%+nqRZT`DlNPzFHq1sXY|-@{dZrR590mT z&4AEyY_K*0gmnCAr>i|BTtW`s9bCw{RnmpGaOzluCwsclP#ZR@UWyVYr7gO?Uhn=} zFG6POs385fSjc{-U*%4qVzvra5_SNLEt@KC=+K@$m=j#Rp@=*L{ku~|)`sUOAPE%7 z*=X;(A5E^yIhZSmHU-U>E&SJ+`qLZs47tUT)yYgCu{sNpwi&n=UwDhid_Wk$l#;2g zH1`qM5T5=5{VQcPV9+)It9BaDqvLd>jfhb80ej4N#c{02DHZm|$g{TgC%hI!41%cS|G9ieik9Pm{Mu1F!jb@AzJir_wbIcFZ z&|-YYW**>Z{j%nteAwOvQQshqWzi7_7x8PLgjWbCfT}&p?)`*brzpk=V8FwyLly*l z3$TVh+?5;n{>L)D{YunytNn7`5w=%j7BvAHZjvhcIG7IFJP z>3$yN93X#mYZYjtOWjP{;7}#}8qblrP`fX2Y_fKLs#^wsCmy`ilA0c_pDyXHChEnl z9jE$upE7mvk$5NwgBxVU<*#tyWv%gXpFcE`E_*jL|<(|4(x>kw%XYDl@@Bx#R?~aj*q_wc|jwoR)cAQ zl@j|%drj1>_MFJh>$y`aJls4+__Dx>vp1_Tav(Vrw_7)t?EeZrJ*GugdADLw;U%x7 z*?u=S{O_taqXnVHB&7{A?{$m?eoBrA85R6lpZ*o;Ia~q4Wsi1)N>)fm@q-;mkh}Lr z@dz=c(<1gcPU6t!O7@QcZ2?FG1c}s0R zam0+~qiwSmtLQdHX-4Jn-etcOFmB}__8v+Wyw{xJyq`vH!<$g}ya2H5yCH&{8hAEm z(+-C_B_jAnebTTAu5PYV+dKQ4cAD-l4_Rs)=P+F<+I;Jgh41ltEO}aKnjN=)74Pnb z7DPwXvB-GQVANk6mLl>WwX2f5)MaDIs$!(IDV?53-@{o9`u#yl&}C2!(wbY$K0ews z-UvAm^b>g3*jM!%MV61rL3Z%faDha1vH*0J2)GV(E3UebyH*Z8BA}Z-X&`prh5rM5{iSZ^|{{+aCGB-UXoLkn=><-4^xG zqJdI(u;d$93=9C&5HN@TB`I(DW$FKdgMy*cOwP9OHm?blNF?hf0LR+Rl4KZ8~z=dm#8YMe=5SQkIutEj1d2jr;aG!sh9=v{VUN1k14bd;25d+(3u65oG+cgd}Vw*2`; zts3sFdKCX;4&f(hdz}ql7l3QZ4OTu?uK&oU!l*D*5sdi&=$t8?5UxiSEcmG<~#!^o~%ovz4Qlfpm! z*-a^@PDsC?Wjkkh#zAk=0AM~KDY7=mv(2rIkkt>04Al*2vFmbzDE8|mgWWIM5*mYmu{B+{3;z5h6&-iWE0mMrIEX8> zJ?1&oAZ9z4wFe=KGyQJVgJ#56+V53ZpnylDsI=^U8Z90yQ}6>+eRj|ar29Q)^`za` z=HjM#*McD#HZA1L6yz5U?2}%sIDv2Yb>`UKH!TcSBmNTHvv4C#%J;0J}k_A6E z+rZ{CD5YJBuD`x*^&BI$rb4HW3~@Ce*CWB|%9-&l!NEd^v$sSV;REUUmEe%J0QF2J zkk|#Sj|1oT+88vQbBA}MCFbNRh(CDUR4@y^^?>Rd~L-mZ?^fkF9JPiBc*$8-KQ?e7Y18$@}~ zB(5cVlby=8YRS1SWKeR%9JBDJH_>kTb88pX5HJmTBp^%_9VhbwyR8b&mT^k`u%siO z+#eSpnFSxr03cJNeM~sb{*Fb!7eAr7OBGMvJAN3}j_K0%x;?0AB|}Oa@>wL~R&!*K7X9i3Ox;^PS7hg%C2F1xl*? zCxdV+KedH~8hd>4OM?k_Z6_VVgeWyqWwy_hnm){ox7)?{5G>`ilE;M+`OX!Jw3TOr z1kB0ngg4)aDsmlnoVeaNU7%h3vH9-c`fc^(Zo#8pk!aSu*1+7VF=73QL9kVf`t8_d zXd-m^x=VF#t&wO?j@31I#&GwPZunKrAIvtZ;Tu>2L_;4EXbnmE_u!s>XBTFG-)R)k zY*#Fqa{;2Mga-Q;2gxGwN#nCho>&_y zJ>lyx2qYkduWB=Carv9fx^cjr;p96g(68k8F|L%%85@7C%u@5w=zdX=4)?K+X*Y0;m{J_-DT9KL?Oxu8Ynd-e@s{+P#fywmnbY)7N4}JFO8OehEggLwr zkDVBco@$R^t7QJgU$gU4>=|TrOfnf)636pnkiZ1)x0ZqLYiNd7hNyeYJvZ;l-&5sg)7r*^7k0E$@Sx)xty_{U+@}WeriYK_ zN7vwL!-nP$h~)%UK|_<~!(5V7)nNo6@CXg*DLiQzu-!XdYN{Z&YLZY!2}5hfy3Mef z+<&iy@6@gzU~Ab<2N<#gbelF15Ese|acgS3k!7WImuzg)**+i}@?hUe8`6FN-1m^H z9l5ljh~^;kh&4X{eX?B6DJ*3AUZ77q$;;pA^)33S%~oPl&FvO-T)n#-e>73u>AGXY zj8NK*R)*<`b7KcK_XB|8+Di0oVLpD4O4hjt&Z-U)_569>9AX_Q{hYA-N0P#Fd!=&c zQPKqtFqe4C#kRxpJS63J{jMfg(6_{yru8Oy^3aZ^GVr%%rF%>P)SU^s5DL~<+zaif zMbiO@IZVN@zhTbv)E$i~M}UswAEfu0j{JOP+L;(EbyO+phdmCLe7kpP6_e>8i(3k_ zQ@yfX*&ZtC#U>l-$ET~+e-44Kp%sSz00BC1QXOjRl8L_{14C@c#lCdz*ZYOh<&V14 z2z1BDdI-9eLh%8Jj&BF-#4s#@_ysV&cJLaHL7i5goJT??e%D2q7iXZ!rl5dRIDJVe zPqioX*}iW$F{yICho`azWWHA5gUEzv=?>su1>Z?LOtg0YzYEewJhG}d3~S)rz|StC zQ!<*2twh!Er`d>R%Eo!y(@Qdkk^0yafBE_h_|3tE=PyM_8kx*>MC9dW#`olilp5HO?DyFWh-*1F zcO59zvxZ|X8}uWEJjWhybwLMX#NiwMMi4y!+qKWgZAEL+!DP4YFLkXy0JOx7*4)#C z*t7gwq>MYUXKY?o{#Duz!TGhCbsT;gDVRnmzfI7C6!GOKu zMlyi@dz*}PByG-`?S%$hHu!hAGL(cU=JeH|+D}4^I{nNZQuV6wmm2{oeZNF5-blUS zLJVt$?dPc6tNe;36uBh?ps@^*1ftx?Dnw(jM1IZ#6OYb;)aD&*sPzQTYQ-0}=v5>!aERKZ!N^Y=|4N2$Zb1h<&_QX20oG#^SUw z7kgjh)Y4Fm993}rH4o?F`ckZTFATBXGK=rTAq@4}FW z44e;Y$~Mfr9>gW35V~x1SJ&^hjJvR=|A)v_&0aK}r$P6Uesd9}+^ud{xlJ3t?A+-q z3DF-#-FXhU$l!JV5(8^eqXFn8*|s;*?h8STm&<6|Z{;@Ae{S#O8C6UgA0Kkq=RqY9 z>Hqy7^uRpD-SBodL72$+-aXf5p)(s~Iu|Y1bWHClzW=EhnDv?3nTOGLr+M*A^f&FVO@AAhIHB|lf)D%<)5s=P-M`6W0P*<|;2|j;qY`kMx;?H%(iJ$T z*A08F$b5Vqw;X24Stq2>#8wL{sS3HCf^)|Tg%Z$Uv84xX#0(CX3-ullVPOrnWyd^- zCbI$>*gmobw&W20iSJQSeGE=h;Uo{`tHs)1M6dLLQP?0X-Jq z=>z0g0fXmuLl%cJ#Bw|G!)17i#sU(q0l!2q0YwW^*P#GW&=#C*-38&osYuu62{YOl zJlsK9SFnwVm)}6Pa1CF7CQ#e*li}7qn=gwUg-uE(P`k2W2&pRSI__rcmL`{Qd7*x> zbFe~q+`?z;%%3UEkQBXS=SXVf4gIq=IX;PO9#v0hhn-M?1Y7%NJbH4xl%>1VPcoWijhOf!3cOk!BKG zIX;**km07xxb!&X;*v#KZ^JHJp65itcWx0STdYk}E+_ASB45QeIR1x~*c(9!bmhRy6C378mz|x_@cWocuSGry_~b#!U_Z(mLqzF!bP7nWjbPEDYNe-2dD} z6qu;*YZNu%tG~kuGKyH~)Io%OSq~1esjPHj^e#p8Q-be7r$$T+lr3YT^e zdf79;J`tF?$>CWxMXq~$eum}JsAh=l<@YM(kA3rHS!H;q2h7#~5&%oCLqH5>~d|U-a6sD1V=OLf_eGz25KiKzdZsQz0svMnkXe>Jbd`pGNJ3ZP^P%A z5o{Yo*z1AMdl&KkJ(z4=8_x8}i0;5GWwTd@ zM;?|9f35+UVM_#ul}=dW!^JCAXpa3g|Kg8PGD10<)aXO3_hb zS3f0SAjqwl{+9y?IiMq{v(+%)u}=l-d^NX6V^WCVJp6;OQL|D)dYNNTxwAu&)ie|K zf3zm9VDy%)8x+9Nh!7C^EC|1TEOdH>r8?@_?L&I!Jti$)g?3W!)JMk2?& zL-BZDit z_34?im&Ga(pWO`^E1K-xd_ZEH45u*Ky2@xA!%dEvgRYuL?3Y0xuz8cTm>zjTpqm){9!u8vEo1VhQit{ELs2xR# zwNZ*k-(Sw?#vAcYSNB&_YGn>k@J0%{0Uh>PAZFBsc9uu_i|F!~b(DlFDGw>I6~=e^ z*oq)I#tu+0#A3I;ya8cM$bImUQE+BrZhLD#@qzXkXH&3|kxJ%&9oL>O{4+B)@>Y0H0wTAxGyz$&NIt^5H28&_N3xku?;O-NqDg!)r3K82)SXj*(c zw*Wd8x0aO{Akn$_|8&K%7}~PO(WwObxtI1_&kfm_J)BF*%TKj$7)MO-3#3my!@oFn zKoJ`Is}9`H56&?S=F1Y8E12LTK^EsDan|1uirR$|9noO}85688c^3(dOo;~OzqB7e zLPrih>t_uHJ;reW15z3x@wf;Exms|ss%yC3WiTeb^Km22$sw2U(ETXQNHA+DZSWial%@{ayPQsAEWT*s!BqDK09hajruEP zdgHD5xVx!36+2&eDB4t>^iW!nr*OlUPAor&75A>?FWe!JvyG>PCUv&9+G| z_=NNs&r4SN0Zjfzut6%ijA$YI6EZmf)^#KTL@ub8MJbAq?3e}P39C7PZX+5umU+qbI8b2Q%li+YI@5=n4sqQMvsZBBTM2%%|RC zXdoFzl2v*A`WB`wD13xak^x%5_?KdD)OLE5|XEY!A12y_$ zc6?HKG`@1G=(O&2oO}0vCs*oLT?1B#qeeF_`iQQ}Zk3SG3^b42v`T znNAweA3q|uAmP|}Apo!zMg(Fw7(F%*xDRIi1Ff7-mwfV|Pw2lHy`K!MSaErB>VV^{ zE>I%%%);)%t}%#okL+(*nPAKP4!~Dxgkj%pvD1m%4t5e&dW!fwBpA8m!Sxb;N0@@c zKNYf-k$X>oS_BJlP-})Ucfj%oYO66Y~)_g||mj9q<{a>l*r@{zwA(>Fi(`^WP!Z`BFifOrpx$3U{3DUdXIXt{Z^92aKej+~yYa`utP z27Q;nBDw>9pXek1BS#~*Vm1bCQPP1#M-;^b`5L(ZiOVF*t;K9S=^WVv309t4;pVMIzT7Yb> z*tq_8@UZB^tI|nEBxY-&y4ZQ%vj<+alztI0VDSZM{POQnaAKmW3DBrH5FYI8({Noc zo_^y}FWm9m+~>Ad-xOBI$PB@`z<{1}Ux)G4_$_wap>TMw+aiiiq{423HU!5Y(W#Ng z@H==o?WnK?<+vs%Mwms~_mL~S=BS2uxpvrvz`8|*`~ZRV4KXVy6BPw$NL1^2VN{a1 zhw#0rBZs3lHoF+&W#HahwHkY-p(+X$ZZMq%!_my4sm@5~#dv@|Y<#}mfWQ(PKCcQeoIPv3EqlK^j$J&ZSb@b2ltz%hox z7@jUcn$h3KaVHuQ2N&@e%u9>j?S`4#JLfOpqe$dAcrYc9qZf2fDY&vyWb;)Vr`th> zG^s->4Yom{WZ%3AR!lF5kA~&HptGY$ou}`~**;4Vc-fWPV@}?;1?am4bX!F*pNzq~ z;P#pf-k2=tijn?)?8Q0ZKK@nZS0iJsw)@^h=+`*cHL&ZfVjqt#m9{#P2pNrSphzF$ z{zHmH>j#kiiUII>S7-w8aJpbmdpQ{eAAJ|CMr-HfP5vg{H z`uqhAUG$(k?y3MY$e58}5a9f}Ko#9s-GRX@2yWAxYSzIO+>3h9Iv2Pz;~^WJ^UpcQ4Y7=xhZ_U)B@Cg$+FeL1WP_F<2%5$a}d_56BPh zw$}HTHtO+M!t3|RkJY-01*2k{gS+tAkcFZQh^gdz<_i>MhS49B)?({|dde>e0Vthp z^?NS9a;!m_2X_Z8>+YfEsHWi(7TIcbzcGsvg~iRFN@+<(>*F<0X34!gx9d`Wcq>NbCpcjpPflLXE>%h1 z&<-ZNR6umnEA?=%77l(>c;9-H9|D2Gk$hP4M9b>?BQMbQ%?>#CSaDCC>nBcAO_pxa zI-A~jwnKK}t7;5}quXJ2@fCOO;*xmSjEsT?8TP0i-?t%sCsKnUOayz(#%Bsxz@i!T z(pvwszfdq1>FJ-Q`$u6>n*GjG%GV|1oiUlFk#7^5^A-?qUZqm*yG;J!lehPd)-12lhZE;4(R*ekA7-xG!oQDJ*`()=y^q|9>o;8c+!=eCoj!1peVj8#y_q;HKAdq1Yr9bxZv4)k7G}qZ0~AO3;_Sx~?Y zbaj>bwj`cdZ)QJTin!%_Au!`(SKbnL&inBjJKD)_k+xyxfMJAgIDwMuASa`MCvBVX zdW`SP{)%d8YsTB)=#h%lhA4&McBuM!uCPF;48XBGyk)_LpPI8^$m&1Kmg?i)E9HHa0{Oh z$j4C|ywgGV?+Ux}3NNgcajO8Rwu@?sODxwfi6)V!v|hV{oQneQ&epI*UrhzfMm^rT ziX8Q;0*2uE&6B!Xm3<}1dJZ}e1LzYv&u_0)tcS9hr_5zv7K?x6A;-W!&WH}G>ltLI zKnt(p~m0MsN8X?%8JIX)Sz0tZCd*ox&ZRLus@%-=}z$QrN?D8|aj$ z!uz1MDhIxD2R&_9sVaMu!t>UOBJQ7O7>`ks+<5WSmDXo^V@DTmsx7Ieu-^2mQV=Nn z`SMaHUa7q0E)!1hE^nkdwBKwvMp!81)jjaGrQV9UeJnA@p>Zvit)4T51@lFH@~3yh zfyn#&F)fjD;fEX{lF!~b>54tzFzqP7$gMcOWt(^#T9RmJmIdni`Esu{g+d}ePa(fl z#!V42Tk`^!yc-I&isMxb^01Tr^oN+8JQp~9r@FOI)2WxfA}|+ZIrg8k%ryXPjX)&D z44hF#qU}D9FN8`)q4=Wl+~~v^!NM5}ri`;*whXoArT81H86RHIxjch-)Q>~q5r0X7 zf6Nyi^%I)}WYd()ttVm+ovcYIc}VEmig|uw^~AX!QwRpV{%IsI**X)>tHRi?+wk!h z!eBMKfUfpyb4uv2i4UY;jw^|5nHAL&)92mB=p$2lYQqkHkeC9MRN zeb+Iv^JbZR4}G3fRy}mc645YPgZc7`(5XHajn&lJA6L+AO(Y@rS-k%R_ig~KWMkRa;9Vnu7| zqjL=MgUc;kqH{Za*Gv)``MH<_J9Bp!%>RgExbPWT4D@(SskMH%ihD3!bvk?cg{oUri$lSZnM*G?{X{nfg-NYEkazow-VpQ`1 z{pM0QnoSGK|*Acn7cYR0c`Hnr(ot!?{|OajZ)!oGuDoRQmdd+>?tp)#Db8 zAxXlv+?VC<1t%6v-)Wy#nFiofZgi2Edml9aMvF2H2eN(@XeOm@)Q>;rQbvqky!C=| zbgo`UF-&H&^qmLEGZrW~GY{+7pRDc-O<32hSNd+V_;<=I;vVbAYP3fC>2@G~yWL7r ze$qpjt9H4Z_}q|3aK69=qq-ODWz79T(u}WWSi519cDm+FU$(gp?3c@AWJ$wtjj)0e z7;Dmf+CV4LTe=TXjcVb2J1IjB*lp|f2bJ?QBm?p*&^t!#r<#QuiTZxdpRcRhdx>!P9H zxqCKIZ<|T}aw!V>)P~Z}dA}!fgsfX*8&o~EV>TK)A9Mz}R#Cm~G`)v>AG#1OweCcB z7UyqoHKoF1w9g!|PKKQ5^{p3AS4n&OByMaMIXV{{Ztb_}GPY6wikQ~@K|YP^L6$v9 zJZaw3K{FBhWt20}$1Nrf(sAL^tx{4;0+$)}y)D}@J6n}~gfP9^XHE2Qx1IMI1+$Fw z>ha7CkLFQsTn?AsKk28+KE3S}3ipvqj=?=FZ5v80y=SE+pLCIBJ@qO!^pq9$yiHQJ zzdzaup_zN~WLAg}1YDZLRrGRNMcvX-i2}_nJ-_sum2S8QYnrt!ZTtjuw<*#AL64&_ zy_FQP;msW7rsJ?>3vw9S}QxR_|eSiixh6G%4e| z_ZdbxD8UP=VPoe2_a_-G{p-^x;$|0S5?k+mKBv-5Eh{BBHkues^YZK27IiJ(@p_&} zz!mBbj0mRkS&+KcJ|IWA-5U0V^=^_^XiqNdL5Jp_6GHKpTce&fmHq^29CwcEAE>K` zUr_9lXu*58Y6XZvh4n^zLERdU%ojG7YA_VB5>n8UV6_8gl*2FUl*GxDU4d!Xh{4^J z*e^x$yy}RDv4*YX9saF&4#|hrcNXODk3uI}xA)vl+hF06Ylwg4;+Ps;tKirYDoQhM z(U33-@UWPdIv6Kzv0i2d_ffhp=9De=2-Vk+C5QEd ze%R(754jO^W1Zgv>LDY(`V0>2ljKvjuy-l_o&S}b`6pJS=ZzZX>)mELVh`zF&z?VN z;{#;7fZX-;o}QszqkHre;>{Mz?SZA5k7CSZob^i*7fZkE{ z$+9ND?bRKVmfndybHY0+`d61GCJi6{|HYH>B-npx6L#mdro19jtfgK91%yrC07`)DYfRxw6ETdoKx(i;YpSpWeo_yPUYH_ zBg_oP_;N)s1vBp7!FpNk4yZI6JM9&~Mbf$IU{0#3znjaQihpKSdoqS6QlTIjQwYQT zxHqk!X|)eg{`;ReQN6K9t%d~r6ML$ta{i)UkA?MCq0cF&)Oh&jXM{YAZJix6Fh6XA zyN~Q-$5aL=yFO7m7pQ|40jp3|6M`zX{Ft}49J2_K!Ub>4O{V#n+!pbG zE4J3FgF>*L8+iXDRqBZ;$!tEyk`r{-`Hz-Ff>wPzMq%D{I84neWqsL1*741D~5i1%>LVHNmHhg?6*E z7kL<3{HBG?2eYX^`1z2HVMe3O|A`~W96ac;C$zM9nClhyAG`s#$Yq<^@KWDoeG>qM z?_wy*=~9OCI}h+MlQM$>G$TP$7=EBT0*_@mmi7-DMA->&iyQK6dP80!PWAQhZXfKoQRI$4}@ zZ$D-de?+mX#d|8+bNm+Le2?%5jBXbAp3+bY!|Q7>qQh+bl^=E9$Gh^!34aamT>#JI zc-RHHjmTT^n`*Wx7tNy%Vk7o-mBE;nXG|)ue0wzRv^Dw;xH4mE4T zx<}XGKf@}=L8n^ZhRJ9CbGu+%3(m3bs#QV(lVh|5UR-fG?4 zS2j6Yu6NR9dX)`!6yA8I?7M}X*rzlfp1z9u_;CG^eaYcFLY;11NpkpPE%D%;k3^CtFaZ5Mi`ivXObgi-i zAhfhottqijC)`MnXV-rDB?N^!(awC@fHQ`r?nT-znZ%kO)((BG85%0oMjM+jJx5Y# zOLA-TXdUM({2u+u(ESY>R%~4rIMy?~mW2ZTt^VzO&W!tWE;fX>i%H@Q(t2AYV&K$U7acF+mXsEIwqxk# zJE9*+N~4H#EYWLIV06ZNo#3p1Utct7nX8}uZ0hICkU7;f;h@zPfAJ>UM+(!5Fa`Lo z9*A_h4y!xP7DIAG`O>8=S5gwQIlY_wKa?3CvP+!TJbG$N`5eFys{53;TuzgpHv8zq z(#Zb`^sz-3&9GZjc3wUzgp89yZPaz8#JGp+1^EP6e{D#>cJVF-s2yUmU9y+}Xr{k3 zY&gdF!#ff;Hg~ue_u*}GzusM6@vrxWFGYy2%aPdraaFlrm-a5D@OYJYW9RxbZS@hq zB zkL9G^ zjGLEXI2OPVhq8YS;BYiLl!_0Yxh4Knb5Hi{VwA`&V!u$-p=PS3Pal1&^`4|n2^6zttSTOt{y=%kkWWVW@ zgt`2=#@?f#nJj(->zOQ);J;rTR1QLsHaquf(%ukDwp^sj+@lK?ri~mGYD%$XJnc{SvHNC=4bPwp`H1CeAHIjI<{&zGfZIE)lyD&Z;sy z#aI&<)}VBrXAV|0wW{47XNK4?-}1brB<}Oo4mhQbtya)(EM6_7;LMRa0eYhiTH*(! zXU5jtx7rAsO)5N6yUz*s6~R;Iy+phtsrx>LE3`R7uUcHmt3|;@QzbPxI(&;!HMTVDtYVDI zPAvw1*~FjfOBK~>;j`KdY2GDEeE5NL&CQ%8v=c@M(8n6t7z?vez`x}(c5I6V`!26F z8?S)Wh2xjl?`x#c*;wPa3y~j1v=}yQwVaQeb*_~hqxoHI{!R>Q34M!lA+XQqR?$HZ zS#;lJy{)iU37`qISxnk7%O z*IQqYeIZSuuV$vTuD#+{5EPX_~Ee@961Oft?Kxe28nVrEo_bT+f}`%yn%?wy_ha ztL={%lG!mM!>JlOO7SmyCaPq2@M1)ZS3gT=8mfgT$NYN&LUhY5iINn;9R+Po&y%%s zQ%E#duEP);vyr~7C3aj!q#RDg?eZ+!7WCu1zfl~9Kc<|G^w?$c*kO}m3DbKvBkP@ac({fJbq&T)_nue^EeFRjZj(wh zje0uGK0xff+8hF!!rP+&!R8DSe3TsHuJz^kAEMq`zh@@6ZhprH&UYf7qla6$u4Tl# zuayozTj9%!@vE(H;@&6S;(eL7&iYv_{#lQ4kB>fY8)uBn^=rU*u#UZ3xS(0m-G1l@ zw8#&aN~`7ae8*c=hbhV>K#kZF;##&UTp&KzEqMvu*7(_hA^CGTga!^lzwZidFXy@Pe z8QF@(H;Xp=!&>^q+cd*cHySJ~U8q(b=}Qgq2S1;y*kw{iBR>482IDMAs%c(p4}P-u zq@4QQndH(lJ2AWEo(its8Oe-$Cu~|;_Re3l23#y%^_JKpDRGzk?p30PhIvJel~)kI zqv?@^byLv|KnE%b4*1iMpBmU+-RIiMVlKdm6NCRh%taMK4dUbZZKlz8vSjv1~3;KYa`=>E|i!8ZhH`i$d`e?GX#+Gw=K5;@$>!#TkjAue&e0khw50^irL0 z*>viOeMgbESM*FIuk}`IJYAtB-{W9INP+E3)2!#eRy4f@XKpp`sLoi9$;y4Q!XE7X zgk{vNWtL&FgB2Y)R4veeR7(xE6B$O8qNxlKH=WyBu@u^b^KDDMcfRhgX^%2zJm1~- z^ZOPz4X2>$&EdNzRkDk0{Nfo5IV?@bspYQf<>1V`To0%-oZGNRRJ*spFHYsI>1{9b zTN3`nxz)3k*&~VTAWrK|s7CnY1s5>(M5c>4Gxwx6tdGsm$s18_LA{Z0yXZfbv-gcy z@b)pckpo10+Ck5Nu?@}IfNZXrxc2E7_MIi`kJSr1#8)Y;^ESLWldvoLnMC66fj{53 z=Yv<`C5V*&k{%XE4}UBq&>n44cqOhV(sF~wJjF~|P z8!oEqDD)g*9i@p4!nR?k9!IDW6+d_JUV={P7$hQG46kEYoL3velJo#{MY(sg_I8jV zUoO!1=>UxlQ{xRa>8IQy_*$A-3zTHbkf#82J-wWprsqnOGsx?;T+x_$K6g&HUTot<35>!xkc^T-|T`T>f6bu!Lrk9&OYV?{i+3cI7+K}NVD zY*lhqct^+7>##B~rd*Qn6WsvE3uyGm!R zRS>iKq*TTjbXV_mzFC1iTMa&afYP$fMkeYGDG~j#6TOOmSoDywkudtJa#9kKy3mbf z;1#Z#^a?Ri`);IOjB+i2sMhpa3B=x-6?KzK`0RB?F`_vFZqI@yDgr}fb~q^}WnMgQ zTgYjSQyC{m&H>oT3nfH;BkC5tV25cDb=ceB2x2#O9Y{oj_M#ez=y0gN)m({EyTI2i zflo@Yod0rkm8MaWx{e0jQuNA_=+Ij`x# zyGRqn{=Ly6P8Pzca*Dmc$!2~jsgJez151r6B7GfdEeM6lJC$<-rR6zO5m-O;nT6n` zi^k~l!JhgiCiw~{_av%jp->wG_W+HrdFE6m5+VeeO8YiF>RXw3~t z6d(Vh#CVbOBRQHzrf^r8!Y^7Rv#m<)iSHRQIY)&KJI>r7@^JcZpo9=1uU(uE`6@!y zmRdKrFL_F;6izGvtnBm*91Hy(TWWO53oHVq8#meFXF2liOX`$o3&ObY95GMHO`ZK( zv)U$&UVd?$Lep6&LW^waz=#J{BU7TmHL`0 zbBIJRY!Jxx0q3lWK85dU*vpp-(rOHQSN_7hG^9P23GTi3pQ9A`gB4%z-wGu>`ubgM zw_6S^q#%b%+Ro?(YfNfXB$y9;z>vA)f#sxZ&mBEU`UlR&borblVTeF9Dchi90wGTC z*veX;vdbh~aw4bLv?3NZiV+8)Z}Og13ULK>J&JQ>vPY_pLWOD(Yzp^GA-v{dC% zFsKddLyuGIjxr;X%KJ zYY%t-w(fo}{BTy%2@q^-dAt`tz;TyuG@hM2=Bvd^mbb%yYOU}=mYHv~;QYc$h6$IG z^;i>I@x(o0)abUr@)E23Cj)DNrO?4$p}Ez@FF}-5Yx%+yEa4ms@`A=QxXE`T8CrWFZ@^MVTkPB7=O!Xa` ztxg!0;KH4@p6zQ?I#XwTVhX=7Z`x7z8HAgJnqSJC-Rhh|XmHK`*|lo>+dXjDzcR(f z#Q{Pvj|2CS*?3jzqj|DGDRM5kr6<)}_TyN;8c9Y9TBsbbYB9_Op$Ezh0Q+OmW9S#4 z*^8pVQ+?#V3uiU7i1*fe2mpuYKJ@(cZ#;2a+Tv00g=ZHeBpz4}i{4Aq%ZL0PSqJ-) zOX9;^b()6EuSe>Z;usQvTEgpW2K`gd5i%9Z>Pg=dP0H z&W#@yFkhHr*u!DC<&5u58`d9v^kz!6>sxYI{J{socT*;C=AekHXEmd+2W}ep9N^Rq zn2)I0l^Ei|0=n);e@?KvQ_V+}BH}EE;5bJj{`u)4ShA~=6?_5Nj7aG|f=+N;hkTjr zjs5dA;aUgY4W-rHlWW!~lKrxJUSgZ~=*S36MN*;LL6nDh98L(O;;)E!WWE)i=Scwi z9q>^JBBx2}JlCM8xubY8&lN?oCc#m=9 z;xnqCCKbkq-L9A!ViVz(U{gXTXb(@!1ia4b&}L4%dIs=%ykb8SJt@7MMxlXQ`i^0V ziK7>o2YX866Mqa z(JuK6lS`T+taF&v%J#kG-qcxuPf9@%q&Pm4SFV!5>{=S_!uvf)66dj*Mm})9)_|{) z%bXm2?yu0a`xSKhzxgV9`qaVl1a#+|Ck|PUv?U=Phf_cU)*)d-U9RFU@XJG8a%6fU zUK^SW(FdSF_0wr6^r4{Hii1iP4~t!Qif3MQ{tZGu*5btQEQWJC-sY%N{$2GoecYO; z$`RaO@FWidX660QP9RiMDdB=Lt+ex=zF*r%c0=BSiNaxuE&c-f+4 z-u?$YJ?~H$64%zFs6<_RUe_E%pP^f=6W(Uk7=N>M{s~;;XLw4>3d~OST{xVXU6$7j z*x-}Ao$f@Rd|Rb8Ak~VV9f#}fg9Og^0-Ye$r_1qBA+>SS!>}0RRPbvdV$ZGSFnF&* zw>AKrsTy}8;S~a@oPV!kCu->j1(_J+*B{;xa)~Zn6+Y(%#3}Ya8wK;CqVhnz%14A% zDd04_sSJUFwwfjMW`Wdcd>(Wq)k;cma${;+e2-b_wbgDeol}Kt+afBem4?gG%OE1@ zf|x6xj)ysp!;q5~&NH$2`Z2veyvi_Y9<)gzz|LGu8zE&eqlaKP?xHzDCy*#NjTJ0bXLCGDzXho<~N>qVXHpz1$Q z_zCh2cik$-OVAH(QuZ;T>IWh==8BYm24j*q7{!#JB~Bb- zct~WYx{Tw?_olaKB9U@^Lfm4`>9bdW#u+NTs%17%O8!s+n3rQVy?nW0lmY+=9Cz+^ zEhd=KfMSXsLf&xykQOu$*!@gMeruL6VY` zoC`rf36eo_hMucD|G96xd%NH3?l;D}J^G$8&M2wcwQKLS*P3&FVXkL4m1L<7Fde{P zFjR8auie66_QF5+uv6@V7x&1PO?VNvm%eSUYGY#WbjQ{hqj<;uzNL-5rI``Cqp_`> znT@poFP5KI^c=gXz5RVVaXvn)e~#d_vAxIlNyniX&a(ghbuBv#h9L(1Pa@~=_!S28 zN>1+DWi{v5)BSs2v8ZocoMk?7-8*6b5sD9ANCQ4L%$^J&U#;nPcyRl+8k*@vEL*UAxV9O-}wb5fwcqc28;5UeiglZzfB*Y<0jfOg{A`{4U1( zmbt+k9r_ohmW_lRy}pXX97eD53K$CXdWsW61~1-6XfdSldg9kgJ#Mb^L;(LWj}?EA0n0bRC)LanT#gQxWy%A7?%^ zw786hY38nG;m_#P!)^P(?P@=}Q{|Id?y;G6I^O;KarfV!%pUGNcHO3IS))voGQs-t zvgP}xaJH#PT<|UMUzg>%Drq}LHS4CdMj=1FJ^m{Yk^N?BDa(BZjv)=2N z#ad>Yl$;N}Tn=*sl8O@Z)n>neqriD z+cnzMT)mIK>OQUw1;ls+F$G7gY%R8=!|BgAiBG-BR83b}cL~f*yz2XFwIouIGj6bl z;r!j}CBHvYb}(7Id13PULG4zw?D7N*T3lVcJKT75=?u*L7}u5QsKMl-e!&if&~QD+ zZ}z=r=|US?+dm#3%8D;&;QjdK+p_;^ma~CKzkP>H#Ix1kY`bf++Lrk>3-^S&Pv|$S zZI!~W43BvDE&FYX5BYH{*adub8IlaC(=mE>s=ZB?E!K1Gr>`IvHtk+%9Ly1?9qU?) z(r#^w)XlkyiZ%!H`kBJPIDvr^n=)9|l~fCT)YZ z9j3Z-7GP|JT{n%QZtxbTw7>N|_T#O0p7hl& znp%GI*>{lP=K7SuZggJ!-IsD)h2L#&7+Y2^zQzTMMSDmUOsS#SVjRHp!F0tD7Msq_ zv@pK&){W6AO)cXv6T(b>9c%UgLC>jQsC`-F`~9n_c4N&=xm6G9j<0?k^zF0$X!UAt z{h&bO^NXsAE^}q8Tmnsks-cA!S9)+U9u-x}7q?$kcs1qJvF-f!t%s}BWn|vn9NguC zuw=QiGX|?mrY|pT2cBYPT6EU8(c~G?`D)r2_loGvfH}{q(HK`AW(yG}@#Ric+4V?^ zyt0+sW!)J5a)`YNbo;zvB6&F!!)eOfjD=yIx_eAl&cQr!DW0p?Enk2o{wv|`fXi5p zk2fsFuIq(-vll-nP+X-J8+l$IV^gj*(N}0QT0qdxwtV|(Uzag&$#{}V?wC;kk4Ev% z#uPVPpG$d&(|}95?L=yX23pbebeg)@&oln9%P@n)e7(4Hi;PAelCx&GZ_X5k@s+P7 z*+MALxcbRBTr2I{@%599z3`mv*cYn~T^w27?Hk>y-Wy0 z>+SsdNa&bxxQ=zP4BP2PkB&R`oFEUQNvqvJ(I_m!z_Zx4Q$=?4`FZP#ZzeH9uA4)} zwVQ|M)FtBG=E_6{WlJ@jzCWltb2V_Td^7i5^^qHDRrM70wK947Wn3K!0*XS{0)MZh zM(AgGTC>8+tcQz^WXn!;_6Q)@H3<>kNWkx}JHo(ujzWCTw4dxeYar5a?tS1pLgD>(bCs zC-4mr1{bBStrg9dO#N!vR#lb^+w;LW7v+<3%h}?w=2am`7n=L1m;~E#+Y1eRVa30q zx%p`GZn*nMt^IiH&Ng3?lV8iUUVJpZzxGPD^Ui!!;1tJF;N}^r!Tj=hmH_VuA4#R?gC_1y6dTDnamACisX zm#ULKQwqU&?67FJfPM%}aJ+4cfF2H%R|cRXX^WWwR+47bR2VXoAU zT0WXG%o?(Xc|!aBy5bM6XR;m}6-%yuw@aC_$kxs+-kt`lTM50fa;yp-Xm z`+4;$&G^w(+7p#MYMs*e4?k4Gh21V_5h4xkZB5EFwzwKdal*gyEEfaEcNQI*?{tZ3 zWGaegYo67(V=X7H#Qd!C4Y7}bbx~Z>NJ7rS|Jh-_L|L%y+IorC_LM=8lnI#>jdksL zm~N{S1nDowKI!Sw|5U|=u$^oTG%j(R5x>PR!Z}x!QN1ekc_i&~ZLj>hv~nGboC1wd zb-SN|0?ipdUW)2WGKWQBt&WaDI>%#OscD>#h3sxj<%P6a$%HsKgba5_G6e9|%2N{b-DW6oKfC<%bBxa6eR>OR;_Jo_od>NZY7*}|5WH_m&$djt3$eP)@=#CWI6f5qe5~M^99(sXub&-aN zne$(w-~V_-`D=dP@UjP_kZ3*O+OGl2OnTT4MuC^?(Mm`VL!n(U^`~=J(f#oRz zUp$|b(^Dv?cu_!bk@&FKd?Bx`?hsbkAP~0-calcV>TLMB>=0GWz4A3nNmxG|dTfKf zZ%1C<5MJ;1*D8;p9SnF2(cu?XGG=3Xu6QhNeM^X-DV?Y~@BC93GH;DD`^rNw9Q(#?&N#lr-4lEBq1W83(wjtarzVA^ z<}O*h`+{6@h&q`-5JX3p1v3TPw)qzEH#iuTvGKJV)hQjN26MP_BCofH;$NyT)&Mh&2KR8d3 zPz6g@N}G1&sL_>}lh^w_H)j@c!F+sWOSe!ySj5#neCB*8fRJyRJhW_W-5x)mL!srh zvpyz4>87Ahipi1d@a-ex(u!m|2mu8^BKjPep(WBcZ)AS z-WO!ASIS01S~DfWCB0Wi_0HJ1>uSG~ogt&ZVc9QkQ#pmdxY`DjDKz~VqwB6=X{Vt4 zt22q<{`~qdi3XX}Q-^M?)0!Gk-YOR#c)Map8^4((o=IorKIA+j7tpQmo+J@ASG6ELQcz4Fd^S5~&ovys2*U~+*wey`hfzN|Pz#15!7 z!ghzYgPuFp&}lY)a-|8}WmS3V-Y$1qNb8|Q$zc4vRhYWQ*Z4M5o8(7AS)%k>cVFn9 z{n#$gJ<3=yY)FE^t8qbj5awt7uuXlz#%oi@6{>Q(>$G%IDzO5VJ#PogLRs$I!bNAr zS!6hpOJ&_XP;okvDW;CKZ<8T*lu1fSqGg$2aM4$e#^>7ec_<&#q&zl8*ts9Iu0e7h zU09&{C?`Uf7A#K|SF=f%SoK$^!b{J*AxY+#RqnXq@$Vm2Him?sK06&JYE?R4yS}vx z(~PEs!oX{LS*$bI|1IDK^OViwF)gsTS7C^fbj+W4wT+_sskz_XZFw>b70#1xo)Arh zH{$A?K77c&Zx>zSsq0l2W4rS0-q;Cp{~y9!^n6gRe76!KT^wg+i(k&Co120kjjYh5 zIVU4BS~%!=!7F}fks%LfZ^tLNx*c`cx@00vcY-N(~zf}0Jj=zh+r>+J8~A3qyPL1r{vOmQ#SNZ?95 zLpt{Ji}8AfIq9vSDFS;*rHrwL-(-cN@V6B0<#o5g_xW0clReo~g-oh$9=JPqRQl=r zxdV7d3i`c<1(=3=h1A@C;l7u(ZI>DVKsB?x_S8yk(1&4;zji2KGLIobpd^)k&P7!{ zvtY30;U03~&D)A)aRsaM0KWoP?1CLwXlaUMoCxFL+v{WTW0kzMN2lA8WvPdUb{QU@ z|2&&dGc4p&6avNnIjj7dW5xCS)b`b59C2fF+qwj)h_*J}FARE{hoIEhV)VlbBvY^ezmv7PT^h0tKRwoZLGdwVlnna%*XN% zF$+&Nf@Oa*UAB7?N8>GBVBu%@HnQ`sr9nf{U{VuPKjDD_o%*{Vhhe^JJ!{*mgCX79 z(t8~hWN2xBxX;(nuW#N~@H^W7gd`!A268EP*GH#08SY0(3vVK;*-d0~A{aQ{WL#nC zJCGP|{pI8EAS3L5P zL)WX4gnXo9oJm)0Qt(vCzTm7QW|jz5c3B6uDdrYiKgcO)&{cUxcUcjq?unJsCOb9U zflz3jS-|geoaTPkZ*jCn*Q7goE%IVp;vIX^=^`KtC6GH2elvcj`Xdwr?6vCQSH znARf{5FDTS=bFq!hB5Bi#sCLD|4NGE9Myr7IVyV@c+^Xqu`bui zjOLkS4e35zm#CxQGm@yJU@>aM&z~ht2e>NeYwk4Y;nZvRn5vfRhd=gP(iI+S=JyS( zjy-pZZ{QkPnZrC(x{gKAg+?Derfr%iL2`xWdW~f!Dx%8u$W9~V-)rBYn5`#=`$b4= zgfhD8(xx9raE7C-b=Ia0meZtkU|t_p=`J2g+2(3{YNy!Vuhc%FqIp%jdGyX$PjEvK ze5m<2o1z$Z#E-(uXvm*|ncw6|klf|y7#gwKHg1#e!TY7*p%k&S)=Vuo3bd%*iwXM5 z8GYokg@gv#JsR&?0|{&QouN&-nX8Sj-CHk{k8X-BHVV{@yU5U)+G_@nF~nyURTK?U z>`SKd_iuBaUTc)5AMnXj@yq=@A6@5mc<{wqHkC|3%QXe_qr9w;Mzb+Zj z;#Jz*=2N@qzsh5>mp4oGg468l!HXEv2V9s-0S|y2lVxl)-L)$E=@SOIU&z&-unhHIjJ3p#nMKg zrP@aBC5r*m=4$-mn~BBBwm%^`@x)ahwGTZ;0VX!|tD8W*1=}8P28c{!EmCz*Kn=>L zLY0Bp+>f(a3P3R=Z-JpTGmW#W(6W`vCnR?W%_Ssa_N1o~SQp+1gTwJ>q`5{(aRJSsf zDb7q?xc{TF{>Y7UZ6n+8rL8c;GHiI^MS{wL=OryBh2AWG;Rq2dwgftcD@1bTcIe>p z&|=$n19wgY^suh6B!*X15N&zQeluY5rOU6V(}$#Xq4rNf086~c12=q2nywSd#5nSd zkOx&pX@rcI{#+5#$F%-(Vhfs#m=_AZtwt{G)SfZ@%sejA^ki0n(e&GlC2fGu3k(K| zK{MS<*Pg1YS>%*VX00#B0+Y0O?U)zvD|SNKYWtPWs3;x_)0t z?~eikM7EeZ#_EHXgJq%7yC7$!4E*bJf&k~ov-i0)$kWIOYAB zH@yW(H|ztm*G58>f3@hirw@IkYY{6+lqn=+91Z+%vfLMEH0ZI~w0b-&cD|f`?jHU! z_NOBLtV(7f%$gAUK4)H)u0p8pM^~~kg}DYIGI|5#N7i(!XN(SZn8qXx@#+s7olpFw zUzZ^q#8o}p|1souvDq0ZT38CY);2?)!vf6e58J3~pZukw^6gcqvXoe%dYgya&!_aw z>V$c5Sh-WoGnWFl>Giru5>A=ZpEWGg1csVH#ojphInN_J1uqGvHvGUP;JWW5y_^Rm z+Bin9W0~)oM~!K%N1A~lX~6!A(MM=+U+Xfk?|e4Y+eq!Ew7Bxets^6kf90tH}Zy6;%jAJegX5iCA= zCQlUe`{SSA=yM(p>S#^8x_Gb*RHXr-@EgDmX(kgKfL45zcL;qHsBFcH0j}w|oe(?v z8^70t1ZhG4ri~O7^HzpPZJ4?Owo={Q+3o-mGbUWu_A4Iozfh$X4P>KlV1{*LC^303 zfnV^(Jg8;tFpkg*@4k%-DtSYTJ`UuY;R=&KRX}p@^#hr>QKy0~jT`Tq#VwcwO>epa zan^yYV+oCewFN8%Z7Sej_MFm%`sC6)c(5Wa{Xm*Ui91e5B5oo$9?3Nr<3&KqzjV!i zR6ZP@!S*U(dR{qiFk8O8T(Z5=*KviC=M}I81%q^$m5Gki1vZ0HK=E3!Um<*Ufb3Swn!|l`4PXsWOWh~_)K!PXgGKY;#tuB_1{0M zI)LSiMzq(~?tt4|>^(hnsQ0_^$X;(qqJk|;7l2PqN1{Ya>t=`Y#ni$9S8nAE8sS0r zMSB381UI0-F&M+w=CmX~(2%V3I}I+i-{4CHaK|SJpfB@Q<_P=Sdo6Kh5SWz_!&9b7 zd|smT?92bSOR#>$Ki)ozLX#7i`mfplnILD}5>@-ZA+0gh&%l33A@ zpVXE)ei9De3ZAW~*j~{o*f^9BSF%;%c&0biH2&{1=waS{k z2?6r$ssD6-+d1kttkbm9v!>OlHpbDky$m3xB^~^}S!b`*gS0V=Ce6Eij|! zJUz?@L$mYB#Wm{`rPfEx;4+jUm%9l4&Qar<_-@yth}6c(@5Oe}fV3Bi0teyluCFrE zB|R_z;=q`?m$-N*QbN;EHGpKe33yhPwjM(d;AX>@3M;mGoVY{AnA72HcX437Iv^f1 zmBH%sh=3;6`8-l4)bp?&P3CV*Uo=;f^})LAz~;GjXXi~$LASQp zVJQf?9Y}>eU1FAFkJ2S9gGcFW0aoi65wF$3JgwUjh;*h{K1zs ze}JZ1+F?}gbnGi&sNz`;cl&$uW$h-B^3_J4>ppc&dVnKD;^m$iAKs7C@3yp%cH9BX ztuEDG-1d-1#Y9?+tY&nZ?>t7vk1{6Pefg+&cjQ6X5n^6>yTVHU(e2Y!*@l`UPf}aMr zfty;~xVGF6Df5+QUWV%%s_#h+)+#d$pSs~*^{95Q*pmzZ+mogD%0har0On=z;hm@B zo*po~D4Co^E4Wxb@37vpWTQN14+}!7vz4ZfRRIgH@KKUEKp&cLj<&s`5=*UDaa*p} zQjKVKo@O|o<1BLl?ou{`E0F&;@XO6%K{Qv6{WKWw$;ahl^b8;FwWTwA;{VTm`_GS^ z3(t~Udg{rPlMuC{x%>Tjb3wsWFJO9$rQwum?YOBDOF#UzX3axU z&)|ZvzJw#(a_+EW(AnCzNJnX(qw80X(axjDPhz@n-~F*0VU?;+6jJ2WERk26lDK%A zWkr5u1Rtacu9;$KDaah_dquI2Ii^j7bf7g|IUZY5;anc%U=fdi#$F@)XTjKQ|C_}b zcLQ@99`q4)m_2(*o`7L_3Z5v9q5m#z|T9_(DX7{I=+EJvDz((fq!~&Ooc* zSptH`;zE;R@8RJMp|m?fcM_cg({lN^YW<&Gxnj>j-PZ4_Mw?zu#!#V&o6>mWRB95m zS0^aP$?q*+7As7uZWEvZ9M09&F~@v!iE-7#Exf4hwow~emnkq(U0ojzxhTA$FD-4N za!OokFxwRszEMwzBhtTup@E0f(eJC2zkGh=I7e+o zK$H$n_aj3H@1wUFL#{s_k}r@kV1Mc0jG}%8PHxod=VhD^sIyeEuiGw*)+xkl%^k)u z)=n|s8K~G<&XYJH^9I*;${9j#)9@+bW%1xltX{=>#C9ujA<9_l~K%W81D<2sG@v<$;5yOoH-0V-y*3JY5%P zk+3Vca2IQ$pW`D?-hJV%804bGbpTm8t}cV+9*br(cj=#IXnb7hwK&}aXR@0sT~u;~ zWU6Y>{(Xvp)}*?~p9-N=v1^_XPCB^rN?&S+BIPN**7D16R#rYDBjKmSrk}~3fRvT~ zB5TaZv2Wb^LSoG|CY7hh)IF%A3k<5BXGIF0G)d&)+@h%Rp=y}A$jjNspoiVpk4l_9 zsTtSZ0g=jRb9PU?kY$xakj`ArpMW)K&!G!VpN4D^G{DIzd)9;w2+bhbXdV%LU+3KI zjpZ9WO|DZp5qtWefIkt!_N_ zl^NsyUuh6Ww0}chx^TEjxbGa%)r68{m={fDDTFDpK!5-Ds!C$1C0k(E3B8vP#D$0T zc_9biE}scriV2{tTLk)9ZNHoda-?E4NOD^Mo*_;NQWgI50q&9cVHB#j+{?j@pj*c9 zaB45T?(@^aW@k?p1-4u+9cw!-+Pw*Ne-nt9&&g_xMyEuL7PVtNKL{H$MIi=_=GL4 z9`|E(@4JOzcu!h)5TDF#l1zs=Uqa>$M^uv02rRT0Mbh+0EE;jSeh?9!fbMrXiC=7bXilxhCSwHP3d zrplOFvnEIn!m(D@>)3(;GsL`v{^Rf&%Lk=_o1aP zRlCmJ&9~z}zn)}qe~YcqZ2T#`{&jLT_xswmUr%))%Mu}5r_kCAFoE15$)#3N_t69g zRp9I3IvaO3=i-|s78_Z{a{wAo0Ewdj$k)adYCOAy!cY5{1K(~{>~7E5Luvg>z7!TE zKe-RWJV>tXK=|Qt)$KIwWAa1ICk&tXA&(Fc*-sI&e~uP39*ER5#F0|5i1J(E+hmAV z$N^r`G8HjcB8%dUP)2s~FG5OEf<>me6EIc;5G3%i=0R-{5_Y&oJQhOLo!sX-4qfWm z5wWh5nZ?o85OG^R83Jg|CRjnb6SIr~T1}=X9u9O6WVFq@H_gxy?-4SUL9rk;F;UaQ%B4j{*-2Ud|xA*sQ zwV!-Ypf`joF~uRAK=Zlc@SSg{zC{L8VlkDg14vWm!0u-l$k1Skg)i<>T8qH)WG(or6n*yZD;Ex z?tXc04@TE9IhiE+2f$mn=pmPx34yZr2yt1p)-HkFl?Mu94!s@ZA?QG)eRblzL7#)f4z~BVui&Gnh-K4jRA=@rAxXlF_^7W z{Ky5UJXpXREACLA4je@vLw|1ZGBVh%QCTh!Zk=GQf*Al+ zuB4_I>8|YG6T@h_6!4u;3zTh*0%MH!uu_{k!K-Qy6DF0>Br>sN8f8UQ!_}`gP4;izFt4l-2Dh%P<6mT<(8+ZcdO9eEY{8}d`~2}7rEe% z8^yhR&4Z8yTs%%*LuOm1*aPuNzS?I6KK{w5NE7l=fl_$O8i$d?ny zuQoOYp#c$@if`+zt-vuFp)5_(YTKK#|HUcBJ^!&8U4W7wCMKSycIWX2IpU)EU*o?BjU+cB0*j|_!awl`3!2rz^ zIsr8kTTb0YQW?kxP`Nqc-qO4JI1 zhS-+?^I{b-ND_vW zj!gquXwrk4RYFv9;R{ET)GX+I(T^)Ea*H{6;F(W=lqcU`;=r6(G7laiWu!5nz|;}H zvy?vRfEMHicxA&K`yB|n9t>UnHN)26;4lN)-VEtK>R#YI{X!@M7QzgP0)wEGGnCf1 z+Ge1F$y=B+YuCK0;yz-8_;U;@JSSib7)L#>8KUZJbP*9Kfc|L}1y#f$K-irjWNYT> zs%RDg#Owf#`W5%o$=BHk*PjTU{i3)Ilicn;!_Ps05HUz@gkVR(5F+*~(tQWsVm`4& ziE_L++^L$Wr)n8|gS4PHTATpPrdsyKxjDx2og6j4?5fb??n{41WmM>y@XhsqTs z-m?4r>ljxl?APmolnit)TYHK5#G6KqE9}_1za>1C32p zM5x#)__i4k<(bGsgxHXb^~`WS13pr~X}b49&xp!^Y-6d;cTH7)^HQU4hXodVA{p)~ zNQ-=@*gi=S$mj}`buSc6%t3Dz0)Nt{#X9!d1EvjJ@t~i&UAm77PNST)Yg!QacQuEY zN)tZ5+AkJL3JgbZYs)izA_&WK=v_oPcE9Xx<1qECh&*I8UV?)S;vN9G$KZioF+flW zrFw2s3uJpJe#d9Rh_qEY?EpGpHOvWY-epRCT@p zhd{1MC~YLbQhui^xB4d(weqK5-~W8ZfYvngZ!Xd#LeEhS2>{wqLD`Pekq{vFM~T~y zKSzX}gy+DxC_?>~l~BXpkG3gaw@o!5^Mj()lDa*Kc{WI7AX406Qmsr;ZB}}(L0}90 zWdr;D`T+MA-(j`BAm%77+B}Q%AAQ-`-b@EG{$B?40%Uc=@er)m z#hRh?HOnpjRsZnLlb1-Gm~CgiJPzVhtPL%prsok7o5bM}6CrH&evX8|w}K|(h*yxM z5Kk=lJJBnsZAy22hns%=IG3Uz$zqmPSps{dU?4*Qnsp|iHf&feKzaf)i2AyzJS9rT z{5B9`m?s>eg!=O7Cony)tBNTO+5k0Q%GyRJ!pnC222X(+^%zPS)DSZ5t7&A_@-Cph zAyCm`KR{>~gcV#G2W+2Hu0$YjbA+y~-0T%yZq`i%^l1aP)`mmYi>s$A>yA6=mcWCT z-E?>uo~DXQ1c~W?vIDI(3nZ0X*YaDLozP6hB{Xx>YEw;_cn0kP>f_=mD>J(`tB$P+ zmzVXACj>Ds?e->`MkINd?)3=Eob>4&7HzIf^fYxXZ6G0d$8Xj{8vTGPQmOor@+B^YeflAW|?E@t2 zaPk~|wO;$FZXTk~c}z#s5X3cyDUBQj>pwsK>NDzEg6Z}Ofeq+!N#N$^zl_9MUr&{F z!pqMU>?jK)2pXkd*h#uD08Db@frAQ%vEMH#thL?pJx+`}H^uX52lA8k^t}y_8N#pr zj|a};J;@KCKVIK^>R&;tB$_yojhG?Kuyo8pOgt2!_P|_?{U%D}^>3Q?dZXH;7ZQIv zXrYm)LO><}_%Bcs^mB>nw^i-HibR4cvYW!ISJLU`fIc)VHCHA10rW{Y^1tALR!V`A zDEnW(r5CdwLw=Cq`3{&$)2;hUQX$g6|NBGQYjvG6EMS4j)XdXI1&AVq1BWtAa&%r! zKTu|+epl|9NDAPUN2x^{45?nL+t9*R_H_zc5VCcRd%#gMxGf6Grj*`ko3T?*#PgNO z_+8Os?JZ=hh_$~lxVH$>DY3&3=DY{%F7bAIU$$#vfCMPdbebEO08FObp(xTXTHI1p zp`uQ%1eTrQk$iATxD8TgIIkxtP1BMD#?9+(gD_mlu$YId3!XrhNv{G#p|~Tomj(>V znILl`kREl=}BCQ1e^t2w- z^BlS#fH6%$g={7y(_{y>{*O*J3njpNae+UH2P)cCr#!H2RVWj__POj&{1af$+_D(- z+;VKUvLpEcB~Q$*5B2eADWLuk&z24u>;%#!iHQQimom+K>5bVh98+ZnT1^xI?{>Oxrp5jzF+TX>{@@^2vK_wEGPE~J2y zvZ?FOrbNT+z==^f>jBq;yfZC)U!WA$n_ETm2U8ohVrBem5Y0I%Rte{nDb#C-2Pl~i zF4Po|%*AxRSgLJ#86OZkRHrzAa z1g1KUEicI6sQ_2rv>78`%o5yiI@%ooxEN#7oD=BqS1n|e8018mTnAW~O0s9)NJSdz z8_fad#{{VRuP<33LLD)K$P^*X0ApH1VweMVm{Qh5#K&r-6cD?Yf%r)(op&PaJAf(&WuWkHPJ&qy z5uC{lHBU|e?K@9qa1kAN<}oNMKyVvt14`&XhN;tg&6LQz(Px*q4`2Y%{6^hdX^4zD z4%oyDJx@^Wj3qVDP8S&Y9A1(i&`S{yRcJVJNFy@J^$mI?82o;RD6$doq${9S*}scF z223;1+GfD^E<2l|qfAvNk6@6^L=tLdL}DyEI}_g*fcFtnD?V^PR8d$C979%;8Ylo@ zdY_aikiG*gKMmHEKtJe{2o=dgTyu^FYD;x zpS?-;v_loyJ@St95;~u$G0Bh72m}2PUvy92!s&2NU>Vc-x4%QZ(Qz1Bv=apUC;x_5 z&l~kFg{-_I`nJG$CrWwW&Y$rScbNa5W9rXS{zisFG^hxh5>0}@R*3L`TZ`6x8=WWN zoc74MN}?MLmRSdKpa5GPw*P}Z`t+tSkK>W&2%Vvx&Iz!3WUXeQ4=!bQ8SSYIK7Jt$ z85I#h^>=7BEzq{aFhI}nD^DoxT=R%w7_ZGEjB$qswhq)}48bSdHIKNW@Y?A401;un zPy%ps>^G`PIdr!e24Qon_)?xg^VMmoSY#CQBbH4W|kCUiLo zy6~i|Awwzd^9D{`9*N@Gu%J8MHSk{eM4&>))Th9hLJ6tz&zROIk^Y$_?FTA-5YX=g zJE*%<$u1(>m-SDfZ=HEsqJe?!FH+5bnrccMymf;oY`4S-HlF|lj;Qi3yDoHG!qLfN zm{~B=nt@8D1Xixgam!A?)-`)Q*n!E}LZ~B}MO*7hE0I{Bxr@ZL2{`@+f@m9{X)O+- zHLnCKmSvv|-4CeH)u2}sPVIdw;|2!tR;g&bK_%m#0jCd2HCNfC%D( z&{n~$Aq6H(L;EaBy^6f=znlyhF(y9O5^Hp|H#r>usDM!fd2j1SjiPQJLvNc(ll(y9 zTQ)}aRQN=9P9K@pY4rY?WSF_xZ{EqrO{^O1ZfJ%!yat}#B?{WetsdnvqeEWF)wxPQ zOB`!-E`*n*5U(u`;;H~BBL`hPZsoJ!)|LY$#F%x{=A|5`oG***bx93k9+yvqCxdh- zYp}85p(iN~>{VFA*mLL_WU4J#Rw6^R{=Y8K@HX(q9WV~hiQo)Yx5P5ofeoY(^%|zB zF9xEdQnHW=F}Sso?x{RXGDsuK@&`J2zf}11&|isS(53(#Ks806*v}d9Ozx6V6jBoZ zxg|Z)O0WxG^mT)1s~6f(?En$i617;J{%>ZcJJMqjF^U*w!Q#d@@0+7hXUq~X!^*7= zDCm2i&xDi!lMC4yAtl^Si#+}(N}eM50fcfmz_;bAb>~5njANf7WJ84zjl; z>+*(2fqQ`Si#TyPFm|27=z_A?fy@Y}_n6fq+X3PSOV6Pv(dtiH35kmjxh|25aeWI~ zltX=hblM$q*F8q4IghyUe}*0>$+~pNYwcE0-qOszm6?L38$>kGCOEM>{ygmGI5U4?eIpT8iZdWlU0^|Ru8~eP$NwRGL zt@GdI9y}PE^Bc>9sY?HkD6|Zmpc1#z&q*VU-SZT4{}VKgI_!P|8js9=UTk z5k1cYMVK5#0Px3VE1DE{i0K92j^XDdLHl2bgIq8<{~KLws|h;Y={V<;i2a{{{Z6h! zNHx$#s{u_qXq}e``3_sFaIT3H3lhfvCT`l&|M=IUYU1U?5aWu*QH)FiJqbHH`OVLm z-~QaW8Ff{}ug0iG$X!D3d+)#c2a})Dh~TZ%)N|-!;ITPEOhKkk-(y1L8;`Iw{Ox-n zdyhH9CMAJ{y9ro_a(l?B6@XjxL`dPE`6k^=x&@Mb3ecS59#DKC(K5<#iWs?!J5HV^ zfw-en0y?~{&=24dU_PJ`@z%(e-sgP>nR<~5gZzzCWnAb2Tg?l4F#PuLbRY<%4`dU^ zFGnMQ3WGevad0t$U23Xb|0|u?fN1pohKUObq~{YQch-g{TD*4bq}UP% zkwz#EopGo+%pS7nuL@}77t?e8`4rh&mi1v*DJb~w^)EL?&1{YXGS=6v@Z)Qwuz ziw>+@>j3IY1&mPA3lp$xROX1^_kJLciM+J|aeXlW3w3#LpC*ymntTHdCg9a`C_xsN z0GeSk;u?y=KhyG#SpelP6>Ua^Hiw7u+zczecn|`GaP27p@&goVj6+2$2UHNAWSOpaAvJDjT-|)Nr@!c( z86a82!0y1F2kF3>j{LR(J5k;$7jaCTJJLuiuO5;VM1B~ttpMu^w330CM%P=LEli7H zg@O*P#^!#{r(Wn?Je`%%k>{w-eH&=lMxq3757R@anoth!vBkHKzI&aTSCqe4Dpfu=iN3A z-N^rE_q_YBi~ob!{y(=_;s3%1_oj)=8)YML=w@jL4{O9R-TR+9^Ko^Zuvdfq=l!QU z-H4T+Vd+F87k~yL#!0w#zas(_BT#rm_tW^ z;aBwYE^KZXHH{#*x3IziA^`n&Rb)XF7E_lBUM}cEgx%Z15O)9_X)*P8EAbs&KzB89 z>A1Du1ilXa@i+G+;N1Sr!#P1|UxXzaq7Q~Wat{z)Vz9p?7pCsY|BJ9XyiG(x-X(vm z*8kp-V`XyWIiROgGe8w~?E+>Tc6UXP5E5Uu>pB!u(P%fpk~bW1+n7KLD^3imemelZ z42}nu7rft~2D1xk9wYJNi}0~7pnTQTTbcjr0yI8&34QiYM*pv1^KbGK=bw7{KmX>U z@BsfSOZ=zb6}DFw10FV!G{jY)zBklsK$KR}U)#!~F3vJdV$_3nT{@y}YlhfIC|vA` zdJEb*l&BM+Zb@X}gbn(@+fAf9hSfmuuz{Um?7Ow@rGiO9f@mfNdP!Mi(A^Hzt{smk zc_R_EY(Mfsw%bkU@5)$!c{CHA;246f>$?XtC;#=aOr63c8We1%3@@W2|}=G-o5~6`1!dtFEl1F5DPI zN{QA?GY>(k9fl$l_%eQdwq!*zZ(77FF2o!=VHkyvH_w9Oa1+pyRi6&K-6j*BA2XXg z%bByv#UGC6!|&^j?7qQ#H9%y2X<2_y?aP&?;heT9Ah4L0O35H~0D76-#bI*_&Im^E zLQA3cU|AP0uuZZ0T96NGs^m}FQJNZfcUKETh5hhBx~{RgVkEQ*6LCh{$oU$Fns+D; z-`oeizOuz` z_to}Ff%VP|_#qfYRAz&GmU}Bxt#(H556F~`$@&vuKgj&eHT>TRxcrwvCK5$xK0Z&4 zOWy;-&_d=7KSD;6I0%M?SfT?B=pDB%5<^mAdZn!fgG)5pdVm{)HuNGIC<-jby2xd8 z@XSySIEX`j&EDOUjIKlGaR0ph*FCkq6uW4T1fl^N(W~IHs->g~e92=w^YF9s7<^VrYPv#7R)s1|;JzH!f?D#9-n{wX(!{Uo>=?_Q3+{?i6;L>@=66Op1S}eEye$N3ni$iLR zEBJI}il`IBYId4YOi`YLhGF*&Bqaei#{(pvQ6{*Mhz?FbTB=g57~%z-2Lj7n8{FK) z9Y~y7m0`DNJZzDp8RAJ2mUwpnn6U*F0p(=oeLl`jEu6#kw^EQ z;3KUNWoIO(9l~c0>_;psu@eJ!0BA=BH)WFOso0x-A5uX*0B(7~Y?`)aN5!(EdgTU7 zYbk*3Tgc=J4{Wq%FO1B&8lMH|rijmlW;Q%*+M0qK;gYZ#))$bPcpNP9Vwh6k;%r9; zfxeGJDn+p5O3$QMAdv7|;<5pnj z=U=>KsQI=RX4c|9d#r%IyCM!J?%0fr8sE`bL##ye2F#^cqB99tps`VhRFVVOJ`?>x1h6LY ztOMbl-2gl9cx5Y{Q&Tu_gq+>K&d(|#L)a6xPg!)N!8;tEDkE*xxV9R8=FS`tqs#9o z=8dk3W^Eu@lGwRoyE0BK0M`2v*wWY}s`P3(XcQ?(OT+S9X|z>;U8|CVoco$oG9ocB znE6UAMP1+F|;Mtn>r5pVicEEQUqEhc7;S3ijz_Z8Oz_*qbc6 zXIPUnmI5y0A>{o+eAC!|Kj>zsu{rojP9cP8VLFtu zc1oeFS(`bem^76M*^0!>Fm0r?h>{Sd82eIYMj@rLWXUo}*+RC6@Ad5c`6IqRx!um` z+&X4nujli+9@pb>zZCj0LnzI(-rFia7RxJ{4fXR6`+ZzNClSxFA0KX&=k>7H9}&C& zZ@uqV@d(for?zp{_rmC^Eo`GBUMIJ}Ol*aqUePYj$nXL#n(1kDLHCiV8N*BoMQgU2 zU_kJ0J26gTNLV#d_nycQ!X~))$;mfxY=UtSUJ9ak#KMG$)}5TQ>GgOqizQGQkt5i= z=uf=joZ60DM|Wjm>Hgu6VDLCtmxyUG<~`*|Hs?D@0d8s?@$IsA;7FN197K|7LQ-07 zb^CHWs~@F)D>qV4SJ^u*!K3l5PiPJEsIssXQSBPxg=(k%hu=Hztv2aM_9Lo&rk$?j z%v#h=geep(!Ap0ilRo?rfslskt3+0)i(AB`*<*Wy0t!<21% zNayN~ZG0+pA}5hduah(GBHs#wSNaf=+yz~T=9f;qwNccXAH9c5QL4z_OPwaic%GmE~y&E@Gxdtdj7%xg~2o+KXEly{%iAL zzQGi17VBMn^fg?B)2Lk?>p<;@Iqsyjf`wK`Vmc}AKx)YZobbBJF#xzjw(1K4iD~*B z$!NyW!xEQ?K}h>O`z5*;qt0si^6Rq;_{bkrxV^LotKcVf6Lqp_Es&S$%{N-6GQp<- z@AOKhAMMobt^8H#*ZKkG^yaChytA(S?kDME)LG3kNpm+1Ow43KZ3IAO23@lV0ja}h zFCdlr{y5kG^wzt}TNfb`&yVJD@o2BB$kF}K@ZMAe6y_>Z&r8J zcRq3)yVZ6&*RVCuVkXexBYV?c95IkCwbA%XbCpH81n-KK-31i>jTW<)QIIjRbdwyj zcfdx%WTjNzCIwM~MT))~%b zCWi};N?^GzksQk5xaHYd*dh96_Wk9$5 zxy@v{;>N{;MINCUi-CoFOs}Jbt1M@{%{e|Cd=yTVFg959a#bDcFrClEn6jnNl7BB& zfZ6;Tyf}ZEE2aJ(3YMqlqoqy#m;y@jAK?onKj>9Yq+Z27)Zz9~m8<42(d-KBmVzy> z*b)h?IX@9p_){$L7n|WW`pf#?%8cO;zbW-J!y&K=ZG2RpC-O7h--9lK8byV*fnaSysUntdW?u*cqqq$W2ZNLt`ehTc(PMJLThwZ31$vasjBjRI z{OtC;Qpf_FO_e4ta|F$rS<-WgV4RO(-apc%(UIck-0y867$qUgzxC+Yz{YhtOQ$$h z&0u=Jp*b4&;NRx$s$yc`THlIHwhf&dn~RKQ-B5Vb{f={7_>*9I`|K9(&eI2&;V+>S z8u|o-@UX{Z8(tOFEFP~cY)qRK?NEdFJahP?{N@1=ir{im6Qtv|P!HVg*K%T(;xejW4|z5FDKuV@Q%H~W2ygYp@?`?a`*FM>={$- z=nGI4b&cuyGHI&(>AUfLF;Zu*ITxpQAC*x zqHq-+Dp)$?Z(!oS(MB0G;*3QCFi8D)nnI|zzp#d@&)KD^{X78^w4J+ zA;x<}I2l1&qF2G7m2ek2v+TK|zF7GoI|DJp&$6bUXZ@%L`t3>1mh#e;a%6Ayj(hEy z+IsoBw~BdNEw&ZhmMPpKleP_pG)vl|@`kz4`7j(3L9eS{t$EK56@62KKx`%Ypmylj z=$Dsg!9(JR=27+VExVnBe6SBREBRq?Z?rg1LZi`i8`b4#d4XC_8+^WQ-+?bu6$T!P zXSz_LFZ_nh7Mn~dlk;%2+j zcSO2xJwivV^nlS$8@O4@>+_`z47mdrmZ6S%->HN&fdi{#qT^V68=?ASa@qrf=09-=|^-JX7c(Ad)rzm7I| z*2qC$pcc;U&;ZMa(!vYyBht@AGJ7Q3^nv7{?v5f--x|=jb!t&^(WGeXD`++j3-F<5 zuP6gSd%=*o^{B2gwNn%bdASd^-%7w-Gn{}NWs|8Gb6*9#>W+~g`U9crd3Fyz>Lf@P zNQ$k6X+%wF=+&*7)L`e9n^!&<6a8!X0#DwL1bQ%TE74%Pl(54<1cyVt0er*Mm{Em7Ma+aV}<-x!s1XG($DYEoYbGSHc#+Nn-r*(q;G_@?Jm-H8lGaZkEo&qR@L6Z)d(*MpN9! zK?wp12t(|H=6*|%VXrq{Kw8k(GpB>SYUbs7N;4_Pkl%6Xj|mtXrY_abFB?hUM(M3x zjgdfQ*;x4ypSFSTc-V?RhqYfJF0Qx0uNrYoyO!B3`$sO-aE)_X`7j&(z66 zSLW^@=n0&WA4@^WB2~l&Klw+$>fx342tRD@&OyZASlmWdW+t3LH+T451nh-uX3jtZ zFU_Y}3AR%h6dxil06pQ;aJW^J+;Oh}VM}k9_P;^Tbnkxf9xC-2l2?$4QLiR0on}^O zL{5dFb9k<&vY(D%RH5r+t@JC;B^S1RMmr`^{geHIoT-+z0@-3`4eiw3_IXl?s;5IE z`y|;xM>{kvv8igK!7=^SqR0E~4%*=;}(8fLIpGZJj_( z3HZeROoaUbr}f9-P^+NBxu2jXy=OE`Tg=*B|25yMdh+KQz6^f{;{kHPB$$9|MTECP z8g~J(Vih;?^*gSka>}b1ZkNRK%7~YhCvq3iT+?XD#D&wwILmJ#l52QG)r?$_1_N7l56=K1 zd)dCZ_SvIhOL50Ai2+L$mpK0T;e9hGjM%&=hUBQK9+0?Lp{AWy2fzRcr(^3hgkrY(eov^LH{}+@o$OzW z`w(RFTY}fhR$0W2TXxyTd?WAS3ew5v2?{k>EmJNlj{KXTK*dh0ajTKnOx`2kB> z=_~UlV&LJ=`^V5b_d@Q`=U8=J?ivA&TtY=vEb&~cwuT^L`c30Q(Pkv#ET{1{&dG^h zDaV`QoQ#_+HfwB@CoJ7dNhtQqAiQRz*a49cyLb-Hp!Whpb(qCIltX)36n&;{-ks@Q zco6z6-v?xqjs!x*J(oIY6n6J$jBjt%rvqe*lM+?lAR(>GYe3glBZ^FJgTKpC2oiO6XOSmy)SLa>CeS?8O4V zIawF)OFyPb`1oVz#&muzvhlB)eG!B-M2(KzVkkwuP_B4$mc7T;rbCMO2l-J$jMLf! zskjwnrm3hrgEqOG?_s`^m{a79F(TJP00`DM^sprHJN2ML7a-mD(GRAu; ze&4VWyu20aoo8k{V;^}*EHe&_hSfDvaQ+u;`X#oH9tK~nde@1#gzp4_;J-}w_T&0` zjM>dXbqv-fN*66*D5Y|{#-I2oeKWRS^hTjD34SX#shbz1`MjAgD?z9q+N(PLG}tDs ziaK#(G^Q@blvyd>=NYy_+c$awt`Zb+cnq3=Zs2=%BANMIJo#(Ud&Rct9&5yNa&aeFMTHfqX0Zs3`Bc@xLMfblV4*&L+5#J=xH<7qilJ5&bG2ycyG z6KaioAJdeR`s-e{Z%J$bp~p2*xl9eCOZh--?Jn7qSYa*V(l{V*nvKXadydI&|IU%f zZZAkwX18Bo0W0KD=ys<9UXvf2M2OOq;mOBhR|rHS-=V|_Ux(MyGFz{7e9*sK=x0iJ zYGRnC{vS=TyTf}c6w}|y-N=(^USg);pmmwu&m;;MF0a51WtANp&(bwaCt6qHwd@Ox zVZuQCD#B2o1?}wYPK<6IBVoKfg>fnkZy<4s!Ln9yZHyJRDY-voSGZ+ejyJ)#9#Sbf z$mAa)uW`8=?x3A$3)q6O66pg>f|82K(41%ot1Z#8Q|4*W{P5(T-18G_JYb2zcJ7+I z&mcV#S)Eb0ukaB0jH>qKQyj6dfR)cIOY)u$SXN@vQ9L{;F0@|Br2d;nW8oJEEFWFI zNSH6x*v0kNj!;n0J^}jYju7~Ui=F$Y{LItW1cJrrZBiJypzoTw*xTEvL0yQbMrp-n z`75&I0(c(&f-{wM;{<5-mM)F6@+6ype0R72TUZ&#-NZ@u|6_$3>Wkfz?X;LoVeUTa zR>QZnXJsW>xKNGxQYq=&q=2K2;RAICn%y|6hpqDo8Agf6>aI`Ye}_Y~+~GFk{>c2R zHIucU>;(BIYh_HYO=d`+GHUqdDBXBVvBJKOR? zRUmDds`w>cF-~ALbE|!4hDz0lWzkPK$nND_w5W1SX!IT<{`9olbn!G={|cCos!C6nNEdtUPv9+Lle?saOfd zJz|aNWXEP4DezM-+F+oRK%npSyH2oM!eR}DI`r8!ZNQRf*nUD*V5p7yWvt&2kyr=e z0>}}&p}5_LJBaJ_ArcHaQ&t6{zT;?<9CcqlQKJd$8Q=I=C}98NPb>8mg;+pJ^NHun z<}I$E8OCG2d?L3#MOWST$L`^@Xl>+ldk}L1`SgqLKsTs^QYAAKTvUX61YAWs|M_lP zyBG<-=Cn;eZZD0yQX+X3d6s_Jo1Jiv;DSFKwwnzW{tYIe+^J9*u-ub ze%xPST$=XRtS$j(>eQ@7STXU%L-C%86a!s$KuWWxMWRsvg zH|+5HAX3}|k%#P?LY#rioVm*RraD#~0&d=pUle7oNG|0xI6`$o|Gp_tM6kWK!3p9OFpjr)y_YD(+(EG|F17He7V*vDkbV$MTHJ zID(tCaOjgsiYEW34Fc*P%bm6kYYiRf({jZ$>fMeR5h(di?Aa#UO-`3TL$1COyrku& zpI9TX)w)?xql{l}%+g_q?XnZWr}942w9AK`=H{wbb~}HzY`cS}DviBD6wWcBqhS*n zW>05_Ddv_kWQBdG`8ylrMVG^REK`|g8U6UO-`7KsK@yUy3bQo)R%i;0C9HJ8T%Q@O1X4N&5k&%#wK?y-L$7NZG zCP_1n8cW-Ys!LX8q>sgawa10oJN)Eavm(0&T-?zr#5NFWp4KiCKm%I!Uzt===5SEU znOvgx>*=u!%lLH-T3Z;(v07e%T6&CxZz8`5zVEc`^&>u0m0zWVY?tQw%b5D%3`plM z#xu{UWzkk+%mV(i4ha=aQ8xa=`6H7)awgz>9g&;1c%?r=y{i-6m~(CYT%k%QuGm)R z(SSPc$pE{$`FotXon;(0_grJ)29GEjMYV(BCzXDmYPh+q%Az>g?tDBgVTwR`cZ~VU zU0#&SN5@WYFB?Qa=TiB@G2zTf1&^CVVii7NCcBapu)l8fmC)NC(HsP*@|~(Zo~LcZ8NPf{1B5!vv3A^%Ex8(0T4qSw;D_r{gXFMDUEB zAcz}$n@J=U_yrPn6La^HL~XE>;fbtArP}=|&y2)oIRkqnmdD^=-<3j}#Ao#DYpBuB z=u^cB#>ZjCnD_O<5{l+5r0ug2aT=Yoke4B1xR+x5l&P$z?M~0Sed9gtp`zahTmr3% zxF4_aSrWyxPa^6WKYl{;to{;VbS*+gwWt)>{p#ZXVT*YqC}Z3}j@d=YhN`_R9l7M+ zv;=cvPl)CzciW}Y4lbohNBgu1T_rnY!z8;!yGndcPQI1dtkAOK(fskPhV?vaUADs% z$Gekh?;c_wml9z)`a-iaALE5G+iikW7aBmj}tU2@a^LAow%sT7y(>wq^{JQ+=!&G zY`V_jF&@9v>8L+k?Ikr6nkQeghY=+zV=c+1N@C1vY;FhZf?5D=O;5+E6mek;bykD@ z*xOP+Rs_SVAqFtme3$Hu8!m_{iYGPLha7bn`kd#V>)a@<LEE?R7$*3QmvPk-JLarrsLCw=Fox9TAqKuRl23N zxgjm9#NS22-C>Qji(G$1e;Fb533f{BaydNXvzu8Ugd4xM zZnn=O?9t%w6=Ch@NtHw;&U(iflEe^4x6(dS$+|$TfcE7vLEXsk7H@y(sEBp1Yy%fB zN&cmCpgrT_0UXgI-~%_7a;vKDaOh;TjQqQ1NBy$zT<-ZLW%;6_Zb#~xDy>I>B{&87 z@yXha-56r`DNE<_d4YOOZ7=GqY6smfx*ua6C$Q!bg=lD)n=wLS_9@BNSzhr}xWh0U z06nA8!v{{fS^dke{Za{ak&uFJl-c`;HF8LSuEqVsuR)(5mo?@T^~|(4#f>Fp-G~?0 z7c=0lG|%Xo?)hLRugi5gCNae!a^X%r>aNAQ3R4D=Rd@YusfIca^6v*>S5E0H zWPGbZCeHOjTtn+$Dsk0*>(?;vi<{HfF4efEV8PQc7IKrQlqHyIa;NEfB^&aWp%8|;a z9?;8Ww1HTEEk6}>2yIX=DODyB3jR+s^LK(D&E$DFC>&kDM(`d4=tBM|sMHIS>X-^B z7_EX8q7ZMOSRP?ZBXHoVcwP_h%sqYD$i`4`BIq1FwA1l4s;uv8c-6r8B4eWpztq>^ zrPjZMwDM($8aW`qQpL*(v|1Q>BI;3!K0g(HFCE0W`8u>3tWD1!$Qpl_Xp~Vi(52+H zL2=!c*ad;)(%pW#o3cFKLvN-t5d)C3rX)zjIO*%NUXFtT6NX7QJQJYR+CUNeypm>i z99Ej{`j19A4!vLQ5|og*A4$wM zE`hkitnf+lF>Pg4sxo8#RosEwS!uKkw<7s$q=tHGMELdB<>SqqmN1;AbM8F=>$TD$ zQngd@|H#a3ND7^dBsqGI@i>e=i|>V<);2XmNsA1q&iD~i-*@W&E&o+L!zOs; zRnVU}p>c=8XPKTJl6@@{@ zQ5&2kGLgI$mXG>PGko-{XT~ou{i(gQo(0*0h$^{LMV`&GpV2NJm?r=rLnJBGUQlAu zb=^fnne=zmPPG0+-QClKixxHJGqMNl9M&lVBXnU3gbwu;Wns;YZu)8Nf>E$5%G(%S zJ>2OQSpUG)lh6s+CDbWLCi? z*-^N^p8@Muy_AqQ8}*;V=oP{VBQY`Ao=MQ+i(pIq#m%wnwuH*=ki}SNVbLI<#c4@4 z>5_zQ>*<~tzY|I@kgC^64Y;x1iOX^%84ZXi!g8^8D}g#f_r{SBKDrw$AJd+vO`HgO zpD~=~XTA(zK_}!Jx_-HK_`7tQ7w8`9-S0tAa0ai;xoq=QVK5A789}s(um$AheH|*@ zvqIP(1zoA{;~6nnW_+}2&*j%04ZNuROEM!Oh3hoj9RUGGO+@EBHueEU z`GtGDfctT_lL9izQ)CHSzwBk~azQj#@ns+1R@(qucH=EvO}MCwGPKI2MjB$YiJ-^H})Z$qE*pK=S?;Kp}AJC#WhU8q2D!nZU>x%mVyVHxCviT{Ugs$Njw@x3RAe06hB%nm;)(ye*Tw?xE`}i#?dE|Qs*t?%I5X-VMv?=G_c*a^-{cb*2NtqCfT9>!k_cU ztk4`k;Y^7AW*3`Adr{Mm1vy=C#NWxpJj{_?SaGyLk>@CS%Rz4Z~*dQM{SboP;&Ie^0-NNlnQMm{ojZ zX=;aWWZBomNOj8EUqJ*RQYMVI$a_L1*_78{gl_R`Iq=J&1iU{VVQ7-1GxZm+my~dIx@Fp#a1mKKucV)P9$KoWaQl!R*Cp2Xig)6Q z91ZP*;9?if1cveUo=69NOMU9Rd{X}|0ja~V5$hvBmG0?8qn)XRNDx^m2KRB7p5+xU zhb{zRM_YgnK^gH~r1Eb1U89aPG!>f5L`=x38D@aPGJ?L2!Iy9nCNPI-TUjor*QWm{ zb{ch-A?^S5n(9NW)}H-zs@OjLIepDQ);O)x;tFo>OREhMM?%s5@ICx8GLbk>&!eGp z;~swlvhC8i%)_-CF4n>|cw4hGj{M?OT}j*^?~Q|v!pBi$5NgY&YG7i_d$dGqzNmS$X ztfXpbR3Ct1lB}va3F~RBPNQG35%fMm^+S zUx=k2;=>Hu|5ETnd-^^Crst^QOYJ6v9y|%7>L%Wrx><9Fo&b2?OJ8S-x8UGcBnBM@ zcEQcQHl3crt(gDdOpbQ zyr=rG{B9wPuIL`ONT+P#s-Vf-G(M^*y*pl;H!KqbzLde)T=AATdmn|xqqFf*J?5Bp zE=Pzgy2I^OW*cP$=BU@VMEI#Kp#=idLeuDIF`FwXIkaTQ6`Di->j@+N>V9jS(A|X7 z9MnXhjRo1aul6s)E~WXfFECj#edQ&L@g7C%qyB;{3N@;=j7^~l$imnwICM>ES_Akbrimnn7Bx+UwRk3At9ia{_ zQhr~oqp^C6^oYA?q+~bUUtb>-(Jdj0wAK6@j|RB$ZFOeqPl0K&^mn*kM=t)2osFz? z_iuC3HRW#^{(;R+()+w@2*KcN8SJe4W9{4rO^=Zx7&uA~p0N=_b>aYJv&L@v2PWR7 zfR|`Iq{nF%I%)cUpOh7a{#433he|$sw?m7eoC}6s^#w?o*YlQS`s=h3OX{pb6kZ)W z^J>-fXl2oyoM|j)G_1%pnKLt4D}pK9GwHUB%y92L<&S%tLl0QJ$aD|5V38%qTVJ=Y zmY`XTPl%4^lpM2{kb2b*U6*5bUL1+uzE4a7y23XEdUbp9Jbc864zlOtKkURHhaWP$ z30Tmgl`TMIHG)95u5CAavdSP)wV;?Fo5M5*!hECZYYRd_JMDA68y;pc`)Z{*R zUMz>@`n;Bg*+SDUv;!p@Ew% z9K3H=@+%iZif{}bW|^M!LR_Lgamqhh&FAVUX+YVoEGIST^^q4pjq|SLwjbMn+?uJf zrTafsV~*Lss~f)mc{cIs4sqOO`#I4Ei=i#w4d)*0n%nlN^!@a5M8vO${=&37Uv5X& zIyC;5>U%8oX=dh+k(1E}u_@OI99ss5c5M^(56kq-J9O6kL8;62k?R^SOV6HrVE#6s zUt#?q;KH2luPJ>H^0RlJvZ&STT9a-5{yQ)VNfX`LVwd{rSrg6F-e$Gjm8JYg}ey3Zo zuWx3V^pUX(5bV9n3|g9>yz75&L)5e0w?T(JjURc-hIN&YGrZKc>9tvI2;moXlQ;js zU(p<;e+<(uW@4oiIsM?qlYN~mQqJ5nr!S&EGyWty)|mdj+FMs|IRQkn>{XgiM|Vxu zZwWtkd3$+3$Rj~F9(*xoD4UMU>U(xQta@6wTl3hi@BEmd{H_6CA+NmEACt?&7N&lO z7#Wocn%P zezYy$tBr3c+cBq1PT%wM%)bL6tNA{8%7vU3)1IF2J$EJ|nZ6x1IUIJJc)Yc=(s%LP zKxxNM@8tuQXSHWqb6t;)m1RswTwi?bFgaMDn!&eCS-SVcQ~qmfrnb2oNBezhy-{Da zBzbUcBP{p)vB^a92mCF$UNJMKO_x{A&jDueP>n52ZNT&hf?2OfK{K%9qJv2f=@R}$@+%FTA1*Lx~R`9-J4Rb{h|T5pN{C4wBa+MzKcE)S51~c_oNq z=D&oMSJ6X8AUIeQq+NSz`pxS3p?9hNhL#->b)|SAjFnHOV{C_JVdu*ewn1wibBsFf z^-=zO;IC|y^em^4ihd`ao`9jHzoN;|-AxUa9U~SwLN+D03nqc)v-4a|CB>|wQKzZq z&m^gG1M!BBN=QWNr&R`e>H^!^Ob!}DjpdTmknz0gi<#I@9SO|GhG$NPzfEKH&k1AZ zjbZ!ulCPfE+VLa*O7Y}(emZpRKt)XGS~d~Ecsg4Nm-mA&JxUw7`URm_gWC9H~#zi41-&82@aeP0Czg*q`4;*=zVP zL&kh6z|5PTv?6Z+3f1N)KI=Ogeew!CXT{jL z+~y@(;ey7}L6sCzH7kH06Bda!Sfyw^ufSkTeU-Zg^o$AEz1&A%6ZV6X;!D(I)^{{{Gk{tr4K@PC;~zW+gN|Nr{WKdft;sMR^J U*Qm4bFT?C#%fl9VX5N?oAKPLa+yDRo diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/charts/locomo_comparison.png b/benchmarks/results/smoke_legacy_compare_20260423_105447/charts/locomo_comparison.png deleted file mode 100644 index 7f5f7c91e81ca4dce1506f440423e88b57b953ca..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 43742 zcmc$`by$^ayDvIWRCFqeiXb2eC@M%zK|%y2rKP(?3F&TB1ZfLIQex1lbW05-C5MvI zC7pxZ_rv_wxA$55T6?c^{yFoy<~2az9dA6(egEnn{tx9OPMx4Sfx%!-Nj?x )rV zmxnKt9EBJ6h_-Eb;kOr4w^y=yZ0}@XYmAXKu(y6{W&hOF@Uo+^t(~cr{Vp^~^sOJ!A6)o^%IjJL3vHvSm;3L|sp&^`2e zorCBydOaV3IfGtL$Y4m&>k%dlF}!#mI)^y~ulJrE#}L8mr)pwvc)>i>KXeaX{{Q^* zPlJ5p@0Ao2#e+L`R@(INeur+opC2fjT!_(e8YsvX`W{p5to%Q(&ikUGhK5Gz?s{LRFSTXFM1)aDSEj}nuI)cxQeyAAC7WlLFa=+{ z^*EuAQqbL*f1!NEcGM^2(w*dwPh$C6?r!v(hN-AZro_6MQE}*3D-Stw-S%fX)joBl z-)*77G^?PeF~@44)M2J4OKVX4PvWD+-(RPv5<}$zSqs(8etkC9b(+m8%3j)J-de0Z z-6^>J=iLR3M{jm|Z(H;xUHzc__a&P;GSeTQ?wguzPN&2z6EgEVq|!wEND2I{cNWSC zrLkUngy|@&%CvQdM2S#2H5~_)nf62p!Ia_wEe|R?~U- z<;wP%?NIZs47Hrq&CyW##Vys{$db{JJ01M9neTE`-Z9BCbMHi`pL?BCxy5|R@uz_) z9J>`qhj-5|nUHE?D|^}tw@|6dbC=1P_FAm7;T~1+N~ql{qdu3(m`?MYirZh)>pC3J z`*yw}W9%?Zi3tzaG)wDBSIOA%thZ6~sKR)srJlrmisd7Wy<;TYfZIe!*AcXz`th0e zuIreCS*gQZZ^0k9`V1SoHh~qHU$8&~zglFLOhhVq4B5!#$wiNPZqKQW-d^uD&ne&Q zv#Qd$JsEAMS|-<5XwkH{J+$BB973VzZleL0W7aPbc!9NXzG7qW^9>ug?coKM1Fp~U zMjvz>FD^L1owfM&chk*VU-4OATKSD@eNSaLR$(2oE7p5I2lEUTH&{eF^k}&3;^@4} zpN4BZPIHWLnfQ2(uFYrNu7w-_9u|}4xgAZy?#58YK*Q%IwmV-}g8C}A=e0(;**sdJ z?K>om@66?Q66aXaNXX)Mem#j}HSbRECJto`SDP%EjAbz&s@x^kb8F_rb!}FtYEF)S z&|}kbU#d6nZ-B+6JL-VJFvUSdh!{Rpy@uDT65x1sZw$U*--h3`rF8s8@AsbDtm``a zEG*|=dS1KSgR@4vuoFrIMDL6S-D(_!CsrN|w>kOG8wuL)!`j5sN^V0~jwfG_P~1H2 z$uJgZQNGf8$C%n8UxBdB7#+Aqp=lET8Sd*{zknCVPP5H+rpUWY70#q*^iVd36@HmY z2n-*vYr9J*cyaVpV-EYv0NJS9tZw6BYD}{prTcr^O>I1dPT#M7c<-^Zgfp(%+cN)E z{p|3{5{|p~Nt_3Bg=JT&g0W7i{r4R6{$iV7@qVDB>B{CbYk^)2q(D z&KJLXGq?H>VbA+8>9>V7_;>B~nT(w9wVCb=_n64Syy@hq@Jk-6?Z&6?u0H$)3r%Ys z-X`2*cMZ>{+-DSecbWSYUeBRhrMt3nmAkXVZgSb0ef<*d4F1mZ3jZVCMz|2xD5pV} zmCsOe!gV~>x{VJXJ^5`x#a*XjvF0zr_T_8^l{k-e4FXzJU845%2!unaYR!D4;WHvIis+)XXBG;HHg&S{~Y+}>Ge4T40MF&rXC zyf+4%^jspWe;W9yY;XO=R{Y65Mw-1MzvHvbXhRrl$=F+$UzuN8HP_oE=(?S0pyDYNN4o@iD^Y-l$1SmD`3T)u6?GD7wRu;agg1uv`rPxG@>6_;uCp$v7|F zo@|}6@>bg>R<+w{;z5*eUPN~YGAUbVtnrH*8&3aZlu2*Bf9#TSrJRqWib^I`*;Pn##k` zW^iTNDauS&viS0m{egy&+-pGtr8~^GSoNAO!-w<*%hKs&sit|@uFLq1xxS)?6{xOu zjd(Yx|BXyy^_ahX@p1u*1#10j6wYD7SP~PX3fbGg9+i({J#xyI8@cD7G_e-l)MHP_ zWGHb`y6q7r&HLN5?OIeU#GbsWW~s?j6q_9LN#lN5LSFUQki=3}&*|J>n9*I(Cyt$4M|8wXXyw>M8V^LwN?B zNBey~@wY#{9LeIdG3dsA>f+x??!Vs23w~U95^0*8UrTyN2JfMk54CafC~3Dp6ZzHI z>@v%O*=DG6B{t%ld)o^vtZu_EDV9CBMt?qfCu=5oK}9oK4J#t4fMk-~|G|-`?&vzPJKapax^?EG6Cm%3r2qj7g({;u^o|dE8mOTiyN* z*RXRRD#g(X@Y;8agqp>)RlpU$!4i_!wR#j2QA+HMsXl4+GuMFc z-U;cOd3D72V7Vgf@A=uBs_d@gx8IkJ$hCQ>osUs7KRb+1%H**f`?kbi5nhuuwXFGW z>bK^JxP&{iN!N97Y&xYf9kU~@?@PCP*7^6gW^)KzYALtz*PeGo)|3U{OJ^K1O_u5@ z;(q6aw_;0YouRr-WS3Fs*ekfMwD8zn*H*&+ju-LkDGohCq0J$iy{Rb##ddjAP1ohB z({AwGTw$MOntYrfx1$-G2dix3Hg|ju_IQOJb^#}}w`s)izZm6A>BGD{;d1Z zd06h!8T1hyqj@bJN*e=B>flRhBHmhUc^M$%BIxZ#$b{I1BgVs zKVj3+DYE&t%E|ra=3c*B@qV2zZ#1*GHHr3#NO1`Fr6!+nwNoc@ibn!yOjw}~PeYf= zzFo(n5-xXI%d*6{aukul-$jI=jpkfLNm~gmtK7avcwA)AAOkullZsw=U&W?WNmP zP^R-4*1eL-Nz3YR$eh!*y=rq;YGto`oxc?z&iFGUoixphYTa;K)aM+6%&Kys8}z7u zrLYr|v$}N0u^DGS=rTEOR5c#1wY+^+l?j@4Dw^H_7I!%^Zi-&J~S zhipXlGV9>1s&)&@=^t2#zAEiflpe=z^xMe?&(^!UV~yB#HcD52zc@XS^KoVeAoVl= zg{Dxzk=;^FQhL|8TL=`Em4pdpd;-0kX>4t%&|dKlK}A&q>;BVndbX4wZiOqs{n@*% z0%DAe+o5|Hec#WMv2GLFah_W@VytV9cW2w7{;b9_2f!#SeC^M#F9P2D0JLqxv}My< zaksH>*0%wvcBLt$EZb#fpR;M+cU=GVWD*AjPkW}fVD`$K5!4Cx1>th&_>+8dOl$mm z8&&(VAM`w33Uc`7i$^=yt9BGq@OcRfmD|}nt}5gM9@1gq&~V+b#@P6L0I})?@k>hb zcf+y)BXDJjJ-icd(5!GuYv#EC0M;-ex&@oGcdOiZ?RSGvla_S1p+_YH8sHTQk*h_h zii`I!Mqi2qL*#u?%7?5y^KvvG?~*`Md(~jMJWk@J+*;D6XEIYgS$)pK3d+l3qc0+w zZXQZDd3EM1v@lKzE#j+i@0|IkKE3F;p!JkfW#evz=;TlbM=J?$V!$l)g(i9(Vht#} zM%n3sD;B-^ror_6K6TbTP*mCU+I~v-yh0t^ljfO`{kF&NXUCSg`4_7Xvs5?&JZD{? zrTMDBmGWwq4$qlA3EUu>t#Wgu+RE*dX1U({BjcestT_!CBrOcpFBGFD}ltTn?4(OBGT0@_e%_Tls;{`^ddP+FImNK2hEN7n86vIVn# zT&KK{&TT|0tf_&FO#sD;R?0j#ss0WMaizP~xxe!)bl{AQWh1%LmUy-DO%xm#DmKDp z{Q*^0WEwYxpDU*=1CHXHL(OIK@IMeZfbM@I@DWYfdAx!}p^B#CTDRH_^y%wtXdv$by&K`E|q!an>N16@B~AS!O2vTpbfr>oxsL#smVt$Gb!S4 z4)v_v2LZxE`NUwHRgsQThPA1?suS+lk14fL6# zXg`KxtdBlS-$9E_hYpEK7$t?t)|k%x$E5ercmJ=om&l$qB*I`iSg7sK=EI9-k z0hUQNa%_9+!GUyhgu-A59!H0f1fkR<#1RIe6uVELyx%=V;tY_G2|pFEFrk9*I8 zsd?BuprG>Y98R|QAqYKexeqM}DxPe*FAE2f&!=in;3&(BITn3|X+XJ5Cq6#OR_m60 z=(A`AjLLL#aoBX~M?Aret#SkQ^y`_!M3>U_@^lOA3MhO*!i6~QOC6a&L%)RHy{y4^=rwc3n z!G7$GL^5bG>F#uS0(Ev+7xHUQJnIwwx$9icsLyL!RjhTu7m*pgs3LQLl0x&eVzon> z&)N+dlH!-gEoKf`TkCUCfJVw_XxIw66!{)(LwWCrb({Yv2o(B8>EH)#Tgg%&_O_pj z+r_D|z?ua>N3$bbXkw_#oQ}wbJDC5|Fc1rJ2TPnevF$K`c3BV>oJfxXCT`E~w{60L zn8JF@=1Gi$Dipo44wjtqx4`ELCHZ@REnmB2`;8tgc~jsG&D+o%C^}K|$q~`Urz6?S z9u$xyptIPj-8n1MPFWsVMu`v^#H0B^1M7S%&GxajisT-kzzzGhyBlM775h%f|9|*OW=;lIwtGMO&7Pha)2OUdRaqqlxMzPyyxlH*rF?Q|jHIv%O)1 z^^y;rD75Hiq=^9NScC$Fi|S`rVEYbvx@1?qljFZp<*B$jKX6CwOhbWrN0OxHJm7SB zmaKfmQb0qtVPlOfIog$;KA94i9F()S%9a|aJE21Bn7YwZ=U+7B)K5>7h%v9?Ot&WORZ?U{+i81Jn)or{wU(XX7S&;5f;^)h1@S z(h}Y-u6D|^pWYYtdB=H$S9D6f9RZaEMw2Y6y_Z+R)frd;%Di)@ITdr_pWDyUqE`f@ zC;YU3*7fHFlr@@EXQ@{X^*nY{5h;Be(Z|L-%A{6anLL?Dl|AF`ezo8t19feTQEW9V zU4FHFXba<$MpG-$J>Tct*ADCrFvHDEjIz9RMiL#i@jK<;H1v;@AO5`tXb#=dO6g(n=gZ|lgwJw|M%L$kk|8V+S^Zm zUOD2;?QlF~;%dC1n6XcYK`Y;6^C|TR*ONrZSh)ULv0GP#RWm$y4Xw~E*E8n>ut6)rh$QO2a=cV3C7{4e;MdoOOoM)|`&35kIlHo!&Yt;?`0X;~zF z2}U+2n=J6z1I|r}ql-zZd$Le$A{ZT}JFcScvNqkBqMGAha$JNb(lp}GZg=Zhss_Xv@s?7aY+v1Rbrc86b2^-$yZZty$Iyq{d7{`=84=iw1l0YU&laaFu8#Xy^0BDkj?Nb0~b zmG>54ao+XwR*2sO&?r|l8r^Pk2zB(N;Lr*?-?9^@V=G#ef63YJ1!FAWf~A338`B;w z>yTo>4%hmnq{Mmhs~yP({4+k>Ro4tG;h+-VnJ+ftAA2p*QMJFjkdKJfQHFMdjUOVX zWl^DeL}&HX4nVOq5?eGMy%FmGP7$jMB3}6)*g^75H@`GrvB;5p$26VvPEN}2lB0Sr zlD2`x#$M32{@VYZvGfqmMF2|DbVg34RA(;~(%4pkm1g^zyd|$eS(J1_R$sy z*R~jEXxSf}$U+Pnqn>`2^xO~Ear~9vYnFixXVsc8wizG@G5@omtzMF=J^(92j{mp! zVcW|WR2<2AKnmyfdyaHUf^}!_PsctBy8=C8KeFfnD1&^U995CbCyNq$I}JFfWv`=h z-?6<=6^H2N0ew7UWh;8!1?nLd$l=vflpL3>%mFFzoh(Q7X4Ssm1~>kjQ@FMGF|Y?K z%ZFzTh7=?&g}JUyDVXlU~no$)SQ<*HJ%K z%CCT$eFr%@CES}xl7OaI(*JyWrA)dm-MPL&kR*yN9$?A4qF-Z+})Dh7(I+G`6$mZ_$8>oNq{UE zg>*qz%h0(N8%n?!rJ~M&QNp2hi1hKULJO@A`QgOg?+kudKO4=6ETI$(K4rktAXEimVY2hm}ckrj^j_>+h006p!s#PJXS`D!`!Tj0}W ztW%;3wge2pB@`YG!3h< z$wjt&!KU1%wSa7r%7bxa+v?kEdcwAqFc4Ut`U++y@dI}8T5$`3IB{x=_b;kUmP;C$ zGsb=}AIbrtHf3YTt4aWK(*@vT2RN2*kq1Y^h6;fE@xnpZSuC&-fdhdMBob+_{q3Pn zfUz-y9vcHRY+~M~#YHmshtIj055Sxc?|GDv7BVh7D+N zYI+`FZhUofs5&D9vLGjTV*&v*knh{>>z_9iyL>TNf3z3p5TS4!b4|qb=K{1|Gwn&Z zC2bq<*rl@yFAkZU%OFL@-nFF%fF?}7F+B7?k@_Rww9}pCZJLW9871mCd{Y_#W;*~t zysgc6OH##<$~^Z)S@vSsW1azfCDJ*m105Bl^ki84cl^}Jh{b0 zPz@f&-F05Soks7r`1M#P5TiE}(RQsg_F>8E%>MLg)K7E2I_LDkwq}0mMK7F0vY602 z_l+_1CO}GJY1la0gAzWjE^;g3in45Eq!iDL>YCbv&VQPIw0VR7Qu3h5VQIww4 zI1SF7zsSVJi(P*if53too}-9S|BRRl$X68|4+*J%f-FE&RdFeHvN%+|Wzs7VbmaWi z&k84KMrz~vv6JRhZKvTvyXc^6cr8O5!AeMXlZU?hr7Guhc3)C_zWW=o3*R?5_jSeE z1h=17=2*&Atl96C5Yq{^UtihMuTP{hZ0nd^02JnjLVNwhe_A zzQnT_Ip4=Ek>i}p!@%jNtz6z$H&8Aqnw!DNe)0?)8W(A{;khvp<1}a%$NutFH!FMa zm7vDbndDvTXgciDqD3~Q9t;-(QfXd!H1>hXkRkyup2q@Te*q^>lpNn1A0Qp5xrnK&q~iAfB8dpL68nGqMMf5T<^(wn%w zL>8NqUxFPNKLu@1h0_oj5C?Q>j=HYv%F04}cL#W|OkwjK;lc%=-ZM~}PKOB%<$(1* zC{!n!#;Ge*c_t&^5kJ#x&^(7iEA=S;j^m$*rUyuzjDCvSv__9%k441`+20P1Cjl=w z4eo<3t}(Y985>7KRukqs)9|{_q@T=g6B(i|y0u=Q?VCN6Ux)8BDZ6xH)F zX(TH;6K~Y~fpUXWfH%_OJ&1wIqzzfh7EWNe8;3c8gbEAc#8yH*Ta3 ze;;Yw4~J4Jk0YUxuGheNbX^fHO}(uB_@lAyRl_m1b4n*pGxBE2#R+;Wdz`$*rYZ5q zA0UsuJDDi;g#Q4u3Ri$ zVV;Sp?hL%MU0wac4%U~lQ%dH>g_mOBGuY0jYbiWBr(~35puAacf=fSonpS)82UqEK zc(nWS_Yw->=1b0?yNB?2={W<}LU={X)P5Xbp3q<^kYP5fLPZ-o1)WV0hC+)5Qhj%T z|KhZ&nEYsLACRsmL@tGES*GRx0VLO!c&`=#$*vz?;kE5# zX;rC%|2FJv|2+cqrm-(HzVfc?>Nf+E`WrtOWCB}DZ2}u;!1okI8kC*W5TJJU(Ci?# zuSkFLlx-fzL-`c=`)}dyh`%k6HWj9(`#t!+Vd^}u1Tdll|$Aa1===tXxaz{xF zr7t}*GO?w>)!hWV=CW5aWnJDWcW2z3yl-d5t3@;M8%sb1py`;d9|x=(v!`Z!LB(3( zDn+nR;@JgY?u`{hhR*Y-Ft+4$BDV|l_-K*5pI*(TYBRTT;r!%nFwO|Lf#b&OXJbRn zZ*M|6wFgd73WCZOMFWnbr$U1RzEM_EtN3%0YjRD#nn;P^caTDa0zsygGb3+FPB7L5 zVio|NH$T&dl#g9175(FA3F3^aWW5MAx$qaL^Z4rHxI>3=dGpCpmYTCGfoo;<({D3Q z80~Y~7lUG2;w9pfj-_uoC7by9l~5Xfgu2S`4%(`z!IZWHKqTWhvaV*(k=bavz5suD zv&|L3_fOJ}{w&gHL8Se781V`Hqvv&O+aM{Rnd1eb)mSnmm%yQW@B{CQKYo9G(FwUB z_1U#VK0i#Z!9Rc5>!0Ex=DWx$b#=wwJZv_fGQ_};)zvJ)8R%MDYtGSr%^)^0+TMDWw2_v zTcC}YK09*Z0SIGy2&^(^K{{)XXW_|5hBEyne&m#=pEEbuHRj819yG@Z<)ZR5CE~t9ly~Em4yj z@zNjIrU9kM0E1Q`2pYP)nv}wqj=1Ulqe^@4qtu#Gj&8;B=&9R3FZ*e+SF9?6O&Vzo}00MB1fTJsHNyxtrGDSUo`ywMFyObGKt_pw;ZMgiu=A*(wd{KF(9vcC;o10W!YvU{YoP-9y1fp3gifcNv! z6xdnTIyV$@T}^01D~lt5DzLB;xagXu+7mkgZ^fbCE7uh$0z-TPN2V&SufC*eV%+71 znp!1b=jtRR-!vlX=N^dby2>^Y#R>Um*Kx^c35S9)$5a3e(+M>Y9O& z5x9e?xS7k~q?lRDAiuc0au7iSkCB;G3tYCQ92_ii-MPXlU|FB9#kqfe`sZg}SS%f< zq3ADU#OjqH{dS|Yqc9FGP^xVKqI=`p0y|{ApN6r z3n-4Jw|^BZ#O7ICLE3j{7;$)iqe$8zr%uKV&SRHfyAG7U$#^KgIxrnyDhSzKlOJ;6 zR~~r>8Mg*hMjO$Z~azVfe;f2NRB}EqbZ76LQT}7)zoT-TwG+V^N7Dd7s_?D zbUV?fPmeg{J%!!W)>Z+BFdh5$#uQZW##!8%UzD2tz+bKp9JYb7mot+6&i~F@V_qN>(}uQpup${jC@Ja}REwpd_yJ z-&px|KQ>&G`SkISfW}lGZI|&d7L}*Isb~6KXR~m9y2^JFAf^NbvV^ixQ{|+aj`i0g zD-Q=zz(D{L^ogye1n!Yn`C?8I7hczt(x%>^9Q5`sXAyU!h&f0shUt3kuC0PWGeI~u z1g63&6#B8@TId3iV@?^#!&yu2#NH~Cqb&X$wuU3+hQ*f63X}sL`#BMRkH;4a?XD_q z!QxWfUY|4f$g~Mgj|#Oqz{)ne(f|XZ{Z8WgeGr_^+Gsz6VkH>NW?W?CQ_+>6ML>Ds zLKJqRFOi`?#B`-5H(>i!$q_haCl{FQ`OumtFdXY--F}>EL(n8*;sP<&QWN`sVH+s8 z?+St2WcC3T>khZ&tqJxZ!m1%@_3?}U=*;hIcNy(Z0!Mpd1}zJ{_dmhyy=&>;&5%D# z5R6n6#r-;-#-GVsQ%-_0qO=CMdiYJMXCQqQ*;au`I|F+leYO%6MouQpEGM1UH}Ve< z2ELuZ;bP=v2=0??J%-R%xqz>j#7anH;a%=*p#qdo0j26BHoltdCzLkhcdNZ5D~8$S zBpVDj13B`}FV=$xtWG^(p7?Ga;7{C(f~kNzd`C%X)Gh9TaIH77Ju|T!^_q-Pe9WMQ ztf$_i%yF(ki?-G-r9iPjpEwxg5|tyoM20HxHs0UQ*zLbOM(g~k<40HTtE1FppTxdx z{4@#;Yui^lYoO0XgaUQ8p9+y`3!6%|XWJg=o7(mcR2gMK`QfBjxfZZ9UDPeR7ujLC zvA)wL@PLA36hiY<+_^VJK2EbD8KQ&)!E6f=hJRfeNLno`zr~*XW-a`gP@a&TK(b&7 ze7(OeF@dBtVbr4e4aI$JP%2=ElXnK0xsHKs=RQ=RwhHJ$29U+Y+Q1 zxg`4n>6ga@*lK8ffX^Z zMw*Kr(I$QlCXeO(3^l!SH)$o=j9Af7Ds86vuG$HTC!mT65Etjqj~%GgAnAT!D_^YX zcOQdi(RRtV78PBT;gamth5T}d(p(!}wKbE$ zm1O~8plOJD%An3RuSr91R(|Y1+%r-P;Iyq}wi?d&uJ2@Tez*_2e(x*CyFO>I9qrqhBkdx9@k@CL)tpT)Lz=DR%CS(SN_*-VhBWZvARMl& zxeqm>b=b_Okc=4g!O}${{lw4HoY$jnC`JZ-NiOXFab{maPkC+l7?{BUjL(BppZ}vv z`}HL3=>DF3@_6+?n2tDrasP9`0zv>Ybq%<|=|((b%ZJoiMtHKAWOAa*0Qs6W^8cI_&_CJ^+Jw9LH_PH-U-wgv0}l97!S@$^|6_>b9D=33yN7N zLNS=#dia>~(7k_tK(=#ypDCEW8hy9F0>WHUAbTY*GNBLbQUK3#Nb%?e`qBw_S=BQJ zfyv2uH;7|(B8%lVC~#b0{bJF40{9@ePz-r|c5hN>UlK{!edZvdYJ%z1sby86GRo}* zySx(=zBtfcKi~l)j4J^xJTGAq_YXJA%)8AQIG?$N511=5&{@~ zSGmgyO?6gY1JyJlMUK1RQYItXk7VC8l?+uEA?U$sNX70%>lb2)I;OiwM@8`_lvhfF zEq!3ILzY>;7q)#TXw5>Bc9=@gl1+hZwr&SnK*oc!%1d5gNwi=V=V<7bH}LH6vE&%X z-Wx3t?jb~sx{nHgdmi@q(Se16QIr~TUy(iV!+~u?Xu=AwO($8a1ms0C1$@uV6gp>) zeHJa?4;o7GAAr%ki4J$58c2r<{xmcZxnp$h*0i5X0YW7q*ZMA8INy~ZW5hZQh0@P_ESBtqBlvn*pxF_2M%72Q8b+a_=BrLRf%KdVf*}vG{1EUwR)Daq z>r}ko^4xnRIZ1#Wwg;l=YZ21#_btkWpgNel{@afCfCN!TkkkQH;Bp2b6hcPOaXzsz z_c4?+`!BLQ&rWA6{{7q@9D3{}r~bU1666=^z;H=PBb#t05|;h+>1#4m?aKh)cU0}I z<^T-t-PZ__8^p~Y2Jpk?kcQHHCHy_HWEUD zFk1*;wt^(Fxq=3T$_ui5z_$l#^qyEMMYr3n7A__1ERBH~apqOBT(M0GygM8^HGiLk zN(l5$%@{43S-{zGjg5`VZf9{i%P`@S3@GU4G4G7b&r+l&88eW9t7JMz1&E16OZXIl zeV~`!9As@Pe&wH%2({4chuSbm)-qP*OX88x%hX%OjUf3I=t42tO_hGIV83fgLYh>vy{CnNs~Cd);kFfcq#bji6BOMt6)fm|M%a zg8>ZDfx|ffL_D$ZJGsNW6KIx42keV+B7IXjvd=0f_k|&EV&|Y^5qiUy3b&RTTzuQ8 zA?W+R1Wq*EG7b1u8UgtNqTjHcKFGm>rg01xC4c;M7Uqf?!=i4PVN4FfPM^L8x@-xb zTmmCs_JKouHw+F$`u$tX1hY^FJ82~N^_?DD`8cw1qoy<$ye=#}!JBHgd#ZrXOXL_(+{DrUCCK?-;SP%h>`N`jfFUo^6v_zp4=^`iqX^G zPQ4ez+?1YB^Vx7y?K1$_$?epMXprd-9DR1*FTN3@mWs^~2I&^mJ3*GAMH?(E+}&`N z^R(Wp37YP-X@VpfxxaZvPA`YrRlO6VT>`7AU|+li?Z|P&wqa zf7kD2p69FKY7oZlw?MbCGx-IDN?=I;q<;wg*E~xxh)X)6Auv*Ea23|vWjOuxxrS2b z7KZ|ebY$_SKouJ|$dtoF3T`hL)5uRd++r5-hEv5bDG}={eRg*ZEZIn)w)F8(Sff0} zH*%pi&giFYbG#Fr%cKL#QHO#w!JzDts{nxHEE~u~G7{6-BTehHpV_l_!3715ilHe-ac=c3M6Npte9t zNSESCzf<5;xlsY1iqS4ov;FRbcY(fot={YrID+!z|>KZ5-gSI8t z63N|UcML`+f=641C{T{v9)(f&%k-)|r{O~BtOkG!ec;hbD-KO%+kpy%>$6lA@!XnD z8O_prCql}1<(Qj7i!&sjl{MF)8mAy@TnP2ok5rMqoViHHaOZz8Lbb0Mc$K6Zl_Kx$O%9)uEd&2_2 z>}UKP$XuNI_8|Ombr6xbDCN4verTbdk$Nf6ByT&+_w)G~=Iuvxvhh!euML4takrl6 zHwt|nbl`@13rH*SecA>P8VjwlC?6VRFpt1(W(K28OXaKLhu@gAa>9Hg%An^1q)9n^ zT7cs;J)8e04&ic=MI<@UkQ73`&o7r8OAnlZ-bF9;e_yGcoy8ToC1xceiuk%#>XHa-o_Vo^w5#C%sTewVJb37wcx`Fh(sfMmKL=uwW*A|Z z?3gMP$*zExfaEKC`_&Dw%F@=ha5Q!@mI3!`8UGnOH7o zPCcFe8sv(qYTMLzK{V)KFc-34Y-ong1p=`eX#TVVT2ho|fms;Jq9Rzl3Q?*a=Tkxe zsq(=uOZtzI)`N&9fJqmjZhx1R5p_e_3S{$J0G>82{PkEGQ4!G3AG?Cp+KAt6^V)mo zy8h=Q8kG%2a7Pa1p`82B(CNx;=%agFFzR9t^dcDz>mnK*SBuimf}rl0g-(JDFO5=n zl}vWcoDya|nHn>&!VcgyxDI^Z_3kW1z98klu2=hW0~$&+lC5Tma9nWFp3{)W2ZYt~p~I!1n2&ieh;d^Z9N8iw^aoFR zGY-ZKZ=(?*zU!Kxeb|{GVwwbaf)%-pNSA9d=!7-$u?Gy&t|ZBK_TXJi{GJ0UN43BY zd3K&-%E&JuZAI%iA1;0JwPi|-qZCRO1mpK0%OwXz%5@76(8@YworYIIBA&z{Nvjlo z+m3e=Zd5XIz|p)kpLXEuMOXwGHf3b_Xg=gXhXSUmL3(ABmyyy9S{hDkzj!o6nnK(9 z{$Lw|7<%WP4WRLtP%H#--mR{E0nR%y+YVw~K8%Vc2jA|!PFD-3AEv5d9EKvB%G-!c zvirS8_IYJEq#PH9^ap%wKL~N!f??!Y7JEu*RY_ zC_&?|*_js@x_S)*SsOoy6=*8|(18a~GGYbbiFQGq!9Fn89G=BZulZMuArDYWjFhwr z9txV5o;*>_0riMxzP=Jrt=G6+8uR)cNNT%>fEF%8m_Kp^9QV+F_^xePa_dOzB+ z*K~YX@6{P*-`C-XDu75Q3GDpE>Ei%b(G(izI+G@aUqV^|jO$B1iMA0(sPi_;(e3aS zYQ&IqGy8fC!^O63kag61O2d)N>QZ|$=GB=4W~vl7)m(Ly@nYT-LHATXtp>lf4}xYuM_?W^z5t497AsryS2I&0_xy8 z3xokqriAD_Kxn5?X*Wj_MdY#K=|p^3~n z{G8``C+6IM)2*E8PPL5snO7I>M5ZrR%pGr(<$P16p2k&0tX_}ii84h){|gvA@xYGf zRZ+g(fju!9{{tKU1x1HkF7w?O*)()hkmZBq^fF-+1)`Vd!dltW2@t-Bp&6Nj>~KM^ z=TkRfyeAZuRy;HFu~8Z%YGWDxDX=?jXn=aK;MHq3QF3h`ZZClD4J8ehjL$_vSYB>U_F z8;EC2NL3zhsZz6X!ydh9vul)H-3v8^ec7Wqo|h94{<{N+^eH}-SJ_@MT!oz6`&@c) zM*2v!Tfx9k5=~oDD({usrSlCtE}SwzD}r49Vss9gxXwAL%eC@qyx@{Jh^-sRD0}1= z!MRfib3P>(B~KQF%+;Tni1=V76+Gt%tMB(FGZQ=a#>ACu zKl^mT4e|c;j6(+n*u#F4)xUv>tU(OlZ&IV6VoG9dA(k?9hTI#a#j^_4((QvRzj108 z-&XNfaJbmV4_s~K;jFLRfllI`c9VIcN7rpmDqhWl?2<VS|bs7~@RjLcxXJxq!)&H#81J0V<4X?8i!)iD0QiR!|IYq)}#U}&2dyq6$JF&kG zY=R3WB_(^EOx-x`d4+LrQ~w0Uci8B~C{p~MdakWAMcE=C2$!<#1tugj&cw_Xrt6h^ z)g)=QQKzIj^slq{k)>6%?2U4dEh<)MGPf5pw03@dBz@r!h@95MUZ^!y19#2+X?wdf ziSDUIB0>>6XH z8$1)+BZJTv!)akSzn_b-rq(&zYk%(@FnTgZ8G&J+a~+__Bc76R)% zo4qDy&dQ36Xg1hLj6br!8)D5qaV^N0{6L;U>LUFRighB3q5o0|oERTTFh{OofzcZX zvvw3R%h+MR^JEkb$G)*}|Fhr1a&PyCyZ_Rp*C$KxZ|Fb7UVPiOwpIJYb$7(>QmQ#X zaXB=T;7Mc92@?j&qos$c4FiwTkyVOeb<%zvbJ+Lbsj#K{hbR8;fJ^ostVPTTOXyDW zz4y4Z{}HbK_uo5MEts{&{{@)o;{6m!4&8%42V|ymz(Z5j5CU6;f{4<@;M$%SevyIs z10pONoyAb4hgo;J%1j4VnjaWuCm4FyT(lU)gB~E-izc%>A?y4W83A45+MQR^{owmMbzIGuThH51;p6y{CFa-uhRC*^~(~BsANN!7}*lmn5 z&;&s`h&)O)Ayx2jE_inGimD3j$P%^lM`I-}ck(Kr8efBH!do!_5xOfU05C~|9^aMpkoWQkY3n-od;-8 z2R?(N(|x7o1CnIjTdoB_XFd$eb!WJ2+xaA@g;1o2NN^v#}O}d29TIGH{I=&xbA-1>?>b1iN z@Y?Wgn*q$!xa1oWyOJr36Dj+6FD)SP6b&^^$L;St!QxS19z_tPq3df!B0&P>O%GS% zb=_F?-XZz)F1P|Nt46Qx-O6r5Bu+vs8^TuvcSh~NPwdaX0DDn z48X2TeJ7DPc8co##xDGL;$F_}ghdD(7D~bKpA4#=e}rbu`4(KDzIUR+feA4+H;eKz z?)P7Ukpsgto5{ge9q407yhskBF{ulhv>P5ebix(ZAdHTmNi7{3hax{f^OhCnt)R2( zvwlV$kms0ph3gu#USW@>`8b9{xH6sSH`uXZB8344`lcGw1@A^7EA$qRL0}MnZHimC5!F1Yau!B6S7g zSOpC=TPWNUiwi-wS|320bjLa{{s}TyrbeHb(hG>#)|#R(;LS(uN}xwJeEJOx{P4wM?nkya)R7yg>M8D#WogFT=mAo>1DqLA|?t1r(_+o>p|Q|rB7l2t4C zQ^@ot8ZS;91dFXYh@N@WjCssIK1-#x>dbxJRf_9tw5h4*N5tMdlvE}sg;_YYU-hYV zaEDgqWaNil zPfZ8d31T8%^=@)NSjbg{@nQ%ybj|dm&+S1IaM~Id?;F;`b%oc|r{2qyL$jVJ5C8}3 zIP?1#cvL`t-prS;_5Ay&lMi5m`N+dfoSJVUQ^xN>I)VV0)oS49N zsDSc>PLxrAH>dBE_>rNpS0zNXY-N8G=a$hp@(|8OocxgrI6IkzU#y$m)R`Gr5{_~b z^D3@!#J6%{KX-MMo;xSvWklod-rnHzZt2vep=i{z_QZp|aLy~eC$AlZ>ii=1vzd_H zN%}n@-bB_g!dy(j^aA7Q#6yb^{r*<_it$fiV*$xN`XCs5;H066xeVWhmD&0(Lvo0$ z){RaA=tb+-wV-Bd13x>el{7h~_>VumqE3z^^-kkfx;1j+_eC}YB5h*xB40QhS z`J7dHCtrjdrM{Y{dy+b2C8)73_4*&^RoOdzXw1dI*y8=6(#R9#3{n1vakg?>@7~+0~u?BUEPzin$clNoK`{~=_S&~m#&hAs1djj|xyPSJPN_#Yl z_blWd&vr$M1`L9Zw8VVs$)ihJYHatDExFA*+O;0apP$~2_N||E-5tJF;kks=9f;^m z%x95d>OH!LKLxQrPn+Q%6kpSA-lq`JNZh(P@m&kt4m<1f_ze&YB!OW)qGixpTO%tL zyG-%xzCg%pcHgrSo@a_kzA#V*;_J2D=nqxTgd`Rshp^v&1NyZrBI1=g5TqbnAkt}e zpuk;{x)i{8Gc%h%JK?%L^CdOqmHPf?Z$j(J{qsU$ripP#PLMDrz;BBN)@H>iDRF0& z^6UA;#UAw=4pv6cairgXLv~2%mBp9NYG?}P&7NzLo(;f6$Raw4^^Ae3ztl-9%QoQ`7fRm1#-*jbW{*Qt+bTb$UCPcO7#ZRolyg6 zAgdaZ*rbqUN6KYrk$^aHPm;0?h3qZEnLHSAO^fr$I4~$=b9A_U;GtXwLg>; z-*vhLx2KNxtKR?LSHjfh7TAN6I7k<@@Gxa2o1g!x_{t!=JIfAeC%!oN+lhn@_DBzu zfF+brp3v50=f{gqG2b(_^&h#EigNh<2MmFEAGTiek0nCs?o z&L;-tYFC5z)eI;$x-TioZb#9O6^0j{6M37MZZDkdu%Z2g$3X%@y5P_hT0QzFilf2@08u&TA^fLDmqDIQoF06HVG*Rd&tGpq0 zM0bqg4YsaQJcQ=sixTt+d6NvFxoKiwucFTy`zZ7DjxNP1F+&rFa<)Sq5Lr%A6$wc! zk>t9__dnWu^JuE~|6hDl8l;d6A%rL@oXm4YY$0Pwrf4#hOv%g`P_Yn&%NKzUF+WUTX%ie_qV?1pL0&_={>z(&*x)$0`fScH+Wse z7l=+2^$nb@kw|;hx7ea8Lg?R8B3ZujN67D*5=vh={NXv!!rHE0tfbqh@922KV)H|| zJr-B(ghDlcD6NLl)!EG@FK%x6z+YN_%q6Oo`?^#%$A~feWNcEnR^`4#irCh<<*S`y_8zqGby66Av+`fv0sx=eBS<$U;dM>!@<9-mi)SPqP}NB0kchF z@h6;wht!re<$M}B-?X^?twqr}3)C!t48nper9a4d{*8Xp8RN7%X6|D<9d7s+Hg4M` z|I<&ynumV@5ygyea#US#;8w!XiLNr2kL)a0%F%g$#~f@A&bD^9&O{Mq?L}*IHRML& zg5AZ=)@-Y+U3-?|6J`0uG4NKQ*dZxn2wi>zmNsazm{QCx2bMtTDAjGZ-KGb1#@_h1 z315AoYXlPPh4sOAfD|`!9j@9&)hA!r0sPE- zB~JM1kFNYv8enf$Z3Kl>r%sTf~puEg@qw4Ilpdzh2lX{OJD;GxJ{{DfYjAJRh8b z|NFac>p?24GXiOncK;06@p=Hykl$D6R9Z1o2oql9!IMN~ULY?@r{j-vn%MEbpg7vN z(oz*uu#YE30)p8N2$v)Xafn~3r0&a?(0rwXX6P?WnMAR%*6{JEZl7vSSjvQUJo`QBPQ9=&)@HG(W)$lfaGuz00wL^x^Yegu)2S&)ql zk+2g(J$3`d+!^rOK|S$7_Vro$Doyv>YH~FGpbKzjDtFj3(e(U-YuhMa#^Li*i2eYB z**?EFptDIsv%d`EX@|lZ=lO2bQ!?)9g42ZZzgSE%4!!amG!*FNvHVsZFL^`kwg7`$;FSa@%M;|LV#Hf;Lx+rnDzf`}kH6 zm}KYY<>jqRVO9|jh^!TF$R4SNbj36f1@p;l49%$o(Nv~x6f2wq|KR+?MuZo(Z9?5q z91@Rf-L#HgM=OY_qu3ZIr98SS!tP7jMuV;~nVR7Px81XoBv}wyho0P6%N?vf)jvT| zQA__N#k*`{&Pz*ayYjO-bGOx!?deC&KWrQSrH9+s+Uy8HeG>V(lxbyyb>aX$4J}`q z){kj@2A%OYi_~KjnHz4x_gqTH*IOi*c@}eR1xuDCP%ot zOAIGNg_BH1!8Qc{B(Py>145EVUxT$n4tmYwC7xoz8#Ro3VB=3TExIH@{B{OKrr2zO z>^vR=kX(^a0Mc4M#24fK5U|YnQi30-dRbD(4Q|}CC?Ty2_>RRx$TuF42zz{i>mH3x zCaP0K>vi^7sF9jpe*Fwx*39K2ROXEupU2IGpR_}S;0|#^M@Wa2*UKp^|gh;v=w8s;GuvmXIQq-;gL{+bWm{NIMn% z^#uOYxhe{?o&E)UH=LpVumt^el_)pH+b|JuZoJKHAwVT7_b7h3|GS*$yDPE(<&?c@ zWj@Y>pU-&%4tNL?;X!p>`c(=4OmdSO8v&uelKu&xIYvjQlIus4Z~_&ao`lG zHR!1j>NN{Gc~M6dNrxg#L%?7$DmMYCiCWO```NWTuDL#1tJe!%QlV}n3x3Omb2_8X zXrD@M>;VB@GvowvjcTBA=AXDb z)x!_`@zn&nCxkpZOriqfcage!4nAzVO1&= zoDPebTz4?T&S=!~a&I`s+5;ED{8R|_2OKJbbBxyKc98PT!s?X<3>qe>z3~2moDu3A z-sC|>j)(c5ub?Nu&N^NMkq{ZM=Xbo)?R;WBBagD2f4ogK07!(YM7CwlI?;r>=q3cv zR$^SpDH>hvyf%R6$j;jMiF%T7LD%|uDen_~u>s=jlmnf^j67hT51){*4<@L$6g|1- zjhSIUP#xL@^hY+x`D{VOp^JO9#c$xK(jb*UqItCw3F}HL6cP`TQSEAows^R; zjB}6e#$TnX#6UQ@-WzQgXpV!+Y)j2FBJ7YQtU#bEH4q=eOgTG4h4=~~S>I35JPLDk z2w{)mS7C({;bV6JWjZ>3?sJm?M_k>Uec zM`p%oA%3T-e*0+xZkk0se{a!sEi=h%-^GF7kZ_o`5*?bg5yIumCPTPL93-Jxh@fr= zPdURX%D03|E%FZGfyUe5$?|`k{haU=gaKrL=9(H+?T~ZDL^_cI48NkTEzk+aVa2%( zq!TNcp@Z~q2MfUVy!`%AV1rfFzY#C-^c3|z};IQlPNk!Vba za~=J3U=EV}Py@dTvarPxsABU^?)n?gkH04mAx^DF2JWaK63CLwJmIEkMb<8NT~GiX zl(@%8-Un30zVGmZJ-uLd1zGIZYsdPdFrfhM4$l&aP1IMeqw`0yj9L$4y8GI9xaaO5 z@`WY|#f?2!7bU^fL4jBMKi+^)u&YpGa$t2l=y&k%a!ike+EVTZJkGclH=MNs66POH z1&x*D$}sf9=|ebRg6h9TJv+qIxBewuYWo~+B~r7LuiCErE!H`>2{}2lbz8U#PI|@86hu zgYMBZAkf`&{_KG86aInLC4v`9)UO;&jAY=$-$#i-juSG5#I9@Te8H6;wVP(Asn7SE zI#lY|1)}(Khy*+5>h%WH=SV`BRvMI<10Im=hbw9B8YbFfII~W%4en6>*(rzD`(tTW zj;ekX21EuT*q*oxc9=OcQ*`If&FBV?<_UEYY;n8OoL|wtJd@)U8dT< zrFlWGCy&AfDOPpTr_w#0!j1P`sbxxVQC7Luz0#?R0m|-~QpuxpAkyS{!4by zn`b-q+n!17`>tv_?AXmaj zuC4iMrxk4_i1t;w{h%(EdSYmL($xJ3HcS(#&#yxKRPcrnaGE{$KK+4J7WLI{cjDQq z{AmmA8kQxiSiL>)HtO8G06`7!=iz?Hox66tAiD|FB(1RJF&q@ev`=)i&haK!-+=KX*%3E0oQKS4;9>j=jLzFnXr4d&+Voz|3-+j9LG#B@7xF~>MT_A^9{Ac%81=y{3otrYXlWW{ddQt z5P`be(*BQ4lzZeOP$E?@A>h$R$6}y;zHDMk_d_+s_0RHPt(1L^bZ&3HY>PG!^ z13LjF`EC%qQVjeeKBXACiY=|}>~Kp=Q6e-93W<=}gIUDT=ICNdVf{CZ%tBd;a@{w? z9Q|!mZ0%IHfe?y{qC)S5U@R3)#fLUEN?BeB)&nwzNkTAM!tgZ&6rj+G<0VO-G&vi@ zO}h)Pywd__#%n5 zs9cc(3>2_0D+Ao}AUHP6squxo`v5>m0uMou*-lw-vt^)YYZsVk+h!l)hgw+!Hgp|O z(t%~j^2&JQDF*xLZVa|soC2dnTrjX^Etl2lda*P{5CY**tp(nM(!Bow0WoKyE z3JNBBQ_MFrzX&?Ef}&q{Z=S750j$O&8GohTsecB*Izwn@EF(=9#FEr<@guNDXfq5P zrd9k88>?Ya0m#y%+-W1n69LjJ_BH>vZFT4%hGg1{ghl3aLAbr%;palT%%eQ5HN!`8 z?`CuPu?@Spo6eM2_$_D`SpGp$1NNV2s6!rP z@}b$c1rdDq=ute5S^d^82t54|CtAlX4n{jXHBmYPVZ;HyuB$<jiC+DvTVa?l$cM@c@leniu2Z#US`4;e>MOZ^NZ zsL!fUYdVp7XaewU<27$1x?jLnFHT|4_VvZ(Gv#>*4B)H;fDgDGy%6#TbTr?2!M(%y z)Mg6)`Yb@_S#Cc?t^*{)vS_{qB=ux{iUmZYwxS#nv67Pz7Kmc;Q=W}LKu|j&I5nrq ziy(WEK%+DnBe6PysNx>uA)q==Kr5e+F+|-?vr{(ur$8YqH~bnMI94){F-nEvFaQr{ z?V%~Y_-hk<*c}Xo?<$*z85em@KR}Sf>ZaUs-$^cLpzX_zSrh+msISj> z+d{1y+B>WU|7^^`e+CfzHzNq^%71c^@}FBG{pWo9r=lm=!1*uP{HvRCV{=#S93dlt zUKJm=z2a0fRPV&Um@3zXj-C2@=fC|W4IjSf|5erazn&6KBf{sGj@|9hnzX|zilZv2 zqZa_^%K%^SiBG?~035=ZnP3ylL|CV&l6FfPS_DyFY`FTFTIV|h4>S1@V#Z+=a--%c z-~c;RJIw;TNZq~XhFb(8AR0XSg*&tiNL23YZiZ%sDv1ZIN*8AdQNcaX_)7mPwVHc) zJ$28xnsScW6tmRjb;=aWs-)2 zAaHopKr>?fsANRyii)Iy&@ zBcxn+yeiB3>pP%()BOl#(??N}{tWHjb(L5+bt4U2Ley}J180ENsP21UKO2VTMa!;e zy`a|rZ3gg4$Oai)hJ$=r>-L!TX-FP9l@0%$3d&$iD5DATtUc|y!Yd$ikjw~@{J<^X+!L2;S$%gvBE$3?%|D6u~ z)0obFow;XGv0zr9;+@ecroLoGfz#`38ETt7Nn^z<3CTEqyVW+C>=^%6lm**x^iIku zj^UeG1%u~l7?!&g9VXQm<;Bm#`(+OO9x6?4JKM)Ua}TP-)*_5*Z{H9Wd24wV#7wBvj;+ab^{0Hs=~EBLdIW zN*Xd$X$O*psSWCgLFzO3M_vibJD8?p`)6GweEgTFu>YMg=szrt)CKpEJg^`d7%_n3 zY1%+s_5j>HLhLy2NGOnA^^t`qMl~Z`k2NM5+E&cKtZmMV=+lR*>-|*|F;vJ96eewO ze;BV}AXhvT_KDnUz~!7-EP=YDqZe`GJs3zLVYLAwxklPCK?eiV5jc~C?1kweb`z@r zs{Ix~*#o#VMF8Kg8^z*Z$I|f)s<`Y~N5X~g9}8yh@fgk#4sGi$R87Q0>$FF4kxl%Hf4Ev8% zv|jXFx#f#94biu<3RRe!&!tMWO@sh>mI{s+U7D|gp6@g|c22)eK) zu!;fpIQ6XvQ0u|uAq4&$8QX1WRUb9sb76}r670q>2eUpI@BbOGTM{37aI6qMcZod0 zsz*<8{=&J4cU9PFEZlZeL$7hT3!~OvL8|ip#DvwRovJ@Vn*Lc zg8^zd|Ar@2yxNDRI;&b&PDmMFzx(%F*mYI9;f}1rp;!YAU@Dp7-d4-V2jrEVZxdYT zQsy?v>Wf!DG{}SAX~pbA_Yy=49Ky)F#qOzwejwYEaDd7RB>?qLyAZ_2yzk^iYF<8K zx1#Hi(C>*m5rEIY&bAp{qTo`pL;fOlU~I_)B23iO~3m>LK03025&Nw1no^G z$OV*;W|b3Ukoe#Z5how2=#u~p!76K-mj`$Cl|bT(9`j5%2)PwKmPKf2G72trF;~^} zvAGSMPSgMWA9^B@mZR!OP*?(D+6l4W36*R-6b_bdRm%R z`)E3|T*vCz8)O&(bSPZuh8xHVu=&L4A$O3C)+dXVNsG{+5Ja(EX6-YJOGy)$hp7`Q zFmO3}RKIs?y9%wURoLL1DBX3|8Zlp-Yl$V;+8!tDWT|6xQMVy)kcGxnH zKX4wh;XBTst8LAd4g={UZH%h7hiO8Q2bU^wrh6g+nFR0(tPynY#bEDv)J zMqn^9DxggKqPKCn2Tgx`)k4i*gMndwBwp9P_{Lv9UJup3&6?g|^>r=J;)jvqlm~A^ zALHcIJs!VHQOVD3mJ+l2L@x_mnOeHk#7^Zp69GY#I|FY`v_*l1IYm{7cVnQ?-h|yB5Y#FZ16&mgwQqEOGz^i`Iy&g#IEbhIYU5t1}vq51qhh^lzVLb0G0?`QZb4| zl(xSZYK>Ei(r|O#_Zz;sih%%&a$p0_Fv~Wb01;7>-^0-QsI2yIm3pj`fv4DFNj< z3tyJu!yDSh)PlM&T#v%Rl04455SwAQ*QZ=VoFl+urFRg(n!MSKkC|NDNP8bL?FvXu zG3=Oh1bWD$G=u|YenII1&UUZkDLl*s+G2wx8oqZgIg+?4)+2NxR~=4ii9b@-YPrd# zwP&FQ$B)P{0FkXU^U#lK$$iRRwZ*#WHfRm=S{V<5b0V0r`gh;P_~?ZILEr0(u=37W z3LRG|BIqeFH`Rz;HbxR2rJv15ogwOWcx9p~QDI1=eBoB4C7Zx-?%uV%EN$}DVxjtE zeFJ*h{zJZi9yL)WZa%q#2HW&bjpvz{xSzpD&NRS1a%**`2-+({GhxQprcU#^T)^cP z6N>7R2U@a~);?};8>@c*@Ksdva+r#}MkTx7D+(Z(BCBSW+rOcGvK1Yderzh>s%ZtV z>Y<-dRMXq3kfem|eONqhef&7=X&?9Bs!Dnr#F^1*Igj?aFHi>Pe*!JhGq6f$je$*b zpBJ3SdK}&L37b2AgJl8PrDu z4Y+x{$pbw3Jk&mJi!)gJec2ttZGdsfD{{Qa^TzXXAi;YEx+u{KUnp@!a8IqrE$tc9 zrm+b4o^O00BX$PuhbUQicD5?mW0@etuWTvm+Z7C2k-Hd5d>&X?%SD%|L+&8HXh8F^ zR`Pr=l)n2$fs+4%UCgyV*;qJ3rx~N4=KCwI`3KtyE z(TP3nM%0<-<=ap%)M4yIXC^ zU~@2w$uIrZ*^TLP8P>}vv>Cw>W(OU>>uym{#O)uN`}Kg5$nq0NMqK&AZTwyT9=ox( zgy8-`=Q60wM5G9zDP@=`i8YbS+#I@}LT2J#d;tic2or2d($IITC}WR7Wyi)2-S<~J z{~q8zmvl{|95rKi?cIg2_8Teq?I5tQn}12&E;cx@1;x8-B?=-7)^>>ohoTmCEy=33m5e)1g<6%C^(iaa?#`UJy>ma{nj$NBj%bwgQQDCV( z!eh=O&=W4V#y=ZgtCimT3kC$FzFBtkzDCk}vM?a-OT4A>@vb6dV|rXSSur@xxq76uI4gzc4OjY&#=WgxW-N$$g2y2%l1!-8en&*dqSr`-tY$9}odl=f(2Xi!W zHO}_Alu344YWFiG*QHp`L?o08`wB%p8G``7=C&z{KL%h^4lDcCiqQp4ZM>II-V0A` zlg?p&!YzJ(EKnSU*DkRb&Z(`t!!g3eubKCG^`&$Wo$8NCE=ftNDV4#6WY-lvCYHH1 zxOL>R&H#YxVysaUd8onGT zuiL=z-p}J!2~kT$QEk2_fj39!or-^GxX1kCXaV`>8pY;DlQn{q5{q6n7r3W9cJ;X^ z`QCd2wQ3upErF%5ILct&Fc}^JZLv3#M{xREy-HhZord-%-eyaXRC;t(LP@E7DvTtS z24WzyB*8UGvAgiclW`sWBug%}<5!a|ncMCZ+1AUnEsH7P^y9tjZCmSU-nPEm$^DUu zNsnQtpyAVKRVS^8r%X~VCj4RhooFTR6{*rmsJFK_`T5LK=DEAQWZ4fsa#8|_VDyQX z+`RUC=j8b>&Az`cye$Q-P&IoU=|rv4{R!;G0z}hEC*HhY zRZ3EV7anZ{WPLmPR-?z0_qmyStb?j8b#!zjHx1IJ?wI&?kS;o>F_U%v8ceE7_%U->&)GT(_$(=?gZ!vZdg>(@Z7of_87y1Lzwf(JppR^Ho84v6`cjCDsfa>w{Z8~GL0oaNw^ZTbd&CF^N1-nPTew^McrqUD)6 z_N&7r%j*)cDrL>R&SlNbwgNY|dYVq@F;(hYNA8SK{HZA}!MpF;P{o6V7*1tTLOMNZ zqjOF|!>Mtnyj&n|TKdVjPy0>=@}2{+OxUg(*CKy<<%686DvWF{MnShf*wCE*YAHh8 zLostTgZra}5>+Fp;dlDnv?8dZZmA0>grNlB%lwt}j^jZ z252<0zua3|Z&!k}tDsN$X_~#p9MRq$l4io2HTGn zFG^ewRR2CpZnJsG$W3RHqQQ6@wm8`<{rtOQSA@9*gq$|9MenC4u9vp|v>zv=n)!bR z_k$Dn?gM%r-sI)DG&{BZf+@#3=rk+SMSc}$GicB6iOaNkS$*;D=!}--Nw#1HOa8@p z&o|G8QzzOU%DMDy66@$&d3fTdlb>|-=doUeI+gxXK_BVi0xgx3X%S1#9qtZap9eJC zJ}@fLSrJ|&)iULC%H|)9cAWlMS1j@(bOOcCpFoYVz}@q_&fbkY)4cwX(b-n32=#Eb znq*?RQ(AZpUx$wsDM~8}0#|BV0u+}NT4ro}-vvz#?K8}%c4xcIJ<(UlEnua8-%}=C zBpM1oWd)dxA<&y}b>&ue%>E0DtFYs+6g;qeG`R7Orh!x{ujuN%rTGb7zl@jLjduys z+-pD6t!AawRLFcZeUsmyOTdL+9=7X3f_SoMck3tg$;a@Cfn?X@YHN7_odBsulP#`l zh;gF%)j?u%b&dD5gU>g`+m-h4s0q%<607vP+oQxNE7flF1Rk&#e4gCMCC@NqBhI4q zGJ9C4+_1y^R0H4VZzpmXcxd887#e-aqol_e7;%%n9BTQ!3g88QoVigpgHCIeJWjeA zg$l(S-PVb{^O>_102&smrk&t5_7a>Prcpn!cj@NUS^peW&P_2!f`{L)FH}6JX2AG% z<(zm_0I^m0W^uI{n?sSUl|V3kB#5Wft#5zYYu~X`sWLyNvF&qykB1p-hZ>zt$ayj{# zXAHC^M}2OZlTk8331Z*Wj~$(c$rE6AsFu8@OS#JXWSO`oF&SEaC0LD^vyF>&?5UXX zj>v9<{FlxH^{+Pd9Ib$W(c)iKv{661A#Y*3B{^#+@u;WTR80`g>wt822^9K3xc^fJ zNDRz$n<9&Z5E$BL^sj+52?bU<1DhJ>iaK_^a1?F713=tGhgg+C0J$^T(VJu zJayHrH5R43ZG7ZqHLsgD0{AdsZg%cWyn2qHu|lw$xKI4aE!Ahf3c~K8%zS2sot9>gA|n)~wkMy-uizL^=H@!y5!f@_$-6cg!m;5<5PgS;cKpwT zdxExet@=+}?Mh%mxq7;k(*s#;+S_M!s;_SM2X?|@h*HGh{+RQjdN&$TxJUHuruu3M zHPgxoYwD8Gg+>`y)M!xeS=69IIv~5%{KW-SI^DZAApTaLO6}1*zf^#@C@A?(# z>#dU+J5py+@9J2&XJ)%Uyy zuhppc!E5`!l%z#2ukSdosD6ZHgO0pDwdNb|Z%Zk;@+Gh(UW_EV&v7MXZ1_kZlR9&tzQ ztdt{u$g+jcu3Gi6hilx`{naPS@e6Ragpmi1!{x5zRugZ7otjeh2$4JNu>4gv^?G*M zdx9Ar4g-gW5;n&qt%mlpmCavzP}z5TghAxh&bVrVuy%s@?sCHe&y)*eM}A=(?U|=m zhXr?yh7A#Y-@SG+kG5+domjf)k{9>03j`Nz9p8bVw~z+<=M~$KLfP|0rNUSm9>I-r zK}WfGp44>ih-v6udbqE==bUa>=Rn#})4bqaBOh&o78gfoNzr>P-VfU5+HLN~S={Fm zw|xSXX5pTig=g1_yU6;}ms-Ck`uYQtervE2(2%G#!)b4VTkzi|>;2-k*~ETJ z!Nuv-gz1~j`vq;^szzm@#-%CM-2M>5jcQ(>On6Ha+08ugkT@Ayy@=2e;113()IN}P zvYyq^IXWpjxSTlQEo}eg+4-Wuln7n23x#~bFRYO_lA#-5wb~2BWI3oFwhA8Dj?=q8 zm!$GX9p*%c@p*yY>ZsF>{W@RG<3&`XZyMcuntC13hw97er0c{5Cx(c+a@56NgKF{| zLd$^s7pON^++ArnwO7laI3ubk28W*t&(36O?$U}FDk?xEGL`EvHU zirMQM{Kc}$Nr1Q3edxVMAub&hZ#iL_(PA@UUZG5?!_E5$3yj3()RvK@8>TzlBV1)v+T|;O(5J= zw(;hhp=s2$@(qbp=WqOBTi;Ye7wbPu*iEB3z+-atuio(RP`B()Rx*F-aHS3z`iN+> zj!u^AtPpdKg|mxA)`dyB481zIYC!xNTdUSLgI7BbKsx;zc&;XfQ|r~b0Ynm8QrQDz zxWi8}YM;-PT9-^`A65vG<>S|YJ*^$R`jY{A)+v@+o7rwg zo{1Tc4*)I1uK+AwpNZJInBm^)^_Ce=8F`fzUxRhUuK2trDKvC$F8`A(bfEpqVA8d(fGZ`Sv}${JZ+6u zokTK#S1gokPL`|x2B8!Sa=r42uXO2pV0?ywy-g6*5JkUxp9gMT1s3@NI)SQvpEN+X zwl)@Dv2vb!73TQae|dX+K-&4{vXL;_FYcuMbfMS%9cv^;M;sjkKgB_BAVqS^n!$6i z1t&jXqG!!F3y76_hpT;7cJ{sx$tpk-d*F6X!eJ<*b7GQ!VZQ@%pwOauH?gA+xS#zD zT+;jd#HJK>M`D&ljS8+3PRUi<6!In^a4q$dOP*QDGbFJWR&Q)(>2t9rx26Zd9;0yO zfuya$RB`mm#X3|%jV_`P`7==B{e_sdn*!pipimZJw&eZ_n2j#t7jo~=#oD3w0DK#M z1`MLV5HsL=0lcHMubm+p9My1WB$>Ve@p};TxGz7=xdS!Jue;q55Y47kD?gasu$w09O+QJo*=EE<j~ggS`^YqoSZIK zfTyt51`Gjx#yqDu+q&UrzJsGT;{fYg@3-!?Ybj75PCW#jg9bQDNst#NtERZ<#y&Mie_vt@Kz$Gp*F z+Q1Qp4HB2(hw86t3~cD|Hww}FL2|N;Y{gi}mWpg>O~1&LPeqtElg1TEp!Z^p<<$N= ztyy$k^6(xh;5HZJbSyHR1&kf9`fZ35-}#qRP{ps((Z_y##=S;vegmSG`^rz2N8c78 zaWf9O1b$*PN}t?`hBgG~YJP8zHmbWh+aPn`L^Rd3Zv_T1FuHkw0nz4^_g;S4{kZ3q+|#Lt~u)yZDc7#NcXJw z6BpiJEBo%Xbxl|vp!O~AWqybq7?E8IhvmnzhBOYMk}pvBF!7FY@9OzGbq70_nH47g zIlVpNm|aZ%cJ~BZoYzEj+6n9Prh=`Gj(ydkQ!|hZxg%V0X|iyiOEt4b_4xh|c;h?3 z#&oMoXLFr`P}ZTMb=-`(@DQx6WvZ;&a&ujfc;C{C11X`p`_VUJwKPj^>5L+^x#WQD zx`s5hmLPvGO~B#J#f&X3)$<_cyP3utdgP<2H9S{NMxRs9M(1pnIAp#qMpmF?o}{uS zVST{B>&6Y6VrXw{f^YDgd}6keAg?n z2ddsD#fJlRe73ur4Gm4iROkxp%K{g;pv@;&=i>Z5zWMfD@w;hmXKD3JBig+Ly4z(O z;ePu9m1t__#@D{_X3oU#VoE~EPqBgZ9+J-Z+vyH8=2{W$T=e^#-qXft9Zp-{bDzzL zJg2pMGhrnr%~+7yb^#1QdcqQn~=`^9jv?H zSve!vf1P{vp~+;6tS>-=VVylvp{s%f59OTAr*F6JO^V>lSzFkD*2x`6KsJ>_S~lL~ zb}Q*<-`~%Z)mc2FLyrbC)al9%+%=sz1!t?Xnc)(@ga&Os2u=$_k8ZyLwvI1jU!Dsb z6J}_8BqJKzZM3lZ%e^A}mF|c1>4|p36A>hr$3n*5FnAX{!Z6j(yxRH65RBE_{$MG+ z!@K8DpvB+bu~&ZHwa7dq9@faMOW)#tJK~|MwPy6E5UA#F&Av@M@z}}CKYSgViP4U+C|Y2H7MQRjahwr?^1nt;dFoD&I? zb3h`KbZy8IJ?*nfQr;VztAf@U@5TUk?dZFL9Zwhz+hweQX-r*FSb{XTt%F;gXDybg*9tQgKF!2!+e#BLOp4;Qlm+iIJplYsBpk@M zc3i8Kk}~B5X}$D7OotDSAt!nk>c+jdb{-P4_6;kP^l=1z8G}QgZ#qZy;aiv6GIV;u zscD*S>wUy#0k*~`&I$!pUiXx|t}$$_6J_t-L!6s|PW9M#U$~<7KYVLgPpN&splXoq z`r|u5Rqr5l^Q1|kwfdJKp$muhibRnnKLdX7W#K^f;pen^A~vvdeF5%<%hKq|)8p@% z37iwHQT@=`85>QtRE4HiKvZU!^C}#!czIT9YFMC!fOnyNg4WuXss4FxjaLve|4i9u zNo~d|M<%Pie-6#4^j zpe#IT%XwmdX6fzDo;hC_*v#&AJUxdchcP{dK1#c8 zFtED`nt9W_u>@!$a6=Yb`vrM?BHp8Cx`TDl5!%MM3BXhZ2yeWDm?UR^Z$8Bn@c z5krT|otZG*p%YkDr1OfDPisULT3#QKE;V25Ar-&nY;{h0}MYy zR%5E^GVB?-a|%hxCXFtbX~mq%e!$e|I@Cj{B5we%)mJJ9wF(=AR-DW|B1rbTX_72b z`j*VuC)GKyXp;162Kp9OG?l+P0qDHI6_z&2bbQ4Ql(k&~Aj>S17cww{U{6aMvyZdr zKW=c;hVGG3EjAJnUu~~pCwzJn#Al`$=fOrxbj$x zPifuZz~9bFD#HrzA_dE6ZDmhPWZmd81R;r|QCM0=l~Jyq@sSvJ&)a%#NBm-74)Ou$ z+0rRV(K!Y=_XXArq;yWGEXhi=2lDvM{#ZA4Hz{}4)f4n%;XX3}X1MZxJQPN1IUc5* zm(93iK3)DCgi|udkO#CFGIp^L~{Wm=OJh8sn&30h} zo6Aa}?yOB{^2H8s*2=RigXby3b<`hO^KHufyWXjGe2p zzvcN&lv6ziPL4vM*jCJ@Yc3T&O*=d_!dZLVp!gB%5jgJx*PiM1kHT+^4F2VPXGhu| z+NrqRJPeNfGz&0Gt9E=qD6_3E;KCFLDcKU-@2qv@v<#7PkF219Xf0hd&qg zZ&>2eY|SLW;ckfKvf8b&r^nHE=E3lcS~BPb?};2sX7UX9SP`|JFH{cixI#V$*qz4s zo^2Z75A@aBo7mS-qcL(=%Bio~M70SxPPUBfX1?%-l^XgOp~dJlJH0C@ag<#?XTZzH zOMHb{csY;B-5OF|co@SW^%J2a?PC{N(7hf-Uk>~7F>fq;?UfwEIzZw9z6TWB(NX`Rqwxo>nDC4UELk=x2;4Jh*i7X2($B@xX8-9-pyl5kSkT;ZCi z!_Z8ckeW+~g$p{orz0Rpph^ibE6o+BmH7Tw`+2hIciWPJ2ps~F41<6B2km9&lJvY- zsR4aps;xMQ-e8YrN%#h~+hc!CuZ+nXn7_A<9Z_M+vh~+A-(p3~k<7MT9@Q~Tcb&Q^vct4q9v52ckDTpq_VYJdCkOl(fs_$3FFzt*ar#KQ6xW{WUdhYxpWpio*J(Kl3ynmNHF!b% z0Kc3jt4y4jC6D1JGR^y11}^z($DCq0wv+es`lYU0e# zNUVh{w#N5`+mz0dmj`7k?lmSUDR}h+Y~-g&up}64KGJ+UZI)LcAP#o;aXEtz1Go~#w{gRMU!E&hzZ0s^>&WwU9ksvZ0 zPQSn5kiKmra{;9D)LP$uwN_l(Dx`xgM9l=~X9BtA_J> z`VM(C9>{L>SZ|WEj;K%CDSyv#037#xmHy}z`?-$a5kM=6 z{5ux;Mw^_!hpSomifh2kI_viv$Jsu=CmP=*YH9uzlo-TcSFzHrNi?X6lkT@|t{Ng- z_PRL_C7kIJ5FDSgdSdg8*eLcN_g{ywKPNT+pglfEWtD%!;w=0c6qWt&4;qWWl?b7* z?y*(6Vn#`2w;|5pk|#^WeRXqR|%ILS&jg6k2Y%(#OsmL;Th9JeGzRg5))4 z7aLBoK6`Id6)e5Q$adyp8NjtQMOhm_2FqeNbqs2*EMvxxN4Nm7)H^pcpmW0L4tsG8ilnMDYD3i@MX7X`XSt`Ivu?B;9b zyqZ#3tsz3`DWoATIWXDFy;z2Ih;^gqU26^v*j}H5%ji5#cKrlqEzLe0IpqbOCMsm> zC+xX=XC+l0IPqisdd0Gzkakbm*BMODBax` zA{i&FV4+F9AJ}gOGMKXt?PH+Y{{nhhR;6|E2hBN2KElw@lqEwx()Ji%+~qXIKe2w> zLyS(-I#2gVkZ32)QA;B<-}*pIrFb4kHsCC(1D55A(SBfxR(DF+xcLqE5iX)UO#6JY zg8P;4$E?Mhee>Zre&hB{D;iYq5 zGPFDWC-3GX+x?ze=EA|`+}W&3jBbBq2I*c?KCoMV0i*SH6##vfzW}$=cT2hIy#OUgLf8A2F3h{I2R#ZR z@D3myn(8LvFWu-x{iQ*6WvCY5f)M*p;O}&R{r*+}kI_74*f^<~z$*<_fnX&@b3UZo z!gdZZH3qtS@@{rc1kFBiM);`gHNfiOn5nbB(NOLH4CHB`&2dh+I;$1=Bfg55JH*5U z2grw7c}&L`ulc6EhJRt-Opiy-ho?B$-vJuUFVTCBOV5701E)p#B{wZUe7Rr-!WI%s6Yc(cj@NN1sS)+M`gE(a3*9Qt0O)C{z`*l-OMr z=a}C^B<^x+yNwI-3|GFqmOI13V_}nifqOOOR#t}6HEF!Wri4XLinL$YN2Ta~w=O*% z^B!phZ9@jN9sfD?Ije?BuGi5q6>>qC*|rKR@vfb$b8o`gLNF{j-{yBPgQ%(*yMXyIn|o^uo$m-laF)?-Dy{o=;{{ z|At&CQT(JwMR0KNcx$x!;S-d!ckkZK5y0bknB<}u7-hmjLRggFL`O%%hq&kZFA|r9 z(fVu5r+drL7?*+ix91soc{L7`Qe8cC?A&Q}b@lDIi?=MVlAOBy?#W=0)xfgrsZ*zP zdjwn-zy7R!&d9*P&>tT_FY*2>bG-XUf5jEmoM2%u&zYg};)90|H>|s}Yvzk(sA5vu z!dX+D7e)y&J~oW#Z322@v4xXSzsh}dboAqz)o?{=jF1O^zw2 zpE-TnpzvK!<%2-x3;ZsNNjsI!YePd(7X;MxNnA z^&GuI=L%N!KZ$y&aIV&JRVhhQc*2-Q2yC&#qk* zQT%7Hl$%XGTZf{>*3RzOz^BK-LZ<8UzaBa|I#R4*^eSC@m!{g-yH{X_Jk)nzc!@Q_ z(wj7gJjAHyykB)k&vewAUx?*LcWlxKx%0AFbZ1hmNd+^>_5Mhd$kN%!&4s>wY4SDT zS7^^_u)NSwx7FX8FUG327G;(;o!93Ru#4DRh5r4;^6^4sMSe8GJE1w4%PvckC(E4| zKGx@Z5oVY%J$F(#^sA`UK8pE1!atn3h*?|8FzQH^r&)y;(QqGH{WE~~U{%dHx7xy4 z{en?GZqV>ThBZw8hs8D{q}SB5W9mJ++WLO?6(oAZ!u;^*8yL`LqetVq)bNC|9+s_b zWc@t+{{6dJ+Z-l4z3jHLo127o_{-zxnZCMFg0&?>R;QGt1CasDgl}FyVL{xo+}+vc z4^XI8YKs$Wy{itZ=}Vk|i{Xm@l+%bu50y@2Gq1~{ac^q8r#Z*2*Y0L~J^v@`q5cZ_ zmN53#Mw&a0vb%S)aRJ*0zEMX+pg*);PbiZQ3} z3fc@;xb)5rmA9g|=f9k4ok$a@Kcmc>g6qoGy+fTpRAPHiSY=v0MfO7FP<=h)@AZYT zA90*$PnEB!xF_RZ19QvJ_$Yj?anqT~YS+2y9Rh~*dbWP`xE`lbuKh(yG%`+ZjrEhNzm0H!pFx)=)p^&1v1odrCZMycQCw1?)TU`_|vkwsa`u9 z5+wCFr!Orn*P8R_#ot}8E-H(|cV~4rrZ1%ubCahTBUrM!_4yL``_rC;F+C|xCodyxog92^1Ght@v18tpB^c` z%bro!_b?e=?K6vi`DdV*^LKxt1;GOoFM+nKbX{4V6BDiQAwmV5MhtntiwahhPLXBg z#wl%KVd1xp{uj6#D%z5yf-`p(GB4kIbs|sW72%7nl{%e{G47OA=K|P=y!x>(BwRlZ zd9zCLp>Zl99x5LU`x|R(@9rwvNP(eS5t^&s^$f8B^Lx|hXvY1IjI0J%^{Xm|C0@GZ z@ay06A@$1_ym|8`o5XvnNmtXBFgX&guxFQ)`BLTM-{)6?fpd*M7B4C)%GQl;4q1?YRYxGZ~N_@tpjHv=|qH%Jv>)uP{Ky8gzrz){LYNyC0#Zx{rceX{B z4d=MVK09<=%Q7~_b5_p8;ZHx$Z#jMD&h7hxDh4id!-5!-N?cUNlLuKgU1^G$xbV}n z4^b!vm4n{!f4ZDwD7mawmD`FkcYV_D81HOtWE~bv5sp(;FkzWiP{B}l()4C%n>w?G zDBxA6KWe3Fer#5qXkK2u#Jf-ZUf#q-0Qn+C|DrSF`f{%*p5J$JxP^9Hh8Fp|{ zwovTtpAEv+Qd^>xZ(5mP)U#MV`$-J}8uQIaZnMm8<=C^znAD?ZnZ$Z;IsVqD)I(oW z%_5~&&MJ`AeV{6nSMsPkGZJgD%1`ux|8~mh9B{>X{A!iwJMkG$`Y$SDuA@m+-rljO zjY)o@rdev2>_S6+Sj*^GgAidl@#?_m>hItVGsAwA*cw&(nDJe z-9$$VvwWObQHkak; zK=1i ze3Ge}T7rX4`ba8-Lo=I7;1-f(!Z}?-^Y;&EdH)NI&t|hN<{SjaS;^igZ6<) zvGZqMj$Ga;nb#v1d6Bw7lRNHh+ zvMV?;HBQxQ;>u}hF#;}1q}2RL@x>NB9L=^FsyZ+|&}m6psV6C~i@=2`&(D+hrp42( zC}f=oHEWA&icWe)7aVc;L!ejH$mE=uPPVOc-p0^T{hr${v7bqdZ4z za>`9xFY2bQjd(3O|S;Ul3b=-h1w|=2=fS%LAd)X-`BCsb$1yHs*)%%lg9XqN)dBW%t`6xg|`$ zMI@9bVY0E%%^kY^sx7kE*i8s2I?A2quFf&u(B(v-hAFg=n4LN(qG9lk?P%S#w%05S z3cB+Z6&1tNeQr~+PKGy02cs`Da};$&JcSU$zoY&p9T#+;mSei@kumAQIxat&)CQAH zCoVoD9MwzJ^4B!qK#zYVt5gxz?-ph3bhccmy=kqtURI|9$%N6$9kLxtZN(vHRSO}D z9Ri;7ExK?8o!u3?=u5Wuu`B#tG=ye&y)4r2IXge~3(Z4YbF};}@+{jm1u@=;N~%$l zJu8}K9Rp#vS6064khkh_HlDk$${wWHF0!F_Rb_=`a*ic;ji0=6!HI*TtF15{t3Pg? z_#<5*;hA1WS!5ZXwqX?IJ@h4rE;DFpIRbM`1vtywi_F7fi2aHkUdqSys)}v!JY`y~7u~FU- zcbp@XmNkTlGGYbMgyk7RDULXcsV0|arQ9eTlZ_K9cbHMKkBEk(=%9V9;ZnJwIT}OU zGtceJ{Nmrd6RszR6AW=jn}q*mb{fz1gv;Es5U!N9#KgzP^DQflHPtuLv_aSxM!~K% zJ=K{a*Gcp)g8%+yq~&a1Yk zXf^p!@hM4%T&0tXzEvnCsM>NByC{clQ_U*wk}dh-b1GbTcS~7+K)cv_29Igbwt%#2 z_LWtZwtc>FQxY2AJ*j4s5~gAv+i&WNdtiR-ao52!IJAnl8{}&q72r8Mx2H`#J*mwr zL|3>IwY)s$Yf1RODj7AWuVsJ!DCo92J)2~Hcxax>q?zG4$*Jx(NTAa1eLs2W*7TT4 zOCzKMGu|t(D9VoGAT(w!O8KmTy)+v9V&aPyWV}zCWtVmSD2wZbP}s!k0XmarlS(k4M`kjTh|4Bl}Aa$N61ss zM5F{W6{lL&RHjq4&^#91vOfH*B{yG&hgIPEGs27##r@v2-yhIg)4%^xnw3`dVYaz$ zZmG>hl zb>Fay_TNoSTUh978$632;uiHYr0J;tn)#%J)?BlaFLF(8~RFj={Lii-Ec0lF1E$ZRsbinvX`rH%M!JU5>(gUl_=qulXN2RpSY zUxs#WSrx9*YkQ@~Zt~lOrS;IGcT*q4F3$x-mGa4iu@T%cVX>^|REG4k{FEb3X9s^2 z6({Gy;6ot1_sGT;D)q=7%FSpgEIXC%#k}8Ffw~cdHz!djS~A74y!7h|Cr7=Orxmr; zNpkskJX3Hu-E<{J>K#B^UPvHbGaxOk>6XfQ@%w&RBq$W>Cl&#!M4o*#J9bkwP2sM) zhliD}F8qeJb`S45`aUXMO)fj(a#)%a>(=)6%0Xx%3Z>_VSZao=8;cX2+~{oGa<%s~ z!W_44hC{$i%=DpE=X*XqR^EVQZ{fYZCz4cDRCV9KKX`TG!lKrM^^H$S5bor;wa`em6U)|z zH;G5$h1}fhh~-IE%hH-$;o#ulvY(W7K`_f}HOi?EW~^)Xf9uW85J}-}(*X~6zz8Qp zMyyy=z4ZN#Qnn7?wmT#_HkF=#uRKa*Ko!GhFMe6jH7P!vO|xTEC}kxBbAA77gKG!9 zZJ+%5e5lgF-ac3*T}gA0S-Z%RY!%^8;IO1?dPlU*bclompA2>j>!Fk)n;s4b-woDI zIB-RPvd&z1H8)s#7LVk9aM1p8g-)|*QzfG&7OC+*ocYHH&q~?f9+haNd+2A@PrWnM zCE>IzzdYeHo;H2|Z^}#1%!smWA@}umoY1Mu0_`nb>B^01-@LuOA8C`MKo%Cpb6*LK z-#DqQUfaH|E2dOi$>3?rc~|N4Hhc=-{rh)b?7H=Ksy_X&eyw-DSzC5E)*+pLk6eueWwPpQz}JD_ST=uqzt0Eh)X=Dnwo*Ur|} z0Z!dA2E>WBUE(occ2)noJe*G+^v<~sfFm-`J|P)gk>@tSH#RnQYq?WC-|>&u5&;~G zRO5T<3CKb_lBCXh2<$(^;F^@_nE=3)q#(_2-G!%A=sk$OCFi-j?X*OIxKtWqjFlyb z9-APD115n@u~}Kyt?}Ij`bxL8Fu=xSlBGi#lv8B$jT%Wy9sfklG6?{fLR5VcCL1f9ER9dSVe88k(cWkH5v4+iD+-6!MjMw{>z&{{7H*?A;(Z8vggrJ zw0T!h2fa01Z?ai@;(*@P$AQZzRM8%ef%MUTX%8X8ygyLA-H8u1&te&$%y_kS zt4gZulbEz*WB!gP5r=8w6Z@C_#{t%Vp;!9vNB;-$f)oczr{qb(`ue(L|EI_GSgb1W zi&r483j(~np)Awq6ixH|uP@2QO+i8YXmhDJfl@fVkL%95c1voojo}oI|@v*2k7LbD-QQCsVul0R^WX8-Q$~qA!m%0a`GD zWI>oVfLhhsiF;llr;k_Ji6lX_qI7;Eo&Zrc{O8F z?YW%eEeMEwhD0ldq=Ny&=J%r z8llYajAXs{RD5i+iA;+}s%;$1u^GwXKpThLahetDO%bgFlxX_sU4Te-qY9BM431F_ zDOqAWBp6s;T+A=?Hz%U>9GOnbt_D=D(`dKyZ-BAHKx&VxFk#pujnbYcfU!55Ma{VQP3`KRUL$KKSpX(n)`LNA4;B>i+znZ3HcZF?$qzd`d6wDkBygP0R^2T@emHYF~yC;-^xCYRjnF-*| zvK}nj{;VOYWYeo~{xqRm<=(e7sP*=4=Qxe{#o|-@Bjzg)vqkm17M4dHuu;#DKQKAE z*ip(!l5~X@J@1Pu=7ZgS50wKQ0qpUQ7An3oJ>C`jKfNBtK!##9FVZuqdrf9H-A=dU zie=4(XprCnxZ^kWwGbqJsdC>)f|*Cwytuf?Me+s^tB|+nImLU0oMv?v*4BF3vCK(ZITMJy7MEv~ni;x~$6urISPr~tGc2Gt^0y;T$1;$kSXo2fKEB?QaF1j=m&tzeTbWFr>)(|D@zI3? zfI1+D^ z(c_hWzD07|EsWkz*U0BvyMR_eBv{o>_IB@wk2_PyL5iX4qnIe(1oEdbofPu~jcQL1prGE8 zvU3BEfC#~Csx~J5u$TTeLWSr<)I7a>oWPISwcmZ*E+A6607SMs;MZKP z%hQzMed)@~Lmw3mdBY2ZgN7}lxYBt+6qvTP{*}GN7xGe*<09lW)B?^5#We+!pwV&N zss{YEBJc)DC}iuAUxe?!bc8(mM;j3Inaqs7zR=VrfK41Msz*FG*+th=8Dzp_!5!(B z|4W-spoD=m6Fe#j4V{4KQ?UpW=3-96!=0;DMT+fs0x~6n3q~#Lqa`OONQ^-%B6{zh z$qEj8gVi1)l1b}&aN}NChzb!oUg3!|8=67K6*(&!&29F81>{qDISo_FXU#!Jp$hO7 zZI*U1g}(ksk82iSMR*g?(=dPzg<_2bC@P&&yZdt((I$15ovl)9qBKO0KoW7Bf-L~L zMFzs|B;aNc^kHZSgjL8P8ASAf%WKEIDfC+SI~rj<`Hu-Pf^KTUU6y4K*_6JcyfIp+ zKlS2?Jq=|`B;=Bm}w(tGMV z(ue|QoDM3;(iwyVTpvoGLtb(Xkfg#Fh3svWXnu;wKOa#V?^u;G9^J@U-MdCK;|xW zQ|!OBRAei04X|4{3Ok#7AWB?b1R!5+?FRan97y!uwJ3w))PQvNbFt;`!O{YUx#6L~ zLOy;XBBCEfq$);~7wi2`1W7U6Ca5BCL?2a)=`6jpbN6eeyJSv02 zJH>Wc<;c|B9E}N0p>Au*CKGYZtCN>-*R8=kjsE$=4oMecFsUI5c9KWvf3OBtFN_e= z4ZRi?5D;*6BK^WU)mUKdIpaME>9SEbrH0++1o;5WqY`r0m`lV_byB?i`_Kg9n%&Ln zT{nwh?+8Gi?~-_oc!I0DxEOixby`o@NDVxm!I;|)+u37dfRFS|Am!rB$K z&#cS%zJgTHZ_y7AAOE}FerluoMFEIjp=W|g_JJk`w78cyHm2f76#ndAf_g4X{CCi6 zYS>Zq<(}JR2M->co-*(7%hW9BFO%NCPUNH65`{OAhav78i!<#xvXNYbUo$i5%v})> zU}s_eW{C{uw@#w&pm7ZG?u)(%vfQ?c$S1cI+OWUN&CCA@c_odBeuC{l&c{)ya?wnrq@?)|zZ^yod1QDi z*a6fWotjE52K^}hpDdf|u2It_0AdkI@ zFxrm7Aa_0g_f@YPLlqb|ebgorm-rJ5YP4b{=KXJK9$8YL)F7E?TfSphcLcL}f<{PZ zP8nnb#ag%jUUHSna^*9){AY{xjPi^Va2 z;bzbQF&ra!0SWQ?D&2~^a`g3ebaWi1Q!j&L zx#Pp#1AT=S;h(Gqy2Die7Gj*?i7rIlMCKzhDIoQx10{4f9??uEUU>Gmvl-@5?P7;K zL3rHNR%|uE1U@{_Y1l=gn3Unf-}{Uh;sv*6(mpa{0?|0D&JmBGL11#nj&EtLUj z5(9{BCef_Z1qB7c5WU@pnujY0-t+J$*ak~gx&4lyRV?VE^fv!|%Eo5y2p%924j@Lt z5L4-827nzaux1cBeNtp6LcxIf#_}@B(5*VI-pXp^ZoC=Pi}2m=nt5JYd0jpeJ;nK% zVR`2G(E{{+6iVbXDGE#Sb>3+6<*QfC2%-WG;O+GppD0Q(xnnSD^ZWua1ybKtTo|HT z?u;6kZMw8)8$V|7l>=I?E=?=E#T?Jdz%L)~(=;*QKk=kSbqYrrt2?=qzty*5V zVfdmeLsbS!Ino+;l?!&nFJEg1j!X>;V|w_}M(XG46WJF4qSOp3=Y3t7ScyAssoWPQ z5LVY9WY2LUqdI4(08M@-JomcG&Tn7dmjdUe&iN<13k7~?YUH>3@6puXXC3!6EEjI zj29@SRLwVjM?{IYH=Uy;egFN6os1H{aPXj}4w8yVegNw?*&HeZ?v>G~;awo2O8`1C z*=D4QY!w!I@HrMGF{qg^LgtdUn9whui%at8O;z^YGt%oMlrIUfc2Dfc!vjvcFh4J_ z)@Z4t1*wI=Eo!22k*CC5pn1m(fOhL64a_Rr1k+8`vz}=;b{&fKQ8qD^&l4G(BvZ5t zQzYYLMdP@t7-4i;y*mHEB%pj(In>fFe|sx&f#$UQ&gMkEd;{mjf9eBJ-X-))AWGVo zeIOI|e?lPYnjmplVatWF$`ewfK0lxt3G%gU|57M+U^r(Lg$)u4Dsy2F^DwK(9e z+sL^sRDf0m$%o%*M1Y9$RjCPW;UC0od4jIx5WJKDX;C)vr~nPWi^530!!Ua}t|%Lz zEN|6=-tdJ8710gFzZs`rY$1uIKt>H(M z)-a%q$F94nbLa;N2?~ei*N+4f^A$*A?HG**(-IaLZg$}|?|3f1cK_9*ni0azbjZF` zCjA*-vYy`?F1xQHM^BU(dY&xQRm6AUUwXkiMzkZUQdIu#SKIVX0(YtXT9vblap>Wy z&PuDpN${szA`lGTQ>YvX?A8q_{hN>iW%ne-cXV8!jg zF!%6}L8VWK?O4N%NFKxv2$hb|>Yiy<5RVmWrGgRK!x*&+&5;B_D)7q7VMkof*~OJ` z&mazu!%Yj;94rcHuMZAYGdKEgvnVtB2lG^OhL5Oa27}kC&&X!p<>$C}cqRPih#^MSYZZe2LD64Jy zAeyaoX*{%{Ra?rpV|a1;$34wr*W{9tk~cPgTx|ssfmoZ~Lpg#<+S{2sMuV&PxA246 zk?(d_k0X4?@akGJD*eaSSLagHY@|)VKImqVn z$sJlm&hk`^)a-Z|_g(RdyzW|Z;vzve6UXm1d2^MA~#rI9Fq5ZGwO@*#E_GKN!RX5cN;Sl#{qi#BkVSK z;-jRb6o_Z}6|Yoit~xVVx@3VZzWe+z*=OMiP^Sk#Le>K7)!~|m3}jb^H@1}XjaS+q zYOT-9+d4W1)_xIx`~3OyOx?mO-Dq}=wAahHff1@57MI0J=93q1ceXkG>7{tM>uV%| zOCC9Y;NuG~>1=@w2KCvByln6RlKqc}fCUJsvl>8)a7N!wV*w-gXLKOVQB?{brZqc! zJP6d4(l9giG~+H>$oo=tM1BHRfJrvu^b{duEGrAN19=-O^cFyJeKMip&y^?rJ;1Jh z-~hw;f)Q}$o%mp8r;t0vHci`ZYYcwT+VAi<{xJf! zvOWQ=TAzFuETA97d=RM`t?%&)Pf+U{FXfi#9?~$6K88AQ;6NffCoSzy6Bj-^jX`TG z583KP;TSpFB*VsEpka~WqTrAUr#gJ-(Nnnx!h~FDlU)LBts>LUoPtEfI6EdB|gl{Hlb`MBg}_W zuOi_7PFS0+aY}%aoF`a)gOfzGV93w(e6mI_EPiEG&lw33dM+VxB$z{oG^FZzK7!7LzyU!+!dlGHIdNL}O|$3YnPvta*BawGApgQ{>}0fYt~E2DACy1@0s;!>+`E zAb;B9TL(*QZL@P&3m|afg4&#>*gM5B!d`l-vqP9-zsrJJNGTI$@%NuU69sx6CcQt2 z7Vqa|bJ?uQHcf`=X6u~{_(HNOwcXG${uuSWk+Gua#$?XvjPtDT$A!u8XjGq)A0FH^>$OX z1DKJIe z+FiXV+suwKY8J8}yOB{q-VW48MLB z?6Z(1;gomQ5)Q!>M~?rLqIFO8AVM|x7AGMZ1gKLUWD&sd^_Q{29epnn(ZUBD2LD7$ z22DU)j)R(M^w+Q2qp-vf`4n6r830=3$9sw)N@87a#3I$cb9tNm$wo#hwo|)f>KZ3{ zz@5pB^s=90OcgSEp{%M5fS$V!A%UF`BHu+SR)9!ngmh{YDu$5+1&N`)2GZ|UP3M1E zT3EOd?gf<}j|WTgjK2olh3u6Mu;_3=BP2C6DAmcqnCb7e=lZ65`=gn~`qSWEf(TVY zL9K#sGl#Z()^8Fd0m3q0kxg&U0*e0(6%`xMgCWwPtiF|%mBU3)b1?%L4V9p)%CO56 zb)`N;&l8YVcI4;_Q(t`b>Qw`QHO;OigoPe(vmhK6(*mS4r`h+IKp_vp>-cz@{YE$t z0dno#=WkEZ#J~GkDcSw;QrQn>&oIw>4J|%2o>xJ;9t`-*eLf>Z#Aru5)bNRY6uJ}w~F1G)c@97;Dk21%A``3C}o(k^lL=~TfP}9=ZL*(gB#gVb z*q&m#z_AslDR9Xus=Jfu!$qeyB~&kjXX1ha)M+8c-r7be;T|kLDpy`eYPe~ zHAhbn^E7eK7aHk8zV1!7*YX-RpKzc0g23&O6~I@+ip|=rSJ=R~sX?I<15m=bW%3FN z3Yj_2!o$PsSInQ0)%TBIftE;}4EBZ(+J?;5Y>eq@p?D#w$}jz)GPDU~!*=*?c9+F* zUyf@W!CnyzG7S=%@W6-TDs3iAD*p)ZRvfGkzhKlTCkhpd#{V_Gk{r& zS|l*lX+0MTS>O@hCOPy>7A(K%Zk6j1*4OKBmP4iX5}c5!Ccf}0gV2s|6Ruj=-_pUf z)08tLt9(_0mwwD^>;5W@5C`JwAT(^iSxz(sOCYwlH_yoROu#(ARxH3*kNt+^w~l*H z=~a`dJiw$>*~hF@C2PCzHAwM6j1XtzgE%bRFyt1%X|Uz-Ya`+mh_0Jn96gJ$^-2k% zhlVaHv=ZEa*5ANka$u>@>A;fob{0D>0!@qXqaIXeIyJ{xTYJiQqd&`z z*tf1=m%5kc6G%+QqLvv!1&oVYjvhNgY(c5W2FPLmz8r-&+$g|I-9MsDbjq=HnVyM5 zuY&TA5LCY}B4E6Pr71;`n&WtOUEs-!HzkGP>iIKBNygss|Ne#+K{}|CC%;4&78U|w zX1)O}ERqQ$gaH)Ld;H0Vx_@?PJ3$fZ)Nd>4?>F-&njsyP3tx3qxY%z1ikIXt)1g#g zDhCxQK=e(7!`=ntE?euiF)dX3Jwv`KoAju+eJo&0qXwqo#ym3i>+fp z&!O=>PVgEiHiKgCI5opw71UFUux0d|f2{WlSpAd@bC>Ak{;qe(x_bnq`jZdfJ;6IFNC{IfVWfuAd?wnsNL+xk*QHk_p{lAdREo5`YJOOq{n}N55BGd z98OMrF^Hj?R{_D6fu+xI^QMab*!fE!66wg=K{U9atZKc4;f5xksZtzE{Tjtyp8Vv; zZj}L8K*>izVkp?H;9;d~I7cj$(|;Ki0v+tTTpL*^<-G5Mpw;If{JO264ZmlgfD}B4 zTm3Q^?K~uTM+G^eoc{+sQ-nsY!BaM^Pm=z-u=zsiZ3ShxZs2GM!GD?Ry69pkhB=#0 zTBfXGB3X2Mb#aXI?KpUdJPsQ`t(7op?6&syonzNAoH}N$Q7huOj8_JVmAwC&m}}vi z5{lCE4qzi*-f{dL0Q`zn!@CQNn)$|y0OsR8E(;u?*xj%nFL>}L$@Qg@5`+A$?KJJI zig(m}V$U*9oHepKA;L2prTs8ckXMXYDU;Kx-s#Sq09*hY9k{H zi4x<-tFx7oEpMhJ48)?av#nR6WiuFa3l=1f%P2j@m#RFLw3m6-j zB{pRjv^a787RZPZy!;>O5uqRyGQ;dIrOM3Xc+PDG55kEcdNU3dm6F#mmF3^lY%8ZcxXf&! z&K}Nqtw#zn*efAcQH^zUk&fQgIQ!9~M+0Tf?}kNoGRaiAyD)B&WF}i0I8}HK*`B!L#|}aH?WODeO~Be`i-|I>Z_+}YxNkOV`58~in+(P8R*6eq(0Z0 z1@BRrww!s?D_&IuLoOjkU(|IWcPQqxTbtDt49Q}L8o(N>06lGB!M$Nhmx6`_lgEGZk+HNZFn9&Qb+H}D_C8d z?k8dGZzQB5BKq?mF=05_EF;G#Yuo2I>4+Ci+%8ccv{3!k&R34scaUSom1HjA6;JSJ@+VGM7l8P_^zhZO3nQ0Y8b?F zV$N9*aYzDqk3KHMxMh1InYa#6)%zN$pKHK`O>>&1%0_+VTkO4QCki9%fEX_t$$CFe6VV_u(QQxS7@ zAJ*4<9b2*Pv{_y{5`5S%x+%xrE9}oekyL`=?ji3nasMuv>Z${E#Sgw5F=6eqpX`+B zB+L)c&si*P)qA1yi`wuC6B$eMze~Sn`8-^_e)C(vTJVc^ZA?c=;^p?_mIzZ09J4-X z{1MPJJY&F@C&enoBk_{GxFYv)2|(~~4>tp0)H=|m@f2*Gi=Pu1)ze-@fA~Lyw_fTB zImhzRQO;VWJR=D$!|pV9Xr_(#ho47gL%%X;^Q>&$V1Bg{8R(<|l%h`b0dS1}5J|CL z`!(J~J@NgIw%zWqpw!qT#aJ0ZWr>F*;Mc^rSP9PEjj1S?q1H>gUz^T0F4GYGD~`ep z!qoaVl~hz4APaE~IwiwRM8iA3_}m66Rg849XDDR=d`)dkQ0$6ehn62z{kamJRbcn; z5f`Jug4zzVZ1pVv^y%ip?t)QO{pH~*v~rn8$6eCdktL|k41QV1gmQ2y7Wi)89Kz|A z_TtrMOq!?!qfyGYAlAc`urC?PkoDK{b6<&!<5uZPoaJiY;)R8qSHY3vD7;yek^%s? zlh7UpiSKf8ZP;bu?d9NMiIZ$rTIYk+Uc#@Pxn;9fuh=^}7P)@&uBvocSI-n<*?2sB zrO#tMBxZfHET+^>TNtD*Tg~}dQep3W+hE1~0b8LkGg4ix_nz{GC~P%gte5kbL09K) zNQN?DQFZ-JHe0yA_(b5u#EC2~{URZzh5s1CkzSd0RxM3g2sV=TJ6mbha(HOPkznb? zqMnzZ)E4kVfUG%D;?%@s>pIPs??YcY2g!MPuU;M22cU?7T8ar}*_)fN3@EAmYQo{9 zY*kFyWfCTE#CBlg1;gaVHUIp>t%G$R$V4KUg)F7MAlK3-_55H^{qjBfqFT!~bD}h| zfcN+juAUyG_dWV{+vqCzu?jU0F9@sD!!8X-^>>*G@fg`QtE4laOE{m)sr@V1GufiA zfN*nfx7g#$zsk~0z_cDMb%YT8h71?chmVJ&SJ&3Ukv$U-9t1XA>K-|bQF|JF+s4zR z7=gGG&9#cl5NsRMVO2RD^hmAIvOUl9?8k?Hs?Yo@9>fkH_Pbm;x%M02 zwR$pgaLEZwn#XN^e7x^Ud>YwvzbCsQEtWH@thj&S>k7 z>&E2hy}!hx(w(J8X0lnzojN|)*E#8xDieP<=nj8#R`l)h;h}PytT=cBTd6a_-Y|VcnvWP9#LqdU@Zb%bwa&n3k@+gC_SKM{=Hw)Ai2Y{FYcT~*rcP|}m%M%3n ze0?eG3kEoTGo&JnK(Ce+H$auqN)c>EB?H@V7^%iVcI*!CapIDINYyk}b+R>%rLQ8t z1&blFTdxsHA1R5X>MhqMK9DWxj#0#9uY24OY4Ex&TjsQ*Fx(ody;!NPAbRg|3&`ci z2O~SV48Jinh2F_2J!F8z@tFL{An*8K%E+SFtR1tnt>o70WE=nZ0^pf4LFNdwi)xW3 zXTM6)vnkUzgr_eI;sfFJ3j*&?Opom9Q+{LeC6qZ-r+CmYt0H`i10_@t6LQiJv=_qdx<5_|Y>Q z!xOCHy}QxBjFRt1le!j}V$GlYcvgyN!wd}1uiiR*v)1CqNn`5M1|9~pbt%KUR~&O{ zqbEW}7JDU~xlQ%?BGMit%j{Yhs(=oD&#SFwcYMPXI;S)-Wk>2VL{_fM2?g67uZhTS z(-M1!v#g=kR;T*`m3pNDWp28})(4v^gtMEzv`@XNh-WZ%za_=HZiy7?)p5=;HhlNk zhuWYc|DUM4q5Y9tP|%4;r|YPQXyF5n{}uXF07V-uFJ|nM4WE++{i-}R8dMolp`8(S z^HI;e$+mc#;kL#`3IFZ3H@OJG#;5?*C|oZDzap0ix&<@7MER)PU#h7zp5Bs+fnv-GH8Yo zEedH>;DX1SYV1 zuFbTu?X9hE0}z!n0O91Gv2Gs1+$IGyoDu(ZpePF!CDks;5Z~ zdcz|E(ArKj1??XnvP;e2&Wv#zvYbOQz{{uF`&R)W!B1c@>hziFV@Fvb?k`%`f`W%B zX;>_EL`1aka|qp9u5;*T|IXvT1S%DZ9!%4nOe^)XFRw!33|Th`sGm%b>X@*{R)TQ( z{+0kJThKn__w<7}&3nC(nG3#n9WJ5fLGd8A{82%{uV23s!q0PRWj`-mfv=o-h((Yq za^y8VD7M0L0)6~14~)N4$xxBvoNA3Wvn`LZLd0@;5_ zr`h|w*XZaHki83?1(lJ`HX@lFuGD6F5JPG5ECAq3yJYzNXbl|)3A|a2Mwf=iTT{Q# ze)jU_{#Gx^_ewm;Ix~hrLRJ@HCMv(*L2C_mr1%=-C+uoe z)2DTn#NSh0m@~_`|CXYxFdlF`oiYchFiw3AWB^x2U`mxDp^dG`K0K7{fDWvR_ei4& z>@#CG^vo75Q}@kIMzTDt6+pM)P+QVK;DsM(i0}OX&i+e_5B~=dx;GujTXlc`{yjcg z|29bB(_=OLMPnIGAYDRUm^axhp{c8vH@|hO!l2k3A)-m)4mdPMh(Us&%t^NgmXO4q2O~R`<;hDv zSxK^i0zp03o?oVscbTWv*qUk!Y~wEs#*s@)1q$>nCt4RSaygElK3e7_+@JqNdF)U6 zO2*c;a89ZCxOnNl(K)H7F`CKdrDwQ+x&nH{d}W(j*j21*9?z=UOcDI7bTC<^HbyB& z{Qd<#;|~W9>)fb*)t{Y)#;tq?Y|qFNs6=sBl5AszQ16q)LxBN1{ z*jqN(CN)DnhXX`$2px+v$N}Xx|3TOkEaTiYHCa*`AcPs_5@|*`rxW2 z06N4b=z&6gxr#Lg5$&G4svt6lDO#ESG3Iv=w#J05RYCzpjP8MQM;TDLC&R98$Zn}p zza}qFlx*0WHbHokgx-9wSma{b0>YCrv5FS80OO{e%?G-EbGLK7Xs8?HC#+k1SzY0Jm^x%$)r#Y57#ag5^gPir!AzX9SkGC&g;Bg2@lKjO$-{`?v zi065yFbol|*ze!VOOxyaI9mH5TxifOUH(d#HAK}bh{zoZ>y7kmjxE5xl?HnM2Xk*7Q04lqi%vvZ zK|x9-6i`A!B&1PLN~A%Ugi3>SH)0}4he!)ZOqxZv3P>v*!XyRh?l|LPt$pvgd;j)6 z``ml}xbq)aESZxzzxR9JF`n_nHuxKT2Rd9D?3k*bq>1R*vvM2cVY>ojqruDcjhY~M zzxvYW9WjdrMso9F)~h5IDAyjHIgYrYE#~-vO8h`AE?&=QCI2U)O@(>oTPHmdD)y_@ zds~8UeB={6m*rkGwr{NKiIm(M@R=A0b^o*7;LAFm7{;L_4ppp8HX=w3e073}^I*^^ zPa}$e$&ykvKh5?gI1xFjo1-{mK@9LiZCn=s=8ZD z)?yUqIS%>SPJuXs{MuBI{md3;xzBP!s+~iCot{_L?p)HX!npGF*=exGx|OG<_d+S* z_GhW@CY(dxcK88$>XJG+>27fcXH#|F&r;8IrzIzUs=p0=)O_099)>>89(c;f`d?cW zfUJ{)#!mp53?RT5TuJ^rGCm>?>o*_6ez3`4sg;|Uj~lwbLO_El--d(i6mcKkXsffhucsDJv7+t@b+4$Hzl*7IktugtfBmb{V>_o8R8#|0 zC!3~pD%PS_+KHY@{SrG-y^_QsGeSZ~be+wj>RM?2^4AtP>8I)ux3uFDdkk#U@~{^g124;c4Z)!2X3!wJEVBg0@z6H7srV z3wFk+wXql|g6jCBv@9u&ilL>M3frR-wa?!7SpATu7k6d6)OW-M1SCp=H&b1i3NEW$ zS~tHOVb3sda1J^Cn0ZuuQTMOMx!!QAOaj&R5^ls;@dy0Rq=bY#Z86!~wQom^S4@Pa zOGW$6j|L8EkzUJE`_py&kp@xj7{B!J52(iQty^#=m>2q6no!G`*c?_W>7ed+iqB(A z_Y~WncAq+T9%Nv;^dr?=ID&pgqYV91Pi7sYMwQ5U#bcQ-dkXj){4E(boDu(*P8p{p z;ctV-f3Gxs`@%!zP*xT!XSiSth$se6sy#*(CZuaa9%tI>`zVobZXi5bgb@wSh_~;+ z5AvR$xc3vO0*4noa^NRYNhCb*&J5=xc-+$fQ*AN}c9MgDlfTC#k?bO;UJcDvSR<;~ewo5@)6nAB<8z7k z8JeN=%t>IQtl)n(fC#$Mpou|*7|`GK4@1Rp9Z`CNG`>Y-xhiFw0|bKLHm?I3xpb+o z;5qZi08hgjHvB1Q)m5wAo!r2%7CEyKnmm*$|NIdrmGM>HPg`ZB?(~HcZV`-~`~Pb~ z#1gX8a-v|k#bAu~CucE&I~|lfbWaX}s+9&EOriIo`^epVU^EwG&4bsR5zv{t${kGC zE81fP6+q}=1oyu%;l{>B;6D_Fzq=YIi(zKya5w%8-I1qi<^~h^Xy8CR5=A3tD$Nn1 zJlBT}Q!Z8@2N3~D7?jyQ0h+-`dC;^y&R}#e(Vy0l^#A99*bx z&eiS=U4TVx;y|5O=qm7=>qw8=%hU6zQwhu(Q=*|xQ~4sH2j9neK||nr|8HW5c@cF> zpWZ%GT1a z(FF=NgU@BIM$N3(A0|c{UhkYcX9E==u!Eu=#pfV*0gR&&ZxMujiukEopUp5i6my&n z@!af%djI=5gz7CR`U#H_zP2@zE^&mv*^V9?BPjG-)I=Dro*dEPoC+p$&7u=`i`Qa> z96q1)B=n-JfF4s(Q&Vfh)Y)@?vGkuece13u7hz=DOp_fdP4>=HKzS1A%pf8nvOAj+ zyul=Jiv-OfVy`vv9slgbR0eoiSO1gx5Wf5WiOFBo)Tugf>VFp;9GnUn5W0GLNNI%_ zWw`Bam}j>jS|da~g}`ozrP@?;_&hJ-uLhbouk2Q2G94TFe{;sJeNY?ztnJkt&RM-M zQmsaw#Zuee?{v_$eCuQE)zu`N zvohGW1v-nXVGmGc&jxwvjxy`4L4N!j1!_aI@EP^w5$KyZVRH*oabuB>_)-4^q+l$F zie?j*B?a8(gQYf!8%rZ-*c2&kmWkjM{97|t)uvV_WhR!!%@OZLUuNcJM;v;AXCL&p z=KDK=tNiaBVr0-!7I$31!NVL@>dO@+1XNs7&A#y{-#51oo{~qc-k?L*?c3nst0LI5 zM?Zoe2R=T23&3le>j)e!#$IEwMXyr7lGyBUL75+ z6*r27BVyCE26lLavhApsz6^x12;l5GUOr91^r74b6?03~|0|usq&zuD+ViDU7-W0js3^!s-JpGf?YG-n z(5`lLbnMB4D;DzT%%1f-&lNd?!z#_lwJ?!4(TH{dL7d;1g@PTP0wTPDQMmoe*d4ls znAfr~fI}QTNJX>bB#f1@w`=WpmIqbb%NBp|QwUi-;L30|kQ+vcjJAE9u*-sdE?+iz#CToiWsSE-DABpXLsOBJ`E%>k3d zgNq#1;`;*=Cu#<;0BkM1cjW%GBCFRDq1LVeP#7a*Rk$#+$y>##GKBWD?C1B!x5 zH`RH1xL3=@%O)mtH{;=HicUz3`uUoVd-c=xXZ6;vwMC4-^KsY|~ zE)OVZYhb!Gr`*{iBK69_;=%7zc*L__y(IvkSB0XKdkx``=&EqCTo(obrukrD8hsJK zU@c)^+QnMz)@Rzno=e@Ao8O{Kd4^CFypNpxU88dto5K#b<0~#=Z5|(s3kSyjO9h0u zK}(F<^)xqJgTXTE;>+S?2Ja$SjYoaD_xO1&W*Ox|zsTJW+Sj+}x1>p~axi`9KQQ=b zrX5|d{16QwGIb{_UIwLZQq@RFrQ{88I7StV`r?Xow-Fzw? z7`(pmm-x92IKnq>e05p+c7&dK$*$HC%^-bV;_~A0{v?UbsF-Ohr$0s3IouJEtCL_G z6llwBSLi}TP&xJVy7L5MGmG}Axq}tam=tE|l7ez56^lICk`FuYd3~m9>H*S}cVA27Lfa?Yrdw#_PfA$3w>3>& z)>7*h18et=u;&GS5Nf!&&&n7zt5tl@Kc6zaZ5&`cl10huDE$rfL%l?bH*|`7U93Ig z3KhGq^xoEd9u_0_{M52s%I9{Y(YQq8PKIIN%pISqlLPIBaiTYF>YcA|PspabSNl3g z*MPPVeyDZba{uM_1a^D+F?U#f??c`2VcP%5bsp#dV)9kbWEuwGlHN3t#(DE5&3$$Y4xb@A=-tj~h_I$PH344A$ z7ZCk;3(P#*_JKd&T!ol2xqm5!?iEprOaUmq1}5UN;P$*3{i&e0#0rg>*Fa@_kx{Vd z&1A3|gx;uo5;B|cWdVC4zxKr43jtmB|7XwzqrMI^l$Z>EsR*B3gA>fI6;}k6FkSQ9_$jlu3)JJyGxY`M&z(DGH808Z z{<`aAeJfehTZV_7OhK711zQ(a=g(5El$8IQ*T6}0Rn}6JpDrBXc1?*wokVh!T^pj?D-40M3`Dujmc|?AQC{J9rR9S z=M(l74=8*C!79D-&FrA56p;64VsCAIVY2DU@5N}Y z+Jl1+H`Pq+oE5E#bH`s{n}&`$4Gy%@KSX41sV&bBI)tLhzD_+c`*3CXIuUK6A~~se zEHd3(4e;<^{N^%W3&ckg0RFj|I`2{^n8%u92b8gwF{_s9;q6e4$Gnh)QGV@|i0!(jm&K{`{6R^?bLxb%3k%t$XD{DgDQ-LtAnJYh zlN=Pen5Hwc!wPSFEJSPQt7}1NPa;3ZXYEc-n&#<$;C}aDxWi(eOkvVyU`CXWzSG`4^rk_xTNUhEv=dEV&@lZ229A1Y2^>a3wChIEuyZgWN9 z#M?bIe(VqU@H{QZi{WIR=lU|Y9y|95R-Z4J2tDo zp<-DgzI^!tbLPZAI5ej;24DUVPz3k9sZdSRCyf$?=8<){H_#yqFKI7d8~B4X(Lmgj z7qcy%BTf;ob6)zj`@RnZ`HWWpgOAeBeN=E)Y2f@Nu<-kd^O1PB$B6MWSIsr{GwO*2 zJ*gHJG}Isj(|S0&#a?@`7SOvIOUb~H_$V^{i@@fz4UjoNyw&oKg~V`Vjhjoo-=KF* z`Bi$r!A!#}jT%YAz{6Cig~&@0Z!&14`V}^>tihD`_vRdW8k(q&rdbLUOH#j}G(lwE zzn^D)MOV8Q^H~k4`?!7-c48lgLR;2eotuO|Hh1r;M*FCxg$i{7@iMg5>DN~YJKNh8 z5%M9pmI%L9GZ@I~)Vub%&-ZiGtGw<*Ew9h${Kt1*Iw4|g7^cp1H9^N2tl@?n*B~zh z)uoqq$$-R{5~ep>tsl-~TZX%Gv<#{*3tD7kqpPpBY{7xjTwavke*7?6`~|a%V+2~z z=6iLe3sy%Agrrt1{#?U6yg9(b$dJAHuC?U1VU5DEw^ud zx2m5Sz`y&zSc?P;q1mKmP+9ap9|07P1+*537z1zy3K^Lun#lFppG-t&&+>O(6a*DF zW}oyT-&+L#pwWc76*(j)mY2B^llgGB%R=_jowHy!r)fS?QHIYuVe93>z?3eJ5(S5YCX8CsD>*Q>h2ST-)n}6$39z(xrUKkCH=V+)pQP&y zO@EI;BS0GGus5qlOU>mMdsed`%B(WT^=pga*V|eiVL{-u+*~%6fXX+Hb6t8>tr~&( z$AVv*-l2ya3%y$Jk60*sUDHHHnLJ#T`K31{iSz@^A6rlgT)bC(d~bp4gp(X1K15X;ZQgK9-#lLe*{1Krx&hU z_?gng`(#fyb0F!3k!zk`nGrY3a#R+}*`du4AaKWSgZSqsB#(fT5)k2$2)aCfJHh+9%kd;QbaV zrAr3ji*JE0Q5Bbyj!R^1#{nl7;AJK1DsqW^il_3hA@Td9&8tO+h)ehRAZ^HfLCltb zJe2yP8rEYgFcLS{p|Wa)*#~lKBepaA7Ck2!kFa>0WHU;Tq>8(4|45}<0CG+Pm0ra! z06f0K=;{WjT3lOA;m@d}IT>X$B1%RRhV8y_gH8G6*0^RV7Q;Qq-?u2p$=Qb=j=r=1 z44iZ(K!HPn*72d=3noN}xDNpy5Q~w%JcB$mxET)YkG&v64A6P)r(_WZ*lro1Y6{Tg zZi>&jLgdGWc(gRCV5qH6$ zoo>1e#9+wS4hhLXDHY={fWfBFff`6!(!l3H7^R-9?~3@*0C{~xhsSd#J%bmNOE)O4 zJ#dv;266@CpT~&u6!pC#r*cEv>}S4a#LBY9jaPmky6T?9(HTwX zZJ|tW8hCS%w2C^+$)uXG1Hhv#oncr5=Z<%z5flLgqFJ3^G99R`oCN^c-B_Er*XopC z0<9A&?{6?`b(0O#>36)wzqazzFg7j}CO&_rAwcVfwN}wtSn7;he_y^$Pp{NR^`7>d zdy2bb;o|oD9o2-UpkY%7B!D8%Gz$WA+!yNLaA^f+i}Gt!J+q{D_xQ4IS^2O);B3;rlcFO7#z1`&D zgJQFvcT2aRWR8RhPcsxBNXlvH(IpY*8(PHhR!RRE+(zVbb!``sZ8GuhXWwh<)R5g~ z|5<$D3<*h5MGn{yfq(kYBL?`~QLX<+DQNJ7|3?Bc>MoVIY(GRWE1-!aIsmTwEa(J0 zhv)w<3=w^Amr?hU8IMkAOz2e>r#5fa7_mLl)^YiBJsLOnj}>Io@`9s0o6cVu&_c>X ztU)Tc#6z!t7Gt`K;N$>TBaWqDNGJymys0pJ;{`|6fc^cQows0}h4{t&Z|-u+5xlT# zWV8b9zY??<4-jP&5=aXH3WxOW&bzW5i6Vvt;y(1^tSPTa!>&1D8`k7z z$k5FUA|;J`=`wWd?fnMwURZh63R`QokXsmaRt;r$=Rd46!O}S%uK&O^@?bZS`k2dw zO@k-B2cx2LpOlvMIgZw+h2zdmOWtQITB=2br58FD*&5c|JG6<*$@12cMXVXhFiL{@XSYSSC?Uzq`&Cv1>(BD;RW6=DOAG5Q!_YT;n@XpW# zHeh4uZ+{J73|2twLV;n25I@skl_7Hyx~^h~WrN&6CXA^MFeDHq9Ac}_B>@a&1PDL$ zMEqi*VV<|;VP=-L=*hT3PfuS6Fb^F$>@^)0=09me_)(2Q8)Wco$9Cv)@e>~=8FWuN$aTI zF^)TYOo!yG*lXbc)w*W!3}9=|F@BRh;3V3 zsq|KE%=LT3Q@zLD2e+jyN*1pAEQn;kD2C&y8OSnI@h+o3hXKE00TDzEqIG@a1w1@~ zXeVHC)2(Gjxj?HO1&3E`DM)nX0DQcTFxjnF;nS;Rb%L?UerKqg7r>B~)*s{gRFhN)K_s*FPj4f)<*hcqf!Ci)M>qvazp z0pUeOZo{89Egd<&U0{ndnkrjPbFO}UK1d9v%Wy~SVi5=}(W^_}&u^(UDH$slAVB9h zmI08POl=1U5{!nnClTfT%`8FaAgcZ^R!SwfaDwHJ+<|Z@KY=^&!1>;5IgG|-wiZTdpA3I1VA|{gDL#x{ShNy6!#K8m`WI(n)SQHJhFu~w7sYvjyfXCH zAB4YLK{@^_(R9{&1Wv$#f8L#rMyrg_rE+`M>VvV_v82kO`%(WKw^?o@umFtOKSn2i zu!uDEpCit0Toi$Ez)~Zok>HGy6Lk>SV>!Bu)&W5l{mVL&E40`KE8Zg9JCWyHeSfWc zgRRh|lR-64r5m!oFy%S1eXWDt=~G9C9%~EO+@ff%qnv2!qmt3-0djr;Y+w7OsbvRK zIALTHjz#FP+Pz^)kuFDZJL;L%*6l89-)_=P@Pjt~T(QaU=d06(6tR!zk@>B{B!;Zh`5 zgXi|&d)}*M_KUGrp_x>^c@}om+V5-)ml`%UpQI>L(yBfOj6sq(G_N0y*D>qrXpoLGDoRWk|^RJ-w`inb+p}-=p5c--Zd+yX!3Ah z{EOu)hU>PwVJ|5=;BVWxL;V;&iEjabvz=J}KoPLF+N)(C2LZi^Js0A6B;>pR#Gli% z+I~QFJkZSb`*=GQgfSJ?FzI){KMrU>EA&4N4GkKN5B5}4U&g*KlUvv1HSOcuo!?P` zsaILVjYSA$rynSlxqFvhVLvF|c&p9)ruX3%okS#1Z;Xe3pH(f^wPVw%N=;C4#g6Jt zEKJ={lrXa@h%|s^-Kli^KOwJ(#V)_S8IIWOxE}*kZt4xKLmGO96uhI3n*%e$nO#Ob zfvjl)=08#Erk&rX6{u5X3&^{Y?A8SxogoqK?meWVNNRi#tGSw!K+x*Y)2RE7tY7}= zk8JH*Y+y-+Si}Q^kJ0B~^$p-qx`?X^V6R~?ww8m<6|3XbBYi1LVGX7aj#^^~rcERS zbB*wX#O-ihx3zl%2F0JyRTkk7F5kNAY(0Dx7-r;xrGKgBehPM6-h2gQ&+D3biS6@r z+9f8Kz+>wS((|RybhzRC3ai+>1bi2RmIDLtLicCUOV!hd_LN2)*?$UsmbnJ=w_NO; z6xp-59m?db?f)u$KEqrL>B-2 z@o**CY(~~%@Z=@sOuwYAR0WT&)OWtD_bwzbYMj5}`}XX4O3Hkf=|RcRT*7y2lFI6y z8IO~hdbz6QLr3=y`fX}mW+aG+US0@)P?K0(crB*f;gRy z?3YfRE5TIl`+5(!?lLt zIoQJC(ZuqfTXe!yk&|*djlY-|A6thN(kQ4>8Ok{}d1ai)b6}NA-#q0DU*I8mf$3!S z58q1P{OAi`80R8cRlcx#0cgd`&I=nN^6+K7|8JG{+oz|IANul{EG^J#mdutr8&*iy`zgB3%}@PSFIR&D~e)O`fM)~Ed=|5ld*w>BVhIbf|E1*nk7Oq@eD^h^Q( zv55ffHn4UI5=1xwF^B>(90mRUcI~khAjWHXu!}XhF2K-oH{+AA~oW!J+P1K9IP@=5j*Zladvt!q*?ypJVvMHL-CO*(|O*-r}@cOJ`_uJMh&q zSX=+&ykC;DHN`}xbEp$}z(2qBr-{B~q|!OQ(Eu9q)aynlfPNSkMrHK>y0d&V%$1|k z@R2{ul%wBzg)tuB$02%;g_}sEDnTElyv#o@fz5UpTAD?Oarq16-EV@mY!O@Ht*>Mf z0|&e7IWWo|$F6Vo8#L|IKnVAEq7xV15y+Gu!!JrK9t(r_Y$1F^17IEb=XnbzuV(0l z?;fOpG$7fe^T2TroH^hzjT-6Qyas&PbV@dzwmE(0ayoI(0dL)^9|>;RrC#g37cTLx z;fk!c-8yu|&EywhX;h**7sHwe2&|P zO06wCwigP*sG~~o-+y$qb?k1GusZssP4D^4!K>-?;eJibUfO0^brGGg%*BICjVI8;?0-1y+1biZ2$64pi74>3Y~q*`f>0o_pP14OpoK z0gyD=$qhnyN$lv>{!w_SW~KWy#Sy!fo>OG;v%xM%yrG%&#_PLK7t8a+j{pU4mfAZT z_%QS+^$>1nR6eA<(fhtTgO4AK*Qpo?`OSNla0OyvD0%yGNCZT~3;+JfkY7e_@Z2LhLC3$EZAlrD)eZ0u9F#ZK1O2C!?EY&R+v z>Xvr|*>xmF-2kkUoACIcBfnI1#|8VsYq5i#?Az_FvFEtno_C)eS_W*+*;Nu8JLZZk zl}j8fcMDZGKC_%`sdl`Q`Q`a#ee==0v*18Cchyr<z9T>t94T zb=3N<^)-8Q9P@xOVM}2|0pTN&BlfMFC6KS<(*YQN4wzInlcRddrDJ?T>m^j2V@q&k z+`)nx?w&04t36E{5;IW5&oKbkg=c-e4E%=G2>>`wAq^on2ExgMFN z{(jZL_F(#Kp6K|%v%w;C`;Zg)hRy!=2bB`jZEMV<%od{O(uWW6P8+yy*zh6sb zlce+DDGq>Uvn8qp8rt)du2vL%0{T`In)%$kbuQD9TCZPt)_Y}L%HXUR63@1(@;`5M z5gfdRi^zc+=g~C@v{#2(tG+6YxL^)5%_<|dy!dosZ`I_d#@H{IAur-H)`L}lBJ{kb z{cEb9?Ui;f{HTB7zgOBGf2-RDU1G^FGr2f~oh@~bAJV)$3?N(6p!RLE+vNQYu#4`= z;3He>7x@cpmzNC;f#K*3pXV^hoF~Oh;N@?6Yo?szCsM&512|SpLAhSs_$S*AnC6O0 zJluwM@+Uxqzc)A4kGTh4dsJ9g{JD1Qg*2VT=G~3BPSkpip|wHa0pU&&F54m-L27m6 z*5Y7;EoqmJVz&ZC!C&G^qVw(IJ(5xe)EC$+sI@TWUAspQ;w z64d_jD1O9UO{7!$#&$5V=6aa$Rwq@?7o0C@M5i2K)E2zH`pa!?Kxk&F7{uDHUiGK& zNE%$LR`BjkcqYuz;N$2666i&Ea*=Mp9Kr60e`{G{Z1ec9YE5k)aQ97*05)Gbp2jHGFu|jOC4#fuNdd zb8=(Z%40kAX6tK#jWGy@5nf*=;g|~8E)N~{q~_Ey0!GyOo=<2n*(HHC$Sd47B0}Lu zY^LV{mE#9)II*bn?2r0Vp5Tl8Ex0j=XTHy2>B!*Z%OS6H`_bLq>OOsC8yn=LDb*rF z|20P7vD=Vd?fsb9cn@#Td+yu*W<>Lra%bhUwv19Sly+r@@%2NmmRWSUXG{*dQn;f@ zgF$)Fr@jDLS>(w2dT8Dhr00!>{OQ{aT8(<1z&W#+N$|)$+?fm(00ErsVXfr31t4C$ zB$&7m!hznZ<^K~FP;wcw`{tA{80t=}7y_2PIm|7-&)i%qQ@n9OHUjItyPMjSVAn`X zzkzXadvThMJGv)#zLcUV`2D(_({3a=3-`v*D|q$0CxuA5dd&m2T97AY9q!HN-K4(+ z83^y`F1bju6V_xdB^0iJTf}VfDnxEEcA}y zkNim9y`>CB{9gh~$s-1}dFArPWXQ~KsxsCoXXb$^v zet}j+Bm2=$>&nHFl~usr3hsQx7A=*!6hW2Lk|`n=j)ozti8UvKGVKB)0^uN!(Xvh8}l}O$o`0gkYn*mXPFcGX-4}EiRD>^23PN&eoGO@jdb}P1XT%$jtD>25|Li5OUXEl_`yGK( zS5^~TrG!)W7-Ov}x>AQb`p5_n`Vz2-8m{rGB;e2>Q%3u0>?7`OHxtx~oatgkojz9! z+LxJb9sXq$YI-WcW65n)sPx{vv$IC`@A1h?0yQmw+gib2UBZvi9GZBwez>fXzk=7d zxa6_eh_){Fd&Nb_;DeP49&k~s{i`8>YGC5t`eC_nso%Styvq9A@{;#9L=WHMzt1c5 zH+WF=#OCbEgKuo%*R{7U3uWKW8FKb&dN`_qj80TPE9>SAg?u*C(1C#um=U-o?vxT; z`BSLJ_ff)-*xhBsf4HTq@V{;a%+vSe{J-vVRyV6=yt{`no-a+`Qjjf$;4G7UmuH2W zXy@VmTY^J1Qca)iMdeyQ#HP}%h$fb^TdM_?oH> zs+D4>rdf7&05;SCrvcRsez-QNf+znp#AFeo$ws}lM@}>3Eqy!r+0&~?weQP^fe!e7 zr!ah*ntXEp!1JAJIN6=yJCsV`$>sf@ekfaY&IeLiyM&R$meGQagl-efA4*7fT^=X( zrSfdqtE25LVfVmo@DwmVk0S}sR!mAK@oYb6spy3a*-+@=@mc&+vidS)i={8#1P_f$ zH7%_CoIG#?OYIjOvWqZ`MuphlEphD3ixRZdZ>D_ zS`dTK)1PxYtA?3}WANk2g`Wg6U8YrQi(r<+Uh(@Mr1E>m2WyrOExC7G) z2^`^2PVxocG^hY>m~a1b2(}t5WEnINfhz!y5E|NRsDUC=rOKs>`DegF)C5vy<=z~| z3MgJ%kOzQ37kaf73t>xvlN@b5SQrV?@A z$T&;O`wfnCQyA$LLs>BNUU1Y5Hq8<+?s&v{LH?%?vU1>I%T9Y0wF7}5`&P@N)gS@1 z7~k&{Sr+w}&nZ3dFkV_5ip2|AOOVq7MuAPTlP&jsgqh_OrxG`;s+0QSv zi9Ap4k{cj$d3c2IsW70B@DA+n*67Yy$8TvcFuEBe2FVv&FX#91Iu%-W|NfW05f?Ub z>R|T0YgO>HVM6Rat#4>kLL0o;S8S6@N4F5dZHGrlSVulsUg?vjp{y&bnydi($8B8Z!M_lB`p5PeHKhHU3mMx zfZ6zFRBHByKosF55&7XUsYCdM$k9fMMsnu8W4gw$3k{FL+_?tFd_ zbKr9t4=3f`Xi*VtU~lc4kGDt4>{75)?CPw8ooAV0$C5X^ai_7~HLQz9DdZ+kR4QL1 zxpX@C1q{TPCd7QIWtzkF1v9l0?9y;4_oyM1*Yl+i2=s7e*Uuqq(nhs|8j0ER3&vBG z3Q21@#(*^mDiyCy5ycaQBBB;!GhSs?`ScQ-i!S9FHS}xpGgSHvRmu1=OiW^@*8g5; zC^j~D75FGOhrvLzVK~ANa6oS~tOm6Na3O&86SP{D8iyEbz(ZCp=GK=!-;ey3d)t5& z3XQJj|2uZi7+iy&b>of-#IaFvRm=1r9+f1Xay3@E^Tv zpj3AMQ2-24wgYa2%K?vvTvDe|aFm(@V6n1L1&sh2pt>A*Lp71xpj|GkaGD>2JD*}w z2)Ja05V7?Y)QomtcEM`+8FZ-|T%549*BziC9|iWMB>>SSzz=qHPO8}cXI*Eq!Al?v z0s&w(RBm)BSiyT1fPpHOK0CV(xP01M3+eq2s~~k$0LG@+-w0o=lSUnX3_?q+0IebF zCxRUZd8n#=R)+hqSL^TpL@_oPeK?^Uz<)C`oovF$|M^eAtAB}T{bxUX1~>MR7U8SA z2mH`CWetF)3ov60(wrW_TsfqzLw1Sclq>_d?mStpS`PyIW^<@0Eo_aPkn(+I0wWZnjp%k-0VZ=fb_xp422cy`Z|21gJbnCtbq3J3`UL&GQf2= ziq?0})X3A1m!S7OjdbB+uu+Q(3b}8X?HmEIVhARzHZE8%!Fq|hHr`hQ>jcu(Ky<9% z+g`RoOi^qwh#CU4XFqC@EDE5I@z75T%yx6L88c1y05w+|kVjq>_FB*?fnDIk5RX zTLi;QG~6MG=ahiwj_p2d#fPha)OErFZrbrX6MROJ@f{xR?LE|Gm~Q6zZT{(Q0!U_d z*t=7BKAR#4&v&PhLh)0g*OoTf5%5sjum3`tS1WAIh}M9_&KPdzscaA)Y2@fcN-+CU z#6iax1wl?ClYzGuax0e#YrOV#Aim1*8Iid20uA*XihK%OT-L@6;4O;%aM>C(-9OYa z@G2Gq3~TEfg%aEkRF~nHXfU9py{IVZ%Sy$(1EpoR4M_N{VBr@de>d#`(-j&?PrmzR z0q?I3QR~`1uU>7uM1K$t%nP^iBPfiALYZV1z(E~h`m8pcw*68y73a{gnk|917a^AY z*dP_4JBkC`kg@r=yqZ1s%PVRtO6%vmnios4)&P(e!QiCbGLc{#mzEL-J|J`Q95R{S z9pnUs8@|2fXm5UHl7RH`N^|M8aRM@WWs88+MBoxj)ES#;KfJ zsV89ANCcI@<%AGi$pwnzwdzr;Vgx(X&!#XM=&& zpiS?h7WE3!bLR-cmn0J? z`NS6EK66pYQ86{?YDnmL)ag%fAx8@gn{25c8!ukhI#lfutF+Cn~oYW|nLrFt3dDeIK5;c-N z1o?A`V$dTc^rq`1>)7@jl2>ZF@hmHwGpQIRe{b*`bB*LwK`9%)Eev`Qc$blBBK(5v zSB94b^ApUk#Sb>_WcdFjejrV~{Lyy)s|Lf%A?K7&js8O;Q&^s^J z*TP4l0~&L&OOGP_Q@T*ADpqt}Y&+@Jb)g&!^$l*2J-z^c?1uYu7#a~2-^2MPN9zls z&$SA@3fum>sV#&^I^QUyyxHh~CcievlWSt4Q-(Sh5B?$x{Z~J}$>1n+zRr5{35?+d zfI_zt1{mZ4A2HjJbj0BwIXHH<4x`AS|)tBV)U_am!#%+GNX z{n&5jm&bB-O|KF|$X*Q~j?PQvGdfTwinSjBKXk*&7ia>^B zX!#SmA6tedftyNA$`~BV@3zZ7#+s3)xcLV~KE6Tpn;Vi4i+}{b*9kH0JF!O^l0#Cc zbFwMwQVM!&j#oJAnML4@{?&^_FE7$c?4bfL=qAm7)x?X_y7r|;zvFr(~A}d$?M!54jveN;4r9a}NM5z6mm@>cS&Cv!_ zyj_Nhvr%A-(f_VUBD0yRYXu3FBp-~uQtyCA(CRQ$rF}M#)hFixANN-VTlx1A+VQ#F zX=K{QI9d2f#FKN|mi%WW9lNIlKQMC?Eq@B&T5FU9Y1{j4A$F#u3cVq&%QxGm7Y)S z!hM_DC>iS4Y?CicZ-kSDcJ;jD3)RG4iI_2&McWTNcSgc%_#-})jIwi&@Wz(uOecs7 z&7UYQdpMoxe`E`0=YUNOKLZ)?Q7@p*MlsRS*0>Pow`>Le=SN;Z>-XHJhxgKWE)cgx zwC_`j^RBoH|p1k4yX0hr|W7Gi?4! zgm?eGgp75#T8Is>4QedX0@j=8J=*A@GqZK~wCATiZ-r607VMo~LdXFmd{La#_D0D& zE;jFJ3z~wy5jLLc#Zjm3=oH^?1E=>t0i>aR*c?gU!h;?~-LtRw{?x2@h*Zm#UK`A- zT6;&mOgcw=N_Ibh*;c|-q+@x%hvuR?Z0Ap(SeTu32r#cApVQHkaDO0 zm5*vhRMKyui!OYm=Fe1`4D_Zzw|qES_u$@sxM2aVa;gaO+kvt=*(MeZ@;z*Nf#k9q-fv{0g!MYtrhpuR^P|<32Fiev zmyEKo#NNlLr8e;VDZV>JVRQd;ZeqWlOML%1ybu9A9Jls6eD{&6RbO11wi=7YJ+m=R<#s2xF6rGGhnF7pZvCPEVZJ8LGPJQfcopIoG!7)z_=(L|C5MuU zim8azNj5@d=nK%++3LLietzmn2$PA@lbc<~hRL27FE$_HJ%Mlm*WE>r2)v zL2)0WGj6%6B87X?IXnubV#`yKa{*^cGKD@ghQPpjp9{rFW2XNp?5agaXv7F{z3x+R zBBNJU|Kvx--uvAU#0BC?KmA^kUwoLpK^~$KibHttUcLW5$ij2da_ojW(j%>%I=i<< z!9#lIO`PTg+*zhj8gG#C1lhC`-;~~y-KURm9((#ClcUboY%c;&$ZUj8TKgHdFcX$e6Rz(a!N8%om!^=aW<% znujU2OE5a6y=s)KFITrk<4Ed_&2Qt6ym9i;<#+MxrybNKlpIP=N^J3CeQ`bMj6}PZ zdbkqx#-29yK-Rs>RsF>@JkVN_kz1Jh3432R5hw5WyJkojvZq=pq&yY(6f{2KeHn5g zr=G(YZmnt4W2n?-x&Og+5dz<17+@`19!tZz^LpP+|3QuQ8SG zoRU4}d-G!;IpCSoswSPdo-TAHSO%@@wu?GuVA~jdS$X?`${(Efcd`t#g?%e#;WXWC zxa9z2t+~m2y2Ut?E#H%kP$AB#uYOB@dYtG?R|QbPD50W!G}mIkG<$UF$7Iw)G91Y< zK4#NHLdB20&GOX4S(vouwVcw?IrO6AY7U8qV(ts8IwXKNuM8s)JohZatWhWl|lgDbI-mQ!3>;J52yasR?<6GJKZn(jqOI2{~7fh4w)P8!!j z6LcU}AY#M5lkxiwl3g1tBrId^EI<(A9gFLq8w}Cn^M>!|j{gYD^()}KSK50@O<{ZS zF6!YNfJ=6xwLp89KAuMWGT9;2ffYE~$-rB91LsQViXmukJy}7ApA1L&UMF}TkxBGH z`nk`8&Q=hovO?*8w4VRu>zAh4oq@dF*`F!NfPKCib)9^#41-Q7s6f^d4HFsaunNny zpqqYaT{iyuhqh__^zt8vPBB5qV-)wXnco5?G!X-mDO76(RiFe?E;N-%-1J+03L2Ya zWI}JCw9w?_-AK&n^hYeCbA13PJ~ZBYfpwYj>;XnDJem$r^C*+zVo1QGW60cbDOxb`Tz!y zAn;BE@Kcqc3p%XcgGTx_z3bR>Wf(3Wh>6&aNlFONghPhW#Vq~miq0oBFt#)D-~=MB z#JKoxX2}U?ARk`cGI9r$E%^<-`*(>QP#;l4w)7A4<3k|IQ)WS+2`ivv$xyGpSth&I zjP!OA!2MnN0XmPvn#W(jcxmDXg_7v`Ebt;Cz{R@Hb4nO^ZXAG1)*BG;TxJKTv(_4q zb|X2L=(cYUKZMZNKGsx*l<~e~$YR=I)&axOKlOVgTxE=W;AC3643XsC`{QpI_V&vm z7Vx@WhAgpSzS=|;>S6Hc8@p<1P#`WEN<-^^cYk|XX(Q2TXJuS`*$i9bWd`S`Hx~ZX z!!O>@izkEdYR5I@)f=cin_P1nQvq&%-55kS?~uB~D!U8o^vzQB_NX#+AzyL60x9X& z44WOE08T0?%az%~Ov^CFCyYFxxADmaBcEhwNSa(9EZc}}fp+ROD7mO>qU3|{q~UX1 zOCln)C6`BQaf(|NYg7|ajv(h^qzyP%fa=7#Z4#&eyd?!3;S!HT#;Vj*n3RiNc%X`^#=5)x~hJG*A z#f!K&ZbR?Vo}AU9e^q%urZJ@6HRtNoO zr;jURyZ}Mh{@$_$={Erb#u#Qb#`AA=0RCO+%?4zzxpWa|Hkf3vv}NA&gbrXUsn8wq zLYXYMBK6w!^C-9t&(^L(7cM%!ThFeukRz#6tabkhEI273b#2c;K--Yo){DhEsIJ#z zUfDS=E%`*OFYd?i!F=4qbvV8T;z7=h(o-AJTPg(6!&@~7bZ8o_p=L0?P>@3{yAP6M zTKBokbWf3lmeUZ^^R4JLzrl~o_tH+Oi%&V?IlP6g?v;zc!A7@|Hx?k&yKq6s5dWzj z@PYrKC^$0J6nK%Jy~B@Te#1z?St0YJP; z`5N5H!#huH289MsqXP&6sU_zKsn#bd7)cwcdiT_mlNfbCn(c? zfaT5C8BsmZBd)BQB^Lo1Pk&qC&i(fipg7(P@El1XZ8wG~cDYdw0sh$kDDO(+q1@Z> zgBfEVkqC!59Ub9>L?khzEay~+7!gfPj%4j1yJ1?TMZ4WtvPGhrJ+8m4UG+#AV5cC zP_~jzlN=L8VCPs53=;D8i(miYH_NYZUO3)$y4=!T03$|vkDn0{E zvlB=?B_m5Ca}4!DrUkf+%HHRinio$zoIMsZ-+^`#pPEMG0)hEP;}KWd_#*}EC|95U zd^{*?bg4+Ra8xo7JCjcAkeH-lY%c&0vL-$&1WH1bK$a&!xv&sviQw? z|A?0gZF{M@Z#EqagIvfxs7m2EY~rY|t8vOsnDE8t20%|6mY%b(Q`bgRnkv@-N`f+US%loW8nu~{h(;*aY1#5JOZHIY$ z*Mf1>c1YRL)$P$}XSM|h!Aon!XIEgy`ATn-MKtt)Q4=!O5N#K(zg|qkj!hmpCG=n;rsbOoH^5--S;L3 zNm_tAg_x7>!XhM*ig@>#eTAJ?e_Z|OxNk=+A%?L-Hm(xm9!M^69hfgWa>w{wObJDD zf*n+=%Oab;Asl8q07MDsp(B?jcWglRWqzh)m$3hg%Qh0gAG{Hol9 zM&z%@Bk9G60D2NdHwS?Q=z@w5CNq3coK^z*!SpUnLb{IG^3a{wYSS6fEwo{n4Pgnr zjKxuj{Kax{i|Z6ujw^o5qY-#Iwb;FgqbT}e`4`CIq6Q`kZO%WW(**p!b5s2rP20XX)`a7KTm26fwloCr7E@EXV zUn@*#L9;~DzPXs8SwutK=tbqsTb&6M3$oRzZ6*voBity=Tk#0(a6Si2d5?u)*`8L0 zn`~N26~PWW2Q%Ofk9;a)gp+8_=;jfNeX9Vp1%@m%{&e%g*o9VAF<4rK-wT4~ONlT$N054z93VHDo=*to-Hq9h@9~rtyLkrB= zXyf?X_$}%OBs};}Y>~X<;F2#OsefGI=z`#ye$%(ZLq6u=G7aiRV^0v=ifGWfrxf%P zw5DL4M>H)sms#VMrWpi*i(a8)?y?7>5_0gUIp( zoB%6UFDeib_(rp>KXOy|K(5NdLrcPUPbXm5)DVg2{|LaK*C(GSj>f^gF+DNn0^{t;-{6ne-dV7IGDg!f8DNUnOG5kEMvop%nzLD|51~^AqauUDNIO8Ud(o z^)*>1EY%NG#~d3Ny}0T!&$1&ADajpdgr>&n@5*FrN2)bNBJD*7JBbs( z3OvO=aNW*eS&awPdbPXH4QGU{yhrT~;l2TIrK>%&DukU~PQfgy*a5XwyX}T&S{zd9 zB`}x}JFf}o75wfSiywerA2GkD)G|rdC&o80n2hFvU>~*C=e}f=Lc{48o!hV~;&`L} z3gs6LUtcVLWKB!ic#id;f5-I=+A2Zd)`^_5qeAw4<9Vz-IoF?W5I#j=cPPpIqH*QZ zn5kW`@nGGUkmVFrSF8U zc&19>bY`la;76n0Xf9P=Kjz#ud&JfK19E0tYmt*C%ma^-VE=)X_odyMJb>kS0#zFY z=i?dW)Z79yu@_63wL}$h3AWy4(*QSi%MT_>xUF&?{HC()_3i}|aYD>V zN&TdQbo?95%y2qeD@ll>Bz8rHw3ihz%ZrfpJQRDR&nYEDBERUvCFW=CH2arTjHC7N zXsyvb<5z(R=@p5K@6T~_Uk`Z)33P20g5-Q+BzB_>hzulakL1OP9^-mDkq(9GWnNm% zCGvPz@#p7l1lJ3ku_Iv=tTa7C^yQ@H5-v{$&D@Eo;V{XT%Prd;{dVTC_f(~_2sTl~ z?N--W35~A@QY@-$yNcP&VIOz(xc2PGNK^=)_~UJN#MkW0W=e(cjzxg1zgN=w;<9rW z*{HG^7!4h03E({l(_%OWjF%kZ95!E+ z;AuV5*Co01`TyDbJJ=*EOw3C$x2XM9#x1LAIx2oDw&sWU1BMu)T(MiJI-?V-0 z6Q#}?;aV4Z8yAC5SyN606FM}>=heON#k_*kJGPvb$-8(7^4LfyCRnaxlfw01xlQeW z4wXW2Li=&3i8u*U6=Z*qO7N&_QxdaPfd-~Sq4OJ$IQ35g`E>PcOKBvnD%lgxBwZME z8rX4Lo!M{Y4QjKG(@igr@{4Xf&`nn$aF!U0FPI}fdot;9r0cB4*@k>j54a~Z<=c5V zRFkPiL)=w+*(Ys-3w@2VZdzPrakxF+Y+{H!%BjgG`M<;%kME}OqG8lp~&^2He zx>TU-_IBn4;3I(Wp0g8Sw?g+it3Ltf73*FA@XwzP?`&*!>a8VvM!fm69ot@;3yjJ7 zJ@8XW2^O)fX|O%Te-`kFZriT-$%H^pOFcVugSM~)Hkd{k76Tq8Q zD`2+#JvH!}YL^U!OmCxs$Z&@mJg_QM1!NXS#L+K;`JtGX17`e~oj9rDBq@D$Fpe4XqADdRWI|*PA zxTS z34`V4q1b099d#M8?VIBFXS{p}N6QdVToUj5-2;oMotgUaJ7lXgJ?~_I5ZxL!QNAx{ z2iF&A{a%9vuQeEI6x1y#iL|Qx$H$iYKUu8d?f>n&{{Q*srkZuU?1AjbyS|Qv=+6FX LWw-yaC58Skgq<$1 diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/charts/membench_comparison.png b/benchmarks/results/smoke_legacy_compare_20260423_105447/charts/membench_comparison.png deleted file mode 100644 index 30a6b2575088134a38bfbd7efd972152367f5e81..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 33079 zcmeFZ2T;`Ow=LMBpaOzQXp~?&h-45YNK!cJQ52M%5fBlHP0pYalxQMI6eZ`JGc>KB zfMm%*L}HU;1G7K8|Noo0^KQNSs-|jcs^(OkYDM}NzOeV&YpwmgyeEJ22<1si3tx;IzGwwT+3T zg%F>h0G}w&>BqLVRyJ4o`OW|R8+?}5#{4x}cFk~=!&d)j*kCYBk?8+@x9pz3!(co= z-TM2QvSZZDz~S3+EU-(VOkP`Gc1nu8wO14zfd) zZnD!|8L|VVcCv&Z);e;m_R&Oz*cjHUwCGVGr|K z>DH6($kLO}H~VpMrl4DQB-&w+)4D$`t1_ai=t(D6M~V_hdc|t*NShr0k-SIWxZ0OC zCX-@is@%70uS}=YO?9S)_4Xzy#0r@8WM5=G)#uiu@9Vuf+m|%)Bl-TP?1ctqUw{5e zQ4(pXC4nuhtlMR-gkxLsaqEZHPd9^G^G(`2f|{eneND3}gh>m8Q~qXk2T!>km6zK1 zxtxAd#3CvvICeV5W$t}5bL@C6X+nDUurHhQ<4yVW{I=VZ8V$s6SUuNcjdjeH<;Q}g zHh*^{QXDxMPi%*`XQq*uub%(?DL9d+b?dx*6jQUr(pOu(h46m2Xvfi)BX1R?I9Hq( z-v5fRT;5(A^37|$!p+Jo{mC(QL1=3xuf>mL?dMwsA6AEc!TiVH&(0Q&_%J69T_eed zjbF*B@YrmRt&K3vd|&CGmRb6w$6?5=4qljXV?)PjJY-Vjc$Q9@YR0l;@8eH_wLgqw zSl>a-I@cbXeFTax{Dq{yl%KVdK*0)j+GU-V35J?Jwp z;w>gtZc#qzObyxL+v$Dy_AaIS#(3DX6s06aTjF3vy<4hEnyLVdzhc&9leYM0nL1^m zj%ITO8^0p4q_wJDo#CD^J$JFHmKVp)QC_kAeQ$KZZmNTmNy^pn%CFS4%-iN%CRMEV zJ#h~P%blny-{0BUV*4gycS*)I z?pw2JQ)2f0IKpw_Ilt)^$+dpP*lTlku-?NqqaD{re5q75O%jd=oA&0L)OhSH#Xe(S z*&{-+x?-$aJANA1S8f33RgZm+dZ+-nnyxNNFAUvuyj z{nuFOs$rNqz7@@zr`(p$W(hs|8g5(pCMZj{!nwwC-~QVY1lS@)pC{V0_3NHZ#ClZL zk5PHVjuAh`6K&=;+*dl)sN8-fMOinV?ORNe-ZptUHoD|r72?fWwAjdvEr&(TK3JE# z3FGf~1LD#yr?dFiVGUEd zZ%!uJM(=EIVJ}(qi<$?UUtoK}Ky$H6+veLd6;+Ld;UMw33+wRfN?NB}W^$f|@@oAm zB6wclJ7rb9kNne(z}x1V>tn%GL06`dNlp|rEX7yFsq~WEwpJ%~v&!a5pYd0ba)*6S zxnE!sHdC)@fDP4Z^Y=kE#|OQ-uD@a!iXA})`t-~&*s@z`fdv=q}DgJ{}lKjR| ztHyVnR6A5UaV~-ST}~6xVYSKY*9KiC)?T*6{bd|&r}`S>Fh5*FPRAjCXM#z@ zLc0b+jYc+H2-|PdBFXN~p;_=Gq>`@BEblI5_~U`w;p z17!|1N(z!H^RNKoTev7}Ja)H+sD$B53kUZxN|vbDgiToplU?SLSw`!?~2NIacAsd^`;aLEk(m;HRcL5(^^P+0*sJ zHdyWvWCQEeV|Jqx$GZ7vSQf;Vr#iU{Mmp8>c-NQP6(AVGI-_KB9De%juE@ZpQ|=+l zl3XM=3kci2{4v}+zeb7YV}y2m7#T2?>!beHYnLWsT=?9=+;=u6mRp|2$@(&~z-&#r zX!gT3b^aO-CNvNg%XQ7ch<4x$7k8K=+hVc)d}c#{yV8JM z7bmFQq!JNA{}28skA;?bK_}(;JK^^Xv0*Bk@?@J7B!qdgK3J@F1 zd8EJZ0pFbc)y!@m? z{lWINgRJic$JSkO+J$oiW&GI+N%GM$VSGAmi&~ZyDi8a=N6{?FljBL3k#@DEFM_ewxokCJ_8+ zQS8O`aJUmj9Ua--Cj9*kck#F*{{9oqZnThOKV6SWiZ`Ys%mnDd^Lo!Q6Ep&FL5(fqryY3OIBY63Gw3`F^*^VlFYB6n;7+D0R24ij0YEk z=^nm&m{HKx$WP2~mrscKBz|)pPnTr71in@KJLiZV zBtnA$(|1(Y#@y&*?>$_fADMIumR$XZ&iCD#!$*z0NBSWus<)aTZVVVii^yMDT<-nGaLTK5|>@!42~>uzC; ztCE@>4xBO4KApJ4*-`Vr7GWP>$l%)Q?`nnY!v4Yr@xK(Eo`a|y_2_W0&~d_v2py z9kbq|-*M$yEXx;=Th#GumfD(eU3aKIb~AqNYplnP6G~g{l?!(81c|dH_X>EPE{yuA zL?9?XWj~N>)YQQ)syc-~EuhK>$mOg#_f8q_^n{9|i%G27b+=Rd9$aUK$|IE9yDpe% zhcpy#rGtI+uc-qPb`Sfp$Jo+sf@_Wo=?l=xw~cE;(R~$?uc9H z`2!hWiHhkD%OC-?EutJ5YS_cbeSJ*SccF|?FTtBI&RaO^$DMg-$bHLvefT+TEyC*^ z-r}nK4MTP;4mb$Adc)zhYibd-ePO;WR2bC*WLGf{N)(HyM-T{Br9b6C!lm>Xkarlz zx`l>PDg1pbZ0rDQh*Or})h~C=76x62-s5*&Z<57L5SktBfXjhqHb0M==7 zqv?quZdu|p*|L?CIDmWlPRI8*ir$i2Zk44SC*0xj7yYH(49Qz_${qLU7ZVwy^dlz{ zpX6$f;l_CFP(tm1iQM!c^Yd0w8w(@9F~Jy*oelP&jrT6AfWlNxN@rh^^`X8+6l5BF zl9s`hUx9hWus2rIizm{&KSo-Va+Z@8o2Uk)RVByI(SmpI>(b@-JL5l2rM>tQq_6B)fTF}6KWPccZzy@qs2q_ zccph4KaKMHE5SG|Hz9MZqQ6J=+PJ40YjvJYkw{P;NsC)}&fDGcR>vLKdt&i>Q( z&K7ZLErKe3v=T?=-^j@E<;9r9pk24lMqvZdF_=xV&DwS>P=rzJynNxYG~-W9f9W|- zCQRD+uWk35W+f1RLEfqOe8Rq|?l4ExtJ3YAmR*v9BP6tT$@&&f)vXockaQH3ejD$d z$B5@R?5_vU28~V>57_s40}Ma29Xe_`MfBKpNmUJ0`LM8iVZCo?M|EV445M*BFP9mE zxu;DE-csLi7=C&%5p$+lJ0h|o|1X&E%w=Dux`;YjX$f)@c5hqUKn{UTYcxkN+yquO zq$IDNL7O&_QM%t{jnyMco=Sew`N+Y^#_iC2;0+m463nZG=!G( z6cIT4{!xRgOlf!5pnA-faQylDrs6znnjj?O8x4iOO3ks1vaL+V8(N839^0nQ42&KM z3DHI6cpQnZ^!G=_B0^&o(?u+8Ju0*4*L55Q>`ZDPPu$M4P$Dy3UELts);F-(u~JM)-2G2)WRo6;P#<5Q+`J~mJW)F_&bi4)7;K5L|5 zz2fD(GL`V&5DKB$4m&X)tCHL0M!Zrj>Tc`6FC_SVQOBL=#!t|5pQF9<`-AIsJ)Gdz zfKIC%Zd|`ekSw-JZh>vgCXmh$Dl<-fr<}>e$V7;#@&0Qk%J7-Fja(^Brs)CVD;-L5 z5}BwlX~}|OchZF+c)Bx9ZF6SSfkR~S&h?X*9^acvWR`w=M5JH9uKk-^m|@Ylw%ur` zmW9S>{F$6w{+-ORiT=d9k+!=FHbGU-nYd8(b-h9Q0ERI`^BT)hV6c)et@Ya|h)yMc zYRmPIJcwzGq`ZoO|NfW6l@OHx`n`wY|IOe37l+iDK0}fHcxAeq*5=pO!vNt@35YR# zbw-*rFPjYS87^k?$XiDiQ%v?^WT@O}iL2}cy3xBlqt_dUFlv~R#;PxGIn?Zx&`&Si z%+M;LVfe=6iT?J#&AOq7L0=3>`tafO*GLimiv4nlP`}092s}CI=JXdL={&j4`?yK( zl5ExitJhgpZJ#GSw?0}|1xvsiNE$B~Up_Vc4}1Jx>Z^^V317>~jq~0fS2rl8g^ZiO zR>AccQq}aNZVc^>!BvJuC_=~Usu}{BeSMBF`-A{nP?Do8q^K{UtU_o$we<+l8Z+ygR=_50YJle@Z?}>3b~a_TdX2-X2;Zy#=OWLGK0hWz==+U4qU1S`3`*V3WoRvpM~I z9hZ7^j~9J)Sdcn!M3?&J#_khFO*Gb??Di$tZMu%Up4#+ zhCN)u$yyZFKjI8!qp#S>j5P$xuFVb90gH9)*SE-`o))u?(XmOpL?IH3v1{HMn@sy3(6tPn2lH|JJ3u-*~lYCW2yT^-=|_ekxYXEb|yVR zQ{}x8q8D;r{@n`9<2`XIm(^WSft;_X~ICGg;CF>pFcABs_u2HNIqIXv+sLlORHUcT@E zVEj0UCi;-M6xL4eFZm5@AuSZUu9Otr8phGtU(QE*0mKv|$EoQ$8KFR33n1x)s;<*G z{>e08C9J&k_A=r+vu6wY5=|@NUL%0?{%JWQ)jhhiT2$rjzP;&bij?!X^LAh*crkE8m=OW~5s0Z%mV-u1Wh;i06xm7u~hp!|;L zzd|?Q{o)*mgClS~sxVz=6Ut1+@iJm16@o96yB5VGK32cp8~-Xg0(srrUTzE(!*E{@ zl>Zl}DQMZE$OwoOH7gph=d|US>(a73Qq*rPCkmtowuh19yW;{L+n$qiQGg?e=+JhH zBOzyp4`>cs(&+D@Dv#9D0WFyj4@?FC1u_-(82HcE(<#JreL;0?l?Sy3%1p{DB0z`% zMRkx~Y^WGFw%F+=cqj1MBWg^fh-FOHqrb5a;Qc(wxyB5zZGcfEy-iEz`8936T~_MU zSO&gQ@+%sgKXiYM4*{P$ivwu zAr-tF(d#I+o9-yA9nn*U;=~GAJ8!s7;zWM20sF{!m|hI+?nfs12ELm6)T~rxKi|l; zxln8R!?m2JGq7EdP4N~tRX&j+ZQ{eL{&l;9thVOn#Fm%kb9 zE8DulIzKi?mbu>=aFtsEIN}ABtVjTppg~;s4!jd1H-^VNbXe5smU8jOi#|c$0UL*& zf1v;6JvkFx_jH}LiVMCI@Z!|WFP=589}y^6g~V~+0rhQGeP*YC;J z=S=q}BFwLD*Qv5(c&1@C$Hn4$lM0nQa1r0CzrOjC>*(9^j9W-|O4;u&3flf|?bt5T zuk$fc^uPYYzR$dUYd@{Tz0pSlBv?B9N;LftnSfX~sgczgaXV-K4#i*A2o z75jH6>sm`}CImcoNLJlC4k44_qQd(09MU7Z|8b{WsF;(c@*p`0N7;30T(6B+H+KLb zC21T;#>*gn7{6}elYj92!?ouEPdbvxAN>rZv+YMk{|d?Q^j^)Qv4$ju!&gVmDsrOz zKVA1{G|{dyX;0)m%2x@b`Unufv`x1y7>B9&%LN7$a1fEHP@%R%1lpF}+cQUI0jdhk zN3+_4?+>74qFy~WL8}`V)2O7zs%|miC$!su$xw)u8Y%3vkQ;bV1RF^J+tGI7$j#+7#|B7c(>ML7`(h; z4y07v&0w}T;IvG%T<26m_JEM50c=P{C{ZisfpO;~H^^z^^RWZ6@K)R~1o&v8g(f4~ zWT~tEK%g|oxGsL7ztAvM#njIm$b^wv{CqB@mxzk+btpvZAXKr&I8UX}kMh1`x-GR( zPp6-jQAjyP2v(SgfSg0Q|KO36^q~nLx2ctZLgL?_(sI|B1%Bxu|mrF@ye?sw}M=@i*p zsn+DDdD4b{#c=gNqonj6ZF<2D@k4%+rn1E)g&AeK16?o8!mR*z-T;j z@0%Z~tFtJZEmV%xN`DfNrS6U0UkwkLDI-((l8z&B;>7-T03K!aZU*z%wEa~;r@PKG z#V)|mF#xe&V%4dl<}E%~{HecBRb_8cdXh`nbv{+uOpySrssA-x5Gl=aBibX4#mVx~ zS5gczib2=O`ez-B2yM=4|4vT?=A=me5JZph6%RkN4P=^Pum>3Gb3*7*{F53;#PlsV z$l6{YarrXT0B+WJvW~@DR3n=Eb!y#u0Dr}D8$G6h>sDx#;2!ezdpV%;sRv)=xLzc{ z?)?m4qnwVfT;Cu}^zzPZ(Ga6XEc0{XD_I)O14yLHf;cZKdwc~hxr ziGuWDKTAM2NUk@o8$GrMN8h~#DV)D^mEk_ikfr`;K&Ag1{6wzAf-PFTw^RPv0-=a& z_D4Qw>JpX6$l$jG@<5)5lc@Dpk;^3=|ATAW0n;w;01uXM!kQax%*a6`FdQ!; z{P1ZK8jb|zN9OtGw%S&h+(hl(YxqA8K%WO|)r(K3H2$U7DrhMJz^k0?GyWFAfHWLR zHwy&&PKsM0_z(U8syj!m0Yip0*=ZJ8L=Q>>tAD)6vhc?v6P`-6!+@E{`dkRenWt4a zRGRE6yoKlJ(_1T3`mj^pYoI)f zts^)NaKyS^48&v8_HeFs$qnj@S*25{wp9@9s6az7cD@UkqpIbEanvy=M(IoKXHPZ+ zu`(kf5fzHd?#thAhRKO1i*^D&qvV<=y))#ru-LSOaZo>wDsY6(HRoF6Q0*Oh@Nfu8 zT#KfahI;2Ri>91-jWhe0E@d%qU^*1>kF zxnO#X>M^pb@KaCf-#_w<1-iM}1u0I$-~S)B!rUXzH|^$APE+N*6E1LuhaV+}-sMK_ zEU)oSz~ygyQ61yMpb~NzLx3>416d)7))YVkHRw%uP7UsoT6TFlo%CE6@YyKPL2>B| z8Qcfz z{|KxD8ecFbC`Mb(^yG|yfrADuz!r(6JBR|TgLHHpX|Szj4n$=fuz(z)T3cvy=ANhU z$PqC@DZh;Ky{ZvewPCQNFlpA(g9@_QUA^?F&ak;Iw0iE{yLS}n=h;im;uGBj{uxlP z-#4F1!$?C#Wc^qYE|$%MA9P<}!0%QPzkj;<7&conEaBgS6)s3tPa&WpgalrRJ7axF zw?i(LIJnD5{9AcSz^+`mM1OP2^qd_r{;;AKi4^0mRP5 zU|j&tAIT;t$hk@5{2qq^dh@O9fXEhYg~XEyUC=?7#cb#L#iDGQ#YTW!Zq_?z5jjOWi(1fG$WU>fk5xY18 z5siIE#DlsPVGAp8gCrc~qHuv=aFa6AqH22;(dQj$0vr?%k+EhE&e2Mfva8GYv-4ti zTp(!a&A6O|nw@gWz>|?^Sh3P2C&>)_X7?oz8586ZAu9tj!$lpDLF4kQe;4Z~X?Q8@ z1pf-yH};NGsqLN#Fr8FT)$7a@R>IoShovZAGlpOg#K1_~X70@5vn;l)*Vx(UsH%~_ zprfJ~fS5iMt#`~J|I6=H=1u%nQLFybtFVFdS?U^@m>AuSZeL@tgQ26Z?*pBx3A@z; zaQieXvmR2dUT#GJb84~$M0Zt(xj8UH&aecl+rXJ2O$oeKm@e*p+xSjd6@VRbpu--$ zHV@ObzSUc}+H1;hr46pI_ClbmGlr2ZCU+KG8h84x*w3gVTNh~)aU2;Qz)mktKl&D- zg3#16q?3ODxI_+>zsqZz5ZkXG^Fat;Rf4+h8C<36b0HFGGO|GbY?P$zgA!}0WeliL z&V_!oAQ3>Kg3Yb=a2q6WDjHxLZD+o4#Fkg%N^H;zvi}lZup9BlGCJwzxB)$>(03z< zMdN2N3cwjzFv2y%HP5ldgJ8vZSGg;O^qF31smQi|08l2eJ&Atn(Y*fsH_3;Ozeq^B z54Ti&Z+5Fo-7xU%79!-hV<5Wlm_aFZy(=1ma~ve65hw_F+Y&^Y#pWSRdLvo@A>5SQ ztZXF5s2p<#2sr{^>04Hd%0k6_E!BpdW%K+qpm@Uy|9JSe7{bdvIFEvcN z(%*9(!m5?8!77mWR&&oW#<#zp2G4!Y5dVmt`v9%zuqSzQ5d=1Gz{w)YmsK7y4lmL- zIoXWIC785+xaPbIMR}ub^SUhOT^{p5A?jVA;r$0x;jKTv7qQf(bTkM*K9ncita1It z*v`bWqNsJ!Xb&#Og>S{-fZjCyH*=PD)2MBYu zL1&#Pt!7V$N%i91)K>d(7sGYc0=*7-SyLp{YQps z>t#}E_YHsi>8u{QN-`9UppzwBpvcXUdAqBCDL!C5;0^_y#*|dm&Zg;W{+}P;t&VYN zvoaZ{gIi%`to-BiTl?<`lD4jJKNCG%8!WvO9%>l&b=_dZU|jQ-b?4ih(zJD(fVZmk zCycR=K=0$R)}EnY*z&E=j+{gMX{|?^u14 zST_rubY1RpNr$YQB14mIallMU>sTUpq2Qng_2wS6N3i&w<7>Duq0(#fRiz$3CRh3> zGnH>HtWVdmDpx8j!bmhR-jw%_t2>NzC2qaAU+BW!>uS5#-6!QvhdB~1Huta=$KxQFZs6t(vEc;XTVL=DZUcSi4&W>u_e zQihDC7t5V(R=cNoRZPFu>pG2+>Kpab3}u%Ir@1Q$t!eA4J(3O@Dy9t|aNq+<#ZU)B+&N&B@6vhJTcjlc{a_HyFRy*FY^x#W4ZaR9GUO`sEuuXA=c9d<5R%%r>5;tx zxP@k}wDdRDG3s&Q<6qRsJ*kd)C@&D~fpy`}&H(U`BJIG^v4pf*cekh!sOy4meQ_Xt z$L{2y`1pg-=9#D%0qmoZ&POdEca2ouzp-r_i>c@8n_H~hkLY>v1tdX)ON>f z@uFjvnYmUbWYrKvC0W1?mm=xQV*O9*`F`+UDUJ8T4rcxj7Kr5>anzq*zDZE_<-~Mf zk>yljd8evYLJoCbp*eG?xB3e^JcC;NL_TN|dm#?xOOzm~0VTLQLtfU}*fvy~0;c`k zxF~L>p=S;utBTzC#(;fH2U{cnH7$8~K!TavhSmR1yDe6}#-`nY@bv!JjChV$esNC$ z5*}!gdobe7mDMsm;0^1LzP%YiIJE(2u=36vqF@^PcGhMKWh6V!_-^m+7G~YS#dyMt zOHs2)h}yZiL77<##8Ox(p`rW?g@vG%rfEh2qo}Ztvzm^bihD`6x)*)fVcHWH|BB}J zagd|KG(4C?sdXU4aIGI4D_5eR3g%4LI&K_=yYU#3Fy&FhkO7p?{61QgjE7#j_D_Fd zy0%pf`J~1XCV|s|Ve634lUmq=zB3U8Ex36e$qH%*#+j#k1-u>qdly{HvZ~75>WtCZ zVf&E*r3Z@h#!E*ZS&RlD)xzXg1k)UB>f;9|0TSVO@aztmvQ3XpjOS!7bT{JXD=zBpG4e>n;M%M%EW(l?JIPyajL zXV0F||HKd5fboe2NS`?}^Wc#ozOtF)0Mkw&tef8gON`ZrhV`T_IOLYi{0;`Zi~dt@ zhl3`S)XL@KA(&hSOnDs&R6|mLA?SZIIKZI#!mbiwasC~*VFv?apXx3SG=h>NB7T!7 zt1nQvApfxAOb$VDrLX$w{v`rnK8F#AU|-Z)q5%kRnAiEz(gHKpr|4#plPXvVp_Lkg z?8uade}H8_JjR&%4l)ZIUwc`jmrg^1aI{!s=-L2vk(wiTbZS-9bQtD}$AYXawN%dR zEoDZ5pV>)i%SySdXRNnB2Azvv5Lr+(hb|c5%a$(+5|28Fgv}KW3r7o(UdZGu@c(dHS25GWBl? z2`^iMz=QQ75d;~n*UBe4}Uu;aF?tIzega5NlFK__$qL(_${ZX_M zz|RRcEW3Y;k<;iTdeV1_uU7%iEBEG^plT!*bm^CUa$0(WPr5Us2ge+}sM10oZXl}> zP?gYuA~m@k=gW3m=%=cWdC`Dyg*4RO>=wa!N5YJ&MW71trg)v;9fzv!lKkiY(d={K z5!fsbAugZJW(Fuu{i%+W>xGEolmJf{!)AYw*st(Q_QTLWQ498b#_-cWfi2|^o6ffw zxKiN(ObollkXhchHg#8|p1>N?duptiz`2}LT*gy<@X6*C{>W!czDvEUv+l?M3`~wK&FSfkk=;NeUC6R&LNc{ z;-b-Q_gGZQP@lYX!FIOyasaEiumljkaUd^EZUY6)7iNDCx7T2$JiNh4Q0(tt9>Qdc zu2xx!&{97*!DQXj9wXFLA!hT-t3`TOg4Y4Wi4JJ}$c;P@m%7`m$^RTmF7)94gH}p^ zWF&%q;F+)n3&IOpfJ(n^5x$6Dv77q&U2PxPRrGJv>mUZS{mp7*2lNmnChgF^)9~Y~ z6w5O|guHI$3dLEKhL{N2!h;rI%bs7YI>P$J1HB?bHpyy0Rffa?yW}^8B z#5i-rXYOSgNfyw<_u7+kBth1(7X_zLyO7C`T%%;LxI)fv#~<52}l8NAfSAK$l*ZZ9o5V(V5mI?E*)YnQsgF2 z*iU!yy7?@KJq-C1edqy5xg(#G;=O!Gu}Q7J0qN36IxinW-X784AD%1rQH&P-Y z?(D5ua!U}531oX&i!vE}tRC^hW??$^hmqNYa*_r_U@JthZfbzQD5Q^alTzZHTN6as z!GF@?=hx@zz$oyhwg!j^@4Y50pw7Kb1^+^=WO9<0)V({4VO3tJ_(zRvoUa0A18RCu z3s%qWGb+AAF#K+%&F3S{-UPjTI)Eg^-nJ@_UHqmL>i(K^0^MH{I-(>IAC+<>SYq*? z9>^56fKXLOaUNTJLyCu-Kp{#r2K$6dVv7;Ol`rjb2s82z^J>+g;)cd zeYXz{RQ7nNmJh;(oPUcW^Tos_pd!W3SZ<<0azTsK*3TmRlLGZ?4iNd0{HkDOx;`ze zZ{_vT`L`^@ieLpji#x2uk&TI11qRRWU;$Gs#%9@lyWAb`!pW!W03!S(n+OQJBplL( z)&Wh$q_P6L8=JaFska2lOfJq?Uyx@86bQ<{*C0*NUk$xZJw2KipyKZ>i9}RH<#oOh zyb##=Qfzm#yGoJ4Abr#SscE`8k6C{)ZiBM(78fu+j(`YzcOu}a;U!QaY+YUL z5!fyU23EE!`A<5*8-?kFBBO;2o-e-SswK!08<1Fa)D^Bw#^+18uFs#HC>{0Vt_2mQ z4iV1uhj%|Z27#MUh?ig!`j^iW$j!fNsrc^@1>^m@zj=GyPrnh+>C9e`kgdKQMzw2A zz)zJj4dXqt9m#dmy?EGmt_!Y{+GBU#rT#_=@;%j@XW5SHVOpp5S2MLTWc@-;kHwaJ z$8{{ge&>`(%Vy9y-!;?qVPlo}z*yoi#gyKas={qY9yCyzcVjPtUgb0x*T4bKN*cPC z@aUWJ5%u0x(LwEI(#BF9D$7C*RdjC|s11u+3_jk*l)f&wco zKYzpD3|Q_R193w;eHNE&%c08ag@kE5--tla(bHG9fX*7<{{?@n58HxG;{=T; z=2S2d5MN^~zk^3k1T57-xif=;1Qw;lC281HRBo$+GXX7Jsx%EQ>abd6L;vW>x4(lq zi@Xc}iqB8QO7H*=AC%wtqghrDR)NNfAHNLL+v^tbIlXc!^IPU$3OMHU+E%r-494|u zA#0~D`%600zkyKU%ayWc57O~6>>g?RIak1g(xmd2qU?n=%2s>@ODlcqt}KC2CT{kHHhUyxup5rP&7LogvKC8l{kF8wOr(D}lIL-qBf_ zDtP!1Z1bz+{tPzA*Ydj6)ajVNsg6bgOUGG(jamPT!`_KcO8@gm zo*7qU+5~A7%cwpCCY4Kqa_LiL$b)Aa4Vr>eiR)^U-(L8AaN>nfcUPv4IAY)Rq3XF_ zN#{*nkI^Iyu0r>>m$QB$*3f68E>;A9^jFdNqprZ&STIAro+pCnNCS*(r-6>R zLuHShs85g>*jw#Gf{5&2zLBvQU*qGn4S4J13NdJ-3gKYoaVCz-B>-RiwYfP8ks+H_ zW6#ZrQKtZnEP^&1)lSMQg&_QSBN^|T6?0u77-_BEtES-R@*yw$W9~0>5~`jW6S4s0 zv=0nfgJ{P1K5#wxQ_$_%ARz|AEtypiwJ&ArY9sTE4+LkooJ@~~6UOlmEWjqD4Z(w`EM^q^tD9{>R;eX-KmcmhOdWJk=V;&SVy3FT3*N1@aPGSgC&3xZc7Tw zA3e@-9j zUbCi6{*#P%(~&|A*WOu%A9EACL{Tz~a`}~MUeQD<%(&%eeFc|vo;8U4bjU9N8U=Fc zhLmPu1o}n~oDy{P53w)#BBj1sm1h*P^_z?uWvzer*b5=y@v4~m_fk2<%il5vM86U@ zyYFQ<5d?@^!7?i+J7rYAMlQM=iRgP@B>GzJSdIiUEdJuQzwjHs!d0wFp6}3h8=G(Z{2jSop6+di{|qZHoVO1P%=#VU8+&NL zL0aOHR?c} z_DSSM71llWGPhZY$plD1(fG}D&CE6l7WeA8I6N13QkBC@iOQzyJ~~B1=kv9V#^*;(EfbHfWH?A)C@Q?C{LR1v9pB&;60}*AbX2C8_&ur4 zRAG>=u;<|?@)$ypWD&K?>JL7%))d*-D?{#UvOU|aO0})}b#;acRYzC*+z6z?K`l$R zhVr%hlpDg+x%%#IB$t)YG_N0d6JB9{XYrD}WnP8h3Eu`u7VQ|@O;E0#q*#-~a%WO`LQ z|JbrG;59hdbPd+Ac72Y|$!n_~!wr4XcC2q`x-U;+{miB-Rv0HGRV&6_H&%Ymch+0h zuJ)N7E1dq$>pH(+%$d%ud#w1ceZz||NEVy1?to2Fq4&JFe2CLzqd;WnFz`bu^Y*IO z|Kyn4IYf&-{^cL_3mY*WcFL(!Tf~mowjuJk(_hzDit@MH`78NyJ=QFkK_{OAnrlM= z>DjUs&UQ--?t+-8jXvVY_axnarZ?yEOizl;5A3?KL|n395}qyJq1;Y?(v^N<)2yhI zY+5U`SK^X++Xa%uY?#2x#N?!9W7vOS^;yj<-Ko5nXLN_?TBzu%-YbHN@pZl6^ALB2 zvbgP#*_iUa_02xRI(f;W+YIi&@~g8l?XP~kZfco;nI9Lqk#tzrplARDZ(>jsm5VAk zM1^Ug(&EBteWF-V!ibu#&QLzl=`3{hfD=V97_R#!Z!PMlD%^slguFd&vK1RMU7p#Q zAHrpM|9Cri*rv*jKfq<%!?>GQ;_yH6ncPJGtjhO=RT2iaJyrujCLe^3SbX!y(JCLg|?s>Xli%%P?~wvPvb6{}rkCQ+4{%$dwPPGbos64gC{mxD~O zZmTNaGppDG_pi52Tl(ajay_27Bf-1OML&?*u~Cw_UUrM-RYWQJyF@pcWtC&bw-?1;E`NTl%L-Hr9U`PNqBrLo=d zpQ+n(&W|6j&osT%5A4)pd@pKfLRJaZ(vaec&L?S!Yy2GYG#;d!;lEkxbY0z2G8M-E zxOsRIrAkDR3cJjKPHeettVUXi?uG|+Y>L% z%U>HC{Zbm9MAt1oL8Tr@rV350S8NXz>{ycUjbHz7Ud}&fxd15^c`9j#rMZISb5T0; z6YU_!a!(q(2CN zUM%?U-7u_{HmaUe@1GXj+mA7#Z%t`4;0`%iMns2~trnc(xJO5T(amb|PCHQtJ3i zUY~J8uH$ei%mly7NBCVT-1R%m2pcOx{;_+@!DhC zepeQ5rMSCNK2jBda`(<1uR8Tb^FNoVxT_cT)P zkPp)-NW88sX6iz^+FnlwkgDqI;B%@4yDQU;0H$N8@$X_RF}I0NZe$3UkK>hqIBhFy zo*(puqJqQQL+~LB_+}h|B;a1IL|1;GpLz%52z*-N8frm-BNTphLspeQMphGWuUpap z7r@`XVKDxHKJ6ex4b_KzNIL+|q%2ZCa$TcbXU4<0mo10L_g?o-yYmY z%kEGEctPcq^0|=dY$fXpSOZ7ryHbU83{N@nV|`{7 zV$4o%WdXMm=dw1paqH@+sOJI^aRzX3N^#U2BE2z;dLf4}#$Ur<4^DSy=}k%DtY7^G zV=iaA*jUNv*BY$0)wN<6JEf!Hd0?0Jg*+_<&dt=X=4K!RrSv7L-xb1ZcBrEJyD^TB}Mf8Xqc zj9D+QXF|u5RkPT{&u#~x?9fpToCjQips54FJFK?)`g;kd#j|Pq{ab{~&u)!$KWnrg zyBc!ec#t+^;D9Gr+!vw|v-}~?1T}uy6Sr00FdFzVkt_p|QUc1*5 zk)k*Q_mX_Sn*N`sG)@+wne${&fT1_U_V!#BaRw3=+am>KlJdS{glw-J#VQ%0G2Y&;NPXo2Cg}YliB~p>w7dexn-emRrBNYqy zYI^6jIh~){AgFvf=!Yz*;UH_go<+?~OcIV(nscDXkq@H4iQ&->2|w@`v=3L*nj}Rt zLc8)&)T%^3)d{C!p^oP}k;1&+e!D<36;|vKhh>LExfi#F!Vbn?G2_FB z4-a0u+@CaLx&hQo^H&a_*mwJ`pY-%oGp^W@c;spHpf%n>!WcOBL2{*wOX*s`|0jV8 zzn67XUjI+4tN)5o|6deu|3wn}@6!AK|JPxU{D1dwy#MzfJ`JGGIvDF}!R&VmY21i; z^Z^5|FYrks=BU-Z4*bZq;OV3T9a8DXUDQVH$snIV;R#ZiE7I!MfkKxB41^B(dknIu z`_}rxSRLwzU<0umbt@`AQbL;518-#$9Z%_<^|5SF?0qNOlYBvG|2JrPQirTNK=tI+PXsXdAw%tB(2OW~0lIW% zgK@_joRsPPy?e?8w=#x_gM@ftY{?K@xDL#nP2&;B;Br!KK2T)vP^5XG#&5{};Sj?` zq4`M$`Xp;X!>ClB*aEj|+s3FUYLzuWveh3`C9-3KXTb}Yw0ClQT{{oFr^x`@jG@!I zc&w|CD_5n1Hv=651P*X%I1BAG>a>l6BNR-wk&~{r3iJ^#f4L*z2DWK`E_6 zkg!5j)dsg;!jn@l>6nL^R8}n9mPkF(dE5;`Vk@{<@}+`WupAu-ebr`F?VLo`(`?kB z)L8(2kjOzmTa3L$mU^R7NHhz-WTMzB;`4?|2uKg*eN5(n8#$|yE?tM#Cer>&Z6}W- z>qo}`YUjkHKEyCd!t;ZDgk}VKdvCve{5P~}YdGV7*ut8du>w~-|B78DOu+IWXab6B zFBv?&*K|6Hy#%8I24S#}?R8=oAV!pp;squGb`Q33oxpS)QR*RlIAxYdz0zxPSM3Kah=;?CA+2 zWq4aQ3d+dNb8#<*NP6m=0|&yE3qct4(^8Cltco;!e(y+j!%=>1{U#_5me5=@0t5;F zRr|2SHEWmeDL&6i0ss&D#2egMbLV>nlePZKaF;m4FMrGWarP-@p~vmT=g+|NPq}p7 zna2$2v$Q7fBj$Vr=7n=f}#IDjM}^U|i^ zv3bVES90fp8Lh1QNQxf>q)lC@{}6nVLvg=*LV3YKHDyA`1L}< zBnfqBW>oJSD2Wo?Pi2>D*t_dXGpsV)Irce*Z@@b$73_a}E2!3y!a#V6&JHHMV1nX} zsUEFQE^TXk>36%QcqgQ`^l1WZz_)MZ{ee0NcXTa4c3-VwT)!To@8N*MXRr% zYThl<_=iGGkbFIr_Y!Fmquec%-BD)Ud-R0}P&Q~yqF){90;6YXi*XsMATcu68$Lqc z0&S46(3MJusS|BPj%E88DcxP2Qx2JUOrkRhUJ-V@sX(PrKltf=w8%yAOZ3xydWBR!G^60tIXamK6 z&%$gsAJgn-1IJM7>HUs=LjmJCRgC^Qu3`nKNtF$u!o*|&3ZYATA(TB~UXAass8^j$ z-#X9Eh->330GcKi7@Ha_ho*Fqq6DLVT3vTBHE+-P%L6s9(aR?80OJa5NGOe4opvUPt^wJNR_~^xOH6YG%0+GkMF`tNfbp#@t z&|S99D0MAd-Dv$Wk_L+#!)Y&%ZVWMSa^|;6$F8KK- z`W1r+-5rb*l$PF**87sf4r#23!2iVx0%JZJLci8SKb!Yffsik8uBQKJhhWU@M<-?l*)ep zlql?E3|XX0`&<$1E@YO2E*AW`tv0KFu@(G$5SZ0Ui!*Jg{)R|_b*`Wef&LSF7Yi?6E`UKlwdV@^7i#8|Q>d7ga|@lfEx1L^nk3`g%*~^4PC< z$I$GEEU&RCAendbOL#LO@->3N!H+7+$q5X{Z6Z#z$A9=8ld$gVKt|PG05>=ofkB2T z$ov`}cfk_11g1oYXW|V#x~)9FTUaFEjtNbDa|nLZjbwO2@YES~<)#NSX?8vD4vVJg zS>n|u;8hb5wRmLuzHg@@OavHy%K79yBN^dC4=SFnzTEQNqYnnD9 zQ#QEMEk3Ww$mhQw?LAZkYq96@$w!7{nGn$2BAeLbo0q?un_Etsh-wz%%If_2^=_v~ z*PzfL%SSd8$$dk8f1BVDp;!Ch^cVq_l_?&`t-qDw+XjFUe0mFOjX zpHZq5{v8PWSO1Fl_#fK(#~QM^s{$wr(eSn+-jgke4fxl-Smld8Wf1ieTPyQ`B~z(g;|5(_p?Z#1c^tes7;FbS^R72tAv+z7Brk2l=lw5!JtQIzH=J z4|*yw5OQ;>TYp~p>wZ*8WAhOxgC+RcDB=7I&{blfQ(ltl|M14q>jRPY$PJL7{b@yq zY$$?)n-|=??HPAi!?0pi0Cb43|FZ)*<6hDZnX1OD1`55@I#0O8w$44_m(?$|egf&;-hyzSZjF+hjlYL3wF1!V_9OsU$BVpvF zvM(qseU!hpZ80S$*dUqorQk9ewWo=1i}Ic#wS%x=@!f^7u`uYRuZ;+spl zjBLi3>*ORmNP12)N2^Hl(X%7ben0x7D0L-`P+liUq}sBPf%YgJ>b!WC@JDKr zEt~;5YE~SeUo&vgHJj2rs`amO1DEz~0B1 zE}1ktR?x`vI=O6xVKBiSLEtZywlQKCmSzDQCbl`&!ja=RDyDmngmN;}&vkorQ7gHcbr5JhGnkqgr zyzJviXH3qOrdaCZx<$L-(KON?12kYa1tO zUXX4>CG4?9_$XZ*aKt)BB&&Hn`4xV6b63XpD#*gAk8(8UI7kMPd?tbIL4qR@Yf1L* za)@=-e!!QPKO`N4F42wCfElutmvlaTUeY4nliNN2aVWVx_Mj2;MHcfLoN`!EM$sKS z@hU4K%-?1;(kz=rukHcx=vgxp_rU0+Va(@%);}8t`FE<+m4GImt3z{` zY0!EEDxG{-^$X!WbI8(&{3C)@{b>z{yXl-n2O9AKv-KTEei$u^?CYpS%UtYM2T`sB zi7QMyYcj-#6&t-L7vHI+YbK@AyEGyN=jyU21g89BI{abf^7V=;CJ)7OAH`}>Ps-R;K>Z`RYkrGe3{J#wm zqk4xSpfW}4_T`l;oX&LGbBHv}<4NCDC-eZy0-j(;=i=FVQ(N*M##jGSXzQY2%C%36 zAGyaq`EC}wRrou#=nx zT7KGd=XKHQT!o&g0wDqFfMeZAtv+V6!hR3b&tc{T-*cCt-q#oa^oa61CUfJHFx%-PjP=AwZZqH8 zYX-skh7gD6PdgYtX>a+J>FCh(q7K6^JhiH~P@;eg1f*!ls`-lLLm8~RH1W?4HDrF< z!p9hI?bu!Ka|wL4BW}4hwwi~P=<(d#h#w^P;lCSjU4&l!9lX6-o5ENwk%V^XY1?Eo znOMYV&}^);o?=j`-O?-75$1n5Ck*LCCROxsm?_yvP24W$-qP`X5OwQeJo}fV7sDCf zCj+lbUp?MPP5jctb!x?K%tamyk2Q_GMY#8>J@d7fB%06(kCqg(OHg>a7~_;4SWYpF zu=HLfuxw*14(6^#Vw3w?hdys9!)USy)w|-z8lnPnjchALI+~ zgSm+u;kUU*fN`|BVYg{Upe*P#xbMh<+x+@ObeXBcMT{O15ZOX?=lQyjBRW*%TTG3} z=Ni^yKefO3I&yn%9cF!vq&PAEhGY6L?imk%2QkP}#QWZ0Cp~lWpaqFTVfK*LBX-df zd*Lxmcn3l4TXSO9@vvD|)~UOJ73YzUBRFi49vK0;UT`ARKGU7Jh-~&~!$n%>$o2UP zACnUxZJ7`xEAD@k13qwG_|CEU1uZa(V8?D`384R4d@I?sW{p?$cqS=;a_`OViY}Nw zjw9a=GBWFV4@hA7Xiwz!6J*Rv-!cN@*GBlK@Uw~*6bY}vWmvXlI%6_5Mz3My2oDn; z7BE9ui4?}<pwypTg?4n$hYM57MwIPG3gs@Dn);XTSvGNJJVtT;k(FAPvmfdsDfX5IL@y z4AZ<9(lxCvq7qyLzwn5=z(`XZ;>H*Id= zLMiwhZw#Xc@OC6lg9#Px=ig|k16HZsiuwzn=V5D{4%rDWde-`l#Gvgg+C?OfRS-qe z%$hwOk?d(R3^|PFS$b3kGk-FCEla>XG3fco`p;lbu&$rw5NMK>yJt}-tj2@iCSu@Z zse4bM-)k{+&_I3J#u1C5&(lOAMDi{LS5g+z%ErzSiXu&vc@|UWBul>^H z4XEe!rTe)UmB*t!L6@x|+XDjWSmA9hrs@tQbij)(D0gFP*klKy=fUj-6tlzh&A@ zpCQYcn+Baej|Zq{6Hp#YTsS<6*_sj>89$Z@=R2Y6WILq}6Q&!T>5zuH?ts*iC^D8< zNSb0y%R|8@_FRE4i%)%W#EF(ZXe*<-lK~?(yXpUNCy&xaate#H3bE9v3^p5%!LBbQhcS!!=r zfm~<`nz~xr2ZUaWlpt^_q;jo^nurQPqt=rg3290?rA_*t0%Wks-3iMzODEfe z4a^ziwbL?LjZft5rrbH}1IB^Dv4ISJwM{sx_YJ<>#d!kJZ|S%#lt@9cU^)M~DXhXA z(~YvL%JL2-rg%tdL^y5VC3csbwKS{X%UOJ!9^#&N*pxKK&@i%8hC*<7P=R-Yz;hy^ z5$QTrI)^;wELv&|BJa-C{Z%er5!{7m>%`!`!HW#w!8dXZ#~Bsz1)yXLsWTtoBu?7L zor9rq<-1wQzY%Q})$uN>Ew?F}Z@qYqe}u}TZY=P~_l4mfPs|D2c)uRy)>)XLGU0Yr z@i*Q>z_|D2Q^GlSCnXaH16As4^i7?tygCd!x;$__8>Vm-~6{qa^qHk)Kod(N!0#>2^o9O z94`9s6yy6!gQMigM6LQXb6GuJfM;?<&!9@8*ijXUjTPI3c!{JI^5w!wS@N6Rxq}bF zrNij1|fL9l5;yUZYfgMxV%49~{$n>*H16|XD7jiZbgL+P9+==b(bp>3a; zu#rGFpMH7p(iQ8=$Qy(9G-ZRO;v3=x0%jSC9g_Jry?hAtuqCHB2E+IKpYbSYN!wx9 z`R2l9RPE@!>ICGLMY>!sV6lI4zKAn;H_t@8k~4q8ce_l9FhoV*Jp|{1WOh9}!dRfA zGIw|IPv4r;tWqA&Z(FU^dFJwo>w8KEa@W4*2sY6a_rL!?_$bG&Bj##w4ZCX9~fY4c7;x$=#r9DOKXe`&RBpNre z3s4`to2bXyeC;C^yDzqx@~xd`yAua&arA}!WShCeqnMl1M$?TiEOQa1!BsZ-QPBKf z)5w|S*pV3)#lx^)E7dI?*5ucEI)bS#G)@UJ0gJ0)bQBXp;L1%%SEWfq*g;YsThH+i z^7+3M8s=hjq_`0q<_TrBF2_kQ_Wf97q`hb)YkW{FZ3Y#6U2O$&Zi3)M80!~_Ji?y| zcc$$}5%%%bB@U8xWT*j}gdmo0uq>Pc-E0PkAPNJVx%Jt1=i0hkUk*nL#)W{Woe81E zvmxT(f5v5Oz3mBSHvm4!?x%owI*9UkjPe;(17B{@7Ga*OPHds ziRn##-9aA1b4Y*FHcaG#SiVIy+45tKWE{v%SITMngpsBwjlzho@g)qSER4-ARgf_mY&ECC4TeR}jJdrs583-~} z&j)~_tIslCr5|Q9SDG{7tKIV>)Kr9SoLe@Ccr_H2@F-bOE)a7!mFK&sn802K#2Zl` zVyd$4>4;LLbi#zV&@ggNuhB7o|6uPz z)%!&5(-nPaZf+w|G%nya-zY}KmaOj+@tG&e)_ZoFl(G09;Zt-|9RboVK;)Tdn zmZPvC;k|6$$ic2atC`+j3;A}OU(1;*^j6~Li#uUc9`hy#DY}==ox3iuqTn-ko(4D8 zGPiKo0YRIXIXQPEZ9?UenvAm#iydD!mT6daL~axlKXCJPnuF0xcYWgzOxU&~B{jTG zSE2{TGI{SuW7ny^-cR1I`QB;X$A{q0xcLkr`YN(qtRd4;UQhbhpElt;GmNJRZWDh-Pe$3w kFf%Xy@e%(" - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "example_count": 0, - "measured": false, - "metadata": {}, - "metrics": {}, - "mode": "raw", - "notes": [ - "limit_per_category=1", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": null, - "reference_kind": "historical", - "runtime_seconds": null, - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "blocked", - "system_name": "mempalace" -} \ No newline at end of file diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/convomem_new_brainctl_cmd.json b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/convomem_new_brainctl_cmd.json deleted file mode 100644 index 7b54da9..0000000 --- a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/convomem_new_brainctl_cmd.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "artifacts": {}, - "benchmark": "convomem", - "caveats": [ - "Blocked while loading ConvoMem evidence data: HTTP Error 404: Not Found" - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "example_count": 0, - "measured": true, - "metadata": {}, - "metrics": {}, - "mode": "cmd", - "notes": [ - "limit_per_category=1", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": null, - "reference_kind": "measured", - "rows": [], - "runtime_seconds": null, - "series_name": "new_brainctl", - "source_path": null, - "status": "blocked", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/convomem_old_brainctl_cmd.json b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/convomem_old_brainctl_cmd.json deleted file mode 100644 index fbd9ef4..0000000 --- a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/convomem_old_brainctl_cmd.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\runs\\convomem_brainctl_cmd.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\coverage_status.png" - }, - "benchmark": "convomem", - "caveats": [ - "Blocked while loading ConvoMem evidence data: " - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "example_count": 0, - "measured": false, - "metadata": {}, - "metrics": {}, - "mode": "cmd", - "notes": [ - "limit_per_category=1", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": null, - "reference_kind": "historical", - "runtime_seconds": null, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "blocked", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/locomo_mempalace_raw_session.json b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/locomo_mempalace_raw_session.json deleted file mode 100644 index 47d4098..0000000 --- a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/locomo_mempalace_raw_session.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\locomo_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\runs\\locomo_mempalace_raw_session.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\coverage_status.png" - }, - "benchmark": "locomo", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "measured": false, - "metadata": {}, - "metrics": { - "avg_recall": 0.6028, - "cat_1_recall": 0.5899, - "cat_2_recall": 0.6921, - "cat_3_recall": 0.4604, - "cat_4_recall": 0.5809, - "cat_5_recall": 0.6188, - "perfect_rate": 0.5534, - "top_k": 10, - "zero_rate": 0.3499 - }, - "mode": "raw_session", - "notes": [ - "granularity=session", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": 0.6028, - "reference_kind": "historical", - "runtime_seconds": 2106.411, - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "mempalace" -} \ No newline at end of file diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/locomo_new_brainctl_cmd_session.json b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/locomo_new_brainctl_cmd_session.json deleted file mode 100644 index dbe14fb..0000000 --- a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/locomo_new_brainctl_cmd_session.json +++ /dev/null @@ -1,4450 +0,0 @@ -{ - "artifacts": {}, - "benchmark": "locomo", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 199, - "measured": true, - "metadata": {}, - "metrics": { - "avg_recall": 0.9413, - "cat_1_recall": 0.8641, - "cat_2_recall": 1.0, - "cat_3_recall": 0.7436, - "cat_4_recall": 0.9571, - "cat_5_recall": 0.9787, - "perfect_rate": 0.9196, - "top_k": 10, - "zero_rate": 0.0402 - }, - "mode": "cmd_session", - "notes": [ - "granularity=session", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": 0.9413, - "reference_kind": "measured", - "rows": [ - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_1" - ], - "question": "When did Caroline go to the LGBTQ support group?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_10", - "session_13", - "session_12", - "session_4", - "session_14", - "session_5", - "session_7", - "session_11", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_1" - ], - "question": "When did Melanie paint a sunrise?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_14", - "session_13", - "session_8", - "session_11", - "session_16", - "session_17", - "session_9", - "session_18", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_1" - ], - "question": "What fields would Caroline be likely to pursue in her educaton?", - "recall": 0.0, - "retrieved_ids": [ - "session_7", - "session_13", - "session_18", - "session_10", - "session_4", - "session_14", - "session_9", - "session_17", - "session_5", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_2" - ], - "question": "What did Caroline research?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_1", - "session_2", - "session_8", - "session_14", - "session_13", - "session_18", - "session_15", - "session_5", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1" - ], - "question": "What is Caroline's identity?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_11", - "session_16", - "session_17", - "session_3", - "session_1", - "session_8", - "session_9", - "session_12", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_2" - ], - "question": "When did Melanie run a charity race?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_7", - "session_8", - "session_14", - "session_13", - "session_18", - "session_15", - "session_1", - "session_5", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_2" - ], - "question": "When is Melanie planning on going camping?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_2", - "session_16", - "session_12", - "session_7", - "session_17", - "session_14", - "session_18", - "session_9", - "session_6" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_2", - "session_3" - ], - "question": "What is Caroline's relationship status?", - "recall": 0.0, - "retrieved_ids": [ - "session_16", - "session_1", - "session_8", - "session_9", - "session_12", - "session_7", - "session_5", - "session_14", - "session_18", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_3" - ], - "question": "When did Caroline give a speech at a school?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_13", - "session_18", - "session_5", - "session_14", - "session_10", - "session_7", - "session_8", - "session_19", - "session_2" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_3" - ], - "question": "When did Caroline meet up with her friends, family, and mentors?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_7", - "session_9", - "session_2", - "session_16", - "session_5", - "session_17", - "session_3", - "session_8", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_3" - ], - "question": "How long has Caroline had her current group of friends for?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_7", - "session_1", - "session_4", - "session_12", - "session_13", - "session_16", - "session_3", - "session_6", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_3", - "session_4" - ], - "question": "Where did Caroline move from 4 years ago?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_8", - "session_19", - "session_4", - "session_16", - "session_15", - "session_7", - "session_17", - "session_9", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_4" - ], - "question": "How long ago was Caroline's 18th birthday?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_11", - "session_16", - "session_15", - "session_3", - "session_17", - "session_8", - "session_9", - "session_7", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_4" - ], - "question": "What career path has Caroline decided to persue?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_7", - "session_14", - "session_1", - "session_3", - "session_16", - "session_15", - "session_8", - "session_9", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_3", - "session_4" - ], - "question": "Would Caroline still want to pursue counseling as a career if she hadn't received support growing up?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_7", - "session_17", - "session_1", - "session_18", - "session_6", - "session_2", - "session_10", - "session_5", - "session_3" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_5", - "session_9" - ], - "question": "What activities does Melanie partake in?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_15", - "session_8", - "session_18", - "session_1", - "session_4", - "session_13", - "session_9", - "session_19", - "session_17" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_5" - ], - "question": "When did Melanie sign up for a pottery class?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_16", - "session_14", - "session_10", - "session_12", - "session_17", - "session_8", - "session_18", - "session_11", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_5" - ], - "question": "When is Caroline going to the transgender conference?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_7", - "session_17", - "session_1", - "session_14", - "session_9", - "session_3", - "session_4", - "session_13", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_4", - "session_6", - "session_8" - ], - "question": "Where has Melanie camped?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_18", - "session_9", - "session_6", - "session_2", - "session_4", - "session_16", - "session_8", - "session_1", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_4", - "session_6" - ], - "question": "What do Melanie's kids like?", - "recall": 0.5, - "retrieved_ids": [ - "session_9", - "session_10", - "session_15", - "session_18", - "session_8", - "session_17", - "session_1", - "session_16", - "session_4", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_6" - ], - "question": "When did Melanie go to the museum?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_13", - "session_1", - "session_14", - "session_5", - "session_11", - "session_18", - "session_8", - "session_7", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_6" - ], - "question": "When did Caroline have a picnic?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_8", - "session_14", - "session_13", - "session_18", - "session_15", - "session_1", - "session_5", - "session_11", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_6" - ], - "question": "Would Caroline likely have Dr. Seuss books on her bookshelf?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_18", - "session_10", - "session_6", - "session_17", - "session_4", - "session_14", - "session_9", - "session_5", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_6", - "session_7" - ], - "question": "What books has Melanie read?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_7", - "session_17", - "session_2", - "session_8", - "session_18", - "session_1", - "session_4", - "session_13", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_5", - "session_7" - ], - "question": "What does Melanie do to destress?", - "recall": 0.5, - "retrieved_ids": [ - "session_7", - "session_8", - "session_18", - "session_1", - "session_4", - "session_13", - "session_9", - "session_19", - "session_17", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_7" - ], - "question": "When did Caroline go to the LGBTQ conference?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_7", - "session_14", - "session_1", - "session_13", - "session_11", - "session_10", - "session_9", - "session_4", - "session_3" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_7" - ], - "question": "When did Melanie read the book \"nothing is impossible\"?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_7", - "session_17", - "session_2", - "session_8", - "session_14", - "session_13", - "session_18", - "session_15", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_7" - ], - "question": "Would Caroline pursue writing as a career option?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_1", - "session_4", - "session_18", - "session_8", - "session_9", - "session_12", - "session_5", - "session_14", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_8" - ], - "question": "When did Caroline go to the adoption meeting?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_2", - "session_17", - "session_19", - "session_13", - "session_10", - "session_5", - "session_15", - "session_7", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_8" - ], - "question": "When did Melanie go to the pottery workshop?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_4", - "session_5", - "session_16", - "session_12", - "session_17", - "session_14", - "session_13", - "session_1", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [], - "question": "Would Melanie be considered a member of the LGBTQ community?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_5", - "session_14", - "session_11", - "session_9", - "session_3", - "session_7", - "session_10", - "session_15", - "session_16" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_4" - ], - "question": "When did Melanie go camping in June?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_18", - "session_9", - "session_6", - "session_2", - "session_4", - "session_16", - "session_8", - "session_13", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_3", - "session_5", - "session_8" - ], - "question": "What LGBTQ+ events has Caroline participated in?", - "recall": 0.75, - "retrieved_ids": [ - "session_9", - "session_10", - "session_3", - "session_2", - "session_11", - "session_7", - "session_17", - "session_5", - "session_14", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_5" - ], - "question": "When did Caroline go to a pride parade during the summer?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_10", - "session_11", - "session_5", - "session_18", - "session_16", - "session_12", - "session_2", - "session_15", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_3", - "session_9" - ], - "question": "What events has Caroline participated in to help children?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_9", - "session_10", - "session_3", - "session_2", - "session_11", - "session_7", - "session_17", - "session_19", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_9" - ], - "question": "When did Melanie go camping in July?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_18", - "session_9", - "session_6", - "session_2", - "session_4", - "session_16", - "session_8", - "session_13", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_9" - ], - "question": "When did Caroline join a mentorship program?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_10", - "session_3", - "session_8", - "session_14", - "session_13", - "session_18", - "session_15", - "session_1", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_8" - ], - "question": "What did Melanie paint recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_17", - "session_11", - "session_12", - "session_18", - "session_10", - "session_14", - "session_1", - "session_8", - "session_16" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_3", - "session_6", - "session_8", - "session_9" - ], - "question": "What activities has Melanie done with her family?", - "recall": 0.4, - "retrieved_ids": [ - "session_10", - "session_5", - "session_15", - "session_7", - "session_1", - "session_6", - "session_13", - "session_16", - "session_17", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_5", - "session_9" - ], - "question": "In what ways is Caroline participating in the LGBTQ community?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_11", - "session_9", - "session_15", - "session_14", - "session_3", - "session_7", - "session_10", - "session_16", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_6" - ], - "question": "How many times has Melanie gone to the beach in 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_3", - "session_14", - "session_6", - "session_4", - "session_7", - "session_17", - "session_8", - "session_18", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_10" - ], - "question": "When did Caroline join a new activist group?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_1", - "session_9", - "session_13", - "session_12", - "session_4", - "session_3", - "session_8", - "session_14", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_10" - ], - "question": "Would Melanie be more interested in going to a national park or a theme park?", - "recall": 0.0, - "retrieved_ids": [ - "session_11", - "session_18", - "session_15", - "session_16", - "session_7", - "session_14", - "session_5", - "session_4", - "session_3", - "session_6" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_11", - "session_9" - ], - "question": "What kind of art does Caroline make?", - "recall": 0.5, - "retrieved_ids": [ - "session_2", - "session_4", - "session_3", - "session_5", - "session_6", - "session_11", - "session_10", - "session_15", - "session_14", - "session_16" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_11" - ], - "question": "When is Melanie's daughter's birthday?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_4", - "session_8", - "session_18", - "session_1", - "session_13", - "session_9", - "session_19", - "session_17", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_11" - ], - "question": "When did Caroline attend a pride parade in August?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_5", - "session_11", - "session_10", - "session_8", - "session_9", - "session_14", - "session_12", - "session_17", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [], - "question": "Would Melanie be considered an ally to the transgender community?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_18", - "session_17", - "session_1", - "session_14", - "session_9", - "session_5", - "session_11", - "session_15", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_12", - "session_3" - ], - "question": "Who supports Caroline when she has a negative experience?", - "recall": 0.5, - "retrieved_ids": [ - "session_4", - "session_10", - "session_12", - "session_19", - "session_9", - "session_5", - "session_7", - "session_15", - "session_11", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_12", - "session_5", - "session_8" - ], - "question": "What types of pottery have Melanie and her kids made?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_10", - "session_15", - "session_7", - "session_5", - "session_16", - "session_12", - "session_17", - "session_8", - "session_2" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_12" - ], - "question": "When did Caroline and Melanie go to a pride fesetival together?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_14", - "session_5", - "session_11", - "session_12", - "session_10", - "session_17", - "session_8", - "session_18", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_12" - ], - "question": "What would Caroline's political leaning likely be?", - "recall": 0.0, - "retrieved_ids": [ - "session_18", - "session_4", - "session_14", - "session_9", - "session_17", - "session_5", - "session_10", - "session_13", - "session_7", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_13", - "session_8" - ], - "question": "What has Melanie painted?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_13", - "session_9", - "session_14", - "session_17", - "session_16", - "session_11", - "session_8", - "session_12", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_13", - "session_7" - ], - "question": "What are Melanie's pets' names?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_7", - "session_8", - "session_18", - "session_1", - "session_4", - "session_9", - "session_19", - "session_17", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_13" - ], - "question": "When did Caroline apply to adoption agencies?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_2", - "session_19", - "session_17", - "session_8", - "session_14", - "session_18", - "session_15", - "session_1", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_13" - ], - "question": "When did Caroline draw a self-portrait?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_14", - "session_16", - "session_11", - "session_17", - "session_7", - "session_8", - "session_19", - "session_2", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_14", - "session_8" - ], - "question": "What subject have Caroline and Melanie both painted?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_9", - "session_19", - "session_13", - "session_12", - "session_16", - "session_15", - "session_3", - "session_1", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_14", - "session_4" - ], - "question": "What symbols are important to Caroline?", - "recall": 0.5, - "retrieved_ids": [ - "session_14", - "session_7", - "session_2", - "session_11", - "session_6", - "session_18", - "session_15", - "session_3", - "session_8", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_14" - ], - "question": "When did Caroline encounter people on a hike and have a negative experience?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_4", - "session_16", - "session_8", - "session_14", - "session_11", - "session_5", - "session_15", - "session_18", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_14" - ], - "question": "When did Melanie make a plate in pottery class?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_5", - "session_16", - "session_12", - "session_17", - "session_8", - "session_15", - "session_7", - "session_10", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_12", - "session_14" - ], - "question": "Would Caroline be considered religious?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_18", - "session_1", - "session_8", - "session_9", - "session_7", - "session_5", - "session_14", - "session_4", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_15", - "session_2" - ], - "question": "What instruments does Melanie play?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_6", - "session_2", - "session_3", - "session_8", - "session_18", - "session_1", - "session_4", - "session_13", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_11", - "session_15" - ], - "question": "What musical artists/bands has Melanie seen?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_11", - "session_17", - "session_14", - "session_8", - "session_18", - "session_1", - "session_4", - "session_13", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_15" - ], - "question": "When did Melanie go to the park?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_16", - "session_13", - "session_1", - "session_14", - "session_5", - "session_11", - "session_18", - "session_8", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_15" - ], - "question": "When is Caroline's youth center putting on a talent show?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_9", - "session_11", - "session_14", - "session_5", - "session_19", - "session_12", - "session_10", - "session_16", - "session_17" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_15" - ], - "question": "Would Melanie likely enjoy the song \"The Four Seasons\" by Vivaldi?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_15", - "session_11", - "session_8", - "session_10", - "session_4", - "session_14", - "session_9", - "session_17", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_11", - "session_16" - ], - "question": "What are some changes Caroline has faced during her transition journey?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_16", - "session_3", - "session_8", - "session_7", - "session_18", - "session_11", - "session_14", - "session_9", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_16" - ], - "question": "What does Melanie do with her family on hikes?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_7", - "session_12", - "session_4", - "session_16", - "session_8", - "session_14", - "session_19", - "session_2", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_16" - ], - "question": "When did Caroline go biking with friends?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_6", - "session_3", - "session_8", - "session_19", - "session_2", - "session_12", - "session_4", - "session_17", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_16" - ], - "question": "How long has Melanie been practicing art?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_3", - "session_15", - "session_17", - "session_1", - "session_6", - "session_19", - "session_4", - "session_8", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_13", - "session_16", - "session_7" - ], - "question": "What personality traits might Melanie say Caroline has?", - "recall": 0.6667, - "retrieved_ids": [ - "session_10", - "session_16", - "session_15", - "session_8", - "session_14", - "session_1", - "session_18", - "session_9", - "session_4", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_15", - "session_17" - ], - "question": "What transgender-specific events has Caroline attended?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_15", - "session_9", - "session_17", - "session_3", - "session_1", - "session_14", - "session_5", - "session_10", - "session_2" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_17", - "session_7" - ], - "question": "What book did Melanie read from Caroline's suggestion?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_7", - "session_17", - "session_2", - "session_8", - "session_14", - "session_13", - "session_18", - "session_15", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_17" - ], - "question": "When did Melanie's friend adopt a child?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_2", - "session_17", - "session_19", - "session_8", - "session_6", - "session_16", - "session_3", - "session_12", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_17" - ], - "question": "When did Melanie get hurt?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_8", - "session_5", - "session_13", - "session_1", - "session_16", - "session_18", - "session_14", - "session_15", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_18" - ], - "question": "When did Melanie's family go on a roadtrip?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_10", - "session_7", - "session_17", - "session_4", - "session_3", - "session_13", - "session_2", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_18" - ], - "question": "How many children does Melanie have?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_10", - "session_3", - "session_18", - "session_1", - "session_4", - "session_13", - "session_9", - "session_19", - "session_17" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_18" - ], - "question": "When did Melanie go on a hike after the roadtrip?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_4", - "session_14", - "session_12", - "session_16", - "session_8", - "session_2", - "session_9", - "session_1", - "session_3" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_18" - ], - "question": "Would Melanie go on another roadtrip soon?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_1", - "session_9", - "session_13", - "session_11", - "session_12", - "session_16", - "session_8", - "session_4", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_19", - "session_7" - ], - "question": "What items has Melanie bought?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_4", - "session_8", - "session_18", - "session_1", - "session_13", - "session_9", - "session_17", - "session_15", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_19" - ], - "question": "When did Caroline pass the adoption interview?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_3", - "session_2", - "session_17", - "session_13", - "session_8", - "session_14", - "session_18", - "session_15", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_19" - ], - "question": "When did Melanie buy the figurines?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_8", - "session_14", - "session_13", - "session_18", - "session_15", - "session_1", - "session_5", - "session_11", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_19" - ], - "question": "Would Caroline want to move back to her home country soon?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_19", - "session_18", - "session_8", - "session_1", - "session_4", - "session_7", - "session_10", - "session_2", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What did the charity race raise awareness for?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_7", - "session_3", - "session_8", - "session_14", - "session_13", - "session_18", - "session_15", - "session_1", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What did Melanie realize after the charity race?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_7", - "session_8", - "session_18", - "session_9", - "session_1", - "session_4", - "session_3", - "session_14", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "How does Melanie prioritize self-care?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_9", - "session_13", - "session_4", - "session_1", - "session_7", - "session_11", - "session_17", - "session_19", - "session_16" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What are Caroline's plans for the summer?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_2", - "session_10", - "session_11", - "session_15", - "session_16", - "session_7", - "session_17", - "session_14", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What type of individuals does the adoption agency Caroline is considering support?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_11", - "session_19", - "session_13", - "session_17", - "session_15", - "session_14", - "session_8", - "session_1", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "Why did Caroline choose the adoption agency?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_2", - "session_13", - "session_17", - "session_16", - "session_8", - "session_14", - "session_18", - "session_15", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What is Caroline excited about in the adoption process?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_2", - "session_19", - "session_13", - "session_8", - "session_5", - "session_16", - "session_1", - "session_9", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What does Melanie think about Caroline's decision to adopt?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_2", - "session_17", - "session_19", - "session_8", - "session_12", - "session_4", - "session_5", - "session_16", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "How long have Mel and her husband been married?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_10", - "session_7", - "session_8", - "session_16", - "session_15", - "session_17", - "session_1", - "session_6", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What does Caroline's necklace symbolize?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_14", - "session_1", - "session_8", - "session_9", - "session_12", - "session_7", - "session_5", - "session_18", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What country is Caroline's grandma from?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_3", - "session_1", - "session_8", - "session_9", - "session_12", - "session_7", - "session_5", - "session_14", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What was grandma's gift to Caroline?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_19", - "session_1", - "session_8", - "session_9", - "session_12", - "session_7", - "session_5", - "session_14", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What is Melanie's hand-painted bowl a reminder of?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_8", - "session_5", - "session_12", - "session_16", - "session_11", - "session_14", - "session_1", - "session_13", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What did Melanie and her family do while camping?", - "recall": 0.0, - "retrieved_ids": [ - "session_10", - "session_7", - "session_16", - "session_8", - "session_17", - "session_13", - "session_18", - "session_9", - "session_6", - "session_2" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What kind of counseling and mental health services is Caroline interested in pursuing?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_7", - "session_11", - "session_5", - "session_6", - "session_1", - "session_2", - "session_8", - "session_3", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What workshop did Caroline attend recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_4", - "session_8", - "session_17", - "session_11", - "session_12", - "session_18", - "session_10", - "session_14", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What was discussed in the LGBTQ+ counseling workshop?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_8", - "session_7", - "session_1", - "session_5", - "session_6", - "session_9", - "session_11", - "session_14", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What motivated Caroline to pursue counseling?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_7", - "session_5", - "session_1", - "session_6", - "session_3", - "session_18", - "session_14", - "session_8", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What kind of place does Caroline want to create for people?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_14", - "session_17", - "session_2", - "session_3", - "session_5", - "session_6", - "session_11", - "session_10", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_5" - ], - "question": "Did Melanie make the black and white bowl in the photo?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_16", - "session_12", - "session_4", - "session_6", - "session_13", - "session_3", - "session_8", - "session_14", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_6" - ], - "question": "What kind of books does Caroline have in her library?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_7", - "session_10", - "session_17", - "session_2", - "session_4", - "session_3", - "session_5", - "session_11", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_6" - ], - "question": "What was Melanie's favorite book from her childhood?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_7", - "session_10", - "session_17", - "session_8", - "session_14", - "session_18", - "session_1", - "session_4", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What book did Caroline recommend to Melanie?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_17", - "session_6", - "session_8", - "session_14", - "session_13", - "session_18", - "session_15", - "session_1", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What did Caroline take away from the book \"Becoming Nicole\"?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_13", - "session_6", - "session_17", - "session_2", - "session_1", - "session_14", - "session_5", - "session_4", - "session_3" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What are the new shoes that Melanie got used for?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_17", - "session_6", - "session_15", - "session_9", - "session_4", - "session_1", - "session_16", - "session_19", - "session_2" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What is Melanie's reason for getting into running?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_2", - "session_16", - "session_18", - "session_4", - "session_10", - "session_15", - "session_14", - "session_6", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What does Melanie say running has been great for?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_10", - "session_2", - "session_6", - "session_17", - "session_15", - "session_9", - "session_8", - "session_3", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What did Mel and her kids make during the pottery workshop?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_16", - "session_10", - "session_4", - "session_7", - "session_18", - "session_5", - "session_12", - "session_17", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What kind of pot did Mel and her kids make with clay?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_10", - "session_5", - "session_7", - "session_2", - "session_4", - "session_3", - "session_6", - "session_11", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What creative project do Mel and her kids do together besides pottery?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_12", - "session_14", - "session_10", - "session_7", - "session_5", - "session_16", - "session_17", - "session_15", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What did Mel and her kids paint in their latest project in July 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_10", - "session_7", - "session_12", - "session_15", - "session_14", - "session_17", - "session_1", - "session_13", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What did Caroline see at the council meeting for adoption?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_2", - "session_17", - "session_19", - "session_13", - "session_10", - "session_5", - "session_15", - "session_7", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What do sunflowers represent according to Caroline?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_11", - "session_14", - "session_1", - "session_9", - "session_12", - "session_7", - "session_5", - "session_18", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "Why are flowers important to Melanie?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_14", - "session_7", - "session_2", - "session_11", - "session_6", - "session_18", - "session_15", - "session_3", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_9" - ], - "question": "What inspired Caroline's painting for the art show?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_16", - "session_9", - "session_17", - "session_11", - "session_8", - "session_12", - "session_13", - "session_4", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "How often does Melanie go to the beach with her kids?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_7", - "session_14", - "session_6", - "session_1", - "session_9", - "session_11", - "session_2", - "session_18", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "What did Melanie and her family see during their camping trip last year?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_18", - "session_8", - "session_7", - "session_16", - "session_12", - "session_17", - "session_4", - "session_9", - "session_6" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "How did Melanie feel while watching the meteor shower?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_8", - "session_17", - "session_13", - "session_16", - "session_14", - "session_1", - "session_5", - "session_15", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_11" - ], - "question": "Whose birthday did Melanie celebrate recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_4", - "session_17", - "session_10", - "session_14", - "session_8", - "session_13", - "session_12", - "session_18", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_11" - ], - "question": "Who performed at the concert at Melanie's daughter's birthday?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_4", - "session_8", - "session_18", - "session_1", - "session_13", - "session_9", - "session_19", - "session_17", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_12" - ], - "question": "Why did Melanie choose to use colors and patterns in her pottery project?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_16", - "session_19", - "session_10", - "session_4", - "session_14", - "session_2", - "session_8", - "session_7", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "What pet does Caroline have?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_13", - "session_1", - "session_8", - "session_9", - "session_12", - "session_5", - "session_14", - "session_18", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "What pets does Melanie have?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_13", - "session_8", - "session_18", - "session_1", - "session_4", - "session_9", - "session_19", - "session_17", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "Where did Oliver hide his bone once?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_10", - "session_6", - "session_7", - "session_11", - "session_8", - "session_14", - "session_18", - "session_15", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "What activity did Caroline used to do with her dad?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_10", - "session_5", - "session_15", - "session_7", - "session_8", - "session_14", - "session_18", - "session_1", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_14" - ], - "question": "What did Caroline make for a local church?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_8", - "session_15", - "session_5", - "session_7", - "session_10", - "session_16", - "session_18", - "session_17", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_14" - ], - "question": "What did Caroline find in her neighborhood during her walk?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_7", - "session_10", - "session_16", - "session_18", - "session_8", - "session_19", - "session_9", - "session_11", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "Which song motivates Caroline to be courageous?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_3", - "session_11", - "session_14", - "session_1", - "session_8", - "session_5", - "session_18", - "session_4", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "Which classical musicians does Melanie enjoy listening to?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_15", - "session_8", - "session_18", - "session_10", - "session_1", - "session_4", - "session_13", - "session_9", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "Who is Melanie a fan of in terms of modern music?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_5", - "session_11", - "session_8", - "session_18", - "session_1", - "session_4", - "session_13", - "session_9", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "How long has Melanie been creating art?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_3", - "session_15", - "session_17", - "session_1", - "session_6", - "session_19", - "session_4", - "session_8", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "What precautionary sign did Melanie see at the caf\u00e9?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_5", - "session_10", - "session_15", - "session_14", - "session_1", - "session_11", - "session_18", - "session_8", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What advice does Caroline give for getting started with adoption?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_13", - "session_2", - "session_19", - "session_8", - "session_15", - "session_12", - "session_18", - "session_4", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What setback did Melanie face in October 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_9", - "session_10", - "session_15", - "session_3", - "session_8", - "session_14", - "session_13", - "session_18", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What does Melanie do to keep herself busy during her pottery break?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_8", - "session_16", - "session_10", - "session_14", - "session_2", - "session_7", - "session_18", - "session_5", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What painting did Melanie show to Caroline on October 13, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_13", - "session_8", - "session_17", - "session_16", - "session_11", - "session_9", - "session_1", - "session_15", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What kind of painting did Caroline share with Melanie on October 13, 2023?", - "recall": 0.0, - "retrieved_ids": [ - "session_2", - "session_4", - "session_3", - "session_5", - "session_6", - "session_11", - "session_10", - "session_15", - "session_14", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What was the poetry reading that Caroline attended about?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_13", - "session_6", - "session_7", - "session_2", - "session_4", - "session_15", - "session_3", - "session_12", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What did the posters at the poetry reading say?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_10", - "session_6", - "session_7", - "session_2", - "session_8", - "session_14", - "session_13", - "session_18", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What does Caroline's drawing symbolize for her?", - "recall": 0.0, - "retrieved_ids": [ - "session_14", - "session_10", - "session_16", - "session_7", - "session_1", - "session_8", - "session_9", - "session_12", - "session_5", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "How do Melanie and Caroline describe their journey through life together?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_8", - "session_4", - "session_17", - "session_14", - "session_19", - "session_7", - "session_15", - "session_10", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What happened to Melanie's son on their road trip?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_10", - "session_12", - "session_17", - "session_7", - "session_6", - "session_14", - "session_2", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "How did Melanie's son handle the accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_16", - "session_8", - "session_14", - "session_13", - "session_15", - "session_1", - "session_5", - "session_11", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "How did Melanie feel about her family after the accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_10", - "session_7", - "session_2", - "session_9", - "session_1", - "session_4", - "session_3", - "session_14", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "How did Melanie's children handle the accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_16", - "session_14", - "session_13", - "session_15", - "session_1", - "session_5", - "session_11", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "How did Melanie feel after the accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_2", - "session_9", - "session_1", - "session_4", - "session_3", - "session_14", - "session_13", - "session_8", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What was Melanie's reaction to her children enjoying the Grand Canyon?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_10", - "session_3", - "session_7", - "session_1", - "session_4", - "session_13", - "session_9", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What do Melanie's family give her?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_7", - "session_3", - "session_19", - "session_2", - "session_18", - "session_6", - "session_4", - "session_8", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "How did Melanie feel about her family supporting her?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_7", - "session_8", - "session_17", - "session_16", - "session_3", - "session_2", - "session_15", - "session_18", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What did Melanie do after the road trip to relax?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_17", - "session_1", - "session_10", - "session_14", - "session_12", - "session_15", - "session_2", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_2" - ], - "question": "What did Caroline realize after her charity race?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_7", - "session_10", - "session_8", - "session_18", - "session_9", - "session_1", - "session_4", - "session_3", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_2" - ], - "question": "What are Melanie's plans for the summer with respect to adoption?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_2", - "session_12", - "session_17", - "session_10", - "session_13", - "session_8", - "session_11", - "session_15", - "session_16" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_2" - ], - "question": "What type of individuals does the adoption agency Melanie is considering support?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_11", - "session_19", - "session_13", - "session_17", - "session_15", - "session_14", - "session_8", - "session_1", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_2" - ], - "question": "Why did Melanie choose the adoption agency?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_2", - "session_13", - "session_17", - "session_16", - "session_8", - "session_14", - "session_18", - "session_15", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_2" - ], - "question": "What is Melanie excited about in her adoption process?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_2", - "session_19", - "session_10", - "session_7", - "session_13", - "session_8", - "session_5", - "session_16", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What does Melanie's necklace symbolize?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_14", - "session_8", - "session_18", - "session_1", - "session_13", - "session_9", - "session_19", - "session_17", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What country is Melanie's grandma from?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_3", - "session_8", - "session_18", - "session_1", - "session_13", - "session_9", - "session_19", - "session_17", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What was grandma's gift to Melanie?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_19", - "session_8", - "session_18", - "session_1", - "session_13", - "session_9", - "session_17", - "session_15", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What was grandpa's gift to Caroline?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_4", - "session_1", - "session_8", - "session_9", - "session_12", - "session_7", - "session_5", - "session_14", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What is Caroline's hand-painted bowl a reminder of?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_8", - "session_5", - "session_12", - "session_16", - "session_11", - "session_14", - "session_1", - "session_9", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What did Caroline and her family do while camping?", - "recall": 0.0, - "retrieved_ids": [ - "session_10", - "session_7", - "session_16", - "session_8", - "session_17", - "session_13", - "session_18", - "session_9", - "session_6", - "session_2" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What kind of counseling and mental health services is Melanie interested in pursuing?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_7", - "session_11", - "session_5", - "session_6", - "session_1", - "session_2", - "session_8", - "session_3", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What kind of counseling workshop did Melanie attend recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_13", - "session_8", - "session_5", - "session_6", - "session_7", - "session_1", - "session_11", - "session_10", - "session_17" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What motivated Melanie to pursue counseling?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_7", - "session_5", - "session_1", - "session_6", - "session_3", - "session_18", - "session_14", - "session_8", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What kind of place does Melanie want to create for people?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_14", - "session_17", - "session_2", - "session_3", - "session_5", - "session_6", - "session_11", - "session_10", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_5" - ], - "question": "Did Caroline make the black and white bowl in the photo?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_16", - "session_12", - "session_4", - "session_6", - "session_13", - "session_3", - "session_8", - "session_14", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_7" - ], - "question": "What are the new shoes that Caroline got used for?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_17", - "session_6", - "session_15", - "session_9", - "session_4", - "session_1", - "session_16", - "session_19", - "session_2" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_7" - ], - "question": "What is Caroline's reason for getting into running?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_2", - "session_16", - "session_18", - "session_4", - "session_10", - "session_15", - "session_14", - "session_5", - "session_6" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_7" - ], - "question": "What does Caroline say running has been great for?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_10", - "session_2", - "session_6", - "session_17", - "session_15", - "session_9", - "session_8", - "session_3", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_8" - ], - "question": "What did Melanie see at the council meeting for adoption?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_2", - "session_17", - "session_19", - "session_13", - "session_10", - "session_5", - "session_15", - "session_7", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_9" - ], - "question": "What inspired Melanie's painting for the art show?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_16", - "session_9", - "session_17", - "session_11", - "session_8", - "session_12", - "session_13", - "session_4", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_9" - ], - "question": "What inspired Caroline's sculpture for the art show?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_16", - "session_9", - "session_17", - "session_11", - "session_12", - "session_8", - "session_13", - "session_5", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_10" - ], - "question": "How often does Caroline go to the beach with her kids?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_7", - "session_14", - "session_6", - "session_1", - "session_9", - "session_11", - "session_2", - "session_18", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_10" - ], - "question": "What did Caroline and her family see during their camping trip last year?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_18", - "session_8", - "session_7", - "session_16", - "session_12", - "session_17", - "session_4", - "session_9", - "session_6" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_10" - ], - "question": "How did Caroline feel while watching the meteor shower?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_8", - "session_17", - "session_13", - "session_16", - "session_14", - "session_1", - "session_5", - "session_15", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_12" - ], - "question": "Why did Caroline choose to use colors and patterns in her pottery project?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_16", - "session_19", - "session_10", - "session_4", - "session_14", - "session_2", - "session_8", - "session_7", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_13" - ], - "question": "Is Oscar Melanie's pet?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_7", - "session_8", - "session_18", - "session_1", - "session_4", - "session_9", - "session_19", - "session_17", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_13" - ], - "question": "Where did Oscar hide his bone once?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_10", - "session_6", - "session_11", - "session_8", - "session_14", - "session_18", - "session_15", - "session_1", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_13" - ], - "question": "What activity did Melanie used to do with her dad?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_10", - "session_5", - "session_15", - "session_7", - "session_8", - "session_14", - "session_18", - "session_1", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_14" - ], - "question": "What did Melanie make for a local church?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_8", - "session_15", - "session_5", - "session_7", - "session_10", - "session_16", - "session_18", - "session_17", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_14" - ], - "question": "What did Melanie find in her neighborhood during her walk?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_7", - "session_10", - "session_16", - "session_18", - "session_8", - "session_19", - "session_9", - "session_11", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_15" - ], - "question": "Which song motivates Melanie to be courageous?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_3", - "session_11", - "session_14", - "session_1", - "session_8", - "session_5", - "session_18", - "session_4", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_15" - ], - "question": "What type of instrument does Caroline play?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_6", - "session_14", - "session_2", - "session_3", - "session_1", - "session_8", - "session_9", - "session_12", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_15" - ], - "question": "Which classical musicians does Caroline enjoy listening to?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_15", - "session_8", - "session_18", - "session_10", - "session_1", - "session_9", - "session_12", - "session_7", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_15" - ], - "question": "Who is Caroline a fan of in terms of modern music?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_5", - "session_11", - "session_1", - "session_8", - "session_9", - "session_12", - "session_7", - "session_14", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_16" - ], - "question": "What precautionary sign did Caroline see at the caf\u00e9?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_5", - "session_10", - "session_15", - "session_14", - "session_1", - "session_11", - "session_18", - "session_8", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_17" - ], - "question": "What setback did Caroline face recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_10", - "session_9", - "session_15", - "session_3", - "session_13", - "session_11", - "session_12", - "session_18", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_17" - ], - "question": "What does Caroline do to keep herself busy during her pottery break?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_8", - "session_16", - "session_10", - "session_14", - "session_2", - "session_7", - "session_18", - "session_5", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_17" - ], - "question": "What was the poetry reading that Melanie attended about?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_13", - "session_6", - "session_7", - "session_2", - "session_4", - "session_15", - "session_3", - "session_11", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "What happened to Caroline's son on their road trip?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_10", - "session_12", - "session_17", - "session_7", - "session_6", - "session_14", - "session_2", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "How did Caroline's son handle the accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_16", - "session_8", - "session_14", - "session_13", - "session_15", - "session_1", - "session_5", - "session_11", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "How did Caroline feel about her family after the accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_10", - "session_7", - "session_2", - "session_9", - "session_1", - "session_4", - "session_3", - "session_14", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "How did Caroline's children handle the accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_16", - "session_14", - "session_13", - "session_15", - "session_1", - "session_5", - "session_11", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "How did Caroline feel after the accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_2", - "session_9", - "session_1", - "session_4", - "session_3", - "session_14", - "session_13", - "session_8", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "What was Caroline's reaction to her children enjoying the Grand Canyon?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_10", - "session_3", - "session_7", - "session_1", - "session_9", - "session_12", - "session_5", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "What did Caroline do after the road trip to relax?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_17", - "session_1", - "session_10", - "session_14", - "session_12", - "session_15", - "session_2", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "What does Caroline love most about camping with her family?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_18", - "session_7", - "session_14", - "session_9", - "session_6", - "session_2", - "session_4", - "session_16", - "session_8" - ], - "sample_id": "conv-26" - } - ], - "runtime_seconds": 37.507, - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/locomo_old_brainctl_cmd_session.json b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/locomo_old_brainctl_cmd_session.json deleted file mode 100644 index 9c0c218..0000000 --- a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/locomo_old_brainctl_cmd_session.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\locomo_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\runs\\locomo_brainctl_cmd_session.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\coverage_status.png" - }, - "benchmark": "locomo", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "measured": false, - "metadata": {}, - "metrics": { - "avg_recall": 0.9217, - "cat_1_recall": 0.7614, - "cat_2_recall": 0.92, - "cat_3_recall": 0.6616, - "cat_4_recall": 0.9738, - "cat_5_recall": 0.9821, - "perfect_rate": 0.8817, - "top_k": 10, - "zero_rate": 0.0438 - }, - "mode": "cmd_session", - "notes": [ - "granularity=session", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": 0.9217, - "reference_kind": "historical", - "runtime_seconds": 445.74, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_mempalace_raw_session.json b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_mempalace_raw_session.json deleted file mode 100644 index 31716e6..0000000 --- a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_mempalace_raw_session.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\runs\\longmemeval_mempalace_raw_session.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "measured": false, - "metadata": {}, - "metrics": { - "ndcg_at_10": 0.8948, - "ndcg_at_5": 0.893, - "r_at_10": 0.983, - "r_at_5": 0.966 - }, - "mode": "raw_session", - "notes": [ - "top_k=10", - "Runs MemPalace benchmark module raw session retrieval logic directly." - ], - "primary_metric": "r_at_5", - "primary_metric_value": 0.966, - "reference_kind": "historical", - "runtime_seconds": 695.36, - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "mempalace" -} \ No newline at end of file diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_new_brainctl_brain.json b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_new_brainctl_brain.json deleted file mode 100644 index 211eb17..0000000 --- a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_new_brainctl_brain.json +++ /dev/null @@ -1,101 +0,0 @@ -{ - "artifacts": {}, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 3, - "measured": true, - "metadata": {}, - "metrics": { - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0 - }, - "mode": "brain", - "notes": [ - "top_k=10", - "Legacy 470-question session-level slice." - ], - "primary_metric": "r_at_5", - "primary_metric_value": 1.0, - "reference_kind": "measured", - "rows": [ - { - "answer_session_ids": [ - "answer_280352e9" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e47becba", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_280352e9", - "02bd2b90_3", - "sharegpt_Cr2tc1f_0", - "f6859b48_2", - "ultrachat_214101", - "5dac7cc2_1" - ] - }, - { - "answer_session_ids": [ - "answer_40a90d51" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "118b2229", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_40a90d51", - "db73b7e4_4", - "ultrachat_392094", - "afe04238_1", - "5dce60dd", - "db50c0f6", - "3ea9f765", - "d600c646", - "a0b8b0ad_6", - "sharegpt_i1iuP70_0" - ] - }, - { - "answer_session_ids": [ - "answer_d61669c7" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "51a45a95", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_d61669c7", - "98c198fb", - "217debf7", - "6e0b1800_2", - "5c44d9fe_1", - "07ba9acd_2", - "sharegpt_3vxz2Zr_0", - "sharegpt_hChsWOp_128", - "ultrachat_282235", - "sharegpt_CyJ3dal_0" - ] - } - ], - "runtime_seconds": 3.541, - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_new_brainctl_cmd.json b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_new_brainctl_cmd.json deleted file mode 100644 index 9820f8d..0000000 --- a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_new_brainctl_cmd.json +++ /dev/null @@ -1,101 +0,0 @@ -{ - "artifacts": {}, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 3, - "measured": true, - "metadata": {}, - "metrics": { - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0 - }, - "mode": "cmd", - "notes": [ - "top_k=10", - "Legacy 470-question session-level slice." - ], - "primary_metric": "r_at_5", - "primary_metric_value": 1.0, - "reference_kind": "measured", - "rows": [ - { - "answer_session_ids": [ - "answer_280352e9" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e47becba", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_280352e9", - "02bd2b90_3", - "sharegpt_Cr2tc1f_0", - "f6859b48_2", - "ultrachat_214101", - "5dac7cc2_1" - ] - }, - { - "answer_session_ids": [ - "answer_40a90d51" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "118b2229", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_40a90d51", - "db73b7e4_4", - "ultrachat_392094", - "afe04238_1", - "5dce60dd", - "db50c0f6", - "3ea9f765", - "d600c646", - "a0b8b0ad_6", - "sharegpt_i1iuP70_0" - ] - }, - { - "answer_session_ids": [ - "answer_d61669c7" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "51a45a95", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_d61669c7", - "98c198fb", - "217debf7", - "6e0b1800_2", - "5c44d9fe_1", - "07ba9acd_2", - "sharegpt_3vxz2Zr_0", - "sharegpt_hChsWOp_128", - "ultrachat_282235", - "sharegpt_CyJ3dal_0" - ] - } - ], - "runtime_seconds": 3.384, - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_old_brainctl_brain.json b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_old_brainctl_brain.json deleted file mode 100644 index c1f858e..0000000 --- a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_old_brainctl_brain.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\runs\\longmemeval_brainctl_brain.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "measured": false, - "metadata": {}, - "metrics": { - "ndcg_at_10": 0.9253, - "ndcg_at_5": 0.9204, - "r_at_10": 0.9894, - "r_at_5": 0.9681 - }, - "mode": "brain", - "notes": [ - "top_k=10" - ], - "primary_metric": "r_at_5", - "primary_metric_value": 0.9681, - "reference_kind": "historical", - "runtime_seconds": 85.439, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_old_brainctl_cmd.json b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_old_brainctl_cmd.json deleted file mode 100644 index cd62ca4..0000000 --- a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/longmemeval_old_brainctl_cmd.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\runs\\longmemeval_brainctl_cmd.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "measured": false, - "metadata": {}, - "metrics": { - "ndcg_at_10": 0.9247, - "ndcg_at_5": 0.9206, - "r_at_10": 0.9894, - "r_at_5": 0.9702 - }, - "mode": "cmd", - "notes": [ - "top_k=10" - ], - "primary_metric": "r_at_5", - "primary_metric_value": 0.9702, - "reference_kind": "historical", - "runtime_seconds": 130.863, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/membench_mempalace_raw_turn.json b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/membench_mempalace_raw_turn.json deleted file mode 100644 index e4a3869..0000000 --- a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/membench_mempalace_raw_turn.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\membench_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\runs\\membench_mempalace_raw_turn.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\coverage_status.png" - }, - "benchmark": "membench", - "caveats": [ - "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 200, - "measured": false, - "metadata": {}, - "metrics": { - "hit_at_5": 0.885, - "simple_hit_at_5": 0.885, - "top_k": 5 - }, - "mode": "raw_turn", - "notes": [ - "FirstAgent slice only", - "turn-level retrieval", - "topic=all" - ], - "primary_metric": "hit_at_5", - "primary_metric_value": 0.885, - "reference_kind": "historical", - "runtime_seconds": 804.35, - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "partial", - "system_name": "mempalace" -} \ No newline at end of file diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/membench_new_brainctl_cmd_turn.json b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/membench_new_brainctl_cmd_turn.json deleted file mode 100644 index 7e88c24..0000000 --- a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/membench_new_brainctl_cmd_turn.json +++ /dev/null @@ -1,83 +0,0 @@ -{ - "artifacts": {}, - "benchmark": "membench", - "caveats": [ - "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 3, - "measured": true, - "metadata": {}, - "metrics": { - "hit_at_5": 1.0, - "simple_hit_at_5": 1.0, - "top_k": 5 - }, - "mode": "cmd_turn", - "notes": [ - "FirstAgent slice only", - "turn-level retrieval", - "topic=all" - ], - "primary_metric": "hit_at_5", - "primary_metric_value": 1.0, - "reference_kind": "measured", - "rows": [ - { - "category": "simple", - "hit_at_k": true, - "question": "What is the name of my niece's company?", - "retrieved_ids": [ - "simple_roles_0|sid=119|g=119|s=5|t=18", - "simple_roles_0|sid=101|g=101|s=5|t=0", - "simple_roles_0|sid=135|g=135|s=6|t=12", - "simple_roles_0|sid=11|g=11|s=0|t=11", - "simple_roles_0|sid=25|g=25|s=1|t=5" - ], - "target_ids": [ - "119" - ], - "tid": 0, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the email address of my cousin, who is male?", - "retrieved_ids": [ - "simple_roles_1|sid=59|g=59|s=2|t=17", - "simple_roles_1|sid=101|g=101|s=4|t=18", - "simple_roles_1|sid=143|g=143|s=6|t=18", - "simple_roles_1|sid=33|g=33|s=1|t=12", - "simple_roles_1|sid=42|g=42|s=2|t=0" - ], - "target_ids": [ - "59" - ], - "tid": 1, - "topic": "roles" - }, - { - "category": "simple", - "hit_at_k": true, - "question": "What is the position of my brother?", - "retrieved_ids": [ - "simple_roles_2|sid=84|g=84|s=4|t=0", - "simple_roles_2|sid=85|g=85|s=4|t=1", - "simple_roles_2|sid=28|g=28|s=1|t=7", - "simple_roles_2|sid=123|g=123|s=5|t=18", - "simple_roles_2|sid=41|g=41|s=1|t=20" - ], - "target_ids": [ - "84" - ], - "tid": 2, - "topic": "roles" - } - ], - "runtime_seconds": 1.919, - "series_name": "new_brainctl", - "source_path": null, - "status": "partial", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/membench_old_brainctl_cmd_turn.json b/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/membench_old_brainctl_cmd_turn.json deleted file mode 100644 index 7af9666..0000000 --- a/benchmarks/results/smoke_legacy_compare_20260423_105447/runs/membench_old_brainctl_cmd_turn.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\membench_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\runs\\membench_brainctl_cmd_turn.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\coverage_status.png" - }, - "benchmark": "membench", - "caveats": [ - "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 200, - "measured": false, - "metadata": {}, - "metrics": { - "hit_at_5": 0.93, - "simple_hit_at_5": 0.93, - "top_k": 5 - }, - "mode": "cmd_turn", - "notes": [ - "FirstAgent slice only", - "turn-level retrieval", - "topic=all" - ], - "primary_metric": "hit_at_5", - "primary_metric_value": 0.93, - "reference_kind": "historical", - "runtime_seconds": 140.592, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "partial", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/summary.csv b/benchmarks/results/smoke_legacy_compare_20260423_105447/summary.csv deleted file mode 100644 index 5fb33da..0000000 --- a/benchmarks/results/smoke_legacy_compare_20260423_105447/summary.csv +++ /dev/null @@ -1,15 +0,0 @@ -benchmark,series_name,system_name,mode,reference_kind,status,example_count,primary_metric,primary_metric_value,runtime_seconds,dataset_path,source_path,notes,caveats,avg_recall,cat_1_recall,cat_2_recall,cat_3_recall,cat_4_recall,cat_5_recall,hit_at_5,ndcg_at_10,ndcg_at_5,perfect_rate,r_at_10,r_at_5,simple_hit_at_5,top_k,zero_rate -longmemeval,old_brainctl,brainctl,brain,historical,full_same_machine,470,r_at_5,0.9681,85.439,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,top_k=10,,,,,,,,,0.9253,0.9204,,0.9894,0.9681,,, -longmemeval,old_brainctl,brainctl,cmd,historical,full_same_machine,470,r_at_5,0.9702,130.863,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,top_k=10,,,,,,,,,0.9247,0.9206,,0.9894,0.9702,,, -longmemeval,mempalace,mempalace,raw_session,historical,full_same_machine,470,r_at_5,0.966,695.36,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,top_k=10 | Runs MemPalace benchmark module raw session retrieval logic directly.,,,,,,,,,0.8948,0.893,,0.983,0.966,,, -locomo,old_brainctl,brainctl,cmd_session,historical,full_same_machine,1986,avg_recall,0.9217,445.74,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,granularity=session | top_k=10,,0.9217,0.7614,0.92,0.6616,0.9738,0.9821,,,,0.8817,,,,10,0.0438 -locomo,mempalace,mempalace,raw_session,historical,full_same_machine,1986,avg_recall,0.6028,2106.411,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,granularity=session | top_k=10,,0.6028,0.5899,0.6921,0.4604,0.5809,0.6188,,,,0.5534,,,,10,0.3499 -membench,old_brainctl,brainctl,cmd_turn,historical,partial,200,hit_at_5,0.93,140.592,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,FirstAgent slice only | turn-level retrieval | topic=all,MemBench comparison is partial because ThirdAgent and noise-extended slices are not included.,,,,,,,0.93,,,,,,0.93,5, -membench,mempalace,mempalace,raw_turn,historical,partial,200,hit_at_5,0.885,804.35,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,FirstAgent slice only | turn-level retrieval | topic=all,MemBench comparison is partial because ThirdAgent and noise-extended slices are not included.,,,,,,,0.885,,,,,,0.885,5, -convomem,old_brainctl,brainctl,cmd,historical,blocked,0,avg_recall,,,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,limit_per_category=1 | top_k=10,Blocked while loading ConvoMem evidence data: ,,,,,,,,,,,,,,, -convomem,mempalace,mempalace,raw,historical,blocked,0,avg_recall,,,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,limit_per_category=1 | top_k=10,Blocked while loading ConvoMem evidence data: ,,,,,,,,,,,,,,, -longmemeval,new_brainctl,brainctl,brain,measured,full_same_machine,3,r_at_5,1.0,3.541,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,,top_k=10 | Legacy 470-question session-level slice.,,,,,,,,,1.0,1.0,,1.0,1.0,,, -longmemeval,new_brainctl,brainctl,cmd,measured,full_same_machine,3,r_at_5,1.0,3.384,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,,top_k=10 | Legacy 470-question session-level slice.,,,,,,,,,1.0,1.0,,1.0,1.0,,, -locomo,new_brainctl,brainctl,cmd_session,measured,full_same_machine,199,avg_recall,0.9413,37.507,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,,granularity=session | top_k=10,,0.9413,0.8641,1.0,0.7436,0.9571,0.9787,,,,0.9196,,,,10,0.0402 -membench,new_brainctl,brainctl,cmd_turn,measured,partial,3,hit_at_5,1.0,1.919,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,,FirstAgent slice only | turn-level retrieval | topic=all,MemBench comparison is partial because ThirdAgent and noise-extended slices are not included.,,,,,,,1.0,,,,,,1.0,5, -convomem,new_brainctl,brainctl,cmd,measured,blocked,0,avg_recall,,,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache,,limit_per_category=1 | top_k=10,Blocked while loading ConvoMem evidence data: HTTP Error 404: Not Found,,,,,,,,,,,,,,, diff --git a/benchmarks/results/smoke_legacy_compare_20260423_105447/summary.json b/benchmarks/results/smoke_legacy_compare_20260423_105447/summary.json deleted file mode 100644 index 0a74157..0000000 --- a/benchmarks/results/smoke_legacy_compare_20260423_105447/summary.json +++ /dev/null @@ -1,509 +0,0 @@ -{ - "generated_at_utc": "2026-04-23T10:55:38.026703+00:00", - "metadata": { - "argv": [ - "benchmarks\\compare_memory_engines.py", - "--label", - "smoke_legacy_compare", - "--longmemeval-limit", - "3", - "--locomo-limit", - "1", - "--membench-limit", - "3", - "--convomem-limit-per-category", - "1" - ], - "cwd": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl", - "datasets": { - "convomem_cache": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "locomo_data": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "longmemeval_data": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "membench_data": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent" - }, - "generated_at_utc": "2026-04-23T10:55:37.887986+00:00", - "git_commit": "c130fbd47174a0932822db2ef7fe64e18a74d30d", - "historical_summary_mode": "recovered", - "historical_summary_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "machine": "AMD64", - "platform": "Windows-11-10.0.26200-SP0", - "processor": "Intel64 Family 6 Model 189 Stepping 1, GenuineIntel", - "python_version": "3.14.4 (tags/v3.14.4:23116f9, Apr 7 2026, 14:10:54) [MSC v.1944 64 bit (AMD64)]" - }, - "notes": [ - "Historical old-BrainCTL and MemPalace series come from the recovered 2026-04-18 comparison bundle.", - "New BrainCTL series are rerun in the current checked-out repo using the legacy benchmark definitions.", - "MemBench remains intentionally partial because the legacy comparison only covered the FirstAgent slice.", - "ConvoMem remains a coverage/status benchmark here; it has no dedicated comparison chart in the legacy chart pack." - ], - "runs": [ - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\runs\\longmemeval_old_brainctl_brain.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "measured": false, - "metadata": {}, - "metrics": { - "ndcg_at_10": 0.9253, - "ndcg_at_5": 0.9204, - "r_at_10": 0.9894, - "r_at_5": 0.9681 - }, - "mode": "brain", - "notes": [ - "top_k=10" - ], - "primary_metric": "r_at_5", - "primary_metric_value": 0.9681, - "reference_kind": "historical", - "runtime_seconds": 85.439, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\runs\\longmemeval_old_brainctl_cmd.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "measured": false, - "metadata": {}, - "metrics": { - "ndcg_at_10": 0.9247, - "ndcg_at_5": 0.9206, - "r_at_10": 0.9894, - "r_at_5": 0.9702 - }, - "mode": "cmd", - "notes": [ - "top_k=10" - ], - "primary_metric": "r_at_5", - "primary_metric_value": 0.9702, - "reference_kind": "historical", - "runtime_seconds": 130.863, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\runs\\longmemeval_mempalace_raw_session.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "measured": false, - "metadata": {}, - "metrics": { - "ndcg_at_10": 0.8948, - "ndcg_at_5": 0.893, - "r_at_10": 0.983, - "r_at_5": 0.966 - }, - "mode": "raw_session", - "notes": [ - "top_k=10", - "Runs MemPalace benchmark module raw session retrieval logic directly." - ], - "primary_metric": "r_at_5", - "primary_metric_value": 0.966, - "reference_kind": "historical", - "runtime_seconds": 695.36, - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "mempalace" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\locomo_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\runs\\locomo_old_brainctl_cmd_session.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\coverage_status.png" - }, - "benchmark": "locomo", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "measured": false, - "metadata": {}, - "metrics": { - "avg_recall": 0.9217, - "cat_1_recall": 0.7614, - "cat_2_recall": 0.92, - "cat_3_recall": 0.6616, - "cat_4_recall": 0.9738, - "cat_5_recall": 0.9821, - "perfect_rate": 0.8817, - "top_k": 10, - "zero_rate": 0.0438 - }, - "mode": "cmd_session", - "notes": [ - "granularity=session", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": 0.9217, - "reference_kind": "historical", - "runtime_seconds": 445.74, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\locomo_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\runs\\locomo_mempalace_raw_session.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\coverage_status.png" - }, - "benchmark": "locomo", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "measured": false, - "metadata": {}, - "metrics": { - "avg_recall": 0.6028, - "cat_1_recall": 0.5899, - "cat_2_recall": 0.6921, - "cat_3_recall": 0.4604, - "cat_4_recall": 0.5809, - "cat_5_recall": 0.6188, - "perfect_rate": 0.5534, - "top_k": 10, - "zero_rate": 0.3499 - }, - "mode": "raw_session", - "notes": [ - "granularity=session", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": 0.6028, - "reference_kind": "historical", - "runtime_seconds": 2106.411, - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "mempalace" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\membench_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\runs\\membench_old_brainctl_cmd_turn.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\coverage_status.png" - }, - "benchmark": "membench", - "caveats": [ - "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 200, - "measured": false, - "metadata": {}, - "metrics": { - "hit_at_5": 0.93, - "simple_hit_at_5": 0.93, - "top_k": 5 - }, - "mode": "cmd_turn", - "notes": [ - "FirstAgent slice only", - "turn-level retrieval", - "topic=all" - ], - "primary_metric": "hit_at_5", - "primary_metric_value": 0.93, - "reference_kind": "historical", - "runtime_seconds": 140.592, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "partial", - "system_name": "brainctl" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\membench_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\runs\\membench_mempalace_raw_turn.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\coverage_status.png" - }, - "benchmark": "membench", - "caveats": [ - "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 200, - "measured": false, - "metadata": {}, - "metrics": { - "hit_at_5": 0.885, - "simple_hit_at_5": 0.885, - "top_k": 5 - }, - "mode": "raw_turn", - "notes": [ - "FirstAgent slice only", - "turn-level retrieval", - "topic=all" - ], - "primary_metric": "hit_at_5", - "primary_metric_value": 0.885, - "reference_kind": "historical", - "runtime_seconds": 804.35, - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "partial", - "system_name": "mempalace" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\runs\\convomem_old_brainctl_cmd.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\coverage_status.png" - }, - "benchmark": "convomem", - "caveats": [ - "Blocked while loading ConvoMem evidence data: " - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "example_count": 0, - "measured": false, - "metadata": {}, - "metrics": {}, - "mode": "cmd", - "notes": [ - "limit_per_category=1", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": null, - "reference_kind": "historical", - "runtime_seconds": null, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "blocked", - "system_name": "brainctl" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\runs\\convomem_mempalace_raw.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\coverage_status.png" - }, - "benchmark": "convomem", - "caveats": [ - "Blocked while loading ConvoMem evidence data: " - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "example_count": 0, - "measured": false, - "metadata": {}, - "metrics": {}, - "mode": "raw", - "notes": [ - "limit_per_category=1", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": null, - "reference_kind": "historical", - "runtime_seconds": null, - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "blocked", - "system_name": "mempalace" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\runs\\longmemeval_new_brainctl_brain.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 3, - "measured": true, - "metadata": {}, - "metrics": { - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0 - }, - "mode": "brain", - "notes": [ - "top_k=10", - "Legacy 470-question session-level slice." - ], - "primary_metric": "r_at_5", - "primary_metric_value": 1.0, - "reference_kind": "measured", - "runtime_seconds": 3.541, - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\runs\\longmemeval_new_brainctl_cmd.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 3, - "measured": true, - "metadata": {}, - "metrics": { - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0 - }, - "mode": "cmd", - "notes": [ - "top_k=10", - "Legacy 470-question session-level slice." - ], - "primary_metric": "r_at_5", - "primary_metric_value": 1.0, - "reference_kind": "measured", - "runtime_seconds": 3.384, - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\locomo_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\runs\\locomo_new_brainctl_cmd_session.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\coverage_status.png" - }, - "benchmark": "locomo", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 199, - "measured": true, - "metadata": {}, - "metrics": { - "avg_recall": 0.9413, - "cat_1_recall": 0.8641, - "cat_2_recall": 1.0, - "cat_3_recall": 0.7436, - "cat_4_recall": 0.9571, - "cat_5_recall": 0.9787, - "perfect_rate": 0.9196, - "top_k": 10, - "zero_rate": 0.0402 - }, - "mode": "cmd_session", - "notes": [ - "granularity=session", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": 0.9413, - "reference_kind": "measured", - "runtime_seconds": 37.507, - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\membench_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\runs\\membench_new_brainctl_cmd_turn.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\coverage_status.png" - }, - "benchmark": "membench", - "caveats": [ - "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 3, - "measured": true, - "metadata": {}, - "metrics": { - "hit_at_5": 1.0, - "simple_hit_at_5": 1.0, - "top_k": 5 - }, - "mode": "cmd_turn", - "notes": [ - "FirstAgent slice only", - "turn-level retrieval", - "topic=all" - ], - "primary_metric": "hit_at_5", - "primary_metric_value": 1.0, - "reference_kind": "measured", - "runtime_seconds": 1.919, - "series_name": "new_brainctl", - "source_path": null, - "status": "partial", - "system_name": "brainctl" - }, - { - "artifacts": { - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\runs\\convomem_new_brainctl_cmd.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_legacy_compare_20260423_105447\\charts\\coverage_status.png" - }, - "benchmark": "convomem", - "caveats": [ - "Blocked while loading ConvoMem evidence data: HTTP Error 404: Not Found" - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "example_count": 0, - "measured": true, - "metadata": {}, - "metrics": {}, - "mode": "cmd", - "notes": [ - "limit_per_category=1", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": null, - "reference_kind": "measured", - "runtime_seconds": null, - "series_name": "new_brainctl", - "source_path": null, - "status": "blocked", - "system_name": "brainctl" - } - ] -} \ No newline at end of file diff --git a/benchmarks/results/smoke_postpush_20260423_123452/README.md b/benchmarks/results/smoke_postpush_20260423_123452/README.md deleted file mode 100644 index 7657f0d..0000000 --- a/benchmarks/results/smoke_postpush_20260423_123452/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# Legacy BrainCTL vs MemPalace comparison bundle - -- Current repo commit: `b9a75f6b975c5330e3d35a2627a5e132b2e06e93` -- Historical reference source: `C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json` -- Historical source mode: `recovered summary bundle` -- Command: `benchmarks\compare_memory_engines.py --label smoke_postpush --longmemeval-limit 1 --locomo-limit 1 --membench-limit 1 --convomem-limit-per-category 1` - -## Datasets - -- LongMemEval: `C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json` -- LoCoMo: `C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json` -- MemBench FirstAgent: `C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent` -- ConvoMem cache: `C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache` - -## What is measured now - -- New BrainCTL reruns: LongMemEval `brain` and `cmd`, LoCoMo `cmd_session`, MemBench FirstAgent `cmd_turn`, and ConvoMem `cmd` coverage/status. -- Old BrainCTL and MemPalace are frozen historical reference series loaded from the recovered 2026-04-18 bundle. - -## Blocked or partial runs - -- membench old_brainctl cmd_turn: MemBench comparison is partial because ThirdAgent and noise-extended slices are not included. -- membench mempalace raw_turn: MemBench comparison is partial because ThirdAgent and noise-extended slices are not included. -- convomem old_brainctl cmd: Blocked while loading ConvoMem evidence data: -- convomem mempalace raw: Blocked while loading ConvoMem evidence data: -- membench new_brainctl cmd_turn: MemBench comparison is partial because ThirdAgent and noise-extended slices are not included. -- convomem new_brainctl cmd: Blocked while loading ConvoMem evidence data: HTTP Error 404: Not Found - -## Output files - -- `summary.json` and `summary.csv`: all series in one table. -- `comparison_table.json` and `comparison_table.csv`: long-form metric rows. -- `runs/*.json`: per-run payloads. -- `charts/*.png`: regenerated charts with old BrainCTL, new BrainCTL, and MemPalace together. diff --git a/benchmarks/results/smoke_postpush_20260423_123452/charts/aggregate_primary_metrics.png b/benchmarks/results/smoke_postpush_20260423_123452/charts/aggregate_primary_metrics.png deleted file mode 100644 index 91ee6a4b5b60cfd85a4fc065daf6c30f817c8c73..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 72024 zcmdqJby!txyDy4~poofzq?D3McdImtbc!^S2`Jqt3ML5BNQWRynn{C*N(+K8r9rxT z5|cRh5Wn?(Yp=D=S$m)T$3ACV*ZYb%V~l4!&wc;u887ar+$JNXBqbsuB9oVsRU;xg zl0Zasi0;@?_)X`hudMKgu;Wc_$9s0>jxNTJ%!rhY9qn!G9Br%~&^enua9<> zol}sV&eGA*-a(j)3;l1e;Iw;W!BwN@^c_Bh#9mIvfr#j04D#Orc_+V6q60)k^0HFu zuJ332kA#Mf5%!i|&1a{UUp^cWa)yLNs_gW&BP2)8-BCX*`&LcCT`F{bUjNjo(DNyZ zZ|D2=O6M2$q6TN@oAK8Rnh8$d-|NIxOhiQ)>@@S?s2S+!=p^Aw{~XUB+(_Q?{reZ9 zQ<8`N{;2OyYWep^U&Vj<^%Ek>zrP+jbGYpv?~9=GMaBd)yI)v(Q`J3ng0d#zLJcLuA*N7HY=uxP!dTU*h&hO}F{2XIG-d?)>p zp{!^>c<|sTey%s)XV3eR(4&;Itm<-U$}!{f#`Mw4`0Hn3(2j8-1Gd#OViP6pXShxKU zG}c{UVV*_nvz=Fg??IWz3qJfU_fb6~#dC9cxt3g_euuCgJD>CJ`;?&58DXc@$-7g( z@{&jEgJpi#1=)33Y1n4#mo~R5$MBP_ZDH*O2W(>KW&G`*5u{j#JG!nNI(p(dj`ePm z?08S*%6E}`yTK|+0=hf@D?6^ync5*v)NOPor<%Xj$nqqW;C-5$`DC1aYjO@*Z2B^c zvPw=w+`X2_8w|gmM&P{?=UBd~U*Ob#8z<(WFi_#vQaZ~*bz+9`DK6W9XJm10E$nB; zy_j%roqW?9IipgGjWl~_8 zVNvNhxT9p0^*u$X+HLP26B&vToa2M;C#V%`{mGPbHrpmWd|>p)Z=|Tk4`8dSg4P~x zFV)e~&*v=cz>QOE2w`H94`Cd~Z_VU1wMy*G7KVsV&MXc3?D6HS^_4orpdzo`W47x2 z81`Q9;gvRT#c=lE;sN(2XUy8#b@S%PQ-gzh+bcn@DftvWYUS7t#5^4N#35T<=1qK>Vgz!8su+-&&}-Tf&37Nh z^#I8k#$(uON?yyin?i;at_K8c`|RBf7W&ICycK--pkVGpyf@`3nk`SAiM}c?4@sC~ zRN`?POC!WKzjAA4c}(k1eZXOLi>wd8nsRZ?zQAR8cy+3yZmPL zBG53X+-2dUTokui?%bNwh1Dg!rYF%n7J+kH{$tz5sjb!XnL(=SF3z)0jvVh9;0h-u zr#Z~17~Y%(r{eN_ViWETx^=iEMxd7O7GeEogpv1@<7-&WCj_Aubd11EV4{P;qc^M# zq!u|U>Vaqfvfeb_5`zvBnN%KRnd!nf2>Vy>Zn4qb4y2+Flxj6@-Cgg10&u7?f@{HX z5NeQ!XwJ^e5=?;;auJ-dwL5E0?u$cGE8?3&e&p}tCMwql99-k{ifx~5iNBy=QSpU| zTf0-cQvSX1tO1*jCdNQ3y4d2sT*kVDUt>ui?!@6I)@CGzpv^PR!v#Ee+ko6Jc- z5%X0`RfxJS=OgZcxA|0r!YM8O9vLw?p3IwmQlZ$cD@*4SjK@LexgN78=KWf=sj%*1 zuB)uCudO+`QfEnWShc4dfQe30A1~qaslsjLAg4i@sqM-{t3avh%{?Ud9B(7D{0CM>oO!@W(c+wp>f%&>;z=J&-l$wo_5-USOo}#--L_Vz4t;xliNb#N*Nv6OzP8o&!=HVjOdfHZ{&|fq z>3}1Cwi}I8H(J;#!{O8-I1Ow}GVdflAz_}=nOWO>xGJns7UZ-I_ddYmHGbh5o6^q- z<+^lSqEWT?fjpBs{+UM;-_cK&67UOyb_2LFWvIl5pf;Ih#~R9by%%#|%{4F~D~+u* zf>viMtxI-mt<0#(Q)s9$jBVU2ea11SIf_T5;=H@~jNdWpSKg=cT^0;i1XsmA&}Mrs zH?WMmNVo0G4${6j!$f*f)HT;FerjV)ETQ#H5v)1Xli1$fwe-5S-r4Sg7J^<~PLA@- zp+JEF*IN962>&+sxm8i(lT^oC33J8hx`$^AXHaF1Q|$+20#1_sDtVYZ=Q0U%@8EeZ zLz_5}nOZ)w5o&|v++EI-*L>r+&0fx}hgYSo?Hnf}>I*>ntE)264wUsCtlJEx*rsXD z??fd=JMYRDhi6Q_U3vj0sj23Tvh{@u3}tx8d|*N@-0!t~epW5vEG1cZmZXwjLc%PY z`^sp8$#EH(kCa?@lb&{28=BclO-$+R@-Ivbdsy`raOMq zA)*Q=(zT-;5^1mY@#*DvnD|cStc-EeeK`HRYBvl0USl=j%dS{z=UgDo#Aw4ibg$>$ z&U#JP1vbs*tW5pVM+xMlvrzcHpAS4*`5nJ-K`dNzn>dv=RJ(VQ_=HqvmQIKBHo6xp z8^5!_LZ-pzNv=?rOsg`X}BKLz;1+kf#Ty8I;(C z(Yx>)JLtLc1}T73TYzpNJ?4}Nu(pI|H_Ony{Cc#L^^;wgn5lPE>YX1olv@o`#muS+ z^W~>6cdFsIW*EKr zO}Fihr8J9N9#|ZKix-Omu=rr-(N|nm*S$(QkyVOegBxEeq-znykB% z0=txlhS9!|+XjJZl)URH zjDN{kR(71v?2QhKLxK1m`y;WgcUVvkvNCnTPScU`J3Bjm%8nLZxykPi9gQ8GgDRPS zzvkddfl9mOk@u$)X_d5F-!Lpx5whWCoX#<-rkSOjCdeQ1v}i5g^}ecGX!)Ez+aI?@ z_xlDd$dEk1QBe zw1ykeo~C@6XDMOVunlFhW0*fcURJFlNO^AIvRAbLs;k+vK4qkfr+?oVUjaXQRM;Tv`6(^et#jM5RDi268@P7 zfTDLw7{l5%bggjaLt^((jo%pb^M0Y&H4~DZotfhn45Hr%_W-{vUsyhX2P{>+DFH zjyFdw1M2JTYKh{hW5*G3bJw6;zombV6GIV>K0ew$4aJ1{#GB@o>;o9hhJe);;?$t- zA1moI)LtX>S<>!vA|l0^115pqTw*@Eed@`rkvC*MOB<$YrfIzCc_~%<0ZL&2<){w5im6a0&Fgw+DK-_p z(H}WAsEw@0cGccHrcL1-%)USD+uUb$Pp3b@COxko>AWMKd7&sD8bK;{Y>RITnszVt zMOLnUP_CgKVdlKrG1?Gltn#Ke$N5svePPY2v65v{c`w*JPH}QjU=s*(d9Xul?KdrE1{kEvju4hMqHsP~%yfKoKeQI9c;U__cetM8muUt=68LY~sk*C=WK%_z>{(BOR;dGgob# z2_9S`RakasNR4&CDQfKQ_dSoMb;Y07FP_QnoxihK(3iNH4F9wVKaE;^6(Skbuv9ueo+WA}U} z7u%{nQvRNHD_SVP>vraS38dD8KEV_>k0~DDUO#?;llNWt&8MAv0x2lku&fuve2c~L z_218FtVyw;3nzaYoE}t zI%Vti>%PQ~58n5aXyZ!!uylD(izFU7nizG@&h8{Ot*!gBS^C(L#TK=M__qP>3@5wH zUoxr>r|kJh;7GcZnRJiPXx(Xar)=AgT>Sa+iuLnf`DTiGPMI3RA3ObF)>5eAGPSrI zax|g=&65{Vc12fkPNuw>ZckNQ=AJz4Lz#8WO9gt7(T86DpQ-}vVnI$x^nJk)k1T=9BFni)Wg(2WH`-`^LooiKI1gbbyne8h#U;fHYyFFIMe2L}h4T9ZYr&gFN`n*!ZCS$s**K4FutMq_m+Bhex6!PjQS7d-*G zgr6Y)tY1MCfam6gbN<-S+^Xs9&U)bkiT+P}D0*->-0JhLfy}!t(`%XN!&#rWSea=( zFBy8w@lEv>=WMzG%~Y%Mbc+I71MTFNZGV~BlLkN=4bTBo;hV89)}GMFpa-k^3u%Ak z8Xc0UEKREP!(WdFm_QZ6PKFY9ChV zu^w%0i4I#F4xo*0YqKsGo{ZM!y_y3u1BSF`x;)GHzIuH)M+^x?QIbwPOYEr?q|tI! zZAe3BF-19@UKO4!>iB3<8NP)fqHYBet)xfc^u1SBla8v|b7 z4~5~68kvm~@jje{^#uWwwl)voQRX1Tj4=wglMoGhtsATq7%J|(y`7UOt_qF)#?DXR zSg31v5_}cw5<3?tr2&h`Sb8a!WiC?_5pAo^?)i`dT(Xp|wII7wM5*J)h7m2gz+;*PXPauRES0Nq{sLN*B9C;ULkTEkonSzYP4puVx)*B_u99mgz+FMV5 zS~Ia)UUcPH0v-JE=gqifW=n zmnYMvPx9f&UuQY9L#rH1I*6=w%60{fx!z1K6(_Zll7}M-X-ZK_M?M^t{4U*?p7lhN z(%qVTa#H%xPw$*Pg;Kz zlPSiWL%SFsNnaYN6WEDQ4hg*3D>CU;*#E@v5XQZ0LX8az=zqj<5Eg0-x#?={<()Q0 zudUT_ujG;D5T;i;X$L+UTFf!blJ=0wvfTT5ANy#mar}nmFAVLdrBkJU!S^05S>i3q zA2@~9II$Lj3~h{m4d@`jz%zfWS~{;Dc+575(uNr_Q1tb+e_N&|#{Jp^EWpQKk0D3? z36Mza71&JqPH9k1K0CWrLl|w1A6#wi!e7shwm%8~`h)l!BE%lMfLfNCOH|r8;n5TQ zHM-(P+3HcK{JyBuh^R}~97rYfAf_!72!+1TVQv6`4)G!`wxHONYQf8hl%$Wri`yz) zmp-dwty?YThSI#eiw1cUfV^5T-+wRSlD%1sH$92|EDf)A4VdIHK~Aktnx(XZgq(wt zeRs7()%?i~*J6+;Vj5OwH@CJLJr*h_ur~l2eJW^sf=J88A)u*HaB><|>09HnTQq9- zc2sXRM}`PFm`?$f$kH!Wd+_z;*G_ng?`a0fY8euE6LLtNO-3fppW}aBd4BOLTZlbL zElKvp=v2pt5Pq~SCrnD@@V)wQZCV4EWnf>mkCFZ}B4Nt5#Z7=y^E7ZX{=7n~ekJ0q z&>6|whcw4lo*lo?Cx(c#$dU70DT=#vn`#xQoPUo0Jt}yffrGZ|OJ83ix83d1QsPrL zZanz%7Lf-%_P^yR^?_A3x!Um<0lP-1i=&|EK5dme+qNiV{P`&|((=Jqju`LD@sbx$ zNmf4qfr}p6Jnt}%Z~D&tiQOFnRu7a0CUd2-i~Caj)2BpR=Nv}9k?C!`XV>8dz2wE0 zR~Le;mFYr;S4QfB#$Y}?BH+<0G#%@@(?2sxv-i)1p3c|%h@M?KFaiZ2?0sU4Lt&={ zZ1qi!AYss~P)(WO zZcG&|s)KSUU}DpY#gqpe{BsUNYtY(dFH?R)m1GTz?w^*Zk&O#Ek>u*|!~_c>ezWW3 zmv~&=|F`Ey;q^~1)$r{9^|Rl3szqHyA%-3(Ki^l%s0|-;nVpy@c;;g2%4Nz0VK|^{G z6chwERQ1t#^ED9EBkF=^MZ7!-__=rd=pH$w&pHHin<-q=ElghpmW=QNk$`NivgYYb z>fqg;{mXL|O852G>M5`Y`k8)L##4J?Q5T7tpDWq zZ@k8z4>v?`#aC>O$BgG$(`D@f8bwrpk~FxlY)p#bp)mKO>0B3wYPenI^}CBu5h)5` ztokNm_CvmoK#J8;~*pfMV21`NTPs6PMQMAI7 z2Z4egLuy^=3?PoX34{{Hs;a7d*QE#dm<-dDF7@Imm)Aj!Ru@C~8l`|O)ub!Pzj+!0 zsK^|tvZHMo_dZ}X9FWosMcU!VlS78L+5C#0$@rhRq*Q`>*2)U6PMg-;*%%301`cp1 zNWiS|Y)8Aaj3!}G$11WP-Y@q?i|xr=tp!3xI>bl7#9@|nLGHrIT8l7n4tV(CrA$hZ zcfJMDav|~~?pj4*i2Kl|$5{9X!E6qpQ2zn^(GwT%&+|=^Q;KIEJVGkByS-T}hIBVL zT#vFQ3_E4*eg*tjRFsnYp(Y#qHdj~8(r7Z5ipx^Y=EuM{-Ahz=%c={;bjhd+X$zRx zIamw(pW|PMyUf4O@IHC$*!8BOwR*tr&3tXB)0(<-bLq4N=dKum?0aXd1UuVyN}sKM zzW^>y)BF14@}%#eq{|a{B|Duct9ppj=#|YcwlBTsAkS%YVrO3 z!;*pS<=p&lzh_H&T7^XC2dlY2-r_aR8_>$sR0j0|HS-KJQzYh}sngjW!DSTQTWnXp z?YaM{uA9eTL=j_h9tz=zeMk+|n<)HcsP;GBO+QP69z4Da6AZqjwUtY41WZly=XFIi zs@hh+2-}+u9p@RKWYu)FM)R@tg|(g@9vNYPPY8q7QV-RswlD~vrYXF6W&hhmX<@yn zAV_j;4yr;5_&<+7D`EI|VN$iU_Ln)m1J)YgE*dMz1uFmVg#|8?Z?8!TRalO_ziup@ z@-6f6hF$f}dCTtY&6SqJPmw7!av}({fT*hP#kT#V=WRiTR21Lac5u`AGmQvqGZr+e zcVOxUC*Kb0*^OmZrlgQYJ;Q@AYS<&Ve?v~)P?DZZo%R&Jo%GI3CR(;T4{Sb*gME~I z*7a~c%9lxvjEqV>*0XK;N`9!{gp=++AL!2vD}&prozbd0KNYDj%&Kwh&q&X4Q(Uy| z#VWh6PSP{n4t%}nx)upV;N5A4TXInR$Kri%HmO;v4#!K&w#l-ib zZtaKLfK9)Hafz8LQkH|EY+2FD(!O!csH%cdDY7lAW9IJ~YL!01TcHLmc=XfbL(2$v zlGMz6&BxAZSW!Zf3iHDjwSMY6EHJ*#@09u|`*PEUGZdDnwSuE{!HEY}!3I<=S^*3^ z224&ZQThn@G*KkM4AL@7Ute8N?9MkEgE1S|1}~c5col4zQL|}agd|mbrZ^PVeps4E zD#^bEaC0zBy2Cx*zrVa_&$!3(;xvOlNzw- z15it|R)_A_#Gmn=VL<-?nd~js0gL_D=Sa_Ayrllm)WZLpctXxuzzGg3jBqhLCc+Nm zum6Dsh-CjYvGB#Y#rVKeoIly+KN#|c$t_Nr{gW{yIo#$OP#?wfp*&#!H$<}6bnQRQ zfUYjD5YB!L43Q9MH;NeDB2#dRp`9D%o4x>Ka#Jl$$toN@P|BC8)V!~_E*o+z6%w= ztP7)4aI4H^fzxMaEdu<~P8+Q~%OBGP((T5>9-t||1|yHYT;q@VX%?F~W0jgEI1Yh;3j?{;nE zRbXL~ww2Q|C>+XHZ7%WKL_!I@4W^j+!2wXgkA74G?=Ooa27uL7NGOmw&%*j{oa-(A zio*%CEdv%%fc0QD*HiQ-j<7ILc?eS+2-D7LrNHaRYB+~pb1Mb2()GusW+3q{kH^?4 zG)LZ09xGhaeLS-$zTXB1!bNT!J=eBqO*!5Z(5m2PjE+d82jTu5^*jzl;eg?SMlcX_ z7j;tQf~CE;mu|1D0L1w$rirdJrWI$S_E?+l90TT%Z`F}zPYe1ma@aF5(?8Yfpz96> zbE64xh4SDS=r%|72Epn)6_oincfQTm-*8Jt!tMeQoaQlff1OL*WiX%*6i`G0A-%mVr z0|x%tsRD8$k%SYMxDOwX2Mem;3a`rkN7D&k0U&uurz>?d2)iMLKC~Y4sln=MPjDGIJ^tc*UzWs!l!nS|ArQV` z0T3GuBi;mXCKaHX`LblJ()Fae_n=grv#4? zq#_P?xDA*_1fqI8RYUj*OqiIr)8Ih8`S}o+W5iWt-5CPVK@kZof!W-iVQ8%?7jqW$$R_@lw9N;vA}+#=c}*iGtPmswO2$-YW(3I>R`ql`P+Gwf#WuivF{^oU zB;SkROrdlrhg3pB!fkQrF-VYK`(^e&N0N7l?_3-7?hsY+O<8ZylZ4z?(+i<4s@3&Y z`&4^ENahTiCc94FgJ(cVZo-mcL+FV*Mz>R4AEt*;<2Q2GgJf|1HaJj|f!h6>_X>H`6qld%lX>F9e^OhWQRSs=F_bzQ}Obq_iXs<3njunCjNh zi$ZJgQ2Mmlp(+tIu@dVt5S(w;)CjY~m=!nZO+XijW`T2M(UY5HSg8w6Ejv_uKCzlL zKXOcH&}$&RkqO#|`XKS72whL|@4-#Ckv9U(H;Bn783716l>~WT zSBHkzF~9%}S1rd-j3*pOG(r%~pv#zx7~mm*6dljUj{DTJ`?vJ9j_`>kB)7x;M{fqGRu3h}Mz{GeA0$&Qi8yC@ zF4hp&bWJgO3qMYqNA01BybN5X+P^-nG4kF_?UoQNY%SBgGgAaNeHw7w@0l64R~NYW zHGYF2nTM`+oKgd@6_nu~cf-V=7YVy8A);ZXZP#@pKgS*O$9T=+kEMTCVQIQ=AQAdT zLSg33sUM}7*M&A2%n*=KrlD)F{f9Yz;Ls{eedeEqA1-LW zUJjKzm@-H6aBo2CM{r@&ShxNoal|HIEDN8#R`~_jX5$e1ITO||xBZZ$-bNNLRk3?E zY7%Yt$@jpxq2F!-e5myT0O0@+s{M7ictTzBwy*yi&C5g zD057EAixMx@Z%JJA9qzF{W^ZzKFfK&#lq|`F?2lS%}7h#R?=n%Dto(BzD^&rKDc3 z|0ocd@quVMuB#SJwT<0^m@(m~2d?;gKwy-%zWaVDe-XqlxSlDi$1_aOj#bh!C{U=SKqjzmq4BtK!>?Gw>q2G0$=99by*>Qwbs_-uYuCPq2ZR-f|L%Gy zV?Fr$xwm~jME6#X-&%lRAu|BEq9Yk)hP@^wx&Wz~E06X=xW{>O^Y2VsJ}r4$5{ZGD zqOAdQo-ED_kAULJ-Ab;X0HtQ!8$yXESuW`pS_WHHCx*=nXBb+tYGzr^BQ8e34A}fv z_y)i<^LN#sS$Z1gU)fWwow-udWobW4+p|4NNX zGzT0HTOPxM!5I^8(Hci>vjZ!sWz@Dy)_#%w4J99UXEBII{{`u;K*TBM3%k3!+z{oI zoN;{eiG+e#*1Cf7`5t`Y~@gOwK0z! z{xeFxgddE8hb4Aa-<5`O8VGBjo3V)ZGMD}YW7_~u5*wfO&+ARJqOSK3%44nreqJ7< zP*tB;5jqpgez~2oG4LFhq3BBAlmox_wrhv~XRsm*?{z~SEaN5UFl8K6WN+|_alBR` zg*M%42xkE#8!r8fG^&B57~RVZUg_IlhTFGXE8LqQ>DKW4Gy@~F@MY)^^&n`sltETd z5mx-EUIz52MVj|mM2>6usD8tq-(VrL4QFq%vd{yNvHh<>}#cINmPAA}uZbuP=fJC3hG<|Dw5%~e6yt-yS3s;j zZPk?(g`h?otqe6L0N1Z2k|NuSfjZC#HJ5Y+d++Fp7aJY3wAlwQ)$($LEth* z5^2~ufy90tuUO=Mt>^y{Pb!XqX4nqjo&we_Xz>ZIknarv9j|u_az0U#Lg_H&rM9=X zQ_Pyev1V}Xd9N?t_^i&3a;*eK72$e1A8H-}*fj+nkVfhb4heP#&f*HF1p>X3yJ8;r zLQHS2vER>ZgWhLa$c=%+UhJSH*Vx<+Wdw&Fv&Z^;7$V`?!$yQJkGoJYZya^WPu+mz0V4<%fl0Sd z?0-##=Z$`Ql}~epss-)mFH6-Fd4}v0oYP9uUBX}yr_=^ z-d9TY-%g||3w<9_z&Z{&6uZxE|DpNC(l)ne-Z9EQm5-(qZtty6$&^~yKN>w7klYA-`AZw@ppCWzk#XEhU&^f63Ul^ z9n;UUYqzOi;nFX;gBV-V_YLN)|7xDzGj0*=I#+~6w95Lvkya3XS+-MCJ zccy-4cwCjwYs-=68x)GNe;A^MK}3|2g^9)${!NtM|N2WS;4eu=ZfCqqYZIl|R5x z5s>^4EeSo*wx$rF<5k4aZghG7;<+Hdv8VvqpZ0hl@y%_rTGK1Rz1dMK#tal8b!K2Ye4znh$(B5KOaB7#D%S>*cY`PGS4-L&w zQ^Zr1rraM+dNhIO5OQ2pjauxGD_H1DMZx~A&|%7%$A^ehpgx8GQOeNHODe5G_zsln zsF?&DUpmr`0NybJlz59scpy5D0>XN| zclXDng8gvYqOk6ZUWh?c1OlJxK3oAs_{I##p-hmP1|gxV9^&BWCmn4N^1g}Gl+P20 z-2%7`J;QJFiw?}bz@M(S!DX$jQ+ivrcDXwoqg^KF?lea?=(X}}BRqr7ZF!6wJ7`m# z!9CQwka{x)U(;pQ_+BS%uk4OzljhsV8+!|{PJu7{_JEf}@Yj`z`pIFL(M+k@&wO3T z#wp0EHW$5uy07Cr*+#zvyS_j$jA31Z2K)<)ds_|7IBnVuz*9R!oFwZ+ptWMkbZWPu zC3hS~oLBzNuKmkhv?(0&#r?20`;Y7yKw`3}`|8!7Bg#?SH$lKbN&iRdbP`l&Vq+x$ioGpI!lCnmj*GASEM!Bm-3}g;4&6fg5<`#6C^LK6solVzcE8lSvhjIae-*F!8il@I6W4&6RJd*okD8)Xx@y{;+(4z;1QR?y{e2NI7olhhWoWwHGW^6>92j-`c*MJB*@*1b(8dLNJ>LUZ%=S2myTq%;SO#pFP5aI{cp1z zcvE=HkOllpED3R}5LCqkbnT-GkirG^7i`?LbxcU6e?QyPI7LWlJ(WEo=4x+*wb8Zs z{9B%9T&`4rRq^9bE1Thh>}uz^k0sF>9A2BtW=N=_sr4^$h=8(c_~Ad0%3#Qd@cb97 zQdqyrwT8;Us7$F|?4w_q$3!5AiTY<)-Ltp4BSqds+A{Xj;9uGq+O97w>Il1~{L0UoGnJD5YeyjJUO>y_W$BJ?I%nw?0dG+jFZiuM9UlItr zZ`e0;Z`)nkc9NLt7&)|Q_U&xLd!`uuGDj)=!j8c-za^uG3!&-j@rkdUpb>HX3wdM$ zwHws0zvYoaQZkTbEsjJ zXDOy%Mw=Ji^HW^}We-452h`*jB!xv=ND?KOGV^F#a-Ax;y_a#A5Q@5>22^TO{THi( zRD%G{)57Mx>mp4h4Lqq|P^n|98SnC~cC6i&H}Xzw;5|0IM+M!YkiEpQC+?fufBLb}^bi1PTps=v;!y29*<Ti-W_{!ONdLqNG_jY?9S9ixX?Eg8ycJHD?U46oTo*}${ zMtXlxiEM9|f7GJ*lc)UioB!U%cGq0k93B}X&|TqH;MNT0$fJK4AE)I1Wt03LN+|gy zl+!>rkVcFZbe7RG`yf!(0bVBh@@OlNG>7%;`zLei2m*uFhH4HWlq;0W$PfuLAyXS9 zV+Ekwrhx7rV>`XS^h9VAD3A=q07&QZVXR?bpveM4zuJpM7of}C*R20|U*=#MqymLm zhRs=m)B*p5PqtW}zsa-H-uha5l~)E&Zs-rVCl{V=43l~}`km1(l#?A3Q^jzPd3)48 z-X}*tyvArff-=mj1&`pXqjp+hQ5`G|kmh@W-UASXSRbM6I_>&?P()sWc!lT`;0iGS zPdrw*188j&Hj}=C9e^=2SwZ)5iVW*O2YQxCybApZaY)R4#55a2ueGejLGHA6u6k>l z%|Wyhs$#U~#&7@<7v(XpTmW>~SLVb94G3xC}j47EA)j8;1RS6bRo94$T!<;8Zq;b~GVD$^%sTo5JXxT<@IG zO_C*n{iYe0f@omx8}%|6G7X3z*4kyVuiHufr1QnAHTm@3Om{a35eh$$Im0W1d>(! zk4(J7Udyj%F`B2h^=XT;Zeg&U|Mq=W9}dcj`R+;{?fW~%b^OSt)lKWR>sCGam@lt4 ztGRCVRk+ENxwxrf?fMxh>Het9Wr1J<_k;12g$>uMw^Y-K)v06nUTZ z1LjT{lq~ZT$c|y!zkUzhG~z>BfKbDns{|N=#2=Dy=Ony6GeESf^(xxx68F^9O#j%~ z2A)ISvsMK*FTSf9ls!Ei3J?fdi2n2w{@wjGz}jboyd9UhkEOUq3nW_SWY@iX?hr?e zxY?7wGB+9BLMyHoy=sf&@fxsyXN(X$((GN3rW)?+OQd(o=KI}5X$T}(9Y2j>F4pc9 zDNBQ0?QdaEErU{!axDRdBnC_C-ZDoUAJ^edQ+Tj97%l|K7lLq+uWNr5qrML>B!7Y( zkNZK>qR=aA$@P2D{%n$UCJCFpQy~QD`o?#m@z=Ohvtg#4Qh+v}F}(RQ?E1Qd=lYv0 zogyWOpk4Tl*W=G343^CNbSW8N@jBWNEf&yMvK#+iXeUS~4b9H0jeGfk%#q8eit4wJ zLIfvzp6@;G{3YEQs?%L-kDLt(qxG#WzfHjPkggT=E8KylEU>Oyxu0R$*A^TQ)gvB0 zz?XCdHc!zJ*$oCOvtJjJYb zU2pT=?I;~7=p@(cbS=+jlsi=+{L66p>@^z+RPRebFHLY0lvOP&cR-#W=fD4nh$lE1 zD|lDeslv8jn1Ua*wh5H6MY%V@(q|I%zEA3E<mV9^6~N6oGoZI1-|nbi~DF%nE_@*1E|q|WOQ&7JM_wdzZ;hs zLOSD7i6KE|5WG+<2$9Iq7Zf7tN{v30=)JS1RqC@>)%)#k-tRr;!#}a-0yj1{Z#K9& z=8MOA=BiY>v{b_udQ-jiTVs35ktNofA#(Az^`bpt>(KZFh>lN7(6hhtlqaFytzAdw z{7E_b9qm0oCy!q1WBwb_Up#>jOwi}U5WW1xG*E9w+@6n}$t(Op!y@ajrnKrVp-gV+p=(v?x%p_t0i;ZHQ_<(mjitS{V64kn;^5l_^z#9lZ)sKM314 zDmkvhHl=8ixhow98Q2|{tDr#3!%oUnpxE4Q%cj@V)8Ah5!{3?zksJ^;%O%>Ct;-pY z-@enHwi3L3>AMUgzz=I079mbVbq&`FN z2?Sn_G|X+-3}(Y2wLcC+-f;I6bRTAR7VE}}z^==C&$2qJK|xRBy9L2;qej7Kel#mQ zYJ`u*AGLPQF_@sQ(J$dWuMcxAnWFR%uS%i-(79+(^V8 zt!Ghr{}!%C9!ao3!Y*ySz`tcHk{yBikU)=|TTLUtlTI`ZeRiBEE+Tt~3rCN=f(S!Q zf(JzV@{MbcuJplkh2)SN3zF6YdGqOv`+oiSRP^BRFP|W4ng-Gpz%8eKJNKL!h8)QF z1gIbK5I#3-+!%6a;oP=+B<9itDxRx|s%Jc3w2q!=O@-zjs^(zDG-5>b%Pd}P@?RW| zXZe3b%{BLS%~y)s9T3<|{Qu)<63qc#UrU)#_$}>i|VF+yo2dh%lYm2w9t_+G7G!58TJjaJf48VhK zLzpH7p8bG`coDB@?ODj;R>qoG5CZUG|5GK?D_DJzr3tYMZQ#fBkdqEamlPky*9 zXb7OzRV*S+O|gt?9@iwaEOS4YECv~I&VJ3e%e$#epTGsX-*th9n2EoBae*stv^nbD z*7^b$AaUod4J23xUPmy(4JNL@^^Mmi{@-Y)0l2~;?F!dC2!r(;_S3`K1fOA2h#W(5 z2qf{KpmG{{EG z2bDHxljt|J+O&`BNHht7Pxqa`G0jKH?pGQhG1s-aW*jgzPQU?D;NdKfcDLQ!;5!i~ zFW5bL)rl8UY1hdJ}iob(6z8tWNW{=wvCxqRWQ`{uZ|BuRR zbMXaAew&LmZ2Rdo--xfMgOf1lV?4;pU@Ls8N~|sn1Pko>WM^SF6m}k_ zg7GkArgwb@_;n6*NUk}Y8aC@!5Uah8Ct=XqqG4IYXE5?a7n>gWV6#r_%@Q4=;&@0^H_serdQpFYD zE(Oh0iU^_PcN8B%_7E79L>}CpF#t5zr3#rq_)614LoKcq{RRX}$iM~UTX_gs z&Rv0c@Z)SP?cyKtn14j0zJI@Prk~~eo>O7&i=DlQBXW`n zYh4|2g2tW!gp6G;_sGRk*#@=tk$0+0J~h&aZmK5|f_ru0ZRXxCTb{{LaclwV1I00a zXLQzF*4pV(PUG&nAEiklH1}vf#7)2recbb=L~&Z%>MO-lya=hQMv8Ltpq59)y4Z?K z{)eHp)4!4r2eToE_#Z!KCNd6sP~8e6RgczA+1_PpdiQCPY#BJ=|Y+Rmwc39W8^x;Zi#(;7( zdlaMGd-h-a$H!X=G=yh_32T}fm4?R&{}*-d0Tk7?wTmK(3W$k~Zc&Vg3W5<(vKSFC zAVH!5k=W#%F<=%I1r-URf`kTz?k0m`Km;X&TLA zj&3@vwdR~-d}-vI^-x=4pYkEqF*TTOaH;|{p?DXsynNPa-d$2CK(i-h8O8-q43rPH zUO((giLLNv)V>aVU#vgj);Y3ZUiMmA{qZ+3P3;+zlQ)*k?(!%Y3+~wD>@jPg;mo(H z@f01!iSsiJhU@2#9T`vw>8YO|(lgRv^*yh(z1`Su{x5c-+#Upw<&=7h{M*b-s86M?(h`r=6D+$vmBAScg6kum>vrP}$v&&K% z|5&#q_B28lzgFCU=Y8N7?Y~Vb+xe7@zI-+knnfv#+nanlP)b#&#IpG-QZJp~PAXdD z`N$7^T2Rk(KhTKCIWtlI!*+S$G}CXz?zKSS>EqG>kQT#Dyu;^dq`NGf>g4CE%l>N? zz(c=`1EW)Y(`KTUMA9KfQ2|_Xl`-$3-4TwU=FX877mVq3X=o@a5&egf?2)`zCb6y( zOCpMZzhLz$5!)~F$RRu(Y<_ii(JNZR!~dWrTObT#;NR3_2Kv&UXiCk3x&u#0BQ$I` zDVNw;GEsjhbByDigA8@y#Q!f|;H&(rYr@370S;9O8}yee62OA7r~p;%?-)9Yd|Jxu=ALU(sIsum7rz;CQ zZQlZJlCC)qMD9o3@ozkXB^B?${?%!@jZ}&{tym%!-IG~#uy`_Y z+rZ`gepZ#vxD15aA=tvRQKUTk!(iB?%0w>NOEKq6~2*r=xE zky;bI*mk5;sgmYNmvI?8Ioo|S!(HC?NxF*EUDQ$1Uy?Zc9*gQF7p89zx9EiC*!0m^ zD`1|)6IL>N@5MIQh#pQu7BWX~vc*&~MJ#crBdXMN*AsuR=FeD-B52$LAS8HP&RvyQ z8UC~d{jVi^E7WnbqzoXRLB~zaFXhr1Ef*j@TWo+q2Cu=y?R@b1?wNk?`B6t{WixeR z4ro9Xq}ZX#gtn}x=o)v(aivgcf8)MOwvpZG1qJ~Jc9`GT@UX%x1;U8*zlvB^FJoLr z>rIZ1W3dfYFZ_1#?avRqf*o#^pe53;Pf5^Z{qg6;$gwX!u-=xI*KFIK9?o&awKdo& zf0j+vGp%Y-QW4=fJ$K;$M%H)(D<=kb-T%-sbqTd8*CCtNqZJK`TDCEB385$b&Cm9>Hdr z>s0SyJZ;cpW83oN(!#0r$JPzu>oo7lruXmGJMTF+;O+Y(xGP#TV`uku)5@1|0NV~b zj#@eWAxIj&|2Lu($bhH->^~v8lZYpY(10`xu%T(d)3Xj#!+aVA^cTDi2%N}6PErdL zaRE31a7ZXUN1c~|oVe=W>NtGu$A&^6CW&SZgyoeI8%X2+IPcGo{!xx}<~$RXr1gL7 zN^^YZm1L$@Fjk!^`b1i~(6K7|-7@pgJFaSkoda~f4r-Jgd60b=T8`KO9u!2RvggFG z@h0tr=V%E(SqzrzT0P1^3?cdHT3B5>ldY!lziRA^*}p^M_IT?s^mWC6@lTH;oiO`S zIb@+V|fw^9c z_)&ApUS}ZA&~~W1(ONd5)U^2UxUZ^Fb-=BjdU}l7)3bbuNH5Wye@&S9O<*=x!fBxT zw-@pyd1cJoEKCt{$=M?UBmy!DiHfu2U~olP?tY*b{Tr7f!G`?E@XnwG>PS4l<{U#> z4ImvVL$Paqfu0h?e$D77)WeyCb@F4~7XjfBG>VcZH!)4SA{LT?Pvhedg7 z@^6+jE#8ObWHn+Q9{{sWLDP&aYUn`na^beTwd3iA0y4Lo<-3g3I|*{!=$8}c0SL5KtGk|&xj_loV_GW8qj3&@u(3mQUI z^#BslG|CE+$1s>R)^p=szo)0?BB{8tktQ_V^D82Im8@WfkY0!AL{5fh???I~sp*$l z_!nO02p+>1;J44Ky#c|;5j5|p8*jW0{qy%`6Lgb51z6M8@#*@*UJvq@{PEl@KWV+t zZ70s>2FOO%qb#?aS)YT)Xb8L7iVZ)b=zgrlk_3WN~S5L^! zmm2$ZBwY^bFqjLC{HCQAbwssDp4ER-Ew1^aT4X=1Ke2t9*k_AtB^F|m3a&2i_xHB7 ziaD`*{i4GIqsQ-Uu=1>=8wTZv_TI>!qy#+K?NH26?SJfWuBq?Ti+xcQ-}9rq&gFgV zUo3rO#j@Rw3HFmii5uN7Obv7AThGD94RwbQ9T-dghk9BP$=JyA)1 zg4D-92;m!%-{I@&ss9N*v{Yc@d1Ev;Wa_w0UoN_<;Q>wbVqGajaKDc^n7x(mZ@hb7 zMC*uVoZe6SdP1TA=vDP{K+UPqbGSv4K$hUQF8~x+z3H=C6Xl{W=oSB0-XIcFEMZlDeY*kr0x&%gK&Zp=3pL zhSQNU*{GUEMVYp&EO!{INiI0@-5;9cLi|^!;77edm>GgvpPCef7lSz4je?T3BGusM zcBW3X{NU6*widE?qukx=i4SivrBhl0H%P>lh zp4yThtmRox-z&v174|IgbWA0X+;ffQRNIM~Wt(&}!$Hg|0Rl8G7m&6+nK?v@B!-7s zAAV!~<4`UdfuT{0R^OE3IqsUAgD{gBn>1q|B0)%%=q{|%fBEkIK~ftaYnVUyTMi+n zUqu_dx?D1@G)Al7=65*lwj%(il4j~(c_c6c(X~H}dGf?xF|wu+|9)~jNgy5P*Wh3v zX-OF!G4YxAAi>rl(KS(MlEPTH&lb@x;$bBEm@`n6xCBWgJv2=%@Z1sm>IqX_x|zl> z;%HP~f-kpAq-qYDJtV2>iPX5!wr76fg5Dw9j4sSuwPR0v`Q)irasLyj8EC3K9`oNg4QcmkqAdaO z41Vb^;Tk0Mu9^ANP^@~&Uptc<7s+eotS zd;wu%7DC$6*FNnD1iPz<+=M|s7rWd$fO3LPzd<=x{3N<7mAv^pZ>y7>R^0#7V#|w54za65me*9szqb_b-*1$8#$My`iZDcNg#an6s+S zVDI_S_y!Tytr);lpJ8-~TTFYg?-1;hXF45<)5G`Q@J*%^GFI{nnX^$TyJ*qc8@86V zpQ7kLeQJlgCtr%PZBZqg)XtHoe(S2VDj#`fxIOIF9^Gc1b>3r)e`gMH@m4Sds}AH; zzJAs;GIg0&kJu=)jf3b6if+YLp$xljuw~>7`ams*j8DH#*_$EE6yU1J#GA&S1ML)B zwzZn;A^rP7bo|6w48w#&WvW9Ar=y5w?B7pq40btb_AK4_m&Nc0?#Y`eReU?`}Eyr3sMn~sjC$O^<98<9MLZ0LoW8HAN?c@tzgG47% zYc3UQ;G_*E>cHL{R=)7eabzg)Hkf@%KmUbachdNVZ;{lV2GaGh>l0*Tx$N42%}MA%o zJp5>2JFZJ2H7WWj^>h0tTK&l+*UOtvC2TEBfROp~;|F;vd48HKDP-zIl{|j{i*R0W zb&?S-x=%SoQPa6VwEfqhyA0iY^402ZBap|>8Y$lgKb^{4Dqj`&9B0$!sruJ@xvn*- zYD>%cb&f}!QG>+pR8?}i62yn=r5*tVaAptQ)e9^kLNeuu&`Z9Q5&p7@nAycTwUaHm zyuBmk1_c8<)$2eZMTIcpJsH_7bfk}#tSDzJD*KUfr1<|^M;f_`-Q(M{qd%f4p%X~7HLdb}BI~B9N5xUR-K20eapv!|_*QsnXk-OqLyjE@kumi& zZnL{D@8y}8!Pp*!RMdPJIQu%wH3{9glZm{dj-FprszYb#By|NX01+xL>D7tFCXCWuo<0;r z`P0ly+|qRoMmw!|fFbclm>r+FhI$_R`D;_9LIy>bSPt@orM2V`w4OKtijI++lm5V9 z`3oQSLm_e=X3+`+0#aGsIu8g?9uS}+xwTFa_nTPRy>f2aGT+BLHr1=M=~2J7PBNqX zCbcD=a|UM5^xELYH6bx;;Iy^tuUVX<2Jj-X#W!u5WWA}kC}_c-4V%0Yhjo9Zq!`j- zyiv|1MLjOfeNXz|9m}5jeo0kvV|YT$c)s`*Y5_Nop=oazThRkuc9Sv~A4(Qzb#hj@3QI*x)3BBO}5+qWHy6g7U* z?;VF|_h|?#aNU&!p9$Cfb8X6t(_j145)UtXwB06_b&>cP@uQHQ!7ha^u5*{@LX~)f zjB@~-Nt{O%;&TzceGJI#dZ0W0sIU*iA}3%#CzeLfk)IoQg)4wfV>$`3w{2$<<`io| zlDd;tdlQuuCrR884fPm+2MFYxAb?TsiX91*yaSM^r=QDxtom8wrC>$egb1`{_ny{! zZqgDvb>H(8&yQtOszvXEIG}i(uVY6b4o%hdiALXEohgnyy6QLX6G4}EIff%za*r(< z?u*nCGc!m9cx9^75Q%`sQ@xzXI*I*9Anx^qj3do`N6Q?d%A`D+yiF};XwQ5k?vX(2 z>U%o$Jko1gIwEtWdZr1*A*?KnvA2!T!Kr=nlb_fKJKzKS@3OSFXUut)LSv%viB6tP zr(R#Ts7~MG$f^8F;}SKKf^RbzCIQw%BQBd zHQC%IcRCK?exNAv2o~TdB=FC#WJEJajhua(<$IHKQH-wg_b9~@3tH!;*?OZ>+^QLjwpDT+t~A7>E6Rpyz`aOhf`nbjmT03 z<}6Eoxr%5FWr98CB6uULp9D^#3;Kks`{nlt4QXG_Z^8i=58qksQxfy6tO&mV$7|m` zF+@}Yrc=k3`5SK;Y`GW!B}C87sh3}^W1}2S$EHt&zCRO}?{wFMzKRUZPzsi!Zw7WR zu=i46;R3}K0Gn^76x`3XDSN#GP@6yUuk%#&UdH!Au}tQ7sSH_Y9Gr%U>()l3DQ3Yd zdl*sS!IsaM_REcDtXar+{_B%vO+Cb8$aE(1ioub$hdaK#%{m?C@nw2TvT6)qlY+vz z2j075NoazXQRXzLk;l}ZqiQ)4M8{wz#-X(s$^s8?OaT9SN93HlPrqsw(`DqZn5GlD zmtem`Fb5@jcks=v&kq*^UVEgl!1@86KS<{_ROlWk=@IR<78aK+ylW9CQ%1YUxqz~~ zgl?`yf_NY8J0bG2KbiQaQXQCwVgJ=1I5~CE*p4;_u%!~uD-v>DkS&Eoh&`tLFOM`h zwtf}B0eOvVNKg54kv_H~o{k#mPrg(Ff(3RwI=#NOK9TgI88V4&X5QyP9qL(wh)G zWC5R3O=V4X_Twj8f8!beU<+y#rJYXRrq(anWz*=D@j_tuWkuwV>GyHwPnuOobi`=8 zAl21BhfL0h!|GP#DS7<4l>`>TTT{Dj-}HA^u%8!3#K|hk7f=`mInE&qJ#WYP{NL^H zxvx(u_4iIc25kFU*{Z_#5GJEW_Rpd4CT1(=n(5bSC?kbDfecsdw*3T3Zt7WKE&6j9 zUG=P4qEMgepo|Bs$vy~AdmeUhGCjj3%J3iW(#$r3ei_`s^+O2W3AW@#h{Pr&+m3-A z-r`^}{bt@KD-=$_M2G~8zuO&CZ&^;m_H1~Q#R?nXhL6L95&VnlA&uhc-(f5GjynyK zGQluseu_=AZwbY|7TP7^=K1$mCR1r_Cnr2eVx)|7H_A^aR1U)mo$moi`(G}fAkM_L z(kOowqy>^KA4UCLa2wUa!2nx@u&QEUCsKltQDl{Sw+|kq>5p>t_6&RUDk?*IS)Lx1 zNN6Id5kBQ}X3aW;lYy8=BToI%@BMB~s4yrB;`~h8tjkc*j5iPk#w4Bx#CR=2IPQ>V z8dmxmahd7YSFrDxK`HXqzW9TL(qirAQ*hsj)GXV0^%LR1O+qf)Bkysy7P#coF&F2a z%NF?jwPnh%PQUyO#EjYbbn2DTZEN@yW_pVtP4wfXJP;1eic@#WS((j)SuRTAo%!9-H@rmO`#`@|`Er&VoJ$Pk)F62811j#YA)lbMVq9wc7@5QW6}EtvQ>;i?mB z3wm(<7EnI0=m>kt9)&~wfIvuqNiyWm!lxHG_lTb?N4peBLKsf)|?d>21o$*Mmem|9ip+wPbwBWzR%K z`3AMrj#{UBae4ETaX$3%>0K$_g!?u;+K13svWp_CEfAX*5#U+m(G*<)h+F=>CjA3; z&wTSP;BiT!qtj6+0c2!C-`b>#6g%S+Ut(tTqq9*-wjj2zzK+Thc)7D9D|4>%<+;C9 z`#%8Y#ejEx_}Pl*A~+i5Wr*HN{wRg~#1ng{6}s_(#jolRzT?iT7`l91^O6yWc2YQ= z=$AqFW{S%c)@-^yMWdFS#zD+TzD^)9|MWRuz|4gu^vTZ?K$r5hTIgqs46t<(Z}@Rn zOG^J!$_&~dO>=*H#Q>e}g67c!rBJP^@WpESn(dG9y@y8e0m1$b=iGi7S#TEuy8i$H zCJYE-IXZ@QYxlPmy;DDqnOuq}wT*NwFMnc|kDubcDIo3mu)-0ymW(73DlIQxdDtxs z_OJg1_>WSj+xWuIV=BuUBFRs*7HjooaeU0fBF``S7DOxfu-}vS>b9N^K{fbG0kWyt z);h9LOT_d{vKe7B`k?f^aM>LpyH9;JT`kACys(f|;%kq}kE`QsTip3-6p=4KoV(I{ zZ*zX8J=U1HpQn%(O4qE&PBazaJhtr;}qzJl_Q?fwvKI7M1 zeg!Mcsbu7HY0`tQ#8s59gzHEm&%Wskq)-*K7s!ttgF@YSWV08N<$hcY1XB1Ylhye2LCfutZMtg`ko>wJn)W-kY~6GUZb*Ts z=q}oRAK%QXUX8uR6r-@Gp_+?yX==Tw+qdqqQ7??|Wmb%%k+-OFJ;}Rz0<;AJUL5BV zrS8u)%QdU!oQ#nZ_bv}98xi-)!#3Q4|HALy>vs-MRC^+M7e_-8R2cw;{O1}bQ^RUa z(bn>b6;--ABF~7I4^08|5FvYG;^_i8sEG5o2eWP4&XQD**sLwErC==UJDMTeWI4n0 zt-LD@Cc7SEbI}}NRq{2Zbk(F7X6iRwXU4Bu5{1(%h|6JW`*W5$v5UB$XC1y(9 z4e}f}7;UW@yjd zbtqp%z>;#(3}SZD{X}JS3j-D-i;eI8k;jo#UB$&T>J{HygcUJ$VjlN9#Fu4Nr7GeA zNc+cKl*9e!NJ%@Y>v>ImPmq zr&syjTCsckF%A2DKP!g7svCMTgnSYsHNJtnkr?-+e^G=`Mp5<$Zj^~RbJ0%sk1SFaU&A}un^{TCNw z>_55O&w6-4Y%Wtc0vnOXwH)Qdp(fas4^76R|ALLHlN?&(-4f`T`4JZ zW581rr!Y?`|Ju#8FE?f<^KZxJE}N_jKSN^#4Sp{_$|M z*yuC55T~x1ZSlkCx~3y*v-O8pERxEN)C9aK4~%82*S8n{0&nsaZm9<9u(w0f{8X(O zW%*(jhXqGIJS*G%Uvcm}L9Z;DH9io;x$>9UMpXCdC<@7B0aFs9PSaO`3=_Pxi4-6^&A5TH*YzbP3SJQNMhgAwB&Q_Doo4d1)0mS!-s`oD!hap;ke zZzQCoTFPS_MNMl@g2|1=83TYCc61qg$aM(pwterOY@~YjCBDe!c(Xit`V*CGfI};Z*Xy!@#FE4+xs?TN$7UmYnF<+Nf&md znwu5n3ZZh`gwlAsKj)H7YVXy{m;>mX@1pjFK-Xn0G4KLT<&cW-{QKz)n&we}m*Rys zu#j6Yk_>dwy>Zdxj>Nu_k8tdAx<{0GeYwrxOP?T~)O2@uj$C3~S7802Mn#U96sruC zvpM$eA-i6m%{`dmn)hbL_;b*TLV^`mqHWK;5r`K-y~QXYgypit@#J9rgE66wh2NLx zTT+a(GEq&CAj;vMye(gnHem8;!CSa9zPKHo*@oS^y3TH7WuZsk-kGdgN`JK`E{)yQ zR@(zfML91<0jcDr*Vl+T#L}8L6-Nd76J!oA9Jczp@AAQwwz3QRi-jB~2FsSae0Hlp z;rzMl+|jqpbz3J6%&7ZqWv|cuZE1gd!eG@H)qQy90-OF7`uS{2W(=@qqkl2c5jNpo z!1J2E-Z)n(a|B`6+mwLz->>) z1vQHFGuV!VaU@|3rnvWnX_NXj0TD**skFlra*is8GYk$06+9GWNjqLE+*Z#l)D}7t zY$~*73{|ZWKzfpYL$b?NAd)f=3O=MhdX9+4-IHkqJjMWysLlxpWGpI*!XLB~F1O<3 zMH7Lpyp>Z?hTcinS)xy$maTldEatMWuh$vb5HpNqd9?3qh~_@W_EIIZCg+8Mxo?on z0387Vol)<9YI?UGv5LR`(sT3E{>^n5y>^ueRMOP4@u_U+Ua2l9tDeNUjbyE@T}bJX zKRr53ou}PG)#=Z+fAYvK(D$dzgP5{=w^=LX&r>Jn7dVtk2)f!G$kGmzG`253ka2lG zDpX3@*G0Rvj*YA^dq|B}qEMSS{qIZ+jzv4OA2qeH-q>GvIstF!Z{A8mVo zqHZ)K`-CTOVuMvaza+W3j}R5^f1}3L3^kx?I(*bwN{ft7Y}RIpu22m#$A2Se4VxAB={4=5fBNIPk0R_QlrzUcQTO(u8G%=U zvy{fJqiYIrlZZ-=1+r*UoXXD{5@na%_6+xFlx4M)oidv&61fnU{8m}k7+A`ej3Zn_ zhz$4^V6M?OXO;VD&rXbLB9mD$m_#xUvjbjnHkTe;<$XM&j8_PQh|Qdx1b+? zO__34v9Z;C5}iStovHJn26Iq$<0Mqx-HMJqmTY3E)w6Hf}Zwsla)1p-zlK717o8l&3;p-7Kjc3^DlT)X|#!BTAL zW@zDoroOwB1_U|)?O4!OXB7Hk^6?m%*Njg16JA5LKVf#G4qt{%^8uwhtM??|CL7Bz z7`zu~4{gz!lI#qO!1{XWD*)ZQs6js(`Ar+b*ktrAoSvv(tS8tD;ttLcax@kIGZh+` zFX{)gfU`toTo!?|>sw8BI2w%~OYRZI6cxrZ6MKcz(3or;^a{=T`eN!HDA1Z2N^|X6 zKCQu)5{UZeGh&^eLkD3CxdqM`zo^5V=%tANNB`YFppFUvDzb=jva%AIU4x}Ya9Sb; z#HV_I!-PZyo&%mp@?3xatRAs;W1}2(+c62LS}~xl9oItZcHS>n8i&tb6abFX;KJ!= zf~`o@yAFxiQI`O;C*EAKTkY(}o0~{M3KG0-RtednVOIQvuBAp+Jza67jPq)uL;z+Q zl-W$W;eQFNV6ccGcVrvE2%zs+58jvo|0;>CUJGH^eO#YPhJJ=ConWLZ$Uu$AzjJ1WVaoK?L$U+!H-9Ng*K| z5()Cwj`Vou^U@l2txX!yq`4|_XA75?G%0*DtB`-=Aiejkgu8Bcr4RK@%jPQ{`0)m2uhrjRZp&-;R7r_Z(7#J{u2_Pden6Wa4V? zzNve(TIUAqwtxHki|Bpt;E>3JwN*9d7M@6D3^5^R50U(z9yPgaLgqsN9_jphyTLZ( z@oYWQ3sH~Cv44qO_=Gld9-|)h`D#e$im~BUZ|AvA+*D*%s0$@tKdoF{L4>6r_h1wm zI(N`rKjxGa1vtD-V>0ztXZ)UIBQ!v2X^xbo2Gh5>fsZE)vRca^^JJ;zE-f z?9|5xTD~B4C1%Sr&pWSy%qx+YgDdp@{e`*P`e*uu5&zgmcTG}P%pj6g!cJha4;ERT zx*Bu$PIX2k>l{fQ$7CYpCceliY@bAY*1@86(?DbTbB-&%Zc;^IvmLRUEI2U~n>jh* zhVUd>Ud|P*P)7POKzj?8$X$7Vd}A&Zue@?v2H4Fts}O}D zSrNIYAewKN%p}hi0f&DctRwtjR%IPFwR0FbC6=Lvicd1*873$HGkS@=;IFC09xjds zhGgKPG*d>IL?kjo9WxDt{yk{%N~REW#FoPT9`LK8a269FbD+)v?jlMeVk`yy^4`(Y z!|LBt7DXA!d$3f92%6COczm}f0n<5**8T!zIV6w5FB%s^^g@PrekNNaL6R2_V-cY7 zTX^!_X)W_AS`;EllrqN$bnQIHHNAbCbjGob9VDZ{@^Ys4taP;*wDA@*{(Ui|d2M6> zO^gc{-3p_Yq2-#QfWOypQnD484xNXHkOH*Jnog&t>{|={rsc}cYjG7&F*xR!{@K~$ zUD>GA>3>EDsRRhsKd`EE|3Ag;2Ej{K1d#j^6cR1qbg)GVT;Zkv6ud+x0_cAHbHZIc zj|1)t@bUoY<)r2XQya7^e#%aNdV;-j-v*uXsJ#`4^e3dS{0=uYDL>psWFio4+n$R z*DAWxc+1l8^+yZ;8O(X}K}mz8tm@k`@brbRKp&i%D|EsDO02R78er z{aiHO_w$5lZ7Tg=0PpADnr4#}9^lrG3<`zOw-8>+1@?+_s8@bO;3?mLcHJrGpC{Nh z@2{BF`8)k(J0vY9_%cAal#D_|Id=XZ=}!uMEdfhbd=)%&M*-BerSNmkP@nE7y@-c| z@)dg`-?}X@4p*6ocuNXuPpF2<)!uxK<@sm!!yA#DXP8iU^)ODvZH>0@1th47RHikV4k!Li zW$F@3@GHMjytB~Ua;xn`wnC~TGN!fSH&{(zDD1+AuspxRnN8@YWq_-ZKy?z61Y5eR zd5JIpI$=eG*{#DS=nuN-GqzeISV7bv`{b2lUg08!&4OpX-wDY*NS|7Z3Onb8Ih=ik z)S!#q%Pb}Tlk&9c(vv+i+fXnEb!*RThe>M|*L{+(34rE4sQ|GHkPxbfVuo2+bPwp9 z$Hy>6;m3JhYG*Bphf{XY)gLT6xN5%=o2=pISkFJ~S{~paL z8xhV#R87yAH7LBWcds<{`K%gNkX0rpY#`9j>5&KpoIFlp0sI&za_0#KT|D~lpk|$A zlp^6%)T***J*rUoFn$~6MW}b?^UH})*8&N1PM)TR(U zEh})9RR%gd z>=I`@o0GLfZs?IiFX5d(rwa&RlU+PjvTw;^9z#~aQ{F*SqdsgCstb&G!unU}cu-lp zP_LW~Fl7_KFhk)J0kO9OjVJ1RL;|<$h65zAmvceQ{sq)~<7w&8zpGIx?K;DeJw@Ir~?r&X0JTXnBy0 z=ja)FQ{jIVkyc~?md*3S%j7`{--7CvhzfW+upqZ7Hm6&6#dpi8( zV8k74ceMMgPw5+`^8LZ7Sc;h{Tj_Q>9pTNE+V+5R>W$4Yv2MnnTFBQsP;!{ z`k1+0K%bvxxKZJuX;&-p27pT-o6H)`d2=+T1*I1K4&@UCOL(5q*+iu2Nmgo_a1OFk;O(*2T5<<3{hog@fri`&=0A*4EjTv#x@zh)d6l%VVok2SiYkX?l;&L@ki|wMjVkzsQzqkcI*Q%Jmol?AuxSyU#yXj2S zWEI4Y(k3PEZSi`G3|1nkIpbHieTeBzAu)D&sDgRaZnP`g6VQH-RZv!qU3F2 z+I6F7+>KxnPz}2ifHxV1XZ(Pp+{)F-Jcf6?J8xw5w74UVZ(!NC@S&JKXxMFXqI?b& zRX|&6ZbbSJ!SaQUB;gw13-1MnNm2HgH0r0f1M~$CF3!%^p~q09)X{>xfD?7mN<-#A z@VUOBsE8_;=2Q*arlD;vRQqL*c2O8KX5o6p9E(v;<7JXA5rCyN31kG;)gPt9zYnn> zwIj)b1vv2Kc?AFeKwBDb!hvmRVuTqVv3RG9hrQ3s#X1E|Xi25@BQu_ekCo~=n{4GT zb|Li6-PgI3-mF%YM~R2NRP|@m>SJA$-_dG=%FLAl2Sf$pO0bXX9Sn9ZEoyld04w?9 zpdILne=J#2IW$PXRoczed|yu)tEzNMW|zEA%?M5DilhT8pG6I6@5 za;GOt{=7%@(LZg_Sk^%2EWM9Clg&d~jtL^toAw=_d)Gr*TR35ZvJpIA`?!T4NT~m{ zUrDEfZ!H%_S>0G)x$Z@W@~6p>n(?mTFWSNWAD0=|#WNq9|Hzl=&EH|St3W2S)y>O_ zEf-^97k(_ASv#og_g6b-{Gc8{qvv(gEvw#K(Ot9WuV{Z>&<&J7E7 zZsJ@jNCtp^E?`?2e4!QRVei|ie;1tDN@bVI4E1?#jc$dg%2c21Dk5;ipdA`6)?@Ry z*XX&lQ5DY2i(d5$OO^#!4uYMED^DDiHA2%&2V0Rx=BJ=ery(DwOmq}#qiXIg>8DZS zCg4cbK~vO$zjMm_a2`8Ioan7MsLZ}+SHO;4(V5DO^XktWm`KezFn^U9cjjaBtg(a= zvBNfZjlKkY5lb86iZOV0ySDa)+s)RoRO6(K@gGda>ADr8My*GbAN8yiao>O3ye9Sq zbdzskOB{(LnrH-~Zykj!hyvkRr|~VzjS&A~5Mv2i=Vs3VO#VIx2}11X7#V}741M2s z>;RZpYKoGV4mIJvpEIZq>X?9Ad)pE?PiOBKhz|-Xg8+=S6T!$&2T02vOIocobQ#ZB zw)M2P>~M=eMDmh@y&FQ^&*F;2kO8iiX)KAs(fr5a%&d|0sR#Pb`x*B3*r8tmJk%la zR88M!R0F=OwJ%W}KsZ72b=`4Y_0_576=I`UffT^i$JtG+@VO=684&@9S4P#Xz}IV- zkYzI1ivX%sdkoeZ14^VVw@NR9w8s!(Y&K`2SBfA$_kO?X!>w;G*Af)qOl)}!C9O5Y zO>bk!$t>1*+2frrS&D7x!#%oX?ext^xa(}~ulF*pj~|sdz?R|u^K9B+h&1bs&v-zV z!k!qo$hgZZ;J zd|an?qqBadh0V=3sk)Jlj@=D2n3MV3H**Y79X~Ug_^}4HVVuevmOdvXJTN zW#u%9?YMRzs{Q0!D}*G4$J(LAiCbJ%;s!ua_!kO{{n`WQ-xx>(n3W-^vANzRN1$`? znF9tBNp?^4Js#0Vs&dukNys~H{&JTs#~?j7P{X*9G@oNSeC<`UHL{uKrnk5W=}AG= zR!lLj+zcdI=~uD02l6AuUgi65@ZP*MkIE*FIg2G&5wsL&Zk>j z{bx*=`d?MEPrT>^tg3gKe}&wQ@}~idul0{wxz*BzL!S+5YdLiBXha`O=?YG&e+BmS zj1w)bS|=$(WTBig-jmWT@~*b`V@9v$>EdG?Z#%`KHE%7FQ$`E~9J8%~eykn97BWy# zi0kq{YBwdhW@a4xIum~voUew#x5ksrBM%0%M}DKvY|pR zwtOjf=daRd+NDBGO*gh2DYY}?K3-;)Jtyw9tO3pBxD&N5=dHV!p+^752(T3aCoAqx zL>BZZ7x2)ucgWM%9h{$&Ha3=WYAmt#7Jtr&bEk|8^;lyb3!#bC<)T5g9%hD5PwVr2 z;IaqaE6`=G{_@yceUT?xH|A+dJ^&(qer$5Nhbr71IlU z9=l2Zr?NXv{dOE*F`|o+Vy-$6coHDn&3?~S=W+&;;}@6j^LSf2Q=4n4*b1gpbE>>t zQf*zfrD;tmV=TSW)!xkNdt7D>e;L(md{pk7@)FUjug7k)Cf%1%t*lOe;A*+r-&69D z$MTW<0L!jIOV8$VM$c!FxEBY}{MMTI=82`{)8n1tIMknPI}N0ZfetWPF0Va}npp%q z&WWnLxXN?guGP2kMIcIWs`6rR4%eExwbajUo_dPrCax@O7l>(yhTbD?0~3T-C;Mie z96wdPw~MSZIg(xuN%~!RY%uxZjFnDcXNE;x;Hht)zvo$$UAM9rb2F@Yp&1<1(Vz4& zmFs-{0$ocp>l-hQu1b{Jb+X)|l0B)65suz@0E3j1Lkd$#?saDm*wfT&E6SSosXL5) zSb8e?m3CPx7$V1|^gYMD+Q|8#p>{<_ZPIe;1Fk)OZ&qREh2yeDWplIwlV27vp~m{! zT)rlycf(HPM}$7FzKe=Ci3cGhxB7o2cOJdE_QUnGJXw9$Qupaqi@9m>M7LLJA&iS&_i$hDPf=9BustUL#|3{i(g*L`ip%DlvMgT zPDKfCyp$-bTP`k;dj-#yVC5Cvj1hTW_Grp}3wWbOL(^{`X)@_~dmyQ<({tk2l`eKa zjhNwrOV@VD2RVL?e-|CES+Rgd9Q)H`&8=p0A0Q1f2*oj#A@%T9-Pd$CEL#)g1-5ayuIq<$hxBF zzgJ3FoV>|7Sm)e)YhS$G=l2U(Y4mOHYCzkeY`#&$N*|tZQfmzns};2i*er3r)Y@(I zONC*d)qqEMr0T@3?q1NNrXUrpWy3B;6^|gY&o0TqHuuEkUSFQczPt00{wm2Xz=kWW15>X4{x z_nom=D=+~8K*{BQ#b|C$+&gE1&TA3nlfC6mvjYl(H-z4DHh<#KRZ|z@HNxq}yro5L zd+>o(tZY|_jnSev;ugcBdPy21-@(z`|3~L;Y@c+bN2P2{MubnN#|mT+bG&=pa6EoD zSl&10AQsVpFn{QDwT=IDA!->nKUV{wb~Q>qmuO?h=*dU;0pJ#3fSIh*+B4AIOL$GV zSCl`4Qq3n|^`B>GR^in{(a9S9)Mz^-vbIIv(HIZHM6C+`XTbSdjiiZ<4wfUQva~8+ z6!Fy$vw>rx=ody^o@qLDX3+;9+AK@!@<-i+!pC13?-(W^6im)e8& z>6LKm)Ei90tgL6Gn3RIy8wc{C($*>Z*}L$tjga$-QI`QgGilLB>C3;R*hMZ^i*GpW zf7f*Tj?3z=6dpzvY?L#*){Pv|lxhV3yoS5F_t$#fAjgRvDF$jHkLA<$r&^DBm3L(L zE}1V>Gklag;JKsyv6b9&mW^~CX;)Ray3@69om}fBg6BmDo*G zk2!)3cWb=7wBzlZRX2YnyMBruq^@*pkBUSHb++N()BiQw>PE2y!B@_|&WeD;k8Mn> zfUAR)jkt(y>6X)`5Vi*P`C;PM0<*s-9zR4i`qOJI4H4uF!}e+k1ZEf07R`NvNPK!v zy2T%cW%aUIg(%$^V0)_23N-5-@H1ss6#e5LNbVNQ6}*Icga(XW7f~>==nu-_NEE^7 z0XGZurUQ7GD>00+15I8ML`CaA2>sz?#NM-!SQwvbyyNw>_vb&|S<|_AGj1PXu&6s$ z>!!b81N*$Nf+xHE1q=S@5d?eLMlh7%$o0T~{LP@!+&|xmbY0WGx?1p`*-LBou<;1; ziiBV5nGJ;Qp2$_HoGN2`TKU{|n zr+Tv=+U{Mi1dunh_kIq;s=l*xGHFI<$D>f~JfVK7$~8sJ26VXRokgvqx^M?T2>;ci6EbPLtSb;M zl)DagqS~Qwr~9s4TH7kmY^a|m2o-Oz@h@WGxToQ~N~y*p2D56@BtNI3Bj#nY z8bUD#+`nybr(@qCW?fY9xI@#WvRM6sxQ(taJQD5iI+#@yf4seHS!?fkPl+#`7M?d) z-!jWAU#iuD$T$Pzl7k<0veC_L?5+XBx>0fIUy^2}pDqueH)kNQP8)nH1&_wHx^f_P zC-Gfb!2TJa;gCI1RwEVa@?)7fnv~cA7cAIVWF*989E7U91ILd;HJJ+L_Upq^(n#Cp z{mjHn_}W^{lW}jPhNjknci}URIcuwsYHNYB$`M8HoK7a3&pdx`eM7ik@Q=3>+~Ky+ z6OU(aM__vO<-9sbe`6s6DMwK0C?HEwT)?=(($0I8K*YHJJ%=Vy$x$J8dc zc4Urrjvi%ibE@}zXRuHe*BR zN(&VA(jN_>cYWm~_t3&gBCAX+wau%QT3WCDpq}d@0UAh&&P%7_h;ra79ZBzZxt2Ww z%(x0$aBWuv$V8v-@~|blp}NyA>qTQ_jx)(GK~Ce|X!{Rrm^mJv(bffzc0|T7rqmK| zhB4%1ZPgo}na?g2^uXUmCK6kqtv{|~wsE88ohOo32Fo#j;(oVwXV#>r*8U9B5}_cJ zEe)Ql)_VQ=FsXBTvkvVOmd<(7+%x0SzFw{}W32(?_8J<5?F$VyXtI`D-CiuGd{LLb z`j(`mef%OVKKl=!H4J$|*E@x{#|caAat{Ei@(ml;X83z0&SYQOXBAU73Iv32ZYW{V z;?^Iuc+ePqa;e6lg9}2xN5{k^EM9-smpQfb9O743SQj4kV4uBNqzw{7(lq(W-pbCw zl{{LuRHv~Ue3}K)+pj0Kz7FW6$XuA&2KVOdfZn&!RW4&VC(#4i0?K{Fl+Qr$@F@O9 zfAluwV(Uj_#VxD0e;qacGE|zWov^Vy$12(@PrHIT_`-2n`*CW!vB%1A&A&-;F^&=x zYnL(0Dn{C}tW>HGB9sKTmr&LK45`rTTX(wf$M zTUw8K33oBHB$ZD_0s^91&F^M??W&hTlZA#$E?u&PN9v6S4)DY@O(u_i)3dD-L*k{n zIuD94l;stUk>c^{Li3j+I;>qG9s}A=q3msTrdex-xtClYpJ1P?$ub_?@>(t=UZI9} zW#{O)Bvx$(RWEaAT?O$phO@W5cc0+OSPwaMoZ>Vd*0?2{-GqMVSb;X3`z6MY=)3>= z5LfngouABE!WWa+Qdl)6$|dz`wca-&S!XW ze3^KDtLmOf&ztuX)kpJNr=Uo0?xXK#6+SLhX-rgXMcJi`ZJ>6Xn9sPMiLa@_tOLs} znL|xkI$j8i)2p7PsumfNe8|&XWIo5o_>2La@Q|3T+LK~`iHXE&JY>y0Pah*$TOR#o zTG-#^5M$%R?TAri6Y&hY)&YuUQ}n%Uo%QtIP_CvLmmVQA4|oM$51;aA%s#6zYz*A@ zbMwL5D{NTX6u)RGi?qpEH?V0GPCb-My{~SPC(fTV1X3Pd?T)}KJJPe%QZ8TTVk|ZX zT0ekOx_agUnbUB33@u3FZ`_v=`rZGi+_hr@Mv(~-L$x4QMTdbPHhc)q@i=tf3oSs&2DY*xlRVr6F`nw z!ts1{xFS+}3XWTu{rbamG>l!p%BI8n$n>5hBoPibb2L1n+co=}G%g#TeR)AgqoAEB zjaqurB508(KCdD6!Zg;m*YAjfh_raGZ)O_?-#jMnzLg`XLf1n!nB$;H76i3Dh$~Aw|D9XZ|;|n@hjH zMqOKlTtc(f8#?z7VI-mju=;H)CmXMe7fVOBW#zhWZg$zhWm#6HxFU?oP#i(5^^0ot)~uk~KTs;-BgtLS_2s^2Ke2AHRXc zq~Kude11{bw%_V7-y5#y;}Cr9DP!(_sAv>C_jtd(b${Z5B#0;*b*zVsCaI224vaae zPPA>$e5;Kv$G>7#GrlFoXr<BI+hXGoULFW(8PgFFT06r+G4!WgwaJC-O-eK1 zW3>Q?WJB4DHun5el&^+vKOPx(yv|vf2;?kv;i;^P7@kxKz_$gc`)#5Z3(ykrC&x2V zC)-?|nCDJ_n5;3h%IMeplr)9$uP)xdkE0JeP+Tk%=(rzFSDh5jB}O!oF@e z@BSwLpjfhvhzV-Wjajn!alYD`HUXy^#DX_n7da~0B8(7Ari=N za$-DPcOcEIbxYPm(;$i1B)H)!gA70FUb(G5SteCoU-H2>t3RaH`W+7g11BU+BM;2m zc#{Mmv)tROlj6BE9l|k;VsB@QpKG#+1<9wF2|No)=sKV`YK>zjLZLUd ztf>8rz$%T3=vDNW;}^to_uIN|nf#brvSi8H@xj&6p5<-q9!nnUsZr(cRdq!U>g?K8 zUH`K8d)w|Q2UzI7~Xm00$7E8FCT`h)5n&%8hVB~~vm6tp4) zpM7ug*y!m~E)$niN3_?jRZZ&r~i=`CwK4L$EF{lvdk$1+pc8p>Bj7BqZi|lRjQ9k*%EJ={RP)RHV6cDz$FxulNtuH@VV@HG~Z^omqXF^q7%YsT(7v)blt%{gxxc&b$ zIoEi%&FG^f)t<+L9U5fu3Td#87}=zYa3w07{R|WhF?uNpvi+qej)&MGyp0S3r_`nI z&SfmWtLk6-bGixBySi4$I{;ut73A&bAe>A9w_JU5*$zFo`5OC)@FYOaW8`_=zJ@1s zQ%8(~z4I^UC*Q6cl!R6HMPePZs zT^eZ*aFG?qrFBXXP(ru!<8mweu4&%#?wTeYYc?WEO8XfR)PY?5S11&dl{WHi#C*cM z%ejvqLX=RxO)eHf_O!xOxOv1*mJ2=r17w|4cF-MqCCdb5Rr{z{4mhj{Xs}d`xlPq| znFZLrY54$?1*uxqjs>!V-(U?4eR)drtFjpVqpEIIMQ;|Z-mBVNBc{T3(1l}5 z+L-+4cS4u?E?8wLSz~>XyBXYr@lqKP`vJ_)Yq&6mSi#>Mj4?WpQ^^J9kfxFjW74>K z>H+lb-iJHWi}%rmKvQoh_Kq`DEsi+UX04R>9K%Cr*juc;r4q~p6$Q1==t}qQJ#Kh1 zShzy}4}0Go)b!f@8Bhd4u^>f}VntEFLKCH^*hN4I9fBymDosiN3s?{=C?G8qrGzFW z^sb1C5I}0^B2q$A2mu0Q&m-J--`~#e?9S|OXJ>Zy{&(+iA^Dc`obzetHVY8k*+y(R z`*iIa;{1U8{}S!9*KWfXu^m*$+ zn2>{-W53GPSa#{1K_1cp`nIYY2;{xFCJONV)A8pFpYcoRHyJU3P}CgP@q6D1 ztWIXBHk|(l4fj8KrVbuh!+_u+P~47r+yF0cC3y1rD5j?!l5-@2n$*=(~qVzKHP0a zhnd%5c#9M+)9|=SP=m2DcyVy~<@9wrck#ia{*+f>bzBC_OZV~6&+xY{4vM>sNRQ70 z$eO8CYBL!0yjXJif7uD4w>yI-)D~dYYy(y6OWWfhC>>9SO$Cd(B1q6vrk~+}zXeE+ zt_b<{@n=QdnfMfV>~M#L3G{(cA6nt^FH&&^I8#2a^L7B5@YOimK-DD>+?Oo(8&3Sw zr9$uwAh~NZru<7E%ea_K#@4z2>;uP{)Hhwxm8yoEsn>vZGal zOU=}kD%Y?{W!H}+<==42N53A(a6Xx>e;b8V>gU;y5nrOL6sXDT=Uji*Mzq4oWd;~? z4x~&{WLgb*+WcXWj~>k4OU)h*sXd>KHV8NMD7abL;YA9`^E2aCDcW8@=__(3Ey@lYzdGH#5vNOMqi|DrhY3*ns$aO484eAn`lE+@R_GCqGn3>zqvt#83 z>Ji^|V?>ja7OQ}S9+k9+e=e_N5pie|F8eJdZBREc7N*Gk`0j)#A6-*v2baX6kBPA2 zdVF(4;PIKNBE?$;aKhFVUQXJpv+}6U#{O&h?8eRi+6wFYTy$k49R~AY_5To>TmDoS zjAa&*AVtD|0UM{h!XzyKgzD1$+%3( zBe3Q#MI-henmkd7)P;`D77o5~l)K-CY&$S54=-RExJ7e;JCtulzAGy13o^P@ z#Tx<&^BRz91NjNYGutLPy^c7guSiz`)Q2u0><2-nrCs9Tg5&_JY9wc^(@A#j5w8!p z7rZXbiXl04eJsV%?T{iN6GpGk*Ab?PAjG6iA5y)?1XHzMcxm^A$F*crjdJG_12gT( zFb7b2E#^DM)>GTon_X&a4js(BX!gWlkUPx|KIh}5R9Ykm5cA0z1RR*VBtSEcU?s@H za4#kstw4N0QjRLsK(+56wHn(XIuRb%D%~aIfMXZFQw~GH^{q#4Iufp_KxfL)Xy(B;B?G}mz08FA~;2n(&WIW zc@Wq4vNl4M(T6cF6?k$84n79J+v_e+#Jj?nS5+VZ6Kx)xbQCD=UGo83_M84aaCOpV zKLH6H;$q#?xTJ8a60kn2g4=_+RXB2$~WH0-MZ znjSjyjok-$kHeAMpmx(RLt^)8-vj+K& zu_;PQfJ4KE`)&}-{}cw!wd`Wp95xw$wU7H^>H?E~;Yp&2ODU&JcjCJ?bG zO|~I{4fw#`r+8@h_JBWSA9)dGZ&u#XEWonp?h`#_J<$A&wrS8LGca9-O3tW=4p#ih@Hp z8e;08XRs9dI}N6}d^#Iz?>Ku9`gx~T16k(HwDDq)Is_0ii<~D0?4|T_MC<5xCh!~9 zR9Y~L&&{uYZ&)yu@2k(^`hG-)`Vlb0Z!QCH2kg_I+<`YNL?f@u)!^x5drt;NpXJuH z^)L96M<(B#U4{~G+sPL1IPITLvt&#&e)nZ3MLoW^2;D}J_IB%(707t%KBuLsG`9P! z4w5=ViDSie)(f@vkI0QK!p)ZjLyen3N#Kom?d>B+VNa-a>&{cnB^Digv=iFCL`RZG zn!;gWDXrPtZYA_q#apG-&|H4XSLY?u1Z8p&h5-C-s%C$JrZ5xwZQJh7_FStE9>*O` z%A)3(Exct*f7BgDK}oG^I+T4iU=uc2MBY)f{@EOmbQ@C1Wj@%W86eO0`wZu|+r5?y zmuh-@ioQXa9QKL>B?o0Jg(QuwxEjjPWS|Y5jR1ZBZax}bn?opS{!GBBo9ZmT!6n-@ z2C!))&4uJO&BvE%7tkXtK@f9T(iLV{7&K)ZlYNtY^HylvPbRaqBb(uKvd~M~J5}*` z?_nTj&PR4k^w^wQ2QWJJNOh~vu{(Hg*7NwK5#Fec09>kXu%*>2MZnf%IFtWY{oWHh ze9x+K**8BAtB$+ImIFa@McI-B*`&eSr*Kk-JAwr&B=(mY6n%;wn)0>z-cU3Vtet&D zu_i*Q?43HHf@Zj6{YkYIcP!TKAmotRg4fAyMNEFJsOl;$y*g07$UZaS);jm6+IAOc zP;a3!7H7r^Y7R3Jr7#$ zg>%tfe=dOcE06{othzzEw$JWZF*u*@RVP;&P7I!KpVBLd z5M&cBiuN~0?#ozza=C}KFPZ!@f z2pXxZbx=TxYHmDJG}ke5d2Mw2%bxVjXlE2)uUBkfzLBwN{8?-pJTHec8qv2gEK9&9 z>9%K?*Vt1_jqEz|>@r~%q+H4f%E=jSWD6-PakmjXKCi~;DwPo2@}D)|xd z&UIn-Ftt59alMd&bVUaA%=8Wzuh3#OR~W1eVCd<(`2Z}+d5DBCa_i!?dF=!Yh*1YO z-hYH@SbXP}4sRNtnDXLuFjq|V)2%!l`S8;Ys98khlCa0n4nZ(jp)*D8X3}$m>ohn{ z#-N|b1103bIl@X8U+2$h*{j?H@(%_8ziGfGrSEhD$zA$Aox4^L=QUa|80}OsG8gxF z$v;TIuFF%=76`E{DRZ^F`|L9yRSC5Hk(g~%zWs7ZoR}KHCat5}43btd*`|I`MqbyX z-6n05b@`;D4Pq_T5^n%O?cjiStrl|27iEAJBpYgL3v|~5<2n<+@-6RG0`^d328O<$ z;??|+9t5ySE)3XhYL)^Z4!0i9MgXgkC|WTbap?NJG$=4YrwlIpGoFS0cV1HNB6CM)PR8bF@|F?m+HqOfD= zgU`{54{1KX{B%W9u8|^B#;<&{h$HVxZ{bHTGN_A8^+O*N!Xaq~aY*AUb@=T$w*L{` zy!@m=PwSzw>Wjt_D;t?2xS~?#kP%-=gu!4CGySh0h4y!_3&02o#Dlk+h5jC1KEHO~ z__vPQ1PpzHMsRZemH)!QF!k?0Rd<|?3|De6c)7lw|7BTm_tD#>4~4J|`$j5s1tDnN zD`B1ojoC=1n};TxihFDlfB}cVTZ35YlDS_IGk^)4(;EaV8j*{LN)mX6hg)#BH4;>@ zud0CpMjhXKqz#7FW^>NsLvG<5oi)lg)>E}IZ31}Tr)0pU$@HSH`m&|%{-v|2QGkhl za^fECaX5|hsIyPj>H4}w%CdD!?L@S)*H8c?*Ea~%P!+vqmx}D8T8)g_(#0cdBul2R z7J&*h4XBR0L#c^H zWbjvhKOEu%=E=@^3d9EhKaz>sQAl$9;cWr+3z4SnQ0X^Mw=dt+-m=p`L582I1B$$3 zy zSP!lPmBLYs#__mLNLlpmwMe^E;RM)^1b%+nqRZT`DlNPzFHq1sXY|-@{dZrR590mT z&4AEyY_K*0gmnCAr>i|BTtW`s9bCw{RnmpGaOzluCwsclP#ZR@UWyVYr7gO?Uhn=} zFG6POs385fSjc{-U*%4qVzvra5_SNLEt@KC=+K@$m=j#Rp@=*L{ku~|)`sUOAPE%7 z*=X;(A5E^yIhZSmHU-U>E&SJ+`qLZs47tUT)yYgCu{sNpwi&n=UwDhid_Wk$l#;2g zH1`qM5T5=5{VQcPV9+)It9BaDqvLd>jfhb80ej4N#c{02DHZm|$g{TgC%hI!41%cS|G9ieik9Pm{Mu1F!jb@AzJir_wbIcFZ z&|-YYW**>Z{j%nteAwOvQQshqWzi7_7x8PLgjWbCfT}&p?)`*brzpk=V8FwyLly*l z3$TVh+?5;n{>L)D{YunytNn7`5w=%j7BvAHZjvhcIG7IFJP z>3$yN93X#mYZYjtOWjP{;7}#}8qblrP`fX2Y_fKLs#^wsCmy`ilA0c_pDyXHChEnl z9jE$upE7mvk$5NwgBxVU<*#tyWv%gXpFcE`E_*jL|<(|4(x>kw%XYDl@@Bx#R?~aj*q_wc|jwoR)cAQ zl@j|%drj1>_MFJh>$y`aJls4+__Dx>vp1_Tav(Vrw_7)t?EeZrJ*GugdADLw;U%x7 z*?u=S{O_taqXnVHB&7{A?{$m?eoBrA85R6lpZ*o;Ia~q4Wsi1)N>)fm@q-;mkh}Lr z@dz=c(<1gcPU6t!O7@QcZ2?FG1c}s0R zam0+~qiwSmtLQdHX-4Jn-etcOFmB}__8v+Wyw{xJyq`vH!<$g}ya2H5yCH&{8hAEm z(+-C_B_jAnebTTAu5PYV+dKQ4cAD-l4_Rs)=P+F<+I;Jgh41ltEO}aKnjN=)74Pnb z7DPwXvB-GQVANk6mLl>WwX2f5)MaDIs$!(IDV?53-@{o9`u#yl&}C2!(wbY$K0ews z-UvAm^b>g3*jM!%MV61rL3Z%faDha1vH*0J2)GV(E3UebyH*Z8BA}Z-X&`prh5rM5{iSZ^|{{+aCGB-UXoLkn=><-4^xG zqJdI(u;d$93=9C&5HN@TB`I(DW$FKdgMy*cOwP9OHm?blNF?hf0LR+Rl4KZ8~z=dm#8YMe=5SQkIutEj1d2jr;aG!sh9=v{VUN1k14bd;25d+(3u65oG+cgd}Vw*2`; zts3sFdKCX;4&f(hdz}ql7l3QZ4OTu?uK&oU!l*D*5sdi&=$t8?5UxiSEcmG<~#!^o~%ovz4Qlfpm! z*-a^@PDsC?Wjkkh#zAk=0AM~KDY7=mv(2rIkkt>04Al*2vFmbzDE8|mgWWIM5*mYmu{B+{3;z5h6&-iWE0mMrIEX8> zJ?1&oAZ9z4wFe=KGyQJVgJ#56+V53ZpnylDsI=^U8Z90yQ}6>+eRj|ar29Q)^`za` z=HjM#*McD#HZA1L6yz5U?2}%sIDv2Yb>`UKH!TcSBmNTHvv4C#%J;0J}k_A6E z+rZ{CD5YJBuD`x*^&BI$rb4HW3~@Ce*CWB|%9-&l!NEd^v$sSV;REUUmEe%J0QF2J zkk|#Sj|1oT+88vQbBA}MCFbNRh(CDUR4@y^^?>Rd~L-mZ?^fkF9JPiBc*$8-KQ?e7Y18$@}~ zB(5cVlby=8YRS1SWKeR%9JBDJH_>kTb88pX5HJmTBp^%_9VhbwyR8b&mT^k`u%siO z+#eSpnFSxr03cJNeM~sb{*Fb!7eAr7OBGMvJAN3}j_K0%x;?0AB|}Oa@>wL~R&!*K7X9i3Ox;^PS7hg%C2F1xl*? zCxdV+KedH~8hd>4OM?k_Z6_VVgeWyqWwy_hnm){ox7)?{5G>`ilE;M+`OX!Jw3TOr z1kB0ngg4)aDsmlnoVeaNU7%h3vH9-c`fc^(Zo#8pk!aSu*1+7VF=73QL9kVf`t8_d zXd-m^x=VF#t&wO?j@31I#&GwPZunKrAIvtZ;Tu>2L_;4EXbnmE_u!s>XBTFG-)R)k zY*#Fqa{;2Mga-Q;2gxGwN#nCho>&_y zJ>lyx2qYkduWB=Carv9fx^cjr;p96g(68k8F|L%%85@7C%u@5w=zdX=4)?K+X*Y0;m{J_-DT9KL?Oxu8Ynd-e@s{+P#fywmnbY)7N4}JFO8OehEggLwr zkDVBco@$R^t7QJgU$gU4>=|TrOfnf)636pnkiZ1)x0ZqLYiNd7hNyeYJvZ;l-&5sg)7r*^7k0E$@Sx)xty_{U+@}WeriYK_ zN7vwL!-nP$h~)%UK|_<~!(5V7)nNo6@CXg*DLiQzu-!XdYN{Z&YLZY!2}5hfy3Mef z+<&iy@6@gzU~Ab<2N<#gbelF15Ese|acgS3k!7WImuzg)**+i}@?hUe8`6FN-1m^H z9l5ljh~^;kh&4X{eX?B6DJ*3AUZ77q$;;pA^)33S%~oPl&FvO-T)n#-e>73u>AGXY zj8NK*R)*<`b7KcK_XB|8+Di0oVLpD4O4hjt&Z-U)_569>9AX_Q{hYA-N0P#Fd!=&c zQPKqtFqe4C#kRxpJS63J{jMfg(6_{yru8Oy^3aZ^GVr%%rF%>P)SU^s5DL~<+zaif zMbiO@IZVN@zhTbv)E$i~M}UswAEfu0j{JOP+L;(EbyO+phdmCLe7kpP6_e>8i(3k_ zQ@yfX*&ZtC#U>l-$ET~+e-44Kp%sSz00BC1QXOjRl8L_{14C@c#lCdz*ZYOh<&V14 z2z1BDdI-9eLh%8Jj&BF-#4s#@_ysV&cJLaHL7i5goJT??e%D2q7iXZ!rl5dRIDJVe zPqioX*}iW$F{yICho`azWWHA5gUEzv=?>su1>Z?LOtg0YzYEewJhG}d3~S)rz|StC zQ!<*2twh!Er`d>R%Eo!y(@Qdkk^0yafBE_h_|3tE=PyM_8kx*>MC9dW#`olilp5HO?DyFWh-*1F zcO59zvxZ|X8}uWEJjWhybwLMX#NiwMMi4y!+qKWgZAEL+!DP4YFLkXy0JOx7*4)#C z*t7gwq>MYUXKY?o{#Duz!TGhCbsT;gDVRnmzfI7C6!GOKu zMlyi@dz*}PByG-`?S%$hHu!hAGL(cU=JeH|+D}4^I{nNZQuV6wmm2{oeZNF5-blUS zLJVt$?dPc6tNe;36uBh?ps@^*1ftx?Dnw(jM1IZ#6OYb;)aD&*sPzQTYQ-0}=v5>!aERKZ!N^Y=|4N2$Zb1h<&_QX20oG#^SUw z7kgjh)Y4Fm993}rH4o?F`ckZTFATBXGK=rTAq@4}FW z44e;Y$~Mfr9>gW35V~x1SJ&^hjJvR=|A)v_&0aK}r$P6Uesd9}+^ud{xlJ3t?A+-q z3DF-#-FXhU$l!JV5(8^eqXFn8*|s;*?h8STm&<6|Z{;@Ae{S#O8C6UgA0Kkq=RqY9 z>Hqy7^uRpD-SBodL72$+-aXf5p)(s~Iu|Y1bWHClzW=EhnDv?3nTOGLr+M*A^f&FVO@AAhIHB|lf)D%<)5s=P-M`6W0P*<|;2|j;qY`kMx;?H%(iJ$T z*A08F$b5Vqw;X24Stq2>#8wL{sS3HCf^)|Tg%Z$Uv84xX#0(CX3-ullVPOrnWyd^- zCbI$>*gmobw&W20iSJQSeGE=h;Uo{`tHs)1M6dLLQP?0X-Jq z=>z0g0fXmuLl%cJ#Bw|G!)17i#sU(q0l!2q0YwW^*P#GW&=#C*-38&osYuu62{YOl zJlsK9SFnwVm)}6Pa1CF7CQ#e*li}7qn=gwUg-uE(P`k2W2&pRSI__rcmL`{Qd7*x> zbFe~q+`?z;%%3UEkQBXS=SXVf4gIq=IX;PO9#v0hhn-M?1Y7%NJbH4xl%>1VPcoWijhOf!3cOk!BKG zIX;**km07xxb!&X;*v#KZ^JHJp65itcWx0STdYk}E+_ASB45QeIR1x~*c(9!bmhRy6C378mz|x_@cWocuSGry_~b#!U_Z(mLqzF!bP7nWjbPEDYNe-2dD} z6qu;*YZNu%tG~kuGKyH~)Io%OSq~1esjPHj^e#p8Q-be7r$$T+lr3YT^e zdf79;J`tF?$>CWxMXq~$eum}JsAh=l<@YM(kA3rHS!H;q2h7#~5&%oCLqH5>~d|U-a6sD1V=OLf_eGz25KiKzdZsQz0svMnkXe>Jbd`pGNJ3ZP^P%A z5o{Yo*z1AMdl&KkJ(z4=8_x8}i0;5GWwTd@ zM;?|9f35+UVM_#ul}=dW!^JCAXpa3g|Kg8PGD10<)aXO3_hb zS3f0SAjqwl{+9y?IiMq{v(+%)u}=l-d^NX6V^WCVJp6;OQL|D)dYNNTxwAu&)ie|K zf3zm9VDy%)8x+9Nh!7C^EC|1TEOdH>r8?@_?L&I!Jti$)g?3W!)JMk2?& zL-BZDit z_34?im&Ga(pWO`^E1K-xd_ZEH45u*Ky2@xA!%dEvgRYuL?3Y0xuz8cTm>zjTpqm){9!u8vEo1VhQit{ELs2xR# zwNZ*k-(Sw?#vAcYSNB&_YGn>k@J0%{0Uh>PAZFBsc9uu_i|F!~b(DlFDGw>I6~=e^ z*oq)I#tu+0#A3I;ya8cM$bImUQE+BrZhLD#@qzXkXH&3|kxJ%&9oL>O{4+B)@>Y0H0wTAxGyz$&NIt^5H28&_N3xku?;O-NqDg!)r3K82)SXj*(c zw*Wd8x0aO{Akn$_|8&K%7}~PO(WwObxtI1_&kfm_J)BF*%TKj$7)MO-3#3my!@oFn zKoJ`Is}9`H56&?S=F1Y8E12LTK^EsDan|1uirR$|9noO}85688c^3(dOo;~OzqB7e zLPrih>t_uHJ;reW15z3x@wf;Exms|ss%yC3WiTeb^Km22$sw2U(ETXQNHA+DZSWial%@{ayPQsAEWT*s!BqDK09hajruEP zdgHD5xVx!36+2&eDB4t>^iW!nr*OlUPAor&75A>?FWe!JvyG>PCUv&9+G| z_=NNs&r4SN0Zjfzut6%ijA$YI6EZmf)^#KTL@ub8MJbAq?3e}P39C7PZX+5umU+qbI8b2Q%li+YI@5=n4sqQMvsZBBTM2%%|RC zXdoFzl2v*A`WB`wD13xak^x%5_?KdD)OLE5|XEY!A12y_$ zc6?HKG`@1G=(O&2oO}0vCs*oLT?1B#qeeF_`iQQ}Zk3SG3^b42v`T znNAweA3q|uAmP|}Apo!zMg(Fw7(F%*xDRIi1Ff7-mwfV|Pw2lHy`K!MSaErB>VV^{ zE>I%%%);)%t}%#okL+(*nPAKP4!~Dxgkj%pvD1m%4t5e&dW!fwBpA8m!Sxb;N0@@c zKNYf-k$X>oS_BJlP-})Ucfj%oYO66Y~)_g||mj9q<{a>l*r@{zwA(>Fi(`^WP!Z`BFifOrpx$3U{3DUdXIXt{Z^92aKej+~yYa`utP z27Q;nBDw>9pXek1BS#~*Vm1bCQPP1#M-;^b`5L(ZiOVF*t;K9S=^WVv309t4;pVMIzT7Yb> z*tq_8@UZB^tI|nEBxY-&y4ZQ%vj<+alztI0VDSZM{POQnaAKmW3DBrH5FYI8({Noc zo_^y}FWm9m+~>Ad-xOBI$PB@`z<{1}Ux)G4_$_wap>TMw+aiiiq{423HU!5Y(W#Ng z@H==o?WnK?<+vs%Mwms~_mL~S=BS2uxpvrvz`8|*`~ZRV4KXVy6BPw$NL1^2VN{a1 zhw#0rBZs3lHoF+&W#HahwHkY-p(+X$ZZMq%!_my4sm@5~#dv@|Y<#}mfWQ(PKCcQeoIPv3EqlK^j$J&ZSb@b2ltz%hox z7@jUcn$h3KaVHuQ2N&@e%u9>j?S`4#JLfOpqe$dAcrYc9qZf2fDY&vyWb;)Vr`th> zG^s->4Yom{WZ%3AR!lF5kA~&HptGY$ou}`~**;4Vc-fWPV@}?;1?am4bX!F*pNzq~ z;P#pf-k2=tijn?)?8Q0ZKK@nZS0iJsw)@^h=+`*cHL&ZfVjqt#m9{#P2pNrSphzF$ z{zHmH>j#kiiUII>S7-w8aJpbmdpQ{eAAJ|CMr-HfP5vg{H z`uqhAUG$(k?y3MY$e58}5a9f}Ko#9s-GRX@2yWAxYSzIO+>3h9Iv2Pz;~^WJ^UpcQ4Y7=xhZ_U)B@Cg$+FeL1WP_F<2%5$a}d_56BPh zw$}HTHtO+M!t3|RkJY-01*2k{gS+tAkcFZQh^gdz<_i>MhS49B)?({|dde>e0Vthp z^?NS9a;!m_2X_Z8>+YfEsHWi(7TIcbzcGsvg~iRFN@+<(>*F<0X34!gx9d`Wcq>NbCpcjpPflLXE>%h1 z&<-ZNR6umnEA?=%77l(>c;9-H9|D2Gk$hP4M9b>?BQMbQ%?>#CSaDCC>nBcAO_pxa zI-A~jwnKK}t7;5}quXJ2@fCOO;*xmSjEsT?8TP0i-?t%sCsKnUOayz(#%Bsxz@i!T z(pvwszfdq1>FJ-Q`$u6>n*GjG%GV|1oiUlFk#7^5^A-?qUZqm*yG;J!lehPd)-12lhZE;4(R*ekA7-xG!oQDJ*`()=y^q|9>o;8c+!=eCoj!1peVj8#y_q;HKAdq1Yr9bxZv4)k7G}qZ0~AO3;_Sx~?Y zbaj>bwj`cdZ)QJTin!%_Au!`(SKbnL&inBjJKD)_k+xyxfMJAgIDwMuASa`MCvBVX zdW`SP{)%d8YsTB)=#h%lhA4&McBuM!uCPF;48XBGyk)_LpPI8^$m&1Kmg?i)E9HHa0{Oh z$j4C|ywgGV?+Ux}3NNgcajO8Rwu@?sODxwfi6)V!v|hV{oQneQ&epI*UrhzfMm^rT ziX8Q;0*2uE&6B!Xm3<}1dJZ}e1LzYv&u_0)tcS9hr_5zv7K?x6A;-W!&WH}G>ltLI zKnt(p~m0MsN8X?%8JIX)Sz0tZCd*ox&ZRLus@%-=}z$QrN?D8|aj$ z!uz1MDhIxD2R&_9sVaMu!t>UOBJQ7O7>`ks+<5WSmDXo^V@DTmsx7Ieu-^2mQV=Nn z`SMaHUa7q0E)!1hE^nkdwBKwvMp!81)jjaGrQV9UeJnA@p>Zvit)4T51@lFH@~3yh zfyn#&F)fjD;fEX{lF!~b>54tzFzqP7$gMcOWt(^#T9RmJmIdni`Esu{g+d}ePa(fl z#!V42Tk`^!yc-I&isMxb^01Tr^oN+8JQp~9r@FOI)2WxfA}|+ZIrg8k%ryXPjX)&D z44hF#qU}D9FN8`)q4=Wl+~~v^!NM5}ri`;*whXoArT81H86RHIxjch-)Q>~q5r0X7 zf6Nyi^%I)}WYd()ttVm+ovcYIc}VEmig|uw^~AX!QwRpV{%IsI**X)>tHRi?+wk!h z!eBMKfUfpyb4uv2i4UY;jw^|5nHAL&)92mB=p$2lYQqkHkeC9MRN zeb+Iv^JbZR4}G3fRy}mc645YPgZc7`(5XHajn&lJA6L+AO(Y@rS-k%R_ig~KWMkRa;9Vnu7| zqjL=MgUc;kqH{Za*Gv)``MH<_J9Bp!%>RgExbPWT4D@(SskMH%ihD3!bvk?cg{oUri$lSZnM*G?{X{nfg-NYEkazow-VpQ`1 z{pM0QnoSGK|*Acn7cYR0c`Hnr(ot!?{|OajZ)!oGuDoRQmdd+>?tp)#Db8 zAxXlv+?VC<1t%6v-)Wy#nFiofZgi2Edml9aMvF2H2eN(@XeOm@)Q>;rQbvqky!C=| zbgo`UF-&H&^qmLEGZrW~GY{+7pRDc-O<32hSNd+V_;<=I;vVbAYP3fC>2@G~yWL7r ze$qpjt9H4Z_}q|3aK69=qq-ODWz79T(u}WWSi519cDm+FU$(gp?3c@AWJ$wtjj)0e z7;Dmf+CV4LTe=TXjcVb2J1IjB*lp|f2bJ?QBm?p*&^t!#r<#QuiTZxdpRcRhdx>!P9H zxqCKIZ<|T}aw!V>)P~Z}dA}!fgsfX*8&o~EV>TK)A9Mz}R#Cm~G`)v>AG#1OweCcB z7UyqoHKoF1w9g!|PKKQ5^{p3AS4n&OByMaMIXV{{Ztb_}GPY6wikQ~@K|YP^L6$v9 zJZaw3K{FBhWt20}$1Nrf(sAL^tx{4;0+$)}y)D}@J6n}~gfP9^XHE2Qx1IMI1+$Fw z>ha7CkLFQsTn?AsKk28+KE3S}3ipvqj=?=FZ5v80y=SE+pLCIBJ@qO!^pq9$yiHQJ zzdzaup_zN~WLAg}1YDZLRrGRNMcvX-i2}_nJ-_sum2S8QYnrt!ZTtjuw<*#AL64&_ zy_FQP;msW7rsJ?>3vw9S}QxR_|eSiixh6G%4e| z_ZdbxD8UP=VPoe2_a_-G{p-^x;$|0S5?k+mKBv-5Eh{BBHkues^YZK27IiJ(@p_&} zz!mBbj0mRkS&+KcJ|IWA-5U0V^=^_^XiqNdL5Jp_6GHKpTce&fmHq^29CwcEAE>K` zUr_9lXu*58Y6XZvh4n^zLERdU%ojG7YA_VB5>n8UV6_8gl*2FUl*GxDU4d!Xh{4^J z*e^x$yy}RDv4*YX9saF&4#|hrcNXODk3uI}xA)vl+hF06Ylwg4;+Ps;tKirYDoQhM z(U33-@UWPdIv6Kzv0i2d_ffhp=9De=2-Vk+C5QEd ze%R(754jO^W1Zgv>LDY(`V0>2ljKvjuy-l_o&S}b`6pJS=ZzZX>)mELVh`zF&z?VN z;{#;7fZX-;o}QszqkHre;>{Mz?SZA5k7CSZob^i*7fZkE{ z$+9ND?bRKVmfndybHY0+`d61GCJi6{|HYH>B-npx6L#mdro19jtfgK91%yrC07`)DYfRxw6ETdoKx(i;YpSpWeo_yPUYH_ zBg_oP_;N)s1vBp7!FpNk4yZI6JM9&~Mbf$IU{0#3znjaQihpKSdoqS6QlTIjQwYQT zxHqk!X|)eg{`;ReQN6K9t%d~r6ML$ta{i)UkA?MCq0cF&)Oh&jXM{YAZJix6Fh6XA zyN~Q-$5aL=yFO7m7pQ|40jp3|6M`zX{Ft}49J2_K!Ub>4O{V#n+!pbG zE4J3FgF>*L8+iXDRqBZ;$!tEyk`r{-`Hz-Ff>wPzMq%D{I84neWqsL1*741D~5i1%>LVHNmHhg?6*E z7kL<3{HBG?2eYX^`1z2HVMe3O|A`~W96ac;C$zM9nClhyAG`s#$Yq<^@KWDoeG>qM z?_wy*=~9OCI}h+MlQM$>G$TP$7=EBT0*_@mmi7-DMA->&iyQK6dP80!PWAQhZXfKoQRI$4}@ zZ$D-de?+mX#d|8+bNm+Le2?%5jBXbAp3+bY!|Q7>qQh+bl^=E9$Gh^!34aamT>#JI zc-RHHjmTT^n`*Wx7tNy%Vk7o-mBE;nXG|)ue0wzRv^Dw;xH4mE4T zx<}XGKf@}=L8n^ZhRJ9CbGu+%3(m3bs#QV(lVh|5UR-fG?4 zS2j6Yu6NR9dX)`!6yA8I?7M}X*rzlfp1z9u_;CG^eaYcFLY;11NpkpPE%D%;k3^CtFaZ5Mi`ivXObgi-i zAhfhottqijC)`MnXV-rDB?N^!(awC@fHQ`r?nT-znZ%kO)((BG85%0oMjM+jJx5Y# zOLA-TXdUM({2u+u(ESY>R%~4rIMy?~mW2ZTt^VzO&W!tWE;fX>i%H@Q(t2AYV&K$U7acF+mXsEIwqxk# zJE9*+N~4H#EYWLIV06ZNo#3p1Utct7nX8}uZ0hICkU7;f;h@zPfAJ>UM+(!5Fa`Lo z9*A_h4y!xP7DIAG`O>8=S5gwQIlY_wKa?3CvP+!TJbG$N`5eFys{53;TuzgpHv8zq z(#Zb`^sz-3&9GZjc3wUzgp89yZPaz8#JGp+1^EP6e{D#>cJVF-s2yUmU9y+}Xr{k3 zY&gdF!#ff;Hg~ue_u*}GzusM6@vrxWFGYy2%aPdraaFlrm-a5D@OYJYW9RxbZS@hq zB zkL9G^ zjGLEXI2OPVhq8YS;BYiLl!_0Yxh4Knb5Hi{VwA`&V!u$-p=PS3Pal1&^`4|n2^6zttSTOt{y=%kkWWVW@ zgt`2=#@?f#nJj(->zOQ);J;rTR1QLsHaquf(%ukDwp^sj+@lK?ri~mGYD%$XJnc{SvHNC=4bPwp`H1CeAHIjI<{&zGfZIE)lyD&Z;sy z#aI&<)}VBrXAV|0wW{47XNK4?-}1brB<}Oo4mhQbtya)(EM6_7;LMRa0eYhiTH*(! zXU5jtx7rAsO)5N6yUz*s6~R;Iy+phtsrx>LE3`R7uUcHmt3|;@QzbPxI(&;!HMTVDtYVDI zPAvw1*~FjfOBK~>;j`KdY2GDEeE5NL&CQ%8v=c@M(8n6t7z?vez`x}(c5I6V`!26F z8?S)Wh2xjl?`x#c*;wPa3y~j1v=}yQwVaQeb*_~hqxoHI{!R>Q34M!lA+XQqR?$HZ zS#;lJy{)iU37`qISxnk7%O z*IQqYeIZSuuV$vTuD#+{5EPX_~Ee@961Oft?Kxe28nVrEo_bT+f}`%yn%?wy_ha ztL={%lG!mM!>JlOO7SmyCaPq2@M1)ZS3gT=8mfgT$NYN&LUhY5iINn;9R+Po&y%%s zQ%E#duEP);vyr~7C3aj!q#RDg?eZ+!7WCu1zfl~9Kc<|G^w?$c*kO}m3DbKvBkP@ac({fJbq&T)_nue^EeFRjZj(wh zje0uGK0xff+8hF!!rP+&!R8DSe3TsHuJz^kAEMq`zh@@6ZhprH&UYf7qla6$u4Tl# zuayozTj9%!@vE(H;@&6S;(eL7&iYv_{#lQ4kB>fY8)uBn^=rU*u#UZ3xS(0m-G1l@ zw8#&aN~`7ae8*c=hbhV>K#kZF;##&UTp&KzEqMvu*7(_hA^CGTga!^lzwZidFXy@Pe z8QF@(H;Xp=!&>^q+cd*cHySJ~U8q(b=}Qgq2S1;y*kw{iBR>482IDMAs%c(p4}P-u zq@4QQndH(lJ2AWEo(its8Oe-$Cu~|;_Re3l23#y%^_JKpDRGzk?p30PhIvJel~)kI zqv?@^byLv|KnE%b4*1iMpBmU+-RIiMVlKdm6NCRh%taMK4dUbZZKlz8vSjv1~3;KYa`=>E|i!8ZhH`i$d`e?GX#+Gw=K5;@$>!#TkjAue&e0khw50^irL0 z*>viOeMgbESM*FIuk}`IJYAtB-{W9INP+E3)2!#eRy4f@XKpp`sLoi9$;y4Q!XE7X zgk{vNWtL&FgB2Y)R4veeR7(xE6B$O8qNxlKH=WyBu@u^b^KDDMcfRhgX^%2zJm1~- z^ZOPz4X2>$&EdNzRkDk0{Nfo5IV?@bspYQf<>1V`To0%-oZGNRRJ*spFHYsI>1{9b zTN3`nxz)3k*&~VTAWrK|s7CnY1s5>(M5c>4Gxwx6tdGsm$s18_LA{Z0yXZfbv-gcy z@b)pckpo10+Ck5Nu?@}IfNZXrxc2E7_MIi`kJSr1#8)Y;^ESLWldvoLnMC66fj{53 z=Yv<`C5V*&k{%XE4}UBq&>n44cqOhV(sF~wJjF~|P z8!oEqDD)g*9i@p4!nR?k9!IDW6+d_JUV={P7$hQG46kEYoL3velJo#{MY(sg_I8jV zUoO!1=>UxlQ{xRa>8IQy_*$A-3zTHbkf#82J-wWprsqnOGsx?;T+x_$K6g&HUTot<35>!xkc^T-|T`T>f6bu!Lrk9&OYV?{i+3cI7+K}NVD zY*lhqct^+7>##B~rd*Qn6WsvE3uyGm!R zRS>iKq*TTjbXV_mzFC1iTMa&afYP$fMkeYGDG~j#6TOOmSoDywkudtJa#9kKy3mbf z;1#Z#^a?Ri`);IOjB+i2sMhpa3B=x-6?KzK`0RB?F`_vFZqI@yDgr}fb~q^}WnMgQ zTgYjSQyC{m&H>oT3nfH;BkC5tV25cDb=ceB2x2#O9Y{oj_M#ez=y0gN)m({EyTI2i zflo@Yod0rkm8MaWx{e0jQuNA_=+Ij`x# zyGRqn{=Ly6P8Pzca*Dmc$!2~jsgJez151r6B7GfdEeM6lJC$<-rR6zO5m-O;nT6n` zi^k~l!JhgiCiw~{_av%jp->wG_W+HrdFE6m5+VeeO8YiF>RXw3~t z6d(Vh#CVbOBRQHzrf^r8!Y^7Rv#m<)iSHRQIY)&KJI>r7@^JcZpo9=1uU(uE`6@!y zmRdKrFL_F;6izGvtnBm*91Hy(TWWO53oHVq8#meFXF2liOX`$o3&ObY95GMHO`ZK( zv)U$&UVd?$Lep6&LW^waz=#J{BU7TmHL`0 zbBIJRY!Jxx0q3lWK85dU*vpp-(rOHQSN_7hG^9P23GTi3pQ9A`gB4%z-wGu>`ubgM zw_6S^q#%b%+Ro?(YfNfXB$y9;z>vA)f#sxZ&mBEU`UlR&borblVTeF9Dchi90wGTC z*veX;vdbh~aw4bLv?3NZiV+8)Z}Og13ULK>J&JQ>vPY_pLWOD(Yzp^GA-v{dC% zFsKddLyuGIjxr;X%KJ zYY%t-w(fo}{BTy%2@q^-dAt`tz;TyuG@hM2=Bvd^mbb%yYOU}=mYHv~;QYc$h6$IG z^;i>I@x(o0)abUr@)E23Cj)DNrO?4$p}Ez@FF}-5Yx%+yEa4ms@`A=QxXE`T8CrWFZ@^MVTkPB7=O!Xa` ztxg!0;KH4@p6zQ?I#XwTVhX=7Z`x7z8HAgJnqSJC-Rhh|XmHK`*|lo>+dXjDzcR(f z#Q{Pvj|2CS*?3jzqj|DGDRM5kr6<)}_TyN;8c9Y9TBsbbYB9_Op$Ezh0Q+OmW9S#4 z*^8pVQ+?#V3uiU7i1*fe2mpuYKJ@(cZ#;2a+Tv00g=ZHeBpz4}i{4Aq%ZL0PSqJ-) zOX9;^b()6EuSe>Z;usQvTEgpW2K`gd5i%9Z>Pg=dP0H z&W#@yFkhHr*u!DC<&5u58`d9v^kz!6>sxYI{J{socT*;C=AekHXEmd+2W}ep9N^Rq zn2)I0l^Ei|0=n);e@?KvQ_V+}BH}EE;5bJj{`u)4ShA~=6?_5Nj7aG|f=+N;hkTjr zjs5dA;aUgY4W-rHlWW!~lKrxJUSgZ~=*S36MN*;LL6nDh98L(O;;)E!WWE)i=Scwi z9q>^JBBx2}JlCM8xubY8&lN?oCc#m=9 z;xnqCCKbkq-L9A!ViVz(U{gXTXb(@!1ia4b&}L4%dIs=%ykb8SJt@7MMxlXQ`i^0V ziK7>o2YX866Mqa z(JuK6lS`T+taF&v%J#kG-qcxuPf9@%q&Pm4SFV!5>{=S_!uvf)66dj*Mm})9)_|{) z%bXm2?yu0a`xSKhzxgV9`qaVl1a#+|Ck|PUv?U=Phf_cU)*)d-U9RFU@XJG8a%6fU zUK^SW(FdSF_0wr6^r4{Hii1iP4~t!Qif3MQ{tZGu*5btQEQWJC-sY%N{$2GoecYO; z$`RaO@FWidX660QP9RiMDdB=Lt+ex=zF*r%c0=BSiNaxuE&c-f+4 z-u?$YJ?~H$64%zFs6<_RUe_E%pP^f=6W(Uk7=N>M{s~;;XLw4>3d~OST{xVXU6$7j z*x-}Ao$f@Rd|Rb8Ak~VV9f#}fg9Og^0-Ye$r_1qBA+>SS!>}0RRPbvdV$ZGSFnF&* zw>AKrsTy}8;S~a@oPV!kCu->j1(_J+*B{;xa)~Zn6+Y(%#3}Ya8wK;CqVhnz%14A% zDd04_sSJUFwwfjMW`Wdcd>(Wq)k;cma${;+e2-b_wbgDeol}Kt+afBem4?gG%OE1@ zf|x6xj)ysp!;q5~&NH$2`Z2veyvi_Y9<)gzz|LGu8zE&eqlaKP?xHzDCy*#NjTJ0bXLCGDzXho<~N>qVXHpz1$Q z_zCh2cik$-OVAH(QuZ;T>IWh==8BYm24j*q7{!#JB~Bb- zct~WYx{Tw?_olaKB9U@^Lfm4`>9bdW#u+NTs%17%O8!s+n3rQVy?nW0lmY+=9Cz+^ zEhd=KfMSXsLf&xykQOu$*!@gMeruL6VY` zoC`rf36eo_hMucD|G96xd%NH3?l;D}J^G$8&M2wcwQKLS*P3&FVXkL4m1L<7Fde{P zFjR8auie66_QF5+uv6@V7x&1PO?VNvm%eSUYGY#WbjQ{hqj<;uzNL-5rI``Cqp_`> znT@poFP5KI^c=gXz5RVVaXvn)e~#d_vAxIlNyniX&a(ghbuBv#h9L(1Pa@~=_!S28 zN>1+DWi{v5)BSs2v8ZocoMk?7-8*6b5sD9ANCQ4L%$^J&U#;nPcyRl+8k*@vEL*UAxV9O-}wb5fwcqc28;5UeiglZzfB*Y<0jfOg{A`{4U1( zmbt+k9r_ohmW_lRy}pXX97eD53K$CXdWsW61~1-6XfdSldg9kgJ#Mb^L;(LWj}?EA0n0bRC)LanT#gQxWy%A7?%^ zw786hY38nG;m_#P!)^P(?P@=}Q{|Id?y;G6I^O;KarfV!%pUGNcHO3IS))voGQs-t zvgP}xaJH#PT<|UMUzg>%Drq}LHS4CdMj=1FJ^m{Yk^N?BDa(BZjv)=2N z#ad>Yl$;N}Tn=*sl8O@Z)n>neqriD z+cnzMT)mIK>OQUw1;ls+F$G7gY%R8=!|BgAiBG-BR83b}cL~f*yz2XFwIouIGj6bl z;r!j}CBHvYb}(7Id13PULG4zw?D7N*T3lVcJKT75=?u*L7}u5QsKMl-e!&if&~QD+ zZ}z=r=|US?+dm#3%8D;&;QjdK+p_;^ma~CKzkP>H#Ix1kY`bf++Lrk>3-^S&Pv|$S zZI!~W43BvDE&FYX5BYH{*adub8IlaC(=mE>s=ZB?E!K1Gr>`IvHtk+%9Ly1?9qU?) z(r#^w)XlkyiZ%!H`kBJPIDvr^n=)9|l~fCT)YZ z9j3Z-7GP|JT{n%QZtxbTw7>N|_T#O0p7hl& znp%GI*>{lP=K7SuZggJ!-IsD)h2L#&7+Y2^zQzTMMSDmUOsS#SVjRHp!F0tD7Msq_ zv@pK&){W6AO)cXv6T(b>9c%UgLC>jQsC`-F`~9n_c4N&=xm6G9j<0?k^zF0$X!UAt z{h&bO^NXsAE^}q8Tmnsks-cA!S9)+U9u-x}7q?$kcs1qJvF-f!t%s}BWn|vn9NguC zuw=QiGX|?mrY|pT2cBYPT6EU8(c~G?`D)r2_loGvfH}{q(HK`AW(yG}@#Ric+4V?^ zyt0+sW!)J5a)`YNbo;zvB6&F!!)eOfjD=yIx_eAl&cQr!DW0p?Enk2o{wv|`fXi5p zk2fsFuIq(-vll-nP+X-J8+l$IV^gj*(N}0QT0qdxwtV|(Uzag&$#{}V?wC;kk4Ev% z#uPVPpG$d&(|}95?L=yX23pbebeg)@&oln9%P@n)e7(4Hi;PAelCx&GZ_X5k@s+P7 z*+MALxcbRBTr2I{@%599z3`mv*cYn~T^w27?Hk>y-Wy0 z>+SsdNa&bxxQ=zP4BP2PkB&R`oFEUQNvqvJ(I_m!z_Zx4Q$=?4`FZP#ZzeH9uA4)} zwVQ|M)FtBG=E_6{WlJ@jzCWltb2V_Td^7i5^^qHDRrM70wK947Wn3K!0*XS{0)MZh zM(AgGTC>8+tcQz^WXn!;_6Q)@H3<>kNWkx}JHo(ujzWCTw4dxeYar5a?tS1pLgD>(bCs zC-4mr1{bBStrg9dO#N!vR#lb^+w;LW7v+<3%h}?w=2am`7n=L1m;~E#+Y1eRVa30q zx%p`GZn*nMt^IiH&Ng3?lV8iUUVJpZzxGPD^Ui!!;1tJF;N}^r!Tj=hmH_VuA4#R?gC_1y6dTDnamACisX zm#ULKQwqU&?67FJfPM%}aJ+4cfF2H%R|cRXX^WWwR+47bR2VXoAU zT0WXG%o?(Xc|!aBy5bM6XR;m}6-%yuw@aC_$kxs+-kt`lTM50fa;yp-Xm z`+4;$&G^w(+7p#MYMs*e4?k4Gh21V_5h4xkZB5EFwzwKdal*gyEEfaEcNQI*?{tZ3 zWGaegYo67(V=X7H#Qd!C4Y7}bbx~Z>NJ7rS|Jh-_L|L%y+IorC_LM=8lnI#>jdksL zm~N{S1nDowKI!Sw|5U|=u$^oTG%j(R5x>PR!Z}x!QN1ekc_i&~ZLj>hv~nGboC1wd zb-SN|0?ipdUW)2WGKWQBt&WaDI>%#OscD>#h3sxj<%P6a$%HsKgba5_G6e9|%2N{b-DW6oKfC<%bBxa6eR>OR;_Jo_od>NZY7*}|5WH_m&$djt3$eP)@=#CWI6f5qe5~M^99(sXub&-aN zne$(w-~V_-`D=dP@UjP_kZ3*O+OGl2OnTT4MuC^?(Mm`VL!n(U^`~=J(f#oRz zUp$|b(^Dv?cu_!bk@&FKd?Bx`?hsbkAP~0-calcV>TLMB>=0GWz4A3nNmxG|dTfKf zZ%1C<5MJ;1*D8;p9SnF2(cu?XGG=3Xu6QhNeM^X-DV?Y~@BC93GH;DD`^rNw9Q(#?&N#lr-4lEBq1W83(wjtarzVA^ z<}O*h`+{6@h&q`-5JX3p1v3TPw)qzEH#iuTvGKJV)hQjN26MP_BCofH;$NyT)&Mh&2KR8d3 zPz6g@N}G1&sL_>}lh^w_H)j@c!F+sWOSe!ySj5#neCB*8fRJyRJhW_W-5x)mL!srh zvpyz4>87Ahipi1d@a-ex(u!m|2mu8^BKjPep(WBcZ)AS z-WO!ASIS01S~DfWCB0Wi_0HJ1>uSG~ogt&ZVc9QkQ#pmdxY`DjDKz~VqwB6=X{Vt4 zt22q<{`~qdi3XX}Q-^M?)0!Gk-YOR#c)Map8^4((o=IorKIA+j7tpQmo+J@ASG6ELQcz4Fd^S5~&ovys2*U~+*wey`hfzN|Pz#15!7 z!ghzYgPuFp&}lY)a-|8}WmS3V-Y$1qNb8|Q$zc4vRhYWQ*Z4M5o8(7AS)%k>cVFn9 z{n#$gJ<3=yY)FE^t8qbj5awt7uuXlz#%oi@6{>Q(>$G%IDzO5VJ#PogLRs$I!bNAr zS!6hpOJ&_XP;okvDW;CKZ<8T*lu1fSqGg$2aM4$e#^>7ec_<&#q&zl8*ts9Iu0e7h zU09&{C?`Uf7A#K|SF=f%SoK$^!b{J*AxY+#RqnXq@$Vm2Him?sK06&JYE?R4yS}vx z(~PEs!oX{LS*$bI|1IDK^OViwF)gsTS7C^fbj+W4wT+_sskz_XZFw>b70#1xo)Arh zH{$A?K77c&Zx>zSsq0l2W4rS0-q;Cp{~y9!^n6gRe76!KT^wg+i(k&Co120kjjYh5 zIVU4BS~%!=!7F}fks%LfZ^tLNx*c`cx@00vcY-N(~zf}0Jj=zh+r>+J8~A3qyPL1r{vOmQ#SNZ?95 zLpt{Ji}8AfIq9vSDFS;*rHrwL-(-cN@V6B0<#o5g_xW0clReo~g-oh$9=JPqRQl=r zxdV7d3i`c<1(=3=h1A@C;l7u(ZI>DVKsB?x_S8yk(1&4;zji2KGLIobpd^)k&P7!{ zvtY30;U03~&D)A)aRsaM0KWoP?1CLwXlaUMoCxFL+v{WTW0kzMN2lA8WvPdUb{QU@ z|2&&dGc4p&6avNnIjj7dW5xCS)b`b59C2fF+qwj)h_*J}FARE{hoIEhV)VlbBvY^ezmv7PT^h0tKRwoZLGdwVlnna%*XN% zF$+&Nf@Oa*UAB7?N8>GBVBu%@HnQ`sr9nf{U{VuPKjDD_o%*{Vhhe^JJ!{*mgCX79 z(t8~hWN2xBxX;(nuW#N~@H^W7gd`!A268EP*GH#08SY0(3vVK;*-d0~A{aQ{WL#nC zJCGP|{pI8EAS3L5P zL)WX4gnXo9oJm)0Qt(vCzTm7QW|jz5c3B6uDdrYiKgcO)&{cUxcUcjq?unJsCOb9U zflz3jS-|geoaTPkZ*jCn*Q7goE%IVp;vIX^=^`KtC6GH2elvcj`Xdwr?6vCQSH znARf{5FDTS=bFq!hB5Bi#sCLD|4NGE9Myr7IVyV@c+^Xqu`bui zjOLkS4e35zm#CxQGm@yJU@>aM&z~ht2e>NeYwk4Y;nZvRn5vfRhd=gP(iI+S=JyS( zjy-pZZ{QkPnZrC(x{gKAg+?Derfr%iL2`xWdW~f!Dx%8u$W9~V-)rBYn5`#=`$b4= zgfhD8(xx9raE7C-b=Ia0meZtkU|t_p=`J2g+2(3{YNy!Vuhc%FqIp%jdGyX$PjEvK ze5m<2o1z$Z#E-(uXvm*|ncw6|klf|y7#gwKHg1#e!TY7*p%k&S)=Vuo3bd%*iwXM5 z8GYokg@gv#JsR&?0|{&QouN&-nX8Sj-CHk{k8X-BHVV{@yU5U)+G_@nF~nyURTK?U z>`SKd_iuBaUTc)5AMnXj@yq=@A6@5mc<{wqHkC|3%QXe_qr9w;Mzb+Zj z;#Jz*=2N@qzsh5>mp4oGg468l!HXEv2V9s-0S|y2lVxl)-L)$E=@SOIU&z&-unhHIjJ3p#nMKg zrP@aBC5r*m=4$-mn~BBBwm%^`@x)ahwGTZ;0VX!|tD8W*1=}8P28c{!EmCz*Kn=>L zLY0Bp+>f(a3P3R=Z-JpTGmW#W(6W`vCnR?W%_Ssa_N1o~SQp+1gTwJ>q`5{(aRJSsf zDb7q?xc{TF{>Y7UZ6n+8rL8c;GHiI^MS{wL=OryBh2AWG;Rq2dwgftcD@1bTcIe>p z&|=$n19wgY^suh6B!*X15N&zQeluY5rOU6V(}$#Xq4rNf086~c12=q2nywSd#5nSd zkOx&pX@rcI{#+5#$F%-(Vhfs#m=_AZtwt{G)SfZ@%sejA^ki0n(e&GlC2fGu3k(K| zK{MS<*Pg1YS>%*VX00#B0+Y0O?U)zvD|SNKYWtPWs3;x_)0t z?~eikM7EeZ#_EHXgJq%7yC7$!4E*bJf&k~ov-i0)$kWIOYAB zH@yW(H|ztm*G58>f3@hirw@IkYY{6+lqn=+91Z+%vfLMEH0ZI~w0b-&cD|f`?jHU! z_NOBLtV(7f%$gAUK4)H)u0p8pM^~~kg}DYIGI|5#N7i(!XN(SZn8qXx@#+s7olpFw zUzZ^q#8o}p|1souvDq0ZT38CY);2?)!vf6e58J3~pZukw^6gcqvXoe%dYgya&!_aw z>V$c5Sh-WoGnWFl>Giru5>A=ZpEWGg1csVH#ojphInN_J1uqGvHvGUP;JWW5y_^Rm z+Bin9W0~)oM~!K%N1A~lX~6!A(MM=+U+Xfk?|e4Y+eq!Ew7Bxets^6kf90tH}Zy6;%jAJegX5iCA= zCQlUe`{SSA=yM(p>S#^8x_Gb*RHXr-@EgDmX(kgKfL45zcL;qHsBFcH0j}w|oe(?v z8^70t1ZhG4ri~O7^HzpPZJ4?Owo={Q+3o-mGbUWu_A4Iozfh$X4P>KlV1{*LC^303 zfnV^(Jg8;tFpkg*@4k%-DtSYTJ`UuY;R=&KRX}p@^#hr>QKy0~jT`Tq#VwcwO>epa zan^yYV+oCewFN8%Z7Sej_MFm%`sC6)c(5Wa{Xm*Ui91e5B5oo$9?3Nr<3&KqzjV!i zR6ZP@!S*U(dR{qiFk8O8T(Z5=*KviC=M}I81%q^$m5Gki1vZ0HK=E3!Um<*Ufb3Swn!|l`4PXsWOWh~_)K!PXgGKY;#tuB_1{0M zI)LSiMzq(~?tt4|>^(hnsQ0_^$X;(qqJk|;7l2PqN1{Ya>t=`Y#ni$9S8nAE8sS0r zMSB381UI0-F&M+w=CmX~(2%V3I}I+i-{4CHaK|SJpfB@Q<_P=Sdo6Kh5SWz_!&9b7 zd|smT?92bSOR#>$Ki)ozLX#7i`mfplnILD}5>@-ZA+0gh&%l33A@ zpVXE)ei9De3ZAW~*j~{o*f^9BSF%;%c&0biH2&{1=waS{k z2?6r$ssD6-+d1kttkbm9v!>OlHpbDky$m3xB^~^}S!b`*gS0V=Ce6Eij|! zJUz?@L$mYB#Wm{`rPfEx;4+jUm%9l4&Qar<_-@yth}6c(@5Oe}fV3Bi0teyluCFrE zB|R_z;=q`?m$-N*QbN;EHGpKe33yhPwjM(d;AX>@3M;mGoVY{AnA72HcX437Iv^f1 zmBH%sh=3;6`8-l4)bp?&P3CV*Uo=;f^})LAz~;GjXXi~$LASQp zVJQf?9Y}>eU1FAFkJ2S9gGcFW0aoi65wF$3JgwUjh;*h{K1zs ze}JZ1+F?}gbnGi&sNz`;cl&$uW$h-B^3_J4>ppc&dVnKD;^m$iAKs7C@3yp%cH9BX ztuEDG-1d-1#Y9?+tY&nZ?>t7vk1{6Pefg+&cjQ6X5n^6>yTVHU(e2Y!*@l`UPf}aMr zfty;~xVGF6Df5+QUWV%%s_#h+)+#d$pSs~*^{95Q*pmzZ+mogD%0har0On=z;hm@B zo*po~D4Co^E4Wxb@37vpWTQN14+}!7vz4ZfRRIgH@KKUEKp&cLj<&s`5=*UDaa*p} zQjKVKo@O|o<1BLl?ou{`E0F&;@XO6%K{Qv6{WKWw$;ahl^b8;FwWTwA;{VTm`_GS^ z3(t~Udg{rPlMuC{x%>Tjb3wsWFJO9$rQwum?YOBDOF#UzX3axU z&)|ZvzJw#(a_+EW(AnCzNJnX(qw80X(axjDPhz@n-~F*0VU?;+6jJ2WERk26lDK%A zWkr5u1Rtacu9;$KDaah_dquI2Ii^j7bf7g|IUZY5;anc%U=fdi#$F@)XTjKQ|C_}b zcLQ@99`q4)m_2(*o`7L_3Z5v9q5m#z|T9_(DX7{I=+EJvDz((fq!~&Ooc* zSptH`;zE;R@8RJMp|m?fcM_cg({lN^YW<&Gxnj>j-PZ4_Mw?zu#!#V&o6>mWRB95m zS0^aP$?q*+7As7uZWEvZ9M09&F~@v!iE-7#Exf4hwow~emnkq(U0ojzxhTA$FD-4N za!OokFxwRszEMwzBhtTup@E0f(eJC2zkGh=I7e+o zK$H$n_aj3H@1wUFL#{s_k}r@kV1Mc0jG}%8PHxod=VhD^sIyeEuiGw*)+xkl%^k)u z)=n|s8K~G<&XYJH^9I*;${9j#)9@+bW%1xltX{=>#C9ujA<9_l~K%W81D<2sG@v<$;5yOoH-0V-y*3JY5%P zk+3Vca2IQ$pW`D?-hJV%804bGbpTm8t}cV+9*br(cj=#IXnb7hwK&}aXR@0sT~u;~ zWU6Y>{(Xvp)}*?~p9-N=v1^_XPCB^rN?&S+BIPN**7D16R#rYDBjKmSrk}~3fRvT~ zB5TaZv2Wb^LSoG|CY7hh)IF%A3k<5BXGIF0G)d&)+@h%Rp=y}A$jjNspoiVpk4l_9 zsTtSZ0g=jRb9PU?kY$xakj`ArpMW)K&!G!VpN4D^G{DIzd)9;w2+bhbXdV%LU+3KI zjpZ9WO|DZp5qtWefIkt!_N_ zl^NsyUuh6Ww0}chx^TEjxbGa%)r68{m={fDDTFDpK!5-Ds!C$1C0k(E3B8vP#D$0T zc_9biE}scriV2{tTLk)9ZNHoda-?E4NOD^Mo*_;NQWgI50q&9cVHB#j+{?j@pj*c9 zaB45T?(@^aW@k?p1-4u+9cw!-+Pw*Ne-nt9&&g_xMyEuL7PVtNKL{H$MIi=_=GL4 z9`|E(@4JOzcu!h)5TDF#l1zs=Uqa>$M^uv02rRT0Mbh+0EE;jSeh?9!fbMrXiC=7bXilxhCSwHP3d zrplOFvnEIn!m(D@>)3(;GsL`v{^Rf&%Lk=_o1aP zRlCmJ&9~z}zn)}qe~YcqZ2T#`{&jLT_xswmUr%))%Mu}5r_kCAFoE15$)#3N_t69g zRp9I3IvaO3=i-|s78_Z{a{wAo0Ewdj$k)adYCOAy!cY5{1K(~{>~7E5Luvg>z7!TE zKe-RWJV>tXK=|Qt)$KIwWAa1ICk&tXA&(Fc*-sI&e~uP39*ER5#F0|5i1J(E+hmAV z$N^r`G8HjcB8%dUP)2s~FG5OEf<>me6EIc;5G3%i=0R-{5_Y&oJQhOLo!sX-4qfWm z5wWh5nZ?o85OG^R83Jg|CRjnb6SIr~T1}=X9u9O6WVFq@H_gxy?-4SUL9rk;F;UaQ%B4j{*-2Ud|xA*sQ zwV!-Ypf`joF~uRAK=Zlc@SSg{zC{L8VlkDg14vWm!0u-l$k1Skg)i<>T8qH)WG(or6n*yZD;Ex z?tXc04@TE9IhiE+2f$mn=pmPx34yZr2yt1p)-HkFl?Mu94!s@ZA?QG)eRblzL7#)f4z~BVui&Gnh-K4jRA=@rAxXlF_^7W z{Ky5UJXpXREACLA4je@vLw|1ZGBVh%QCTh!Zk=GQf*Al+ zuB4_I>8|YG6T@h_6!4u;3zTh*0%MH!uu_{k!K-Qy6DF0>Br>sN8f8UQ!_}`gP4;izFt4l-2Dh%P<6mT<(8+ZcdO9eEY{8}d`~2}7rEe% z8^yhR&4Z8yTs%%*LuOm1*aPuNzS?I6KK{w5NE7l=fl_$O8i$d?ny zuQoOYp#c$@if`+zt-vuFp)5_(YTKK#|HUcBJ^!&8U4W7wCMKSycIWX2IpU)EU*o?BjU+cB0*j|_!awl`3!2rz^ zIsr8kTTb0YQW?kxP`Nqc-qO4JI1 zhS-+?^I{b-ND_vW zj!gquXwrk4RYFv9;R{ET)GX+I(T^)Ea*H{6;F(W=lqcU`;=r6(G7laiWu!5nz|;}H zvy?vRfEMHicxA&K`yB|n9t>UnHN)26;4lN)-VEtK>R#YI{X!@M7QzgP0)wEGGnCf1 z+Ge1F$y=B+YuCK0;yz-8_;U;@JSSib7)L#>8KUZJbP*9Kfc|L}1y#f$K-irjWNYT> zs%RDg#Owf#`W5%o$=BHk*PjTU{i3)Ilicn;!_Ps05HUz@gkVR(5F+*~(tQWsVm`4& ziE_L++^L$Wr)n8|gS4PHTATpPrdsyKxjDx2og6j4?5fb??n{41WmM>y@XhsqTs z-m?4r>ljxl?APmolnit)TYHK5#G6KqE9}_1za>1C32p zM5x#)__i4k<(bGsgxHXb^~`WS13pr~X}b49&xp!^Y-6d;cTH7)^HQU4hXodVA{p)~ zNQ-=@*gi=S$mj}`buSc6%t3Dz0)Nt{#X9!d1EvjJ@t~i&UAm77PNST)Yg!QacQuEY zN)tZ5+AkJL3JgbZYs)izA_&WK=v_oPcE9Xx<1qECh&*I8UV?)S;vN9G$KZioF+flW zrFw2s3uJpJe#d9Rh_qEY?EpGpHOvWY-epRCT@p zhd{1MC~YLbQhui^xB4d(weqK5-~W8ZfYvngZ!Xd#LeEhS2>{wqLD`Pekq{vFM~T~y zKSzX}gy+DxC_?>~l~BXpkG3gaw@o!5^Mj()lDa*Kc{WI7AX406Qmsr;ZB}}(L0}90 zWdr;D`T+MA-(j`BAm%77+B}Q%AAQ-`-b@EG{$B?40%Uc=@er)m z#hRh?HOnpjRsZnLlb1-Gm~CgiJPzVhtPL%prsok7o5bM}6CrH&evX8|w}K|(h*yxM z5Kk=lJJBnsZAy22hns%=IG3Uz$zqmPSps{dU?4*Qnsp|iHf&feKzaf)i2AyzJS9rT z{5B9`m?s>eg!=O7Cony)tBNTO+5k0Q%GyRJ!pnC222X(+^%zPS)DSZ5t7&A_@-Cph zAyCm`KR{>~gcV#G2W+2Hu0$YjbA+y~-0T%yZq`i%^l1aP)`mmYi>s$A>yA6=mcWCT z-E?>uo~DXQ1c~W?vIDI(3nZ0X*YaDLozP6hB{Xx>YEw;_cn0kP>f_=mD>J(`tB$P+ zmzVXACj>Ds?e->`MkINd?)3=Eob>4&7HzIf^fYxXZ6G0d$8Xj{8vTGPQmOor@+B^YeflAW|?E@t2 zaPk~|wO;$FZXTk~c}z#s5X3cyDUBQj>pwsK>NDzEg6Z}Ofeq+!N#N$^zl_9MUr&{F z!pqMU>?jK)2pXkd*h#uD08Db@frAQ%vEMH#thL?pJx+`}H^uX52lA8k^t}y_8N#pr zj|a};J;@KCKVIK^>R&;tB$_yojhG?Kuyo8pOgt2!_P|_?{U%D}^>3Q?dZXH;7ZQIv zXrYm)LO><}_%Bcs^mB>nw^i-HibR4cvYW!ISJLU`fIc)VHCHA10rW{Y^1tALR!V`A zDEnW(r5CdwLw=Cq`3{&$)2;hUQX$g6|NBGQYjvG6EMS4j)XdXI1&AVq1BWtAa&%r! zKTu|+epl|9NDAPUN2x^{45?nL+t9*R_H_zc5VCcRd%#gMxGf6Grj*`ko3T?*#PgNO z_+8Os?JZ=hh_$~lxVH$>DY3&3=DY{%F7bAIU$$#vfCMPdbebEO08FObp(xTXTHI1p zp`uQ%1eTrQk$iATxD8TgIIkxtP1BMD#?9+(gD_mlu$YId3!XrhNv{G#p|~Tomj(>V znILl`kREl=}BCQ1e^t2w- z^BlS#fH6%$g={7y(_{y>{*O*J3njpNae+UH2P)cCr#!H2RVWj__POj&{1af$+_D(- z+;VKUvLpEcB~Q$*5B2eADWLuk&z24u>;%#!iHQQimom+K>5bVh98+ZnT1^xI?{>Oxrp5jzF+TX>{@@^2vK_wEGPE~J2y zvZ?FOrbNT+z==^f>jBq;yfZC)U!WA$n_ETm2U8ohVrBem5Y0I%Rte{nDb#C-2Pl~i zF4Po|%*AxRSgLJ#86OZkRHrzAa z1g1KUEicI6sQ_2rv>78`%o5yiI@%ooxEN#7oD=BqS1n|e8018mTnAW~O0s9)NJSdz z8_fad#{{VRuP<33LLD)K$P^*X0ApH1VweMVm{Qh5#K&r-6cD?Yf%r)(op&PaJAf(&WuWkHPJ&qy z5uC{lHBU|e?K@9qa1kAN<}oNMKyVvt14`&XhN;tg&6LQz(Px*q4`2Y%{6^hdX^4zD z4%oyDJx@^Wj3qVDP8S&Y9A1(i&`S{yRcJVJNFy@J^$mI?82o;RD6$doq${9S*}scF z223;1+GfD^E<2l|qfAvNk6@6^L=tLdL}DyEI}_g*fcFtnD?V^PR8d$C979%;8Ylo@ zdY_aikiG*gKMmHEKtJe{2o=dgTyu^FYD;x zpS?-;v_loyJ@St95;~u$G0Bh72m}2PUvy92!s&2NU>Vc-x4%QZ(Qz1Bv=apUC;x_5 z&l~kFg{-_I`nJG$CrWwW&Y$rScbNa5W9rXS{zisFG^hxh5>0}@R*3L`TZ`6x8=WWN zoc74MN}?MLmRSdKpa5GPw*P}Z`t+tSkK>W&2%Vvx&Iz!3WUXeQ4=!bQ8SSYIK7Jt$ z85I#h^>=7BEzq{aFhI}nD^DoxT=R%w7_ZGEjB$qswhq)}48bSdHIKNW@Y?A401;un zPy%ps>^G`PIdr!e24Qon_)?xg^VMmoSY#CQBbH4W|kCUiLo zy6~i|Awwzd^9D{`9*N@Gu%J8MHSk{eM4&>))Th9hLJ6tz&zROIk^Y$_?FTA-5YX=g zJE*%<$u1(>m-SDfZ=HEsqJe?!FH+5bnrccMymf;oY`4S-HlF|lj;Qi3yDoHG!qLfN zm{~B=nt@8D1Xixgam!A?)-`)Q*n!E}LZ~B}MO*7hE0I{Bxr@ZL2{`@+f@m9{X)O+- zHLnCKmSvv|-4CeH)u2}sPVIdw;|2!tR;g&bK_%m#0jCd2HCNfC%D( z&{n~$Aq6H(L;EaBy^6f=znlyhF(y9O5^Hp|H#r>usDM!fd2j1SjiPQJLvNc(ll(y9 zTQ)}aRQN=9P9K@pY4rY?WSF_xZ{EqrO{^O1ZfJ%!yat}#B?{WetsdnvqeEWF)wxPQ zOB`!-E`*n*5U(u`;;H~BBL`hPZsoJ!)|LY$#F%x{=A|5`oG***bx93k9+yvqCxdh- zYp}85p(iN~>{VFA*mLL_WU4J#Rw6^R{=Y8K@HX(q9WV~hiQo)Yx5P5ofeoY(^%|zB zF9xEdQnHW=F}Sso?x{RXGDsuK@&`J2zf}11&|isS(53(#Ks806*v}d9Ozx6V6jBoZ zxg|Z)O0WxG^mT)1s~6f(?En$i617;J{%>ZcJJMqjF^U*w!Q#d@@0+7hXUq~X!^*7= zDCm2i&xDi!lMC4yAtl^Si#+}(N}eM50fcfmz_;bAb>~5njANf7WJ84zjl; z>+*(2fqQ`Si#TyPFm|27=z_A?fy@Y}_n6fq+X3PSOV6Pv(dtiH35kmjxh|25aeWI~ zltX=hblM$q*F8q4IghyUe}*0>$+~pNYwcE0-qOszm6?L38$>kGCOEM>{ygmGI5U4?eIpT8iZdWlU0^|Ru8~eP$NwRGL zt@GdI9y}PE^Bc>9sY?HkD6|Zmpc1#z&q*VU-SZT4{}VKgI_!P|8js9=UTk z5k1cYMVK5#0Px3VE1DE{i0K92j^XDdLHl2bgIq8<{~KLws|h;Y={V<;i2a{{{Z6h! zNHx$#s{u_qXq}e``3_sFaIT3H3lhfvCT`l&|M=IUYU1U?5aWu*QH)FiJqbHH`OVLm z-~QaW8Ff{}ug0iG$X!D3d+)#c2a})Dh~TZ%)N|-!;ITPEOhKkk-(y1L8;`Iw{Ox-n zdyhH9CMAJ{y9ro_a(l?B6@XjxL`dPE`6k^=x&@Mb3ecS59#DKC(K5<#iWs?!J5HV^ zfw-en0y?~{&=24dU_PJ`@z%(e-sgP>nR<~5gZzzCWnAb2Tg?l4F#PuLbRY<%4`dU^ zFGnMQ3WGevad0t$U23Xb|0|u?fN1pohKUObq~{YQch-g{TD*4bq}UP% zkwz#EopGo+%pS7nuL@}77t?e8`4rh&mi1v*DJb~w^)EL?&1{YXGS=6v@Z)Qwuz ziw>+@>j3IY1&mPA3lp$xROX1^_kJLciM+J|aeXlW3w3#LpC*ymntTHdCg9a`C_xsN z0GeSk;u?y=KhyG#SpelP6>Ua^Hiw7u+zczecn|`GaP27p@&goVj6+2$2UHNAWSOpaAvJDjT-|)Nr@!c( z86a82!0y1F2kF3>j{LR(J5k;$7jaCTJJLuiuO5;VM1B~ttpMu^w330CM%P=LEli7H zg@O*P#^!#{r(Wn?Je`%%k>{w-eH&=lMxq3757R@anoth!vBkHKzI&aTSCqe4Dpfu=iN3A z-N^rE_q_YBi~ob!{y(=_;s3%1_oj)=8)YML=w@jL4{O9R-TR+9^Ko^Zuvdfq=l!QU z-H4T+Vd+F87k~yL#!0w#zas(_BT#rm_tW^ z;aBwYE^KZXHH{#*x3IziA^`n&Rb)XF7E_lBUM}cEgx%Z15O)9_X)*P8EAbs&KzB89 z>A1Du1ilXa@i+G+;N1Sr!#P1|UxXzaq7Q~Wat{z)Vz9p?7pCsY|BJ9XyiG(x-X(vm z*8kp-V`XyWIiROgGe8w~?E+>Tc6UXP5E5Uu>pB!u(P%fpk~bW1+n7KLD^3imemelZ z42}nu7rft~2D1xk9wYJNi}0~7pnTQTTbcjr0yI8&34QiYM*pv1^KbGK=bw7{KmX>U z@BsfSOZ=zb6}DFw10FV!G{jY)zBklsK$KR}U)#!~F3vJdV$_3nT{@y}YlhfIC|vA` zdJEb*l&BM+Zb@X}gbn(@+fAf9hSfmuuz{Um?7Ow@rGiO9f@mfNdP!Mi(A^Hzt{smk zc_R_EY(Mfsw%bkU@5)$!c{CHA;246f>$?XtC;#=aOr63c8We1%3@@W2|}=G-o5~6`1!dtFEl1F5DPI zN{QA?GY>(k9fl$l_%eQdwq!*zZ(77FF2o!=VHkyvH_w9Oa1+pyRi6&K-6j*BA2XXg z%bByv#UGC6!|&^j?7qQ#H9%y2X<2_y?aP&?;heT9Ah4L0O35H~0D76-#bI*_&Im^E zLQA3cU|AP0uuZZ0T96NGs^m}FQJNZfcUKETh5hhBx~{RgVkEQ*6LCh{$oU$Fns+D; z-`oeizOuz` z_to}Ff%VP|_#qfYRAz&GmU}Bxt#(H556F~`$@&vuKgj&eHT>TRxcrwvCK5$xK0Z&4 zOWy;-&_d=7KSD;6I0%M?SfT?B=pDB%5<^mAdZn!fgG)5pdVm{)HuNGIC<-jby2xd8 z@XSySIEX`j&EDOUjIKlGaR0ph*FCkq6uW4T1fl^N(W~IHs->g~e92=w^YF9s7<^VrYPv#7R)s1|;JzH!f?D#9-n{wX(!{Uo>=?_Q3+{?i6;L>@=66Op1S}eEye$N3ni$iLR zEBJI}il`IBYId4YOi`YLhGF*&Bqaei#{(pvQ6{*Mhz?FbTB=g57~%z-2Lj7n8{FK) z9Y~y7m0`DNJZzDp8RAJ2mUwpnn6U*F0p(=oeLl`jEu6#kw^EQ z;3KUNWoIO(9l~c0>_;psu@eJ!0BA=BH)WFOso0x-A5uX*0B(7~Y?`)aN5!(EdgTU7 zYbk*3Tgc=J4{Wq%FO1B&8lMH|rijmlW;Q%*+M0qK;gYZ#))$bPcpNP9Vwh6k;%r9; zfxeGJDn+p5O3$QMAdv7|;<5pnj z=U=>KsQI=RX4c|9d#r%IyCM!J?%0fr8sE`bL##ye2F#^cqB99tps`VhRFVVOJ`?>x1h6LY ztOMbl-2gl9cx5Y{Q&Tu_gq+>K&d(|#L)a6xPg!)N!8;tEDkE*xxV9R8=FS`tqs#9o z=8dk3W^Eu@lGwRoyE0BK0M`2v*wWY}s`P3(XcQ?(OT+S9X|z>;U8|CVoco$oG9ocB znE6UAMP1+F|;Mtn>r5pVicEEQUqEhc7;S3ijz_Z8Oz_*qbc6 zXIPUnmI5y0A>{o+eAC!|Kj>zsu{rojP9cP8VLFtu zc1oeFS(`bem^76M*^0!>Fm0r?h>{Sd82eIYMj@rLWXUo}*+RC6@Ad5c`6IqRx!um` z+&X4nujli+9@pb>zZCj0LnzI(-rFia7RxJ{4fXR6`+ZzNClSxFA0KX&=k>7H9}&C& zZ@uqV@d(for?zp{_rmC^Eo`GBUMIJ}Ol*aqUePYj$nXL#n(1kDLHCiV8N*BoMQgU2 zU_kJ0J26gTNLV#d_nycQ!X~))$;mfxY=UtSUJ9ak#KMG$)}5TQ>GgOqizQGQkt5i= z=uf=joZ60DM|Wjm>Hgu6VDLCtmxyUG<~`*|Hs?D@0d8s?@$IsA;7FN197K|7LQ-07 zb^CHWs~@F)D>qV4SJ^u*!K3l5PiPJEsIssXQSBPxg=(k%hu=Hztv2aM_9Lo&rk$?j z%v#h=geep(!Ap0ilRo?rfslskt3+0)i(AB`*<*Wy0t!<21% zNayN~ZG0+pA}5hduah(GBHs#wSNaf=+yz~T=9f;qwNccXAH9c5QL4z_OPwaic%GmE~y&E@Gxdtdj7%xg~2o+KXEly{%iAL zzQGi17VBMn^fg?B)2Lk?>p<;@Iqsyjf`wK`Vmc}AKx)YZobbBJF#xzjw(1K4iD~*B z$!NyW!xEQ?K}h>O`z5*;qt0si^6Rq;_{bkrxV^LotKcVf6Lqp_Es&S$%{N-6GQp<- z@AOKhAMMobt^8H#*ZKkG^yaChytA(S?kDME)LG3kNpm+1Ow43KZ3IAO23@lV0ja}h zFCdlr{y5kG^wzt}TNfb`&yVJD@o2BB$kF}K@ZMAe6y_>Z&r8J zcRq3)yVZ6&*RVCuVkXexBYV?c95IkCwbA%XbCpH81n-KK-31i>jTW<)QIIjRbdwyj zcfdx%WTjNzCIwM~MT))~%b zCWi};N?^GzksQk5xaHYd*dh96_Wk9$5 zxy@v{;>N{;MINCUi-CoFOs}Jbt1M@{%{e|Cd=yTVFg959a#bDcFrClEn6jnNl7BB& zfZ6;Tyf}ZEE2aJ(3YMqlqoqy#m;y@jAK?onKj>9Yq+Z27)Zz9~m8<42(d-KBmVzy> z*b)h?IX@9p_){$L7n|WW`pf#?%8cO;zbW-J!y&K=ZG2RpC-O7h--9lK8byV*fnaSysUntdW?u*cqqq$W2ZNLt`ehTc(PMJLThwZ31$vasjBjRI z{OtC;Qpf_FO_e4ta|F$rS<-WgV4RO(-apc%(UIck-0y867$qUgzxC+Yz{YhtOQ$$h z&0u=Jp*b4&;NRx$s$yc`THlIHwhf&dn~RKQ-B5Vb{f={7_>*9I`|K9(&eI2&;V+>S z8u|o-@UX{Z8(tOFEFP~cY)qRK?NEdFJahP?{N@1=ir{im6Qtv|P!HVg*K%T(;xejW4|z5FDKuV@Q%H~W2ygYp@?`?a`*FM>={$- z=nGI4b&cuyGHI&(>AUfLF;Zu*ITxpQAC*x zqHq-+Dp)$?Z(!oS(MB0G;*3QCFi8D)nnI|zzp#d@&)KD^{X78^w4J+ zA;x<}I2l1&qF2G7m2ek2v+TK|zF7GoI|DJp&$6bUXZ@%L`t3>1mh#e;a%6Ayj(hEy z+IsoBw~BdNEw&ZhmMPpKleP_pG)vl|@`kz4`7j(3L9eS{t$EK56@62KKx`%Ypmylj z=$Dsg!9(JR=27+VExVnBe6SBREBRq?Z?rg1LZi`i8`b4#d4XC_8+^WQ-+?bu6$T!P zXSz_LFZ_nh7Mn~dlk;%2+j zcSO2xJwivV^nlS$8@O4@>+_`z47mdrmZ6S%->HN&fdi{#qT^V68=?ASa@qrf=09-=|^-JX7c(Ad)rzm7I| z*2qC$pcc;U&;ZMa(!vYyBht@AGJ7Q3^nv7{?v5f--x|=jb!t&^(WGeXD`++j3-F<5 zuP6gSd%=*o^{B2gwNn%bdASd^-%7w-Gn{}NWs|8Gb6*9#>W+~g`U9crd3Fyz>Lf@P zNQ$k6X+%wF=+&*7)L`e9n^!&<6a8!X0#DwL1bQ%TE74%Pl(54<1cyVt0er*Mm{Em7Ma+aV}<-x!s1XG($DYEoYbGSHc#+Nn-r*(q;G_@?Jm-H8lGaZkEo&qR@L6Z)d(*MpN9! zK?wp12t(|H=6*|%VXrq{Kw8k(GpB>SYUbs7N;4_Pkl%6Xj|mtXrY_abFB?hUM(M3x zjgdfQ*;x4ypSFSTc-V?RhqYfJF0Qx0uNrYoyO!B3`$sO-aE)_X`7j&(z66 zSLW^@=n0&WA4@^WB2~l&Klw+$>fx342tRD@&OyZASlmWdW+t3LH+T451nh-uX3jtZ zFU_Y}3AR%h6dxil06pQ;aJW^J+;Oh}VM}k9_P;^Tbnkxf9xC-2l2?$4QLiR0on}^O zL{5dFb9k<&vY(D%RH5r+t@JC;B^S1RMmr`^{geHIoT-+z0@-3`4eiw3_IXl?s;5IE z`y|;xM>{kvv8igK!7=^SqR0E~4%*=;}(8fLIpGZJj_( z3HZeROoaUbr}f9-P^+NBxu2jXy=OE`Tg=*B|25yMdh+KQz6^f{;{kHPB$$9|MTECP z8g~J(Vih;?^*gSka>}b1ZkNRK%7~YhCvq3iT+?XD#D&wwILmJ#l52QG)r?$_1_N7l56=K1 zd)dCZ_SvIhOL50Ai2+L$mpK0T;e9hGjM%&=hUBQK9+0?Lp{AWy2fzRcr(^3hgkrY(eov^LH{}+@o$OzW z`w(RFTY}fhR$0W2TXxyTd?WAS3ew5v2?{k>EmJNlj{KXTK*dh0ajTKnOx`2kB> z=_~UlV&LJ=`^V5b_d@Q`=U8=J?ivA&TtY=vEb&~cwuT^L`c30Q(Pkv#ET{1{&dG^h zDaV`QoQ#_+HfwB@CoJ7dNhtQqAiQRz*a49cyLb-Hp!Whpb(qCIltX)36n&;{-ks@Q zco6z6-v?xqjs!x*J(oIY6n6J$jBjt%rvqe*lM+?lAR(>GYe3glBZ^FJgTKpC2oiO6XOSmy)SLa>CeS?8O4V zIawF)OFyPb`1oVz#&muzvhlB)eG!B-M2(KzVkkwuP_B4$mc7T;rbCMO2l-J$jMLf! zskjwnrm3hrgEqOG?_s`^m{a79F(TJP00`DM^sprHJN2ML7a-mD(GRAu; ze&4VWyu20aoo8k{V;^}*EHe&_hSfDvaQ+u;`X#oH9tK~nde@1#gzp4_;J-}w_T&0` zjM>dXbqv-fN*66*D5Y|{#-I2oeKWRS^hTjD34SX#shbz1`MjAgD?z9q+N(PLG}tDs ziaK#(G^Q@blvyd>=NYy_+c$awt`Zb+cnq3=Zs2=%BANMIJo#(Ud&Rct9&5yNa&aeFMTHfqX0Zs3`Bc@xLMfblV4*&L+5#J=xH<7qilJ5&bG2ycyG z6KaioAJdeR`s-e{Z%J$bp~p2*xl9eCOZh--?Jn7qSYa*V(l{V*nvKXadydI&|IU%f zZZAkwX18Bo0W0KD=ys<9UXvf2M2OOq;mOBhR|rHS-=V|_Ux(MyGFz{7e9*sK=x0iJ zYGRnC{vS=TyTf}c6w}|y-N=(^USg);pmmwu&m;;MF0a51WtANp&(bwaCt6qHwd@Ox zVZuQCD#B2o1?}wYPK<6IBVoKfg>fnkZy<4s!Ln9yZHyJRDY-voSGZ+ejyJ)#9#Sbf z$mAa)uW`8=?x3A$3)q6O66pg>f|82K(41%ot1Z#8Q|4*W{P5(T-18G_JYb2zcJ7+I z&mcV#S)Eb0ukaB0jH>qKQyj6dfR)cIOY)u$SXN@vQ9L{;F0@|Br2d;nW8oJEEFWFI zNSH6x*v0kNj!;n0J^}jYju7~Ui=F$Y{LItW1cJrrZBiJypzoTw*xTEvL0yQbMrp-n z`75&I0(c(&f-{wM;{<5-mM)F6@+6ype0R72TUZ&#-NZ@u|6_$3>Wkfz?X;LoVeUTa zR>QZnXJsW>xKNGxQYq=&q=2K2;RAICn%y|6hpqDo8Agf6>aI`Ye}_Y~+~GFk{>c2R zHIucU>;(BIYh_HYO=d`+GHUqdDBXBVvBJKOR? zRUmDds`w>cF-~ALbE|!4hDz0lWzkPK$nND_w5W1SX!IT<{`9olbn!G={|cCos!C6nNEdtUPv9+Lle?saOfd zJz|aNWXEP4DezM-+F+oRK%npSyH2oM!eR}DI`r8!ZNQRf*nUD*V5p7yWvt&2kyr=e z0>}}&p}5_LJBaJ_ArcHaQ&t6{zT;?<9CcqlQKJd$8Q=I=C}98NPb>8mg;+pJ^NHun z<}I$E8OCG2d?L3#MOWST$L`^@Xl>+ldk}L1`SgqLKsTs^QYAAKTvUX61YAWs|M_lP zyBG<-=Cn;eZZD0yQX+X3d6s_Jo1Jiv;DSFKwwnzW{tYIe+^J9*u-ub ze%xPST$=XRtS$j(>eQ@7STXU%L-C%86a!s$KuWWxMWRsvg zH|+5HAX3}|k%#P?LY#rioVm*RraD#~0&d=pUle7oNG|0xI6`$o|Gp_tM6kWK!3p9OFpjr)y_YD(+(EG|F17He7V*vDkbV$MTHJ zID(tCaOjgsiYEW34Fc*P%bm6kYYiRf({jZ$>fMeR5h(di?Aa#UO-`3TL$1COyrku& zpI9TX)w)?xql{l}%+g_q?XnZWr}942w9AK`=H{wbb~}HzY`cS}DviBD6wWcBqhS*n zW>05_Ddv_kWQBdG`8ylrMVG^REK`|g8U6UO-`7KsK@yUy3bQo)R%i;0C9HJ8T%Q@O1X4N&5k&%#wK?y-L$7NZG zCP_1n8cW-Ys!LX8q>sgawa10oJN)Eavm(0&T-?zr#5NFWp4KiCKm%I!Uzt===5SEU znOvgx>*=u!%lLH-T3Z;(v07e%T6&CxZz8`5zVEc`^&>u0m0zWVY?tQw%b5D%3`plM z#xu{UWzkk+%mV(i4ha=aQ8xa=`6H7)awgz>9g&;1c%?r=y{i-6m~(CYT%k%QuGm)R z(SSPc$pE{$`FotXon;(0_grJ)29GEjMYV(BCzXDmYPh+q%Az>g?tDBgVTwR`cZ~VU zU0#&SN5@WYFB?Qa=TiB@G2zTf1&^CVVii7NCcBapu)l8fmC)NC(HsP*@|~(Zo~LcZ8NPf{1B5!vv3A^%Ex8(0T4qSw;D_r{gXFMDUEB zAcz}$n@J=U_yrPn6La^HL~XE>;fbtArP}=|&y2)oIRkqnmdD^=-<3j}#Ao#DYpBuB z=u^cB#>ZjCnD_O<5{l+5r0ug2aT=Yoke4B1xR+x5l&P$z?M~0Sed9gtp`zahTmr3% zxF4_aSrWyxPa^6WKYl{;to{;VbS*+gwWt)>{p#ZXVT*YqC}Z3}j@d=YhN`_R9l7M+ zv;=cvPl)CzciW}Y4lbohNBgu1T_rnY!z8;!yGndcPQI1dtkAOK(fskPhV?vaUADs% z$Gekh?;c_wml9z)`a-iaALE5G+iikW7aBmj}tU2@a^LAow%sT7y(>wq^{JQ+=!&G zY`V_jF&@9v>8L+k?Ikr6nkQeghY=+zV=c+1N@C1vY;FhZf?5D=O;5+E6mek;bykD@ z*xOP+Rs_SVAqFtme3$Hu8!m_{iYGPLha7bn`kd#V>)a@<LEE?R7$*3QmvPk-JLarrsLCw=Fox9TAqKuRl23N zxgjm9#NS22-C>Qji(G$1e;Fb533f{BaydNXvzu8Ugd4xM zZnn=O?9t%w6=Ch@NtHw;&U(iflEe^4x6(dS$+|$TfcE7vLEXsk7H@y(sEBp1Yy%fB zN&cmCpgrT_0UXgI-~%_7a;vKDaOh;TjQqQ1NBy$zT<-ZLW%;6_Zb#~xDy>I>B{&87 z@yXha-56r`DNE<_d4YOOZ7=GqY6smfx*ua6C$Q!bg=lD)n=wLS_9@BNSzhr}xWh0U z06nA8!v{{fS^dke{Za{ak&uFJl-c`;HF8LSuEqVsuR)(5mo?@T^~|(4#f>Fp-G~?0 z7c=0lG|%Xo?)hLRugi5gCNae!a^X%r>aNAQ3R4D=Rd@YusfIca^6v*>S5E0H zWPGbZCeHOjTtn+$Dsk0*>(?;vi<{HfF4efEV8PQc7IKrQlqHyIa;NEfB^&aWp%8|;a z9?;8Ww1HTEEk6}>2yIX=DODyB3jR+s^LK(D&E$DFC>&kDM(`d4=tBM|sMHIS>X-^B z7_EX8q7ZMOSRP?ZBXHoVcwP_h%sqYD$i`4`BIq1FwA1l4s;uv8c-6r8B4eWpztq>^ zrPjZMwDM($8aW`qQpL*(v|1Q>BI;3!K0g(HFCE0W`8u>3tWD1!$Qpl_Xp~Vi(52+H zL2=!c*ad;)(%pW#o3cFKLvN-t5d)C3rX)zjIO*%NUXFtT6NX7QJQJYR+CUNeypm>i z99Ej{`j19A4!vLQ5|og*A4$wM zE`hkitnf+lF>Pg4sxo8#RosEwS!uKkw<7s$q=tHGMELdB<>SqqmN1;AbM8F=>$TD$ zQngd@|H#a3ND7^dBsqGI@i>e=i|>V<);2XmNsA1q&iD~i-*@W&E&o+L!zOs; zRnVU}p>c=8XPKTJl6@@{@ zQ5&2kGLgI$mXG>PGko-{XT~ou{i(gQo(0*0h$^{LMV`&GpV2NJm?r=rLnJBGUQlAu zb=^fnne=zmPPG0+-QClKixxHJGqMNl9M&lVBXnU3gbwu;Wns;YZu)8Nf>E$5%G(%S zJ>2OQSpUG)lh6s+CDbWLCi? z*-^N^p8@Muy_AqQ8}*;V=oP{VBQY`Ao=MQ+i(pIq#m%wnwuH*=ki}SNVbLI<#c4@4 z>5_zQ>*<~tzY|I@kgC^64Y;x1iOX^%84ZXi!g8^8D}g#f_r{SBKDrw$AJd+vO`HgO zpD~=~XTA(zK_}!Jx_-HK_`7tQ7w8`9-S0tAa0ai;xoq=QVK5A789}s(um$AheH|*@ zvqIP(1zoA{;~6nnW_+}2&*j%04ZNuROEM!Oh3hoj9RUGGO+@EBHueEU z`GtGDfctT_lL9izQ)CHSzwBk~azQj#@ns+1R@(qucH=EvO}MCwGPKI2MjB$YiJ-^H})Z$qE*pK=S?;Kp}AJC#WhU8q2D!nZU>x%mVyVHxCviT{Ugs$Njw@x3RAe06hB%nm;)(ye*Tw?xE`}i#?dE|Qs*t?%I5X-VMv?=G_c*a^-{cb*2NtqCfT9>!k_cU ztk4`k;Y^7AW*3`Adr{Mm1vy=C#NWxpJj{_?SaGyLk>@CS%Rz4Z~*dQM{SboP;&Ie^0-NNlnQMm{ojZ zX=;aWWZBomNOj8EUqJ*RQYMVI$a_L1*_78{gl_R`Iq=J&1iU{VVQ7-1GxZm+my~dIx@Fp#a1mKKucV)P9$KoWaQl!R*Cp2Xig)6Q z91ZP*;9?if1cveUo=69NOMU9Rd{X}|0ja~V5$hvBmG0?8qn)XRNDx^m2KRB7p5+xU zhb{zRM_YgnK^gH~r1Eb1U89aPG!>f5L`=x38D@aPGJ?L2!Iy9nCNPI-TUjor*QWm{ zb{ch-A?^S5n(9NW)}H-zs@OjLIepDQ);O)x;tFo>OREhMM?%s5@ICx8GLbk>&!eGp z;~swlvhC8i%)_-CF4n>|cw4hGj{M?OT}j*^?~Q|v!pBi$5NgY&YG7i_d$dGqzNmS$X ztfXpbR3Ct1lB}va3F~RBPNQG35%fMm^+S zUx=k2;=>Hu|5ETnd-^^Crst^QOYJ6v9y|%7>L%Wrx><9Fo&b2?OJ8S-x8UGcBnBM@ zcEQcQHl3crt(gDdOpbQ zyr=rG{B9wPuIL`ONT+P#s-Vf-G(M^*y*pl;H!KqbzLde)T=AATdmn|xqqFf*J?5Bp zE=Pzgy2I^OW*cP$=BU@VMEI#Kp#=idLeuDIF`FwXIkaTQ6`Di->j@+N>V9jS(A|X7 z9MnXhjRo1aul6s)E~WXfFECj#edQ&L@g7C%qyB;{3N@;=j7^~l$imnwICM>ES_Akbrimnn7Bx+UwRk3At9ia{_ zQhr~oqp^C6^oYA?q+~bUUtb>-(Jdj0wAK6@j|RB$ZFOeqPl0K&^mn*kM=t)2osFz? z_iuC3HRW#^{(;R+()+w@2*KcN8SJe4W9{4rO^=Zx7&uA~p0N=_b>aYJv&L@v2PWR7 zfR|`Iq{nF%I%)cUpOh7a{#433he|$sw?m7eoC}6s^#w?o*YlQS`s=h3OX{pb6kZ)W z^J>-fXl2oyoM|j)G_1%pnKLt4D}pK9GwHUB%y92L<&S%tLl0QJ$aD|5V38%qTVJ=Y zmY`XTPl%4^lpM2{kb2b*U6*5bUL1+uzE4a7y23XEdUbp9Jbc864zlOtKkURHhaWP$ z30Tmgl`TMIHG)95u5CAavdSP)wV;?Fo5M5*!hECZYYRd_JMDA68y;pc`)Z{*R zUMz>@`n;Bg*+SDUv;!p@Ew% z9K3H=@+%iZif{}bW|^M!LR_Lgamqhh&FAVUX+YVoEGIST^^q4pjq|SLwjbMn+?uJf zrTafsV~*Lss~f)mc{cIs4sqOO`#I4Ei=i#w4d)*0n%nlN^!@a5M8vO${=&37Uv5X& zIyC;5>U%8oX=dh+k(1E}u_@OI99ss5c5M^(56kq-J9O6kL8;62k?R^SOV6HrVE#6s zUt#?q;KH2luPJ>H^0RlJvZ&STT9a-5{yQ)VNfX`LVwd{rSrg6F-e$Gjm8JYg}ey3Zo zuWx3V^pUX(5bV9n3|g9>yz75&L)5e0w?T(JjURc-hIN&YGrZKc>9tvI2;moXlQ;js zU(p<;e+<(uW@4oiIsM?qlYN~mQqJ5nr!S&EGyWty)|mdj+FMs|IRQkn>{XgiM|Vxu zZwWtkd3$+3$Rj~F9(*xoD4UMU>U(xQta@6wTl3hi@BEmd{H_6CA+NmEACt?&7N&lO z7#Wocn%P zezYy$tBr3c+cBq1PT%wM%)bL6tNA{8%7vU3)1IF2J$EJ|nZ6x1IUIJJc)Yc=(s%LP zKxxNM@8tuQXSHWqb6t;)m1RswTwi?bFgaMDn!&eCS-SVcQ~qmfrnb2oNBezhy-{Da zBzbUcBP{p)vB^a92mCF$UNJMKO_x{A&jDueP>n52ZNT&hf?2OfK{K%9qJv2f=@R}$@+%FTA1*Lx~R`9-J4Rb{h|T5pN{C4wBa+MzKcE)S51~c_oNq z=D&oMSJ6X8AUIeQq+NSz`pxS3p?9hNhL#->b)|SAjFnHOV{C_JVdu*ewn1wibBsFf z^-=zO;IC|y^em^4ihd`ao`9jHzoN;|-AxUa9U~SwLN+D03nqc)v-4a|CB>|wQKzZq z&m^gG1M!BBN=QWNr&R`e>H^!^Ob!}DjpdTmknz0gi<#I@9SO|GhG$NPzfEKH&k1AZ zjbZ!ulCPfE+VLa*O7Y}(emZpRKt)XGS~d~Ecsg4Nm-mA&JxUw7`URm_gWC9H~#zi41-&82@aeP0Czg*q`4;*=zVP zL&kh6z|5PTv?6Z+3f1N)KI=Ogeew!CXT{jL z+~y@(;ey7}L6sCzH7kH06Bda!Sfyw^ufSkTeU-Zg^o$AEz1&A%6ZV6X;!D(I)^{{{Gk{tr4K@PC;~zW+gN|Nr{WKdft;sMR^J U*Qm4bFT?C#%fl9VX5N?oAKPLa+yDRo diff --git a/benchmarks/results/smoke_postpush_20260423_123452/charts/locomo_comparison.png b/benchmarks/results/smoke_postpush_20260423_123452/charts/locomo_comparison.png deleted file mode 100644 index 7f5f7c91e81ca4dce1506f440423e88b57b953ca..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 43742 zcmc$`by$^ayDvIWRCFqeiXb2eC@M%zK|%y2rKP(?3F&TB1ZfLIQex1lbW05-C5MvI zC7pxZ_rv_wxA$55T6?c^{yFoy<~2az9dA6(egEnn{tx9OPMx4Sfx%!-Nj?x )rV zmxnKt9EBJ6h_-Eb;kOr4w^y=yZ0}@XYmAXKu(y6{W&hOF@Uo+^t(~cr{Vp^~^sOJ!A6)o^%IjJL3vHvSm;3L|sp&^`2e zorCBydOaV3IfGtL$Y4m&>k%dlF}!#mI)^y~ulJrE#}L8mr)pwvc)>i>KXeaX{{Q^* zPlJ5p@0Ao2#e+L`R@(INeur+opC2fjT!_(e8YsvX`W{p5to%Q(&ikUGhK5Gz?s{LRFSTXFM1)aDSEj}nuI)cxQeyAAC7WlLFa=+{ z^*EuAQqbL*f1!NEcGM^2(w*dwPh$C6?r!v(hN-AZro_6MQE}*3D-Stw-S%fX)joBl z-)*77G^?PeF~@44)M2J4OKVX4PvWD+-(RPv5<}$zSqs(8etkC9b(+m8%3j)J-de0Z z-6^>J=iLR3M{jm|Z(H;xUHzc__a&P;GSeTQ?wguzPN&2z6EgEVq|!wEND2I{cNWSC zrLkUngy|@&%CvQdM2S#2H5~_)nf62p!Ia_wEe|R?~U- z<;wP%?NIZs47Hrq&CyW##Vys{$db{JJ01M9neTE`-Z9BCbMHi`pL?BCxy5|R@uz_) z9J>`qhj-5|nUHE?D|^}tw@|6dbC=1P_FAm7;T~1+N~ql{qdu3(m`?MYirZh)>pC3J z`*yw}W9%?Zi3tzaG)wDBSIOA%thZ6~sKR)srJlrmisd7Wy<;TYfZIe!*AcXz`th0e zuIreCS*gQZZ^0k9`V1SoHh~qHU$8&~zglFLOhhVq4B5!#$wiNPZqKQW-d^uD&ne&Q zv#Qd$JsEAMS|-<5XwkH{J+$BB973VzZleL0W7aPbc!9NXzG7qW^9>ug?coKM1Fp~U zMjvz>FD^L1owfM&chk*VU-4OATKSD@eNSaLR$(2oE7p5I2lEUTH&{eF^k}&3;^@4} zpN4BZPIHWLnfQ2(uFYrNu7w-_9u|}4xgAZy?#58YK*Q%IwmV-}g8C}A=e0(;**sdJ z?K>om@66?Q66aXaNXX)Mem#j}HSbRECJto`SDP%EjAbz&s@x^kb8F_rb!}FtYEF)S z&|}kbU#d6nZ-B+6JL-VJFvUSdh!{Rpy@uDT65x1sZw$U*--h3`rF8s8@AsbDtm``a zEG*|=dS1KSgR@4vuoFrIMDL6S-D(_!CsrN|w>kOG8wuL)!`j5sN^V0~jwfG_P~1H2 z$uJgZQNGf8$C%n8UxBdB7#+Aqp=lET8Sd*{zknCVPP5H+rpUWY70#q*^iVd36@HmY z2n-*vYr9J*cyaVpV-EYv0NJS9tZw6BYD}{prTcr^O>I1dPT#M7c<-^Zgfp(%+cN)E z{p|3{5{|p~Nt_3Bg=JT&g0W7i{r4R6{$iV7@qVDB>B{CbYk^)2q(D z&KJLXGq?H>VbA+8>9>V7_;>B~nT(w9wVCb=_n64Syy@hq@Jk-6?Z&6?u0H$)3r%Ys z-X`2*cMZ>{+-DSecbWSYUeBRhrMt3nmAkXVZgSb0ef<*d4F1mZ3jZVCMz|2xD5pV} zmCsOe!gV~>x{VJXJ^5`x#a*XjvF0zr_T_8^l{k-e4FXzJU845%2!unaYR!D4;WHvIis+)XXBG;HHg&S{~Y+}>Ge4T40MF&rXC zyf+4%^jspWe;W9yY;XO=R{Y65Mw-1MzvHvbXhRrl$=F+$UzuN8HP_oE=(?S0pyDYNN4o@iD^Y-l$1SmD`3T)u6?GD7wRu;agg1uv`rPxG@>6_;uCp$v7|F zo@|}6@>bg>R<+w{;z5*eUPN~YGAUbVtnrH*8&3aZlu2*Bf9#TSrJRqWib^I`*;Pn##k` zW^iTNDauS&viS0m{egy&+-pGtr8~^GSoNAO!-w<*%hKs&sit|@uFLq1xxS)?6{xOu zjd(Yx|BXyy^_ahX@p1u*1#10j6wYD7SP~PX3fbGg9+i({J#xyI8@cD7G_e-l)MHP_ zWGHb`y6q7r&HLN5?OIeU#GbsWW~s?j6q_9LN#lN5LSFUQki=3}&*|J>n9*I(Cyt$4M|8wXXyw>M8V^LwN?B zNBey~@wY#{9LeIdG3dsA>f+x??!Vs23w~U95^0*8UrTyN2JfMk54CafC~3Dp6ZzHI z>@v%O*=DG6B{t%ld)o^vtZu_EDV9CBMt?qfCu=5oK}9oK4J#t4fMk-~|G|-`?&vzPJKapax^?EG6Cm%3r2qj7g({;u^o|dE8mOTiyN* z*RXRRD#g(X@Y;8agqp>)RlpU$!4i_!wR#j2QA+HMsXl4+GuMFc z-U;cOd3D72V7Vgf@A=uBs_d@gx8IkJ$hCQ>osUs7KRb+1%H**f`?kbi5nhuuwXFGW z>bK^JxP&{iN!N97Y&xYf9kU~@?@PCP*7^6gW^)KzYALtz*PeGo)|3U{OJ^K1O_u5@ z;(q6aw_;0YouRr-WS3Fs*ekfMwD8zn*H*&+ju-LkDGohCq0J$iy{Rb##ddjAP1ohB z({AwGTw$MOntYrfx1$-G2dix3Hg|ju_IQOJb^#}}w`s)izZm6A>BGD{;d1Z zd06h!8T1hyqj@bJN*e=B>flRhBHmhUc^M$%BIxZ#$b{I1BgVs zKVj3+DYE&t%E|ra=3c*B@qV2zZ#1*GHHr3#NO1`Fr6!+nwNoc@ibn!yOjw}~PeYf= zzFo(n5-xXI%d*6{aukul-$jI=jpkfLNm~gmtK7avcwA)AAOkullZsw=U&W?WNmP zP^R-4*1eL-Nz3YR$eh!*y=rq;YGto`oxc?z&iFGUoixphYTa;K)aM+6%&Kys8}z7u zrLYr|v$}N0u^DGS=rTEOR5c#1wY+^+l?j@4Dw^H_7I!%^Zi-&J~S zhipXlGV9>1s&)&@=^t2#zAEiflpe=z^xMe?&(^!UV~yB#HcD52zc@XS^KoVeAoVl= zg{Dxzk=;^FQhL|8TL=`Em4pdpd;-0kX>4t%&|dKlK}A&q>;BVndbX4wZiOqs{n@*% z0%DAe+o5|Hec#WMv2GLFah_W@VytV9cW2w7{;b9_2f!#SeC^M#F9P2D0JLqxv}My< zaksH>*0%wvcBLt$EZb#fpR;M+cU=GVWD*AjPkW}fVD`$K5!4Cx1>th&_>+8dOl$mm z8&&(VAM`w33Uc`7i$^=yt9BGq@OcRfmD|}nt}5gM9@1gq&~V+b#@P6L0I})?@k>hb zcf+y)BXDJjJ-icd(5!GuYv#EC0M;-ex&@oGcdOiZ?RSGvla_S1p+_YH8sHTQk*h_h zii`I!Mqi2qL*#u?%7?5y^KvvG?~*`Md(~jMJWk@J+*;D6XEIYgS$)pK3d+l3qc0+w zZXQZDd3EM1v@lKzE#j+i@0|IkKE3F;p!JkfW#evz=;TlbM=J?$V!$l)g(i9(Vht#} zM%n3sD;B-^ror_6K6TbTP*mCU+I~v-yh0t^ljfO`{kF&NXUCSg`4_7Xvs5?&JZD{? zrTMDBmGWwq4$qlA3EUu>t#Wgu+RE*dX1U({BjcestT_!CBrOcpFBGFD}ltTn?4(OBGT0@_e%_Tls;{`^ddP+FImNK2hEN7n86vIVn# zT&KK{&TT|0tf_&FO#sD;R?0j#ss0WMaizP~xxe!)bl{AQWh1%LmUy-DO%xm#DmKDp z{Q*^0WEwYxpDU*=1CHXHL(OIK@IMeZfbM@I@DWYfdAx!}p^B#CTDRH_^y%wtXdv$by&K`E|q!an>N16@B~AS!O2vTpbfr>oxsL#smVt$Gb!S4 z4)v_v2LZxE`NUwHRgsQThPA1?suS+lk14fL6# zXg`KxtdBlS-$9E_hYpEK7$t?t)|k%x$E5ercmJ=om&l$qB*I`iSg7sK=EI9-k z0hUQNa%_9+!GUyhgu-A59!H0f1fkR<#1RIe6uVELyx%=V;tY_G2|pFEFrk9*I8 zsd?BuprG>Y98R|QAqYKexeqM}DxPe*FAE2f&!=in;3&(BITn3|X+XJ5Cq6#OR_m60 z=(A`AjLLL#aoBX~M?Aret#SkQ^y`_!M3>U_@^lOA3MhO*!i6~QOC6a&L%)RHy{y4^=rwc3n z!G7$GL^5bG>F#uS0(Ev+7xHUQJnIwwx$9icsLyL!RjhTu7m*pgs3LQLl0x&eVzon> z&)N+dlH!-gEoKf`TkCUCfJVw_XxIw66!{)(LwWCrb({Yv2o(B8>EH)#Tgg%&_O_pj z+r_D|z?ua>N3$bbXkw_#oQ}wbJDC5|Fc1rJ2TPnevF$K`c3BV>oJfxXCT`E~w{60L zn8JF@=1Gi$Dipo44wjtqx4`ELCHZ@REnmB2`;8tgc~jsG&D+o%C^}K|$q~`Urz6?S z9u$xyptIPj-8n1MPFWsVMu`v^#H0B^1M7S%&GxajisT-kzzzGhyBlM775h%f|9|*OW=;lIwtGMO&7Pha)2OUdRaqqlxMzPyyxlH*rF?Q|jHIv%O)1 z^^y;rD75Hiq=^9NScC$Fi|S`rVEYbvx@1?qljFZp<*B$jKX6CwOhbWrN0OxHJm7SB zmaKfmQb0qtVPlOfIog$;KA94i9F()S%9a|aJE21Bn7YwZ=U+7B)K5>7h%v9?Ot&WORZ?U{+i81Jn)or{wU(XX7S&;5f;^)h1@S z(h}Y-u6D|^pWYYtdB=H$S9D6f9RZaEMw2Y6y_Z+R)frd;%Di)@ITdr_pWDyUqE`f@ zC;YU3*7fHFlr@@EXQ@{X^*nY{5h;Be(Z|L-%A{6anLL?Dl|AF`ezo8t19feTQEW9V zU4FHFXba<$MpG-$J>Tct*ADCrFvHDEjIz9RMiL#i@jK<;H1v;@AO5`tXb#=dO6g(n=gZ|lgwJw|M%L$kk|8V+S^Zm zUOD2;?QlF~;%dC1n6XcYK`Y;6^C|TR*ONrZSh)ULv0GP#RWm$y4Xw~E*E8n>ut6)rh$QO2a=cV3C7{4e;MdoOOoM)|`&35kIlHo!&Yt;?`0X;~zF z2}U+2n=J6z1I|r}ql-zZd$Le$A{ZT}JFcScvNqkBqMGAha$JNb(lp}GZg=Zhss_Xv@s?7aY+v1Rbrc86b2^-$yZZty$Iyq{d7{`=84=iw1l0YU&laaFu8#Xy^0BDkj?Nb0~b zmG>54ao+XwR*2sO&?r|l8r^Pk2zB(N;Lr*?-?9^@V=G#ef63YJ1!FAWf~A338`B;w z>yTo>4%hmnq{Mmhs~yP({4+k>Ro4tG;h+-VnJ+ftAA2p*QMJFjkdKJfQHFMdjUOVX zWl^DeL}&HX4nVOq5?eGMy%FmGP7$jMB3}6)*g^75H@`GrvB;5p$26VvPEN}2lB0Sr zlD2`x#$M32{@VYZvGfqmMF2|DbVg34RA(;~(%4pkm1g^zyd|$eS(J1_R$sy z*R~jEXxSf}$U+Pnqn>`2^xO~Ear~9vYnFixXVsc8wizG@G5@omtzMF=J^(92j{mp! zVcW|WR2<2AKnmyfdyaHUf^}!_PsctBy8=C8KeFfnD1&^U995CbCyNq$I}JFfWv`=h z-?6<=6^H2N0ew7UWh;8!1?nLd$l=vflpL3>%mFFzoh(Q7X4Ssm1~>kjQ@FMGF|Y?K z%ZFzTh7=?&g}JUyDVXlU~no$)SQ<*HJ%K z%CCT$eFr%@CES}xl7OaI(*JyWrA)dm-MPL&kR*yN9$?A4qF-Z+})Dh7(I+G`6$mZ_$8>oNq{UE zg>*qz%h0(N8%n?!rJ~M&QNp2hi1hKULJO@A`QgOg?+kudKO4=6ETI$(K4rktAXEimVY2hm}ckrj^j_>+h006p!s#PJXS`D!`!Tj0}W ztW%;3wge2pB@`YG!3h< z$wjt&!KU1%wSa7r%7bxa+v?kEdcwAqFc4Ut`U++y@dI}8T5$`3IB{x=_b;kUmP;C$ zGsb=}AIbrtHf3YTt4aWK(*@vT2RN2*kq1Y^h6;fE@xnpZSuC&-fdhdMBob+_{q3Pn zfUz-y9vcHRY+~M~#YHmshtIj055Sxc?|GDv7BVh7D+N zYI+`FZhUofs5&D9vLGjTV*&v*knh{>>z_9iyL>TNf3z3p5TS4!b4|qb=K{1|Gwn&Z zC2bq<*rl@yFAkZU%OFL@-nFF%fF?}7F+B7?k@_Rww9}pCZJLW9871mCd{Y_#W;*~t zysgc6OH##<$~^Z)S@vSsW1azfCDJ*m105Bl^ki84cl^}Jh{b0 zPz@f&-F05Soks7r`1M#P5TiE}(RQsg_F>8E%>MLg)K7E2I_LDkwq}0mMK7F0vY602 z_l+_1CO}GJY1la0gAzWjE^;g3in45Eq!iDL>YCbv&VQPIw0VR7Qu3h5VQIww4 zI1SF7zsSVJi(P*if53too}-9S|BRRl$X68|4+*J%f-FE&RdFeHvN%+|Wzs7VbmaWi z&k84KMrz~vv6JRhZKvTvyXc^6cr8O5!AeMXlZU?hr7Guhc3)C_zWW=o3*R?5_jSeE z1h=17=2*&Atl96C5Yq{^UtihMuTP{hZ0nd^02JnjLVNwhe_A zzQnT_Ip4=Ek>i}p!@%jNtz6z$H&8Aqnw!DNe)0?)8W(A{;khvp<1}a%$NutFH!FMa zm7vDbndDvTXgciDqD3~Q9t;-(QfXd!H1>hXkRkyup2q@Te*q^>lpNn1A0Qp5xrnK&q~iAfB8dpL68nGqMMf5T<^(wn%w zL>8NqUxFPNKLu@1h0_oj5C?Q>j=HYv%F04}cL#W|OkwjK;lc%=-ZM~}PKOB%<$(1* zC{!n!#;Ge*c_t&^5kJ#x&^(7iEA=S;j^m$*rUyuzjDCvSv__9%k441`+20P1Cjl=w z4eo<3t}(Y985>7KRukqs)9|{_q@T=g6B(i|y0u=Q?VCN6Ux)8BDZ6xH)F zX(TH;6K~Y~fpUXWfH%_OJ&1wIqzzfh7EWNe8;3c8gbEAc#8yH*Ta3 ze;;Yw4~J4Jk0YUxuGheNbX^fHO}(uB_@lAyRl_m1b4n*pGxBE2#R+;Wdz`$*rYZ5q zA0UsuJDDi;g#Q4u3Ri$ zVV;Sp?hL%MU0wac4%U~lQ%dH>g_mOBGuY0jYbiWBr(~35puAacf=fSonpS)82UqEK zc(nWS_Yw->=1b0?yNB?2={W<}LU={X)P5Xbp3q<^kYP5fLPZ-o1)WV0hC+)5Qhj%T z|KhZ&nEYsLACRsmL@tGES*GRx0VLO!c&`=#$*vz?;kE5# zX;rC%|2FJv|2+cqrm-(HzVfc?>Nf+E`WrtOWCB}DZ2}u;!1okI8kC*W5TJJU(Ci?# zuSkFLlx-fzL-`c=`)}dyh`%k6HWj9(`#t!+Vd^}u1Tdll|$Aa1===tXxaz{xF zr7t}*GO?w>)!hWV=CW5aWnJDWcW2z3yl-d5t3@;M8%sb1py`;d9|x=(v!`Z!LB(3( zDn+nR;@JgY?u`{hhR*Y-Ft+4$BDV|l_-K*5pI*(TYBRTT;r!%nFwO|Lf#b&OXJbRn zZ*M|6wFgd73WCZOMFWnbr$U1RzEM_EtN3%0YjRD#nn;P^caTDa0zsygGb3+FPB7L5 zVio|NH$T&dl#g9175(FA3F3^aWW5MAx$qaL^Z4rHxI>3=dGpCpmYTCGfoo;<({D3Q z80~Y~7lUG2;w9pfj-_uoC7by9l~5Xfgu2S`4%(`z!IZWHKqTWhvaV*(k=bavz5suD zv&|L3_fOJ}{w&gHL8Se781V`Hqvv&O+aM{Rnd1eb)mSnmm%yQW@B{CQKYo9G(FwUB z_1U#VK0i#Z!9Rc5>!0Ex=DWx$b#=wwJZv_fGQ_};)zvJ)8R%MDYtGSr%^)^0+TMDWw2_v zTcC}YK09*Z0SIGy2&^(^K{{)XXW_|5hBEyne&m#=pEEbuHRj819yG@Z<)ZR5CE~t9ly~Em4yj z@zNjIrU9kM0E1Q`2pYP)nv}wqj=1Ulqe^@4qtu#Gj&8;B=&9R3FZ*e+SF9?6O&Vzo}00MB1fTJsHNyxtrGDSUo`ywMFyObGKt_pw;ZMgiu=A*(wd{KF(9vcC;o10W!YvU{YoP-9y1fp3gifcNv! z6xdnTIyV$@T}^01D~lt5DzLB;xagXu+7mkgZ^fbCE7uh$0z-TPN2V&SufC*eV%+71 znp!1b=jtRR-!vlX=N^dby2>^Y#R>Um*Kx^c35S9)$5a3e(+M>Y9O& z5x9e?xS7k~q?lRDAiuc0au7iSkCB;G3tYCQ92_ii-MPXlU|FB9#kqfe`sZg}SS%f< zq3ADU#OjqH{dS|Yqc9FGP^xVKqI=`p0y|{ApN6r z3n-4Jw|^BZ#O7ICLE3j{7;$)iqe$8zr%uKV&SRHfyAG7U$#^KgIxrnyDhSzKlOJ;6 zR~~r>8Mg*hMjO$Z~azVfe;f2NRB}EqbZ76LQT}7)zoT-TwG+V^N7Dd7s_?D zbUV?fPmeg{J%!!W)>Z+BFdh5$#uQZW##!8%UzD2tz+bKp9JYb7mot+6&i~F@V_qN>(}uQpup${jC@Ja}REwpd_yJ z-&px|KQ>&G`SkISfW}lGZI|&d7L}*Isb~6KXR~m9y2^JFAf^NbvV^ixQ{|+aj`i0g zD-Q=zz(D{L^ogye1n!Yn`C?8I7hczt(x%>^9Q5`sXAyU!h&f0shUt3kuC0PWGeI~u z1g63&6#B8@TId3iV@?^#!&yu2#NH~Cqb&X$wuU3+hQ*f63X}sL`#BMRkH;4a?XD_q z!QxWfUY|4f$g~Mgj|#Oqz{)ne(f|XZ{Z8WgeGr_^+Gsz6VkH>NW?W?CQ_+>6ML>Ds zLKJqRFOi`?#B`-5H(>i!$q_haCl{FQ`OumtFdXY--F}>EL(n8*;sP<&QWN`sVH+s8 z?+St2WcC3T>khZ&tqJxZ!m1%@_3?}U=*;hIcNy(Z0!Mpd1}zJ{_dmhyy=&>;&5%D# z5R6n6#r-;-#-GVsQ%-_0qO=CMdiYJMXCQqQ*;au`I|F+leYO%6MouQpEGM1UH}Ve< z2ELuZ;bP=v2=0??J%-R%xqz>j#7anH;a%=*p#qdo0j26BHoltdCzLkhcdNZ5D~8$S zBpVDj13B`}FV=$xtWG^(p7?Ga;7{C(f~kNzd`C%X)Gh9TaIH77Ju|T!^_q-Pe9WMQ ztf$_i%yF(ki?-G-r9iPjpEwxg5|tyoM20HxHs0UQ*zLbOM(g~k<40HTtE1FppTxdx z{4@#;Yui^lYoO0XgaUQ8p9+y`3!6%|XWJg=o7(mcR2gMK`QfBjxfZZ9UDPeR7ujLC zvA)wL@PLA36hiY<+_^VJK2EbD8KQ&)!E6f=hJRfeNLno`zr~*XW-a`gP@a&TK(b&7 ze7(OeF@dBtVbr4e4aI$JP%2=ElXnK0xsHKs=RQ=RwhHJ$29U+Y+Q1 zxg`4n>6ga@*lK8ffX^Z zMw*Kr(I$QlCXeO(3^l!SH)$o=j9Af7Ds86vuG$HTC!mT65Etjqj~%GgAnAT!D_^YX zcOQdi(RRtV78PBT;gamth5T}d(p(!}wKbE$ zm1O~8plOJD%An3RuSr91R(|Y1+%r-P;Iyq}wi?d&uJ2@Tez*_2e(x*CyFO>I9qrqhBkdx9@k@CL)tpT)Lz=DR%CS(SN_*-VhBWZvARMl& zxeqm>b=b_Okc=4g!O}${{lw4HoY$jnC`JZ-NiOXFab{maPkC+l7?{BUjL(BppZ}vv z`}HL3=>DF3@_6+?n2tDrasP9`0zv>Ybq%<|=|((b%ZJoiMtHKAWOAa*0Qs6W^8cI_&_CJ^+Jw9LH_PH-U-wgv0}l97!S@$^|6_>b9D=33yN7N zLNS=#dia>~(7k_tK(=#ypDCEW8hy9F0>WHUAbTY*GNBLbQUK3#Nb%?e`qBw_S=BQJ zfyv2uH;7|(B8%lVC~#b0{bJF40{9@ePz-r|c5hN>UlK{!edZvdYJ%z1sby86GRo}* zySx(=zBtfcKi~l)j4J^xJTGAq_YXJA%)8AQIG?$N511=5&{@~ zSGmgyO?6gY1JyJlMUK1RQYItXk7VC8l?+uEA?U$sNX70%>lb2)I;OiwM@8`_lvhfF zEq!3ILzY>;7q)#TXw5>Bc9=@gl1+hZwr&SnK*oc!%1d5gNwi=V=V<7bH}LH6vE&%X z-Wx3t?jb~sx{nHgdmi@q(Se16QIr~TUy(iV!+~u?Xu=AwO($8a1ms0C1$@uV6gp>) zeHJa?4;o7GAAr%ki4J$58c2r<{xmcZxnp$h*0i5X0YW7q*ZMA8INy~ZW5hZQh0@P_ESBtqBlvn*pxF_2M%72Q8b+a_=BrLRf%KdVf*}vG{1EUwR)Daq z>r}ko^4xnRIZ1#Wwg;l=YZ21#_btkWpgNel{@afCfCN!TkkkQH;Bp2b6hcPOaXzsz z_c4?+`!BLQ&rWA6{{7q@9D3{}r~bU1666=^z;H=PBb#t05|;h+>1#4m?aKh)cU0}I z<^T-t-PZ__8^p~Y2Jpk?kcQHHCHy_HWEUD zFk1*;wt^(Fxq=3T$_ui5z_$l#^qyEMMYr3n7A__1ERBH~apqOBT(M0GygM8^HGiLk zN(l5$%@{43S-{zGjg5`VZf9{i%P`@S3@GU4G4G7b&r+l&88eW9t7JMz1&E16OZXIl zeV~`!9As@Pe&wH%2({4chuSbm)-qP*OX88x%hX%OjUf3I=t42tO_hGIV83fgLYh>vy{CnNs~Cd);kFfcq#bji6BOMt6)fm|M%a zg8>ZDfx|ffL_D$ZJGsNW6KIx42keV+B7IXjvd=0f_k|&EV&|Y^5qiUy3b&RTTzuQ8 zA?W+R1Wq*EG7b1u8UgtNqTjHcKFGm>rg01xC4c;M7Uqf?!=i4PVN4FfPM^L8x@-xb zTmmCs_JKouHw+F$`u$tX1hY^FJ82~N^_?DD`8cw1qoy<$ye=#}!JBHgd#ZrXOXL_(+{DrUCCK?-;SP%h>`N`jfFUo^6v_zp4=^`iqX^G zPQ4ez+?1YB^Vx7y?K1$_$?epMXprd-9DR1*FTN3@mWs^~2I&^mJ3*GAMH?(E+}&`N z^R(Wp37YP-X@VpfxxaZvPA`YrRlO6VT>`7AU|+li?Z|P&wqa zf7kD2p69FKY7oZlw?MbCGx-IDN?=I;q<;wg*E~xxh)X)6Auv*Ea23|vWjOuxxrS2b z7KZ|ebY$_SKouJ|$dtoF3T`hL)5uRd++r5-hEv5bDG}={eRg*ZEZIn)w)F8(Sff0} zH*%pi&giFYbG#Fr%cKL#QHO#w!JzDts{nxHEE~u~G7{6-BTehHpV_l_!3715ilHe-ac=c3M6Npte9t zNSESCzf<5;xlsY1iqS4ov;FRbcY(fot={YrID+!z|>KZ5-gSI8t z63N|UcML`+f=641C{T{v9)(f&%k-)|r{O~BtOkG!ec;hbD-KO%+kpy%>$6lA@!XnD z8O_prCql}1<(Qj7i!&sjl{MF)8mAy@TnP2ok5rMqoViHHaOZz8Lbb0Mc$K6Zl_Kx$O%9)uEd&2_2 z>}UKP$XuNI_8|Ombr6xbDCN4verTbdk$Nf6ByT&+_w)G~=Iuvxvhh!euML4takrl6 zHwt|nbl`@13rH*SecA>P8VjwlC?6VRFpt1(W(K28OXaKLhu@gAa>9Hg%An^1q)9n^ zT7cs;J)8e04&ic=MI<@UkQ73`&o7r8OAnlZ-bF9;e_yGcoy8ToC1xceiuk%#>XHa-o_Vo^w5#C%sTewVJb37wcx`Fh(sfMmKL=uwW*A|Z z?3gMP$*zExfaEKC`_&Dw%F@=ha5Q!@mI3!`8UGnOH7o zPCcFe8sv(qYTMLzK{V)KFc-34Y-ong1p=`eX#TVVT2ho|fms;Jq9Rzl3Q?*a=Tkxe zsq(=uOZtzI)`N&9fJqmjZhx1R5p_e_3S{$J0G>82{PkEGQ4!G3AG?Cp+KAt6^V)mo zy8h=Q8kG%2a7Pa1p`82B(CNx;=%agFFzR9t^dcDz>mnK*SBuimf}rl0g-(JDFO5=n zl}vWcoDya|nHn>&!VcgyxDI^Z_3kW1z98klu2=hW0~$&+lC5Tma9nWFp3{)W2ZYt~p~I!1n2&ieh;d^Z9N8iw^aoFR zGY-ZKZ=(?*zU!Kxeb|{GVwwbaf)%-pNSA9d=!7-$u?Gy&t|ZBK_TXJi{GJ0UN43BY zd3K&-%E&JuZAI%iA1;0JwPi|-qZCRO1mpK0%OwXz%5@76(8@YworYIIBA&z{Nvjlo z+m3e=Zd5XIz|p)kpLXEuMOXwGHf3b_Xg=gXhXSUmL3(ABmyyy9S{hDkzj!o6nnK(9 z{$Lw|7<%WP4WRLtP%H#--mR{E0nR%y+YVw~K8%Vc2jA|!PFD-3AEv5d9EKvB%G-!c zvirS8_IYJEq#PH9^ap%wKL~N!f??!Y7JEu*RY_ zC_&?|*_js@x_S)*SsOoy6=*8|(18a~GGYbbiFQGq!9Fn89G=BZulZMuArDYWjFhwr z9txV5o;*>_0riMxzP=Jrt=G6+8uR)cNNT%>fEF%8m_Kp^9QV+F_^xePa_dOzB+ z*K~YX@6{P*-`C-XDu75Q3GDpE>Ei%b(G(izI+G@aUqV^|jO$B1iMA0(sPi_;(e3aS zYQ&IqGy8fC!^O63kag61O2d)N>QZ|$=GB=4W~vl7)m(Ly@nYT-LHATXtp>lf4}xYuM_?W^z5t497AsryS2I&0_xy8 z3xokqriAD_Kxn5?X*Wj_MdY#K=|p^3~n z{G8``C+6IM)2*E8PPL5snO7I>M5ZrR%pGr(<$P16p2k&0tX_}ii84h){|gvA@xYGf zRZ+g(fju!9{{tKU1x1HkF7w?O*)()hkmZBq^fF-+1)`Vd!dltW2@t-Bp&6Nj>~KM^ z=TkRfyeAZuRy;HFu~8Z%YGWDxDX=?jXn=aK;MHq3QF3h`ZZClD4J8ehjL$_vSYB>U_F z8;EC2NL3zhsZz6X!ydh9vul)H-3v8^ec7Wqo|h94{<{N+^eH}-SJ_@MT!oz6`&@c) zM*2v!Tfx9k5=~oDD({usrSlCtE}SwzD}r49Vss9gxXwAL%eC@qyx@{Jh^-sRD0}1= z!MRfib3P>(B~KQF%+;Tni1=V76+Gt%tMB(FGZQ=a#>ACu zKl^mT4e|c;j6(+n*u#F4)xUv>tU(OlZ&IV6VoG9dA(k?9hTI#a#j^_4((QvRzj108 z-&XNfaJbmV4_s~K;jFLRfllI`c9VIcN7rpmDqhWl?2<VS|bs7~@RjLcxXJxq!)&H#81J0V<4X?8i!)iD0QiR!|IYq)}#U}&2dyq6$JF&kG zY=R3WB_(^EOx-x`d4+LrQ~w0Uci8B~C{p~MdakWAMcE=C2$!<#1tugj&cw_Xrt6h^ z)g)=QQKzIj^slq{k)>6%?2U4dEh<)MGPf5pw03@dBz@r!h@95MUZ^!y19#2+X?wdf ziSDUIB0>>6XH z8$1)+BZJTv!)akSzn_b-rq(&zYk%(@FnTgZ8G&J+a~+__Bc76R)% zo4qDy&dQ36Xg1hLj6br!8)D5qaV^N0{6L;U>LUFRighB3q5o0|oERTTFh{OofzcZX zvvw3R%h+MR^JEkb$G)*}|Fhr1a&PyCyZ_Rp*C$KxZ|Fb7UVPiOwpIJYb$7(>QmQ#X zaXB=T;7Mc92@?j&qos$c4FiwTkyVOeb<%zvbJ+Lbsj#K{hbR8;fJ^ostVPTTOXyDW zz4y4Z{}HbK_uo5MEts{&{{@)o;{6m!4&8%42V|ymz(Z5j5CU6;f{4<@;M$%SevyIs z10pONoyAb4hgo;J%1j4VnjaWuCm4FyT(lU)gB~E-izc%>A?y4W83A45+MQR^{owmMbzIGuThH51;p6y{CFa-uhRC*^~(~BsANN!7}*lmn5 z&;&s`h&)O)Ayx2jE_inGimD3j$P%^lM`I-}ck(Kr8efBH!do!_5xOfU05C~|9^aMpkoWQkY3n-od;-8 z2R?(N(|x7o1CnIjTdoB_XFd$eb!WJ2+xaA@g;1o2NN^v#}O}d29TIGH{I=&xbA-1>?>b1iN z@Y?Wgn*q$!xa1oWyOJr36Dj+6FD)SP6b&^^$L;St!QxS19z_tPq3df!B0&P>O%GS% zb=_F?-XZz)F1P|Nt46Qx-O6r5Bu+vs8^TuvcSh~NPwdaX0DDn z48X2TeJ7DPc8co##xDGL;$F_}ghdD(7D~bKpA4#=e}rbu`4(KDzIUR+feA4+H;eKz z?)P7Ukpsgto5{ge9q407yhskBF{ulhv>P5ebix(ZAdHTmNi7{3hax{f^OhCnt)R2( zvwlV$kms0ph3gu#USW@>`8b9{xH6sSH`uXZB8344`lcGw1@A^7EA$qRL0}MnZHimC5!F1Yau!B6S7g zSOpC=TPWNUiwi-wS|320bjLa{{s}TyrbeHb(hG>#)|#R(;LS(uN}xwJeEJOx{P4wM?nkya)R7yg>M8D#WogFT=mAo>1DqLA|?t1r(_+o>p|Q|rB7l2t4C zQ^@ot8ZS;91dFXYh@N@WjCssIK1-#x>dbxJRf_9tw5h4*N5tMdlvE}sg;_YYU-hYV zaEDgqWaNil zPfZ8d31T8%^=@)NSjbg{@nQ%ybj|dm&+S1IaM~Id?;F;`b%oc|r{2qyL$jVJ5C8}3 zIP?1#cvL`t-prS;_5Ay&lMi5m`N+dfoSJVUQ^xN>I)VV0)oS49N zsDSc>PLxrAH>dBE_>rNpS0zNXY-N8G=a$hp@(|8OocxgrI6IkzU#y$m)R`Gr5{_~b z^D3@!#J6%{KX-MMo;xSvWklod-rnHzZt2vep=i{z_QZp|aLy~eC$AlZ>ii=1vzd_H zN%}n@-bB_g!dy(j^aA7Q#6yb^{r*<_it$fiV*$xN`XCs5;H066xeVWhmD&0(Lvo0$ z){RaA=tb+-wV-Bd13x>el{7h~_>VumqE3z^^-kkfx;1j+_eC}YB5h*xB40QhS z`J7dHCtrjdrM{Y{dy+b2C8)73_4*&^RoOdzXw1dI*y8=6(#R9#3{n1vakg?>@7~+0~u?BUEPzin$clNoK`{~=_S&~m#&hAs1djj|xyPSJPN_#Yl z_blWd&vr$M1`L9Zw8VVs$)ihJYHatDExFA*+O;0apP$~2_N||E-5tJF;kks=9f;^m z%x95d>OH!LKLxQrPn+Q%6kpSA-lq`JNZh(P@m&kt4m<1f_ze&YB!OW)qGixpTO%tL zyG-%xzCg%pcHgrSo@a_kzA#V*;_J2D=nqxTgd`Rshp^v&1NyZrBI1=g5TqbnAkt}e zpuk;{x)i{8Gc%h%JK?%L^CdOqmHPf?Z$j(J{qsU$ripP#PLMDrz;BBN)@H>iDRF0& z^6UA;#UAw=4pv6cairgXLv~2%mBp9NYG?}P&7NzLo(;f6$Raw4^^Ae3ztl-9%QoQ`7fRm1#-*jbW{*Qt+bTb$UCPcO7#ZRolyg6 zAgdaZ*rbqUN6KYrk$^aHPm;0?h3qZEnLHSAO^fr$I4~$=b9A_U;GtXwLg>; z-*vhLx2KNxtKR?LSHjfh7TAN6I7k<@@Gxa2o1g!x_{t!=JIfAeC%!oN+lhn@_DBzu zfF+brp3v50=f{gqG2b(_^&h#EigNh<2MmFEAGTiek0nCs?o z&L;-tYFC5z)eI;$x-TioZb#9O6^0j{6M37MZZDkdu%Z2g$3X%@y5P_hT0QzFilf2@08u&TA^fLDmqDIQoF06HVG*Rd&tGpq0 zM0bqg4YsaQJcQ=sixTt+d6NvFxoKiwucFTy`zZ7DjxNP1F+&rFa<)Sq5Lr%A6$wc! zk>t9__dnWu^JuE~|6hDl8l;d6A%rL@oXm4YY$0Pwrf4#hOv%g`P_Yn&%NKzUF+WUTX%ie_qV?1pL0&_={>z(&*x)$0`fScH+Wse z7l=+2^$nb@kw|;hx7ea8Lg?R8B3ZujN67D*5=vh={NXv!!rHE0tfbqh@922KV)H|| zJr-B(ghDlcD6NLl)!EG@FK%x6z+YN_%q6Oo`?^#%$A~feWNcEnR^`4#irCh<<*S`y_8zqGby66Av+`fv0sx=eBS<$U;dM>!@<9-mi)SPqP}NB0kchF z@h6;wht!re<$M}B-?X^?twqr}3)C!t48nper9a4d{*8Xp8RN7%X6|D<9d7s+Hg4M` z|I<&ynumV@5ygyea#US#;8w!XiLNr2kL)a0%F%g$#~f@A&bD^9&O{Mq?L}*IHRML& zg5AZ=)@-Y+U3-?|6J`0uG4NKQ*dZxn2wi>zmNsazm{QCx2bMtTDAjGZ-KGb1#@_h1 z315AoYXlPPh4sOAfD|`!9j@9&)hA!r0sPE- zB~JM1kFNYv8enf$Z3Kl>r%sTf~puEg@qw4Ilpdzh2lX{OJD;GxJ{{DfYjAJRh8b z|NFac>p?24GXiOncK;06@p=Hykl$D6R9Z1o2oql9!IMN~ULY?@r{j-vn%MEbpg7vN z(oz*uu#YE30)p8N2$v)Xafn~3r0&a?(0rwXX6P?WnMAR%*6{JEZl7vSSjvQUJo`QBPQ9=&)@HG(W)$lfaGuz00wL^x^Yegu)2S&)ql zk+2g(J$3`d+!^rOK|S$7_Vro$Doyv>YH~FGpbKzjDtFj3(e(U-YuhMa#^Li*i2eYB z**?EFptDIsv%d`EX@|lZ=lO2bQ!?)9g42ZZzgSE%4!!amG!*FNvHVsZFL^`kwg7`$;FSa@%M;|LV#Hf;Lx+rnDzf`}kH6 zm}KYY<>jqRVO9|jh^!TF$R4SNbj36f1@p;l49%$o(Nv~x6f2wq|KR+?MuZo(Z9?5q z91@Rf-L#HgM=OY_qu3ZIr98SS!tP7jMuV;~nVR7Px81XoBv}wyho0P6%N?vf)jvT| zQA__N#k*`{&Pz*ayYjO-bGOx!?deC&KWrQSrH9+s+Uy8HeG>V(lxbyyb>aX$4J}`q z){kj@2A%OYi_~KjnHz4x_gqTH*IOi*c@}eR1xuDCP%ot zOAIGNg_BH1!8Qc{B(Py>145EVUxT$n4tmYwC7xoz8#Ro3VB=3TExIH@{B{OKrr2zO z>^vR=kX(^a0Mc4M#24fK5U|YnQi30-dRbD(4Q|}CC?Ty2_>RRx$TuF42zz{i>mH3x zCaP0K>vi^7sF9jpe*Fwx*39K2ROXEupU2IGpR_}S;0|#^M@Wa2*UKp^|gh;v=w8s;GuvmXIQq-;gL{+bWm{NIMn% z^#uOYxhe{?o&E)UH=LpVumt^el_)pH+b|JuZoJKHAwVT7_b7h3|GS*$yDPE(<&?c@ zWj@Y>pU-&%4tNL?;X!p>`c(=4OmdSO8v&uelKu&xIYvjQlIus4Z~_&ao`lG zHR!1j>NN{Gc~M6dNrxg#L%?7$DmMYCiCWO```NWTuDL#1tJe!%QlV}n3x3Omb2_8X zXrD@M>;VB@GvowvjcTBA=AXDb z)x!_`@zn&nCxkpZOriqfcage!4nAzVO1&= zoDPebTz4?T&S=!~a&I`s+5;ED{8R|_2OKJbbBxyKc98PT!s?X<3>qe>z3~2moDu3A z-sC|>j)(c5ub?Nu&N^NMkq{ZM=Xbo)?R;WBBagD2f4ogK07!(YM7CwlI?;r>=q3cv zR$^SpDH>hvyf%R6$j;jMiF%T7LD%|uDen_~u>s=jlmnf^j67hT51){*4<@L$6g|1- zjhSIUP#xL@^hY+x`D{VOp^JO9#c$xK(jb*UqItCw3F}HL6cP`TQSEAows^R; zjB}6e#$TnX#6UQ@-WzQgXpV!+Y)j2FBJ7YQtU#bEH4q=eOgTG4h4=~~S>I35JPLDk z2w{)mS7C({;bV6JWjZ>3?sJm?M_k>Uec zM`p%oA%3T-e*0+xZkk0se{a!sEi=h%-^GF7kZ_o`5*?bg5yIumCPTPL93-Jxh@fr= zPdURX%D03|E%FZGfyUe5$?|`k{haU=gaKrL=9(H+?T~ZDL^_cI48NkTEzk+aVa2%( zq!TNcp@Z~q2MfUVy!`%AV1rfFzY#C-^c3|z};IQlPNk!Vba za~=J3U=EV}Py@dTvarPxsABU^?)n?gkH04mAx^DF2JWaK63CLwJmIEkMb<8NT~GiX zl(@%8-Un30zVGmZJ-uLd1zGIZYsdPdFrfhM4$l&aP1IMeqw`0yj9L$4y8GI9xaaO5 z@`WY|#f?2!7bU^fL4jBMKi+^)u&YpGa$t2l=y&k%a!ike+EVTZJkGclH=MNs66POH z1&x*D$}sf9=|ebRg6h9TJv+qIxBewuYWo~+B~r7LuiCErE!H`>2{}2lbz8U#PI|@86hu zgYMBZAkf`&{_KG86aInLC4v`9)UO;&jAY=$-$#i-juSG5#I9@Te8H6;wVP(Asn7SE zI#lY|1)}(Khy*+5>h%WH=SV`BRvMI<10Im=hbw9B8YbFfII~W%4en6>*(rzD`(tTW zj;ekX21EuT*q*oxc9=OcQ*`If&FBV?<_UEYY;n8OoL|wtJd@)U8dT< zrFlWGCy&AfDOPpTr_w#0!j1P`sbxxVQC7Luz0#?R0m|-~QpuxpAkyS{!4by zn`b-q+n!17`>tv_?AXmaj zuC4iMrxk4_i1t;w{h%(EdSYmL($xJ3HcS(#&#yxKRPcrnaGE{$KK+4J7WLI{cjDQq z{AmmA8kQxiSiL>)HtO8G06`7!=iz?Hox66tAiD|FB(1RJF&q@ev`=)i&haK!-+=KX*%3E0oQKS4;9>j=jLzFnXr4d&+Voz|3-+j9LG#B@7xF~>MT_A^9{Ac%81=y{3otrYXlWW{ddQt z5P`be(*BQ4lzZeOP$E?@A>h$R$6}y;zHDMk_d_+s_0RHPt(1L^bZ&3HY>PG!^ z13LjF`EC%qQVjeeKBXACiY=|}>~Kp=Q6e-93W<=}gIUDT=ICNdVf{CZ%tBd;a@{w? z9Q|!mZ0%IHfe?y{qC)S5U@R3)#fLUEN?BeB)&nwzNkTAM!tgZ&6rj+G<0VO-G&vi@ zO}h)Pywd__#%n5 zs9cc(3>2_0D+Ao}AUHP6squxo`v5>m0uMou*-lw-vt^)YYZsVk+h!l)hgw+!Hgp|O z(t%~j^2&JQDF*xLZVa|soC2dnTrjX^Etl2lda*P{5CY**tp(nM(!Bow0WoKyE z3JNBBQ_MFrzX&?Ef}&q{Z=S750j$O&8GohTsecB*Izwn@EF(=9#FEr<@guNDXfq5P zrd9k88>?Ya0m#y%+-W1n69LjJ_BH>vZFT4%hGg1{ghl3aLAbr%;palT%%eQ5HN!`8 z?`CuPu?@Spo6eM2_$_D`SpGp$1NNV2s6!rP z@}b$c1rdDq=ute5S^d^82t54|CtAlX4n{jXHBmYPVZ;HyuB$<jiC+DvTVa?l$cM@c@leniu2Z#US`4;e>MOZ^NZ zsL!fUYdVp7XaewU<27$1x?jLnFHT|4_VvZ(Gv#>*4B)H;fDgDGy%6#TbTr?2!M(%y z)Mg6)`Yb@_S#Cc?t^*{)vS_{qB=ux{iUmZYwxS#nv67Pz7Kmc;Q=W}LKu|j&I5nrq ziy(WEK%+DnBe6PysNx>uA)q==Kr5e+F+|-?vr{(ur$8YqH~bnMI94){F-nEvFaQr{ z?V%~Y_-hk<*c}Xo?<$*z85em@KR}Sf>ZaUs-$^cLpzX_zSrh+msISj> z+d{1y+B>WU|7^^`e+CfzHzNq^%71c^@}FBG{pWo9r=lm=!1*uP{HvRCV{=#S93dlt zUKJm=z2a0fRPV&Um@3zXj-C2@=fC|W4IjSf|5erazn&6KBf{sGj@|9hnzX|zilZv2 zqZa_^%K%^SiBG?~035=ZnP3ylL|CV&l6FfPS_DyFY`FTFTIV|h4>S1@V#Z+=a--%c z-~c;RJIw;TNZq~XhFb(8AR0XSg*&tiNL23YZiZ%sDv1ZIN*8AdQNcaX_)7mPwVHc) zJ$28xnsScW6tmRjb;=aWs-)2 zAaHopKr>?fsANRyii)Iy&@ zBcxn+yeiB3>pP%()BOl#(??N}{tWHjb(L5+bt4U2Ley}J180ENsP21UKO2VTMa!;e zy`a|rZ3gg4$Oai)hJ$=r>-L!TX-FP9l@0%$3d&$iD5DATtUc|y!Yd$ikjw~@{J<^X+!L2;S$%gvBE$3?%|D6u~ z)0obFow;XGv0zr9;+@ecroLoGfz#`38ETt7Nn^z<3CTEqyVW+C>=^%6lm**x^iIku zj^UeG1%u~l7?!&g9VXQm<;Bm#`(+OO9x6?4JKM)Ua}TP-)*_5*Z{H9Wd24wV#7wBvj;+ab^{0Hs=~EBLdIW zN*Xd$X$O*psSWCgLFzO3M_vibJD8?p`)6GweEgTFu>YMg=szrt)CKpEJg^`d7%_n3 zY1%+s_5j>HLhLy2NGOnA^^t`qMl~Z`k2NM5+E&cKtZmMV=+lR*>-|*|F;vJ96eewO ze;BV}AXhvT_KDnUz~!7-EP=YDqZe`GJs3zLVYLAwxklPCK?eiV5jc~C?1kweb`z@r zs{Ix~*#o#VMF8Kg8^z*Z$I|f)s<`Y~N5X~g9}8yh@fgk#4sGi$R87Q0>$FF4kxl%Hf4Ev8% zv|jXFx#f#94biu<3RRe!&!tMWO@sh>mI{s+U7D|gp6@g|c22)eK) zu!;fpIQ6XvQ0u|uAq4&$8QX1WRUb9sb76}r670q>2eUpI@BbOGTM{37aI6qMcZod0 zsz*<8{=&J4cU9PFEZlZeL$7hT3!~OvL8|ip#DvwRovJ@Vn*Lc zg8^zd|Ar@2yxNDRI;&b&PDmMFzx(%F*mYI9;f}1rp;!YAU@Dp7-d4-V2jrEVZxdYT zQsy?v>Wf!DG{}SAX~pbA_Yy=49Ky)F#qOzwejwYEaDd7RB>?qLyAZ_2yzk^iYF<8K zx1#Hi(C>*m5rEIY&bAp{qTo`pL;fOlU~I_)B23iO~3m>LK03025&Nw1no^G z$OV*;W|b3Ukoe#Z5how2=#u~p!76K-mj`$Cl|bT(9`j5%2)PwKmPKf2G72trF;~^} zvAGSMPSgMWA9^B@mZR!OP*?(D+6l4W36*R-6b_bdRm%R z`)E3|T*vCz8)O&(bSPZuh8xHVu=&L4A$O3C)+dXVNsG{+5Ja(EX6-YJOGy)$hp7`Q zFmO3}RKIs?y9%wURoLL1DBX3|8Zlp-Yl$V;+8!tDWT|6xQMVy)kcGxnH zKX4wh;XBTst8LAd4g={UZH%h7hiO8Q2bU^wrh6g+nFR0(tPynY#bEDv)J zMqn^9DxggKqPKCn2Tgx`)k4i*gMndwBwp9P_{Lv9UJup3&6?g|^>r=J;)jvqlm~A^ zALHcIJs!VHQOVD3mJ+l2L@x_mnOeHk#7^Zp69GY#I|FY`v_*l1IYm{7cVnQ?-h|yB5Y#FZ16&mgwQqEOGz^i`Iy&g#IEbhIYU5t1}vq51qhh^lzVLb0G0?`QZb4| zl(xSZYK>Ei(r|O#_Zz;sih%%&a$p0_Fv~Wb01;7>-^0-QsI2yIm3pj`fv4DFNj< z3tyJu!yDSh)PlM&T#v%Rl04455SwAQ*QZ=VoFl+urFRg(n!MSKkC|NDNP8bL?FvXu zG3=Oh1bWD$G=u|YenII1&UUZkDLl*s+G2wx8oqZgIg+?4)+2NxR~=4ii9b@-YPrd# zwP&FQ$B)P{0FkXU^U#lK$$iRRwZ*#WHfRm=S{V<5b0V0r`gh;P_~?ZILEr0(u=37W z3LRG|BIqeFH`Rz;HbxR2rJv15ogwOWcx9p~QDI1=eBoB4C7Zx-?%uV%EN$}DVxjtE zeFJ*h{zJZi9yL)WZa%q#2HW&bjpvz{xSzpD&NRS1a%**`2-+({GhxQprcU#^T)^cP z6N>7R2U@a~);?};8>@c*@Ksdva+r#}MkTx7D+(Z(BCBSW+rOcGvK1Yderzh>s%ZtV z>Y<-dRMXq3kfem|eONqhef&7=X&?9Bs!Dnr#F^1*Igj?aFHi>Pe*!JhGq6f$je$*b zpBJ3SdK}&L37b2AgJl8PrDu z4Y+x{$pbw3Jk&mJi!)gJec2ttZGdsfD{{Qa^TzXXAi;YEx+u{KUnp@!a8IqrE$tc9 zrm+b4o^O00BX$PuhbUQicD5?mW0@etuWTvm+Z7C2k-Hd5d>&X?%SD%|L+&8HXh8F^ zR`Pr=l)n2$fs+4%UCgyV*;qJ3rx~N4=KCwI`3KtyE z(TP3nM%0<-<=ap%)M4yIXC^ zU~@2w$uIrZ*^TLP8P>}vv>Cw>W(OU>>uym{#O)uN`}Kg5$nq0NMqK&AZTwyT9=ox( zgy8-`=Q60wM5G9zDP@=`i8YbS+#I@}LT2J#d;tic2or2d($IITC}WR7Wyi)2-S<~J z{~q8zmvl{|95rKi?cIg2_8Teq?I5tQn}12&E;cx@1;x8-B?=-7)^>>ohoTmCEy=33m5e)1g<6%C^(iaa?#`UJy>ma{nj$NBj%bwgQQDCV( z!eh=O&=W4V#y=ZgtCimT3kC$FzFBtkzDCk}vM?a-OT4A>@vb6dV|rXSSur@xxq76uI4gzc4OjY&#=WgxW-N$$g2y2%l1!-8en&*dqSr`-tY$9}odl=f(2Xi!W zHO}_Alu344YWFiG*QHp`L?o08`wB%p8G``7=C&z{KL%h^4lDcCiqQp4ZM>II-V0A` zlg?p&!YzJ(EKnSU*DkRb&Z(`t!!g3eubKCG^`&$Wo$8NCE=ftNDV4#6WY-lvCYHH1 zxOL>R&H#YxVysaUd8onGT zuiL=z-p}J!2~kT$QEk2_fj39!or-^GxX1kCXaV`>8pY;DlQn{q5{q6n7r3W9cJ;X^ z`QCd2wQ3upErF%5ILct&Fc}^JZLv3#M{xREy-HhZord-%-eyaXRC;t(LP@E7DvTtS z24WzyB*8UGvAgiclW`sWBug%}<5!a|ncMCZ+1AUnEsH7P^y9tjZCmSU-nPEm$^DUu zNsnQtpyAVKRVS^8r%X~VCj4RhooFTR6{*rmsJFK_`T5LK=DEAQWZ4fsa#8|_VDyQX z+`RUC=j8b>&Az`cye$Q-P&IoU=|rv4{R!;G0z}hEC*HhY zRZ3EV7anZ{WPLmPR-?z0_qmyStb?j8b#!zjHx1IJ?wI&?kS;o>F_U%v8ceE7_%U->&)GT(_$(=?gZ!vZdg>(@Z7of_87y1Lzwf(JppR^Ho84v6`cjCDsfa>w{Z8~GL0oaNw^ZTbd&CF^N1-nPTew^McrqUD)6 z_N&7r%j*)cDrL>R&SlNbwgNY|dYVq@F;(hYNA8SK{HZA}!MpF;P{o6V7*1tTLOMNZ zqjOF|!>Mtnyj&n|TKdVjPy0>=@}2{+OxUg(*CKy<<%686DvWF{MnShf*wCE*YAHh8 zLostTgZra}5>+Fp;dlDnv?8dZZmA0>grNlB%lwt}j^jZ z252<0zua3|Z&!k}tDsN$X_~#p9MRq$l4io2HTGn zFG^ewRR2CpZnJsG$W3RHqQQ6@wm8`<{rtOQSA@9*gq$|9MenC4u9vp|v>zv=n)!bR z_k$Dn?gM%r-sI)DG&{BZf+@#3=rk+SMSc}$GicB6iOaNkS$*;D=!}--Nw#1HOa8@p z&o|G8QzzOU%DMDy66@$&d3fTdlb>|-=doUeI+gxXK_BVi0xgx3X%S1#9qtZap9eJC zJ}@fLSrJ|&)iULC%H|)9cAWlMS1j@(bOOcCpFoYVz}@q_&fbkY)4cwX(b-n32=#Eb znq*?RQ(AZpUx$wsDM~8}0#|BV0u+}NT4ro}-vvz#?K8}%c4xcIJ<(UlEnua8-%}=C zBpM1oWd)dxA<&y}b>&ue%>E0DtFYs+6g;qeG`R7Orh!x{ujuN%rTGb7zl@jLjduys z+-pD6t!AawRLFcZeUsmyOTdL+9=7X3f_SoMck3tg$;a@Cfn?X@YHN7_odBsulP#`l zh;gF%)j?u%b&dD5gU>g`+m-h4s0q%<607vP+oQxNE7flF1Rk&#e4gCMCC@NqBhI4q zGJ9C4+_1y^R0H4VZzpmXcxd887#e-aqol_e7;%%n9BTQ!3g88QoVigpgHCIeJWjeA zg$l(S-PVb{^O>_102&smrk&t5_7a>Prcpn!cj@NUS^peW&P_2!f`{L)FH}6JX2AG% z<(zm_0I^m0W^uI{n?sSUl|V3kB#5Wft#5zYYu~X`sWLyNvF&qykB1p-hZ>zt$ayj{# zXAHC^M}2OZlTk8331Z*Wj~$(c$rE6AsFu8@OS#JXWSO`oF&SEaC0LD^vyF>&?5UXX zj>v9<{FlxH^{+Pd9Ib$W(c)iKv{661A#Y*3B{^#+@u;WTR80`g>wt822^9K3xc^fJ zNDRz$n<9&Z5E$BL^sj+52?bU<1DhJ>iaK_^a1?F713=tGhgg+C0J$^T(VJu zJayHrH5R43ZG7ZqHLsgD0{AdsZg%cWyn2qHu|lw$xKI4aE!Ahf3c~K8%zS2sot9>gA|n)~wkMy-uizL^=H@!y5!f@_$-6cg!m;5<5PgS;cKpwT zdxExet@=+}?Mh%mxq7;k(*s#;+S_M!s;_SM2X?|@h*HGh{+RQjdN&$TxJUHuruu3M zHPgxoYwD8Gg+>`y)M!xeS=69IIv~5%{KW-SI^DZAApTaLO6}1*zf^#@C@A?(# z>#dU+J5py+@9J2&XJ)%Uyy zuhppc!E5`!l%z#2ukSdosD6ZHgO0pDwdNb|Z%Zk;@+Gh(UW_EV&v7MXZ1_kZlR9&tzQ ztdt{u$g+jcu3Gi6hilx`{naPS@e6Ragpmi1!{x5zRugZ7otjeh2$4JNu>4gv^?G*M zdx9Ar4g-gW5;n&qt%mlpmCavzP}z5TghAxh&bVrVuy%s@?sCHe&y)*eM}A=(?U|=m zhXr?yh7A#Y-@SG+kG5+domjf)k{9>03j`Nz9p8bVw~z+<=M~$KLfP|0rNUSm9>I-r zK}WfGp44>ih-v6udbqE==bUa>=Rn#})4bqaBOh&o78gfoNzr>P-VfU5+HLN~S={Fm zw|xSXX5pTig=g1_yU6;}ms-Ck`uYQtervE2(2%G#!)b4VTkzi|>;2-k*~ETJ z!Nuv-gz1~j`vq;^szzm@#-%CM-2M>5jcQ(>On6Ha+08ugkT@Ayy@=2e;113()IN}P zvYyq^IXWpjxSTlQEo}eg+4-Wuln7n23x#~bFRYO_lA#-5wb~2BWI3oFwhA8Dj?=q8 zm!$GX9p*%c@p*yY>ZsF>{W@RG<3&`XZyMcuntC13hw97er0c{5Cx(c+a@56NgKF{| zLd$^s7pON^++ArnwO7laI3ubk28W*t&(36O?$U}FDk?xEGL`EvHU zirMQM{Kc}$Nr1Q3edxVMAub&hZ#iL_(PA@UUZG5?!_E5$3yj3()RvK@8>TzlBV1)v+T|;O(5J= zw(;hhp=s2$@(qbp=WqOBTi;Ye7wbPu*iEB3z+-atuio(RP`B()Rx*F-aHS3z`iN+> zj!u^AtPpdKg|mxA)`dyB481zIYC!xNTdUSLgI7BbKsx;zc&;XfQ|r~b0Ynm8QrQDz zxWi8}YM;-PT9-^`A65vG<>S|YJ*^$R`jY{A)+v@+o7rwg zo{1Tc4*)I1uK+AwpNZJInBm^)^_Ce=8F`fzUxRhUuK2trDKvC$F8`A(bfEpqVA8d(fGZ`Sv}${JZ+6u zokTK#S1gokPL`|x2B8!Sa=r42uXO2pV0?ywy-g6*5JkUxp9gMT1s3@NI)SQvpEN+X zwl)@Dv2vb!73TQae|dX+K-&4{vXL;_FYcuMbfMS%9cv^;M;sjkKgB_BAVqS^n!$6i z1t&jXqG!!F3y76_hpT;7cJ{sx$tpk-d*F6X!eJ<*b7GQ!VZQ@%pwOauH?gA+xS#zD zT+;jd#HJK>M`D&ljS8+3PRUi<6!In^a4q$dOP*QDGbFJWR&Q)(>2t9rx26Zd9;0yO zfuya$RB`mm#X3|%jV_`P`7==B{e_sdn*!pipimZJw&eZ_n2j#t7jo~=#oD3w0DK#M z1`MLV5HsL=0lcHMubm+p9My1WB$>Ve@p};TxGz7=xdS!Jue;q55Y47kD?gasu$w09O+QJo*=EE<j~ggS`^YqoSZIK zfTyt51`Gjx#yqDu+q&UrzJsGT;{fYg@3-!?Ybj75PCW#jg9bQDNst#NtERZ<#y&Mie_vt@Kz$Gp*F z+Q1Qp4HB2(hw86t3~cD|Hww}FL2|N;Y{gi}mWpg>O~1&LPeqtElg1TEp!Z^p<<$N= ztyy$k^6(xh;5HZJbSyHR1&kf9`fZ35-}#qRP{ps((Z_y##=S;vegmSG`^rz2N8c78 zaWf9O1b$*PN}t?`hBgG~YJP8zHmbWh+aPn`L^Rd3Zv_T1FuHkw0nz4^_g;S4{kZ3q+|#Lt~u)yZDc7#NcXJw z6BpiJEBo%Xbxl|vp!O~AWqybq7?E8IhvmnzhBOYMk}pvBF!7FY@9OzGbq70_nH47g zIlVpNm|aZ%cJ~BZoYzEj+6n9Prh=`Gj(ydkQ!|hZxg%V0X|iyiOEt4b_4xh|c;h?3 z#&oMoXLFr`P}ZTMb=-`(@DQx6WvZ;&a&ujfc;C{C11X`p`_VUJwKPj^>5L+^x#WQD zx`s5hmLPvGO~B#J#f&X3)$<_cyP3utdgP<2H9S{NMxRs9M(1pnIAp#qMpmF?o}{uS zVST{B>&6Y6VrXw{f^YDgd}6keAg?n z2ddsD#fJlRe73ur4Gm4iROkxp%K{g;pv@;&=i>Z5zWMfD@w;hmXKD3JBig+Ly4z(O z;ePu9m1t__#@D{_X3oU#VoE~EPqBgZ9+J-Z+vyH8=2{W$T=e^#-qXft9Zp-{bDzzL zJg2pMGhrnr%~+7yb^#1QdcqQn~=`^9jv?H zSve!vf1P{vp~+;6tS>-=VVylvp{s%f59OTAr*F6JO^V>lSzFkD*2x`6KsJ>_S~lL~ zb}Q*<-`~%Z)mc2FLyrbC)al9%+%=sz1!t?Xnc)(@ga&Os2u=$_k8ZyLwvI1jU!Dsb z6J}_8BqJKzZM3lZ%e^A}mF|c1>4|p36A>hr$3n*5FnAX{!Z6j(yxRH65RBE_{$MG+ z!@K8DpvB+bu~&ZHwa7dq9@faMOW)#tJK~|MwPy6E5UA#F&Av@M@z}}CKYSgViP4U+C|Y2H7MQRjahwr?^1nt;dFoD&I? zb3h`KbZy8IJ?*nfQr;VztAf@U@5TUk?dZFL9Zwhz+hweQX-r*FSb{XTt%F;gXDybg*9tQgKF!2!+e#BLOp4;Qlm+iIJplYsBpk@M zc3i8Kk}~B5X}$D7OotDSAt!nk>c+jdb{-P4_6;kP^l=1z8G}QgZ#qZy;aiv6GIV;u zscD*S>wUy#0k*~`&I$!pUiXx|t}$$_6J_t-L!6s|PW9M#U$~<7KYVLgPpN&splXoq z`r|u5Rqr5l^Q1|kwfdJKp$muhibRnnKLdX7W#K^f;pen^A~vvdeF5%<%hKq|)8p@% z37iwHQT@=`85>QtRE4HiKvZU!^C}#!czIT9YFMC!fOnyNg4WuXss4FxjaLve|4i9u zNo~d|M<%Pie-6#4^j zpe#IT%XwmdX6fzDo;hC_*v#&AJUxdchcP{dK1#c8 zFtED`nt9W_u>@!$a6=Yb`vrM?BHp8Cx`TDl5!%MM3BXhZ2yeWDm?UR^Z$8Bn@c z5krT|otZG*p%YkDr1OfDPisULT3#QKE;V25Ar-&nY;{h0}MYy zR%5E^GVB?-a|%hxCXFtbX~mq%e!$e|I@Cj{B5we%)mJJ9wF(=AR-DW|B1rbTX_72b z`j*VuC)GKyXp;162Kp9OG?l+P0qDHI6_z&2bbQ4Ql(k&~Aj>S17cww{U{6aMvyZdr zKW=c;hVGG3EjAJnUu~~pCwzJn#Al`$=fOrxbj$x zPifuZz~9bFD#HrzA_dE6ZDmhPWZmd81R;r|QCM0=l~Jyq@sSvJ&)a%#NBm-74)Ou$ z+0rRV(K!Y=_XXArq;yWGEXhi=2lDvM{#ZA4Hz{}4)f4n%;XX3}X1MZxJQPN1IUc5* zm(93iK3)DCgi|udkO#CFGIp^L~{Wm=OJh8sn&30h} zo6Aa}?yOB{^2H8s*2=RigXby3b<`hO^KHufyWXjGe2p zzvcN&lv6ziPL4vM*jCJ@Yc3T&O*=d_!dZLVp!gB%5jgJx*PiM1kHT+^4F2VPXGhu| z+NrqRJPeNfGz&0Gt9E=qD6_3E;KCFLDcKU-@2qv@v<#7PkF219Xf0hd&qg zZ&>2eY|SLW;ckfKvf8b&r^nHE=E3lcS~BPb?};2sX7UX9SP`|JFH{cixI#V$*qz4s zo^2Z75A@aBo7mS-qcL(=%Bio~M70SxPPUBfX1?%-l^XgOp~dJlJH0C@ag<#?XTZzH zOMHb{csY;B-5OF|co@SW^%J2a?PC{N(7hf-Uk>~7F>fq;?UfwEIzZw9z6TWB(NX`Rqwxo>nDC4UELk=x2;4Jh*i7X2($B@xX8-9-pyl5kSkT;ZCi z!_Z8ckeW+~g$p{orz0Rpph^ibE6o+BmH7Tw`+2hIciWPJ2ps~F41<6B2km9&lJvY- zsR4aps;xMQ-e8YrN%#h~+hc!CuZ+nXn7_A<9Z_M+vh~+A-(p3~k<7MT9@Q~Tcb&Q^vct4q9v52ckDTpq_VYJdCkOl(fs_$3FFzt*ar#KQ6xW{WUdhYxpWpio*J(Kl3ynmNHF!b% z0Kc3jt4y4jC6D1JGR^y11}^z($DCq0wv+es`lYU0e# zNUVh{w#N5`+mz0dmj`7k?lmSUDR}h+Y~-g&up}64KGJ+UZI)LcAP#o;aXEtz1Go~#w{gRMU!E&hzZ0s^>&WwU9ksvZ0 zPQSn5kiKmra{;9D)LP$uwN_l(Dx`xgM9l=~X9BtA_J> z`VM(C9>{L>SZ|WEj;K%CDSyv#037#xmHy}z`?-$a5kM=6 z{5ux;Mw^_!hpSomifh2kI_viv$Jsu=CmP=*YH9uzlo-TcSFzHrNi?X6lkT@|t{Ng- z_PRL_C7kIJ5FDSgdSdg8*eLcN_g{ywKPNT+pglfEWtD%!;w=0c6qWt&4;qWWl?b7* z?y*(6Vn#`2w;|5pk|#^WeRXqR|%ILS&jg6k2Y%(#OsmL;Th9JeGzRg5))4 z7aLBoK6`Id6)e5Q$adyp8NjtQMOhm_2FqeNbqs2*EMvxxN4Nm7)H^pcpmW0L4tsG8ilnMDYD3i@MX7X`XSt`Ivu?B;9b zyqZ#3tsz3`DWoATIWXDFy;z2Ih;^gqU26^v*j}H5%ji5#cKrlqEzLe0IpqbOCMsm> zC+xX=XC+l0IPqisdd0Gzkakbm*BMODBax` zA{i&FV4+F9AJ}gOGMKXt?PH+Y{{nhhR;6|E2hBN2KElw@lqEwx()Ji%+~qXIKe2w> zLyS(-I#2gVkZ32)QA;B<-}*pIrFb4kHsCC(1D55A(SBfxR(DF+xcLqE5iX)UO#6JY zg8P;4$E?Mhee>Zre&hB{D;iYq5 zGPFDWC-3GX+x?ze=EA|`+}W&3jBbBq2I*c?KCoMV0i*SH6##vfzW}$=cT2hIy#OUgLf8A2F3h{I2R#ZR z@D3myn(8LvFWu-x{iQ*6WvCY5f)M*p;O}&R{r*+}kI_74*f^<~z$*<_fnX&@b3UZo z!gdZZH3qtS@@{rc1kFBiM);`gHNfiOn5nbB(NOLH4CHB`&2dh+I;$1=Bfg55JH*5U z2grw7c}&L`ulc6EhJRt-Opiy-ho?B$-vJuUFVTCBOV5701E)p#B{wZUe7Rr-!WI%s6Yc(cj@NN1sS)+M`gE(a3*9Qt0O)C{z`*l-OMr z=a}C^B<^x+yNwI-3|GFqmOI13V_}nifqOOOR#t}6HEF!Wri4XLinL$YN2Ta~w=O*% z^B!phZ9@jN9sfD?Ije?BuGi5q6>>qC*|rKR@vfb$b8o`gLNF{j-{yBPgQ%(*yMXyIn|o^uo$m-laF)?-Dy{o=;{{ z|At&CQT(JwMR0KNcx$x!;S-d!ckkZK5y0bknB<}u7-hmjLRggFL`O%%hq&kZFA|r9 z(fVu5r+drL7?*+ix91soc{L7`Qe8cC?A&Q}b@lDIi?=MVlAOBy?#W=0)xfgrsZ*zP zdjwn-zy7R!&d9*P&>tT_FY*2>bG-XUf5jEmoM2%u&zYg};)90|H>|s}Yvzk(sA5vu z!dX+D7e)y&J~oW#Z322@v4xXSzsh}dboAqz)o?{=jF1O^zw2 zpE-TnpzvK!<%2-x3;ZsNNjsI!YePd(7X;MxNnA z^&GuI=L%N!KZ$y&aIV&JRVhhQc*2-Q2yC&#qk* zQT%7Hl$%XGTZf{>*3RzOz^BK-LZ<8UzaBa|I#R4*^eSC@m!{g-yH{X_Jk)nzc!@Q_ z(wj7gJjAHyykB)k&vewAUx?*LcWlxKx%0AFbZ1hmNd+^>_5Mhd$kN%!&4s>wY4SDT zS7^^_u)NSwx7FX8FUG327G;(;o!93Ru#4DRh5r4;^6^4sMSe8GJE1w4%PvckC(E4| zKGx@Z5oVY%J$F(#^sA`UK8pE1!atn3h*?|8FzQH^r&)y;(QqGH{WE~~U{%dHx7xy4 z{en?GZqV>ThBZw8hs8D{q}SB5W9mJ++WLO?6(oAZ!u;^*8yL`LqetVq)bNC|9+s_b zWc@t+{{6dJ+Z-l4z3jHLo127o_{-zxnZCMFg0&?>R;QGt1CasDgl}FyVL{xo+}+vc z4^XI8YKs$Wy{itZ=}Vk|i{Xm@l+%bu50y@2Gq1~{ac^q8r#Z*2*Y0L~J^v@`q5cZ_ zmN53#Mw&a0vb%S)aRJ*0zEMX+pg*);PbiZQ3} z3fc@;xb)5rmA9g|=f9k4ok$a@Kcmc>g6qoGy+fTpRAPHiSY=v0MfO7FP<=h)@AZYT zA90*$PnEB!xF_RZ19QvJ_$Yj?anqT~YS+2y9Rh~*dbWP`xE`lbuKh(yG%`+ZjrEhNzm0H!pFx)=)p^&1v1odrCZMycQCw1?)TU`_|vkwsa`u9 z5+wCFr!Orn*P8R_#ot}8E-H(|cV~4rrZ1%ubCahTBUrM!_4yL``_rC;F+C|xCodyxog92^1Ght@v18tpB^c` z%bro!_b?e=?K6vi`DdV*^LKxt1;GOoFM+nKbX{4V6BDiQAwmV5MhtntiwahhPLXBg z#wl%KVd1xp{uj6#D%z5yf-`p(GB4kIbs|sW72%7nl{%e{G47OA=K|P=y!x>(BwRlZ zd9zCLp>Zl99x5LU`x|R(@9rwvNP(eS5t^&s^$f8B^Lx|hXvY1IjI0J%^{Xm|C0@GZ z@ay06A@$1_ym|8`o5XvnNmtXBFgX&guxFQ)`BLTM-{)6?fpd*M7B4C)%GQl;4q1?YRYxGZ~N_@tpjHv=|qH%Jv>)uP{Ky8gzrz){LYNyC0#Zx{rceX{B z4d=MVK09<=%Q7~_b5_p8;ZHx$Z#jMD&h7hxDh4id!-5!-N?cUNlLuKgU1^G$xbV}n z4^b!vm4n{!f4ZDwD7mawmD`FkcYV_D81HOtWE~bv5sp(;FkzWiP{B}l()4C%n>w?G zDBxA6KWe3Fer#5qXkK2u#Jf-ZUf#q-0Qn+C|DrSF`f{%*p5J$JxP^9Hh8Fp|{ zwovTtpAEv+Qd^>xZ(5mP)U#MV`$-J}8uQIaZnMm8<=C^znAD?ZnZ$Z;IsVqD)I(oW z%_5~&&MJ`AeV{6nSMsPkGZJgD%1`ux|8~mh9B{>X{A!iwJMkG$`Y$SDuA@m+-rljO zjY)o@rdev2>_S6+Sj*^GgAidl@#?_m>hItVGsAwA*cw&(nDJe z-9$$VvwWObQHkak; zK=1i ze3Ge}T7rX4`ba8-Lo=I7;1-f(!Z}?-^Y;&EdH)NI&t|hN<{SjaS;^igZ6<) zvGZqMj$Ga;nb#v1d6Bw7lRNHh+ zvMV?;HBQxQ;>u}hF#;}1q}2RL@x>NB9L=^FsyZ+|&}m6psV6C~i@=2`&(D+hrp42( zC}f=oHEWA&icWe)7aVc;L!ejH$mE=uPPVOc-p0^T{hr${v7bqdZ4z za>`9xFY2bQjd(3O|S;Ul3b=-h1w|=2=fS%LAd)X-`BCsb$1yHs*)%%lg9XqN)dBW%t`6xg|`$ zMI@9bVY0E%%^kY^sx7kE*i8s2I?A2quFf&u(B(v-hAFg=n4LN(qG9lk?P%S#w%05S z3cB+Z6&1tNeQr~+PKGy02cs`Da};$&JcSU$zoY&p9T#+;mSei@kumAQIxat&)CQAH zCoVoD9MwzJ^4B!qK#zYVt5gxz?-ph3bhccmy=kqtURI|9$%N6$9kLxtZN(vHRSO}D z9Ri;7ExK?8o!u3?=u5Wuu`B#tG=ye&y)4r2IXge~3(Z4YbF};}@+{jm1u@=;N~%$l zJu8}K9Rp#vS6064khkh_HlDk$${wWHF0!F_Rb_=`a*ic;ji0=6!HI*TtF15{t3Pg? z_#<5*;hA1WS!5ZXwqX?IJ@h4rE;DFpIRbM`1vtywi_F7fi2aHkUdqSys)}v!JY`y~7u~FU- zcbp@XmNkTlGGYbMgyk7RDULXcsV0|arQ9eTlZ_K9cbHMKkBEk(=%9V9;ZnJwIT}OU zGtceJ{Nmrd6RszR6AW=jn}q*mb{fz1gv;Es5U!N9#KgzP^DQflHPtuLv_aSxM!~K% zJ=K{a*Gcp)g8%+yq~&a1Yk zXf^p!@hM4%T&0tXzEvnCsM>NByC{clQ_U*wk}dh-b1GbTcS~7+K)cv_29Igbwt%#2 z_LWtZwtc>FQxY2AJ*j4s5~gAv+i&WNdtiR-ao52!IJAnl8{}&q72r8Mx2H`#J*mwr zL|3>IwY)s$Yf1RODj7AWuVsJ!DCo92J)2~Hcxax>q?zG4$*Jx(NTAa1eLs2W*7TT4 zOCzKMGu|t(D9VoGAT(w!O8KmTy)+v9V&aPyWV}zCWtVmSD2wZbP}s!k0XmarlS(k4M`kjTh|4Bl}Aa$N61ss zM5F{W6{lL&RHjq4&^#91vOfH*B{yG&hgIPEGs27##r@v2-yhIg)4%^xnw3`dVYaz$ zZmG>hl zb>Fay_TNoSTUh978$632;uiHYr0J;tn)#%J)?BlaFLF(8~RFj={Lii-Ec0lF1E$ZRsbinvX`rH%M!JU5>(gUl_=qulXN2RpSY zUxs#WSrx9*YkQ@~Zt~lOrS;IGcT*q4F3$x-mGa4iu@T%cVX>^|REG4k{FEb3X9s^2 z6({Gy;6ot1_sGT;D)q=7%FSpgEIXC%#k}8Ffw~cdHz!djS~A74y!7h|Cr7=Orxmr; zNpkskJX3Hu-E<{J>K#B^UPvHbGaxOk>6XfQ@%w&RBq$W>Cl&#!M4o*#J9bkwP2sM) zhliD}F8qeJb`S45`aUXMO)fj(a#)%a>(=)6%0Xx%3Z>_VSZao=8;cX2+~{oGa<%s~ z!W_44hC{$i%=DpE=X*XqR^EVQZ{fYZCz4cDRCV9KKX`TG!lKrM^^H$S5bor;wa`em6U)|z zH;G5$h1}fhh~-IE%hH-$;o#ulvY(W7K`_f}HOi?EW~^)Xf9uW85J}-}(*X~6zz8Qp zMyyy=z4ZN#Qnn7?wmT#_HkF=#uRKa*Ko!GhFMe6jH7P!vO|xTEC}kxBbAA77gKG!9 zZJ+%5e5lgF-ac3*T}gA0S-Z%RY!%^8;IO1?dPlU*bclompA2>j>!Fk)n;s4b-woDI zIB-RPvd&z1H8)s#7LVk9aM1p8g-)|*QzfG&7OC+*ocYHH&q~?f9+haNd+2A@PrWnM zCE>IzzdYeHo;H2|Z^}#1%!smWA@}umoY1Mu0_`nb>B^01-@LuOA8C`MKo%Cpb6*LK z-#DqQUfaH|E2dOi$>3?rc~|N4Hhc=-{rh)b?7H=Ksy_X&eyw-DSzC5E)*+pLk6eueWwPpQz}JD_ST=uqzt0Eh)X=Dnwo*Ur|} z0Z!dA2E>WBUE(occ2)noJe*G+^v<~sfFm-`J|P)gk>@tSH#RnQYq?WC-|>&u5&;~G zRO5T<3CKb_lBCXh2<$(^;F^@_nE=3)q#(_2-G!%A=sk$OCFi-j?X*OIxKtWqjFlyb z9-APD115n@u~}Kyt?}Ij`bxL8Fu=xSlBGi#lv8B$jT%Wy9sfklG6?{fLR5VcCL1f9ER9dSVe88k(cWkH5v4+iD+-6!MjMw{>z&{{7H*?A;(Z8vggrJ zw0T!h2fa01Z?ai@;(*@P$AQZzRM8%ef%MUTX%8X8ygyLA-H8u1&te&$%y_kS zt4gZulbEz*WB!gP5r=8w6Z@C_#{t%Vp;!9vNB;-$f)oczr{qb(`ue(L|EI_GSgb1W zi&r483j(~np)Awq6ixH|uP@2QO+i8YXmhDJfl@fVkL%95c1voojo}oI|@v*2k7LbD-QQCsVul0R^WX8-Q$~qA!m%0a`GD zWI>oVfLhhsiF;llr;k_Ji6lX_qI7;Eo&Zrc{O8F z?YW%eEeMEwhD0ldq=Ny&=J%r z8llYajAXs{RD5i+iA;+}s%;$1u^GwXKpThLahetDO%bgFlxX_sU4Te-qY9BM431F_ zDOqAWBp6s;T+A=?Hz%U>9GOnbt_D=D(`dKyZ-BAHKx&VxFk#pujnbYcfU!55Ma{VQP3`KRUL$KKSpX(n)`LNA4;B>i+znZ3HcZF?$qzd`d6wDkBygP0R^2T@emHYF~yC;-^xCYRjnF-*| zvK}nj{;VOYWYeo~{xqRm<=(e7sP*=4=Qxe{#o|-@Bjzg)vqkm17M4dHuu;#DKQKAE z*ip(!l5~X@J@1Pu=7ZgS50wKQ0qpUQ7An3oJ>C`jKfNBtK!##9FVZuqdrf9H-A=dU zie=4(XprCnxZ^kWwGbqJsdC>)f|*Cwytuf?Me+s^tB|+nImLU0oMv?v*4BF3vCK(ZITMJy7MEv~ni;x~$6urISPr~tGc2Gt^0y;T$1;$kSXo2fKEB?QaF1j=m&tzeTbWFr>)(|D@zI3? zfI1+D^ z(c_hWzD07|EsWkz*U0BvyMR_eBv{o>_IB@wk2_PyL5iX4qnIe(1oEdbofPu~jcQL1prGE8 zvU3BEfC#~Csx~J5u$TTeLWSr<)I7a>oWPISwcmZ*E+A6607SMs;MZKP z%hQzMed)@~Lmw3mdBY2ZgN7}lxYBt+6qvTP{*}GN7xGe*<09lW)B?^5#We+!pwV&N zss{YEBJc)DC}iuAUxe?!bc8(mM;j3Inaqs7zR=VrfK41Msz*FG*+th=8Dzp_!5!(B z|4W-spoD=m6Fe#j4V{4KQ?UpW=3-96!=0;DMT+fs0x~6n3q~#Lqa`OONQ^-%B6{zh z$qEj8gVi1)l1b}&aN}NChzb!oUg3!|8=67K6*(&!&29F81>{qDISo_FXU#!Jp$hO7 zZI*U1g}(ksk82iSMR*g?(=dPzg<_2bC@P&&yZdt((I$15ovl)9qBKO0KoW7Bf-L~L zMFzs|B;aNc^kHZSgjL8P8ASAf%WKEIDfC+SI~rj<`Hu-Pf^KTUU6y4K*_6JcyfIp+ zKlS2?Jq=|`B;=Bm}w(tGMV z(ue|QoDM3;(iwyVTpvoGLtb(Xkfg#Fh3svWXnu;wKOa#V?^u;G9^J@U-MdCK;|xW zQ|!OBRAei04X|4{3Ok#7AWB?b1R!5+?FRan97y!uwJ3w))PQvNbFt;`!O{YUx#6L~ zLOy;XBBCEfq$);~7wi2`1W7U6Ca5BCL?2a)=`6jpbN6eeyJSv02 zJH>Wc<;c|B9E}N0p>Au*CKGYZtCN>-*R8=kjsE$=4oMecFsUI5c9KWvf3OBtFN_e= z4ZRi?5D;*6BK^WU)mUKdIpaME>9SEbrH0++1o;5WqY`r0m`lV_byB?i`_Kg9n%&Ln zT{nwh?+8Gi?~-_oc!I0DxEOixby`o@NDVxm!I;|)+u37dfRFS|Am!rB$K z&#cS%zJgTHZ_y7AAOE}FerluoMFEIjp=W|g_JJk`w78cyHm2f76#ndAf_g4X{CCi6 zYS>Zq<(}JR2M->co-*(7%hW9BFO%NCPUNH65`{OAhav78i!<#xvXNYbUo$i5%v})> zU}s_eW{C{uw@#w&pm7ZG?u)(%vfQ?c$S1cI+OWUN&CCA@c_odBeuC{l&c{)ya?wnrq@?)|zZ^yod1QDi z*a6fWotjE52K^}hpDdf|u2It_0AdkI@ zFxrm7Aa_0g_f@YPLlqb|ebgorm-rJ5YP4b{=KXJK9$8YL)F7E?TfSphcLcL}f<{PZ zP8nnb#ag%jUUHSna^*9){AY{xjPi^Va2 z;bzbQF&ra!0SWQ?D&2~^a`g3ebaWi1Q!j&L zx#Pp#1AT=S;h(Gqy2Die7Gj*?i7rIlMCKzhDIoQx10{4f9??uEUU>Gmvl-@5?P7;K zL3rHNR%|uE1U@{_Y1l=gn3Unf-}{Uh;sv*6(mpa{0?|0D&JmBGL11#nj&EtLUj z5(9{BCef_Z1qB7c5WU@pnujY0-t+J$*ak~gx&4lyRV?VE^fv!|%Eo5y2p%924j@Lt z5L4-827nzaux1cBeNtp6LcxIf#_}@B(5*VI-pXp^ZoC=Pi}2m=nt5JYd0jpeJ;nK% zVR`2G(E{{+6iVbXDGE#Sb>3+6<*QfC2%-WG;O+GppD0Q(xnnSD^ZWua1ybKtTo|HT z?u;6kZMw8)8$V|7l>=I?E=?=E#T?Jdz%L)~(=;*QKk=kSbqYrrt2?=qzty*5V zVfdmeLsbS!Ino+;l?!&nFJEg1j!X>;V|w_}M(XG46WJF4qSOp3=Y3t7ScyAssoWPQ z5LVY9WY2LUqdI4(08M@-JomcG&Tn7dmjdUe&iN<13k7~?YUH>3@6puXXC3!6EEjI zj29@SRLwVjM?{IYH=Uy;egFN6os1H{aPXj}4w8yVegNw?*&HeZ?v>G~;awo2O8`1C z*=D4QY!w!I@HrMGF{qg^LgtdUn9whui%at8O;z^YGt%oMlrIUfc2Dfc!vjvcFh4J_ z)@Z4t1*wI=Eo!22k*CC5pn1m(fOhL64a_Rr1k+8`vz}=;b{&fKQ8qD^&l4G(BvZ5t zQzYYLMdP@t7-4i;y*mHEB%pj(In>fFe|sx&f#$UQ&gMkEd;{mjf9eBJ-X-))AWGVo zeIOI|e?lPYnjmplVatWF$`ewfK0lxt3G%gU|57M+U^r(Lg$)u4Dsy2F^DwK(9e z+sL^sRDf0m$%o%*M1Y9$RjCPW;UC0od4jIx5WJKDX;C)vr~nPWi^530!!Ua}t|%Lz zEN|6=-tdJ8710gFzZs`rY$1uIKt>H(M z)-a%q$F94nbLa;N2?~ei*N+4f^A$*A?HG**(-IaLZg$}|?|3f1cK_9*ni0azbjZF` zCjA*-vYy`?F1xQHM^BU(dY&xQRm6AUUwXkiMzkZUQdIu#SKIVX0(YtXT9vblap>Wy z&PuDpN${szA`lGTQ>YvX?A8q_{hN>iW%ne-cXV8!jg zF!%6}L8VWK?O4N%NFKxv2$hb|>Yiy<5RVmWrGgRK!x*&+&5;B_D)7q7VMkof*~OJ` z&mazu!%Yj;94rcHuMZAYGdKEgvnVtB2lG^OhL5Oa27}kC&&X!p<>$C}cqRPih#^MSYZZe2LD64Jy zAeyaoX*{%{Ra?rpV|a1;$34wr*W{9tk~cPgTx|ssfmoZ~Lpg#<+S{2sMuV&PxA246 zk?(d_k0X4?@akGJD*eaSSLagHY@|)VKImqVn z$sJlm&hk`^)a-Z|_g(RdyzW|Z;vzve6UXm1d2^MA~#rI9Fq5ZGwO@*#E_GKN!RX5cN;Sl#{qi#BkVSK z;-jRb6o_Z}6|Yoit~xVVx@3VZzWe+z*=OMiP^Sk#Le>K7)!~|m3}jb^H@1}XjaS+q zYOT-9+d4W1)_xIx`~3OyOx?mO-Dq}=wAahHff1@57MI0J=93q1ceXkG>7{tM>uV%| zOCC9Y;NuG~>1=@w2KCvByln6RlKqc}fCUJsvl>8)a7N!wV*w-gXLKOVQB?{brZqc! zJP6d4(l9giG~+H>$oo=tM1BHRfJrvu^b{duEGrAN19=-O^cFyJeKMip&y^?rJ;1Jh z-~hw;f)Q}$o%mp8r;t0vHci`ZYYcwT+VAi<{xJf! zvOWQ=TAzFuETA97d=RM`t?%&)Pf+U{FXfi#9?~$6K88AQ;6NffCoSzy6Bj-^jX`TG z583KP;TSpFB*VsEpka~WqTrAUr#gJ-(Nnnx!h~FDlU)LBts>LUoPtEfI6EdB|gl{Hlb`MBg}_W zuOi_7PFS0+aY}%aoF`a)gOfzGV93w(e6mI_EPiEG&lw33dM+VxB$z{oG^FZzK7!7LzyU!+!dlGHIdNL}O|$3YnPvta*BawGApgQ{>}0fYt~E2DACy1@0s;!>+`E zAb;B9TL(*QZL@P&3m|afg4&#>*gM5B!d`l-vqP9-zsrJJNGTI$@%NuU69sx6CcQt2 z7Vqa|bJ?uQHcf`=X6u~{_(HNOwcXG${uuSWk+Gua#$?XvjPtDT$A!u8XjGq)A0FH^>$OX z1DKJIe z+FiXV+suwKY8J8}yOB{q-VW48MLB z?6Z(1;gomQ5)Q!>M~?rLqIFO8AVM|x7AGMZ1gKLUWD&sd^_Q{29epnn(ZUBD2LD7$ z22DU)j)R(M^w+Q2qp-vf`4n6r830=3$9sw)N@87a#3I$cb9tNm$wo#hwo|)f>KZ3{ zz@5pB^s=90OcgSEp{%M5fS$V!A%UF`BHu+SR)9!ngmh{YDu$5+1&N`)2GZ|UP3M1E zT3EOd?gf<}j|WTgjK2olh3u6Mu;_3=BP2C6DAmcqnCb7e=lZ65`=gn~`qSWEf(TVY zL9K#sGl#Z()^8Fd0m3q0kxg&U0*e0(6%`xMgCWwPtiF|%mBU3)b1?%L4V9p)%CO56 zb)`N;&l8YVcI4;_Q(t`b>Qw`QHO;OigoPe(vmhK6(*mS4r`h+IKp_vp>-cz@{YE$t z0dno#=WkEZ#J~GkDcSw;QrQn>&oIw>4J|%2o>xJ;9t`-*eLf>Z#Aru5)bNRY6uJ}w~F1G)c@97;Dk21%A``3C}o(k^lL=~TfP}9=ZL*(gB#gVb z*q&m#z_AslDR9Xus=Jfu!$qeyB~&kjXX1ha)M+8c-r7be;T|kLDpy`eYPe~ zHAhbn^E7eK7aHk8zV1!7*YX-RpKzc0g23&O6~I@+ip|=rSJ=R~sX?I<15m=bW%3FN z3Yj_2!o$PsSInQ0)%TBIftE;}4EBZ(+J?;5Y>eq@p?D#w$}jz)GPDU~!*=*?c9+F* zUyf@W!CnyzG7S=%@W6-TDs3iAD*p)ZRvfGkzhKlTCkhpd#{V_Gk{r& zS|l*lX+0MTS>O@hCOPy>7A(K%Zk6j1*4OKBmP4iX5}c5!Ccf}0gV2s|6Ruj=-_pUf z)08tLt9(_0mwwD^>;5W@5C`JwAT(^iSxz(sOCYwlH_yoROu#(ARxH3*kNt+^w~l*H z=~a`dJiw$>*~hF@C2PCzHAwM6j1XtzgE%bRFyt1%X|Uz-Ya`+mh_0Jn96gJ$^-2k% zhlVaHv=ZEa*5ANka$u>@>A;fob{0D>0!@qXqaIXeIyJ{xTYJiQqd&`z z*tf1=m%5kc6G%+QqLvv!1&oVYjvhNgY(c5W2FPLmz8r-&+$g|I-9MsDbjq=HnVyM5 zuY&TA5LCY}B4E6Pr71;`n&WtOUEs-!HzkGP>iIKBNygss|Ne#+K{}|CC%;4&78U|w zX1)O}ERqQ$gaH)Ld;H0Vx_@?PJ3$fZ)Nd>4?>F-&njsyP3tx3qxY%z1ikIXt)1g#g zDhCxQK=e(7!`=ntE?euiF)dX3Jwv`KoAju+eJo&0qXwqo#ym3i>+fp z&!O=>PVgEiHiKgCI5opw71UFUux0d|f2{WlSpAd@bC>Ak{;qe(x_bnq`jZdfJ;6IFNC{IfVWfuAd?wnsNL+xk*QHk_p{lAdREo5`YJOOq{n}N55BGd z98OMrF^Hj?R{_D6fu+xI^QMab*!fE!66wg=K{U9atZKc4;f5xksZtzE{Tjtyp8Vv; zZj}L8K*>izVkp?H;9;d~I7cj$(|;Ki0v+tTTpL*^<-G5Mpw;If{JO264ZmlgfD}B4 zTm3Q^?K~uTM+G^eoc{+sQ-nsY!BaM^Pm=z-u=zsiZ3ShxZs2GM!GD?Ry69pkhB=#0 zTBfXGB3X2Mb#aXI?KpUdJPsQ`t(7op?6&syonzNAoH}N$Q7huOj8_JVmAwC&m}}vi z5{lCE4qzi*-f{dL0Q`zn!@CQNn)$|y0OsR8E(;u?*xj%nFL>}L$@Qg@5`+A$?KJJI zig(m}V$U*9oHepKA;L2prTs8ckXMXYDU;Kx-s#Sq09*hY9k{H zi4x<-tFx7oEpMhJ48)?av#nR6WiuFa3l=1f%P2j@m#RFLw3m6-j zB{pRjv^a787RZPZy!;>O5uqRyGQ;dIrOM3Xc+PDG55kEcdNU3dm6F#mmF3^lY%8ZcxXf&! z&K}Nqtw#zn*efAcQH^zUk&fQgIQ!9~M+0Tf?}kNoGRaiAyD)B&WF}i0I8}HK*`B!L#|}aH?WODeO~Be`i-|I>Z_+}YxNkOV`58~in+(P8R*6eq(0Z0 z1@BRrww!s?D_&IuLoOjkU(|IWcPQqxTbtDt49Q}L8o(N>06lGB!M$Nhmx6`_lgEGZk+HNZFn9&Qb+H}D_C8d z?k8dGZzQB5BKq?mF=05_EF;G#Yuo2I>4+Ci+%8ccv{3!k&R34scaUSom1HjA6;JSJ@+VGM7l8P_^zhZO3nQ0Y8b?F zV$N9*aYzDqk3KHMxMh1InYa#6)%zN$pKHK`O>>&1%0_+VTkO4QCki9%fEX_t$$CFe6VV_u(QQxS7@ zAJ*4<9b2*Pv{_y{5`5S%x+%xrE9}oekyL`=?ji3nasMuv>Z${E#Sgw5F=6eqpX`+B zB+L)c&si*P)qA1yi`wuC6B$eMze~Sn`8-^_e)C(vTJVc^ZA?c=;^p?_mIzZ09J4-X z{1MPJJY&F@C&enoBk_{GxFYv)2|(~~4>tp0)H=|m@f2*Gi=Pu1)ze-@fA~Lyw_fTB zImhzRQO;VWJR=D$!|pV9Xr_(#ho47gL%%X;^Q>&$V1Bg{8R(<|l%h`b0dS1}5J|CL z`!(J~J@NgIw%zWqpw!qT#aJ0ZWr>F*;Mc^rSP9PEjj1S?q1H>gUz^T0F4GYGD~`ep z!qoaVl~hz4APaE~IwiwRM8iA3_}m66Rg849XDDR=d`)dkQ0$6ehn62z{kamJRbcn; z5f`Jug4zzVZ1pVv^y%ip?t)QO{pH~*v~rn8$6eCdktL|k41QV1gmQ2y7Wi)89Kz|A z_TtrMOq!?!qfyGYAlAc`urC?PkoDK{b6<&!<5uZPoaJiY;)R8qSHY3vD7;yek^%s? zlh7UpiSKf8ZP;bu?d9NMiIZ$rTIYk+Uc#@Pxn;9fuh=^}7P)@&uBvocSI-n<*?2sB zrO#tMBxZfHET+^>TNtD*Tg~}dQep3W+hE1~0b8LkGg4ix_nz{GC~P%gte5kbL09K) zNQN?DQFZ-JHe0yA_(b5u#EC2~{URZzh5s1CkzSd0RxM3g2sV=TJ6mbha(HOPkznb? zqMnzZ)E4kVfUG%D;?%@s>pIPs??YcY2g!MPuU;M22cU?7T8ar}*_)fN3@EAmYQo{9 zY*kFyWfCTE#CBlg1;gaVHUIp>t%G$R$V4KUg)F7MAlK3-_55H^{qjBfqFT!~bD}h| zfcN+juAUyG_dWV{+vqCzu?jU0F9@sD!!8X-^>>*G@fg`QtE4laOE{m)sr@V1GufiA zfN*nfx7g#$zsk~0z_cDMb%YT8h71?chmVJ&SJ&3Ukv$U-9t1XA>K-|bQF|JF+s4zR z7=gGG&9#cl5NsRMVO2RD^hmAIvOUl9?8k?Hs?Yo@9>fkH_Pbm;x%M02 zwR$pgaLEZwn#XN^e7x^Ud>YwvzbCsQEtWH@thj&S>k7 z>&E2hy}!hx(w(J8X0lnzojN|)*E#8xDieP<=nj8#R`l)h;h}PytT=cBTd6a_-Y|VcnvWP9#LqdU@Zb%bwa&n3k@+gC_SKM{=Hw)Ai2Y{FYcT~*rcP|}m%M%3n ze0?eG3kEoTGo&JnK(Ce+H$auqN)c>EB?H@V7^%iVcI*!CapIDINYyk}b+R>%rLQ8t z1&blFTdxsHA1R5X>MhqMK9DWxj#0#9uY24OY4Ex&TjsQ*Fx(ody;!NPAbRg|3&`ci z2O~SV48Jinh2F_2J!F8z@tFL{An*8K%E+SFtR1tnt>o70WE=nZ0^pf4LFNdwi)xW3 zXTM6)vnkUzgr_eI;sfFJ3j*&?Opom9Q+{LeC6qZ-r+CmYt0H`i10_@t6LQiJv=_qdx<5_|Y>Q z!xOCHy}QxBjFRt1le!j}V$GlYcvgyN!wd}1uiiR*v)1CqNn`5M1|9~pbt%KUR~&O{ zqbEW}7JDU~xlQ%?BGMit%j{Yhs(=oD&#SFwcYMPXI;S)-Wk>2VL{_fM2?g67uZhTS z(-M1!v#g=kR;T*`m3pNDWp28})(4v^gtMEzv`@XNh-WZ%za_=HZiy7?)p5=;HhlNk zhuWYc|DUM4q5Y9tP|%4;r|YPQXyF5n{}uXF07V-uFJ|nM4WE++{i-}R8dMolp`8(S z^HI;e$+mc#;kL#`3IFZ3H@OJG#;5?*C|oZDzap0ix&<@7MER)PU#h7zp5Bs+fnv-GH8Yo zEedH>;DX1SYV1 zuFbTu?X9hE0}z!n0O91Gv2Gs1+$IGyoDu(ZpePF!CDks;5Z~ zdcz|E(ArKj1??XnvP;e2&Wv#zvYbOQz{{uF`&R)W!B1c@>hziFV@Fvb?k`%`f`W%B zX;>_EL`1aka|qp9u5;*T|IXvT1S%DZ9!%4nOe^)XFRw!33|Th`sGm%b>X@*{R)TQ( z{+0kJThKn__w<7}&3nC(nG3#n9WJ5fLGd8A{82%{uV23s!q0PRWj`-mfv=o-h((Yq za^y8VD7M0L0)6~14~)N4$xxBvoNA3Wvn`LZLd0@;5_ zr`h|w*XZaHki83?1(lJ`HX@lFuGD6F5JPG5ECAq3yJYzNXbl|)3A|a2Mwf=iTT{Q# ze)jU_{#Gx^_ewm;Ix~hrLRJ@HCMv(*L2C_mr1%=-C+uoe z)2DTn#NSh0m@~_`|CXYxFdlF`oiYchFiw3AWB^x2U`mxDp^dG`K0K7{fDWvR_ei4& z>@#CG^vo75Q}@kIMzTDt6+pM)P+QVK;DsM(i0}OX&i+e_5B~=dx;GujTXlc`{yjcg z|29bB(_=OLMPnIGAYDRUm^axhp{c8vH@|hO!l2k3A)-m)4mdPMh(Us&%t^NgmXO4q2O~R`<;hDv zSxK^i0zp03o?oVscbTWv*qUk!Y~wEs#*s@)1q$>nCt4RSaygElK3e7_+@JqNdF)U6 zO2*c;a89ZCxOnNl(K)H7F`CKdrDwQ+x&nH{d}W(j*j21*9?z=UOcDI7bTC<^HbyB& z{Qd<#;|~W9>)fb*)t{Y)#;tq?Y|qFNs6=sBl5AszQ16q)LxBN1{ z*jqN(CN)DnhXX`$2px+v$N}Xx|3TOkEaTiYHCa*`AcPs_5@|*`rxW2 z06N4b=z&6gxr#Lg5$&G4svt6lDO#ESG3Iv=w#J05RYCzpjP8MQM;TDLC&R98$Zn}p zza}qFlx*0WHbHokgx-9wSma{b0>YCrv5FS80OO{e%?G-EbGLK7Xs8?HC#+k1SzY0Jm^x%$)r#Y57#ag5^gPir!AzX9SkGC&g;Bg2@lKjO$-{`?v zi065yFbol|*ze!VOOxyaI9mH5TxifOUH(d#HAK}bh{zoZ>y7kmjxE5xl?HnM2Xk*7Q04lqi%vvZ zK|x9-6i`A!B&1PLN~A%Ugi3>SH)0}4he!)ZOqxZv3P>v*!XyRh?l|LPt$pvgd;j)6 z``ml}xbq)aESZxzzxR9JF`n_nHuxKT2Rd9D?3k*bq>1R*vvM2cVY>ojqruDcjhY~M zzxvYW9WjdrMso9F)~h5IDAyjHIgYrYE#~-vO8h`AE?&=QCI2U)O@(>oTPHmdD)y_@ zds~8UeB={6m*rkGwr{NKiIm(M@R=A0b^o*7;LAFm7{;L_4ppp8HX=w3e073}^I*^^ zPa}$e$&ykvKh5?gI1xFjo1-{mK@9LiZCn=s=8ZD z)?yUqIS%>SPJuXs{MuBI{md3;xzBP!s+~iCot{_L?p)HX!npGF*=exGx|OG<_d+S* z_GhW@CY(dxcK88$>XJG+>27fcXH#|F&r;8IrzIzUs=p0=)O_099)>>89(c;f`d?cW zfUJ{)#!mp53?RT5TuJ^rGCm>?>o*_6ez3`4sg;|Uj~lwbLO_El--d(i6mcKkXsffhucsDJv7+t@b+4$Hzl*7IktugtfBmb{V>_o8R8#|0 zC!3~pD%PS_+KHY@{SrG-y^_QsGeSZ~be+wj>RM?2^4AtP>8I)ux3uFDdkk#U@~{^g124;c4Z)!2X3!wJEVBg0@z6H7srV z3wFk+wXql|g6jCBv@9u&ilL>M3frR-wa?!7SpATu7k6d6)OW-M1SCp=H&b1i3NEW$ zS~tHOVb3sda1J^Cn0ZuuQTMOMx!!QAOaj&R5^ls;@dy0Rq=bY#Z86!~wQom^S4@Pa zOGW$6j|L8EkzUJE`_py&kp@xj7{B!J52(iQty^#=m>2q6no!G`*c?_W>7ed+iqB(A z_Y~WncAq+T9%Nv;^dr?=ID&pgqYV91Pi7sYMwQ5U#bcQ-dkXj){4E(boDu(*P8p{p z;ctV-f3Gxs`@%!zP*xT!XSiSth$se6sy#*(CZuaa9%tI>`zVobZXi5bgb@wSh_~;+ z5AvR$xc3vO0*4noa^NRYNhCb*&J5=xc-+$fQ*AN}c9MgDlfTC#k?bO;UJcDvSR<;~ewo5@)6nAB<8z7k z8JeN=%t>IQtl)n(fC#$Mpou|*7|`GK4@1Rp9Z`CNG`>Y-xhiFw0|bKLHm?I3xpb+o z;5qZi08hgjHvB1Q)m5wAo!r2%7CEyKnmm*$|NIdrmGM>HPg`ZB?(~HcZV`-~`~Pb~ z#1gX8a-v|k#bAu~CucE&I~|lfbWaX}s+9&EOriIo`^epVU^EwG&4bsR5zv{t${kGC zE81fP6+q}=1oyu%;l{>B;6D_Fzq=YIi(zKya5w%8-I1qi<^~h^Xy8CR5=A3tD$Nn1 zJlBT}Q!Z8@2N3~D7?jyQ0h+-`dC;^y&R}#e(Vy0l^#A99*bx z&eiS=U4TVx;y|5O=qm7=>qw8=%hU6zQwhu(Q=*|xQ~4sH2j9neK||nr|8HW5c@cF> zpWZ%GT1a z(FF=NgU@BIM$N3(A0|c{UhkYcX9E==u!Eu=#pfV*0gR&&ZxMujiukEopUp5i6my&n z@!af%djI=5gz7CR`U#H_zP2@zE^&mv*^V9?BPjG-)I=Dro*dEPoC+p$&7u=`i`Qa> z96q1)B=n-JfF4s(Q&Vfh)Y)@?vGkuece13u7hz=DOp_fdP4>=HKzS1A%pf8nvOAj+ zyul=Jiv-OfVy`vv9slgbR0eoiSO1gx5Wf5WiOFBo)Tugf>VFp;9GnUn5W0GLNNI%_ zWw`Bam}j>jS|da~g}`ozrP@?;_&hJ-uLhbouk2Q2G94TFe{;sJeNY?ztnJkt&RM-M zQmsaw#Zuee?{v_$eCuQE)zu`N zvohGW1v-nXVGmGc&jxwvjxy`4L4N!j1!_aI@EP^w5$KyZVRH*oabuB>_)-4^q+l$F zie?j*B?a8(gQYf!8%rZ-*c2&kmWkjM{97|t)uvV_WhR!!%@OZLUuNcJM;v;AXCL&p z=KDK=tNiaBVr0-!7I$31!NVL@>dO@+1XNs7&A#y{-#51oo{~qc-k?L*?c3nst0LI5 zM?Zoe2R=T23&3le>j)e!#$IEwMXyr7lGyBUL75+ z6*r27BVyCE26lLavhApsz6^x12;l5GUOr91^r74b6?03~|0|usq&zuD+ViDU7-W0js3^!s-JpGf?YG-n z(5`lLbnMB4D;DzT%%1f-&lNd?!z#_lwJ?!4(TH{dL7d;1g@PTP0wTPDQMmoe*d4ls znAfr~fI}QTNJX>bB#f1@w`=WpmIqbb%NBp|QwUi-;L30|kQ+vcjJAE9u*-sdE?+iz#CToiWsSE-DABpXLsOBJ`E%>k3d zgNq#1;`;*=Cu#<;0BkM1cjW%GBCFRDq1LVeP#7a*Rk$#+$y>##GKBWD?C1B!x5 zH`RH1xL3=@%O)mtH{;=HicUz3`uUoVd-c=xXZ6;vwMC4-^KsY|~ zE)OVZYhb!Gr`*{iBK69_;=%7zc*L__y(IvkSB0XKdkx``=&EqCTo(obrukrD8hsJK zU@c)^+QnMz)@Rzno=e@Ao8O{Kd4^CFypNpxU88dto5K#b<0~#=Z5|(s3kSyjO9h0u zK}(F<^)xqJgTXTE;>+S?2Ja$SjYoaD_xO1&W*Ox|zsTJW+Sj+}x1>p~axi`9KQQ=b zrX5|d{16QwGIb{_UIwLZQq@RFrQ{88I7StV`r?Xow-Fzw? z7`(pmm-x92IKnq>e05p+c7&dK$*$HC%^-bV;_~A0{v?UbsF-Ohr$0s3IouJEtCL_G z6llwBSLi}TP&xJVy7L5MGmG}Axq}tam=tE|l7ez56^lICk`FuYd3~m9>H*S}cVA27Lfa?Yrdw#_PfA$3w>3>& z)>7*h18et=u;&GS5Nf!&&&n7zt5tl@Kc6zaZ5&`cl10huDE$rfL%l?bH*|`7U93Ig z3KhGq^xoEd9u_0_{M52s%I9{Y(YQq8PKIIN%pISqlLPIBaiTYF>YcA|PspabSNl3g z*MPPVeyDZba{uM_1a^D+F?U#f??c`2VcP%5bsp#dV)9kbWEuwGlHN3t#(DE5&3$$Y4xb@A=-tj~h_I$PH344A$ z7ZCk;3(P#*_JKd&T!ol2xqm5!?iEprOaUmq1}5UN;P$*3{i&e0#0rg>*Fa@_kx{Vd z&1A3|gx;uo5;B|cWdVC4zxKr43jtmB|7XwzqrMI^l$Z>EsR*B3gA>fI6;}k6FkSQ9_$jlu3)JJyGxY`M&z(DGH808Z z{<`aAeJfehTZV_7OhK711zQ(a=g(5El$8IQ*T6}0Rn}6JpDrBXc1?*wokVh!T^pj?D-40M3`Dujmc|?AQC{J9rR9S z=M(l74=8*C!79D-&FrA56p;64VsCAIVY2DU@5N}Y z+Jl1+H`Pq+oE5E#bH`s{n}&`$4Gy%@KSX41sV&bBI)tLhzD_+c`*3CXIuUK6A~~se zEHd3(4e;<^{N^%W3&ckg0RFj|I`2{^n8%u92b8gwF{_s9;q6e4$Gnh)QGV@|i0!(jm&K{`{6R^?bLxb%3k%t$XD{DgDQ-LtAnJYh zlN=Pen5Hwc!wPSFEJSPQt7}1NPa;3ZXYEc-n&#<$;C}aDxWi(eOkvVyU`CXWzSG`4^rk_xTNUhEv=dEV&@lZ229A1Y2^>a3wChIEuyZgWN9 z#M?bIe(VqU@H{QZi{WIR=lU|Y9y|95R-Z4J2tDo zp<-DgzI^!tbLPZAI5ej;24DUVPz3k9sZdSRCyf$?=8<){H_#yqFKI7d8~B4X(Lmgj z7qcy%BTf;ob6)zj`@RnZ`HWWpgOAeBeN=E)Y2f@Nu<-kd^O1PB$B6MWSIsr{GwO*2 zJ*gHJG}Isj(|S0&#a?@`7SOvIOUb~H_$V^{i@@fz4UjoNyw&oKg~V`Vjhjoo-=KF* z`Bi$r!A!#}jT%YAz{6Cig~&@0Z!&14`V}^>tihD`_vRdW8k(q&rdbLUOH#j}G(lwE zzn^D)MOV8Q^H~k4`?!7-c48lgLR;2eotuO|Hh1r;M*FCxg$i{7@iMg5>DN~YJKNh8 z5%M9pmI%L9GZ@I~)Vub%&-ZiGtGw<*Ew9h${Kt1*Iw4|g7^cp1H9^N2tl@?n*B~zh z)uoqq$$-R{5~ep>tsl-~TZX%Gv<#{*3tD7kqpPpBY{7xjTwavke*7?6`~|a%V+2~z z=6iLe3sy%Agrrt1{#?U6yg9(b$dJAHuC?U1VU5DEw^ud zx2m5Sz`y&zSc?P;q1mKmP+9ap9|07P1+*537z1zy3K^Lun#lFppG-t&&+>O(6a*DF zW}oyT-&+L#pwWc76*(j)mY2B^llgGB%R=_jowHy!r)fS?QHIYuVe93>z?3eJ5(S5YCX8CsD>*Q>h2ST-)n}6$39z(xrUKkCH=V+)pQP&y zO@EI;BS0GGus5qlOU>mMdsed`%B(WT^=pga*V|eiVL{-u+*~%6fXX+Hb6t8>tr~&( z$AVv*-l2ya3%y$Jk60*sUDHHHnLJ#T`K31{iSz@^A6rlgT)bC(d~bp4gp(X1K15X;ZQgK9-#lLe*{1Krx&hU z_?gng`(#fyb0F!3k!zk`nGrY3a#R+}*`du4AaKWSgZSqsB#(fT5)k2$2)aCfJHh+9%kd;QbaV zrAr3ji*JE0Q5Bbyj!R^1#{nl7;AJK1DsqW^il_3hA@Td9&8tO+h)ehRAZ^HfLCltb zJe2yP8rEYgFcLS{p|Wa)*#~lKBepaA7Ck2!kFa>0WHU;Tq>8(4|45}<0CG+Pm0ra! z06f0K=;{WjT3lOA;m@d}IT>X$B1%RRhV8y_gH8G6*0^RV7Q;Qq-?u2p$=Qb=j=r=1 z44iZ(K!HPn*72d=3noN}xDNpy5Q~w%JcB$mxET)YkG&v64A6P)r(_WZ*lro1Y6{Tg zZi>&jLgdGWc(gRCV5qH6$ zoo>1e#9+wS4hhLXDHY={fWfBFff`6!(!l3H7^R-9?~3@*0C{~xhsSd#J%bmNOE)O4 zJ#dv;266@CpT~&u6!pC#r*cEv>}S4a#LBY9jaPmky6T?9(HTwX zZJ|tW8hCS%w2C^+$)uXG1Hhv#oncr5=Z<%z5flLgqFJ3^G99R`oCN^c-B_Er*XopC z0<9A&?{6?`b(0O#>36)wzqazzFg7j}CO&_rAwcVfwN}wtSn7;he_y^$Pp{NR^`7>d zdy2bb;o|oD9o2-UpkY%7B!D8%Gz$WA+!yNLaA^f+i}Gt!J+q{D_xQ4IS^2O);B3;rlcFO7#z1`&D zgJQFvcT2aRWR8RhPcsxBNXlvH(IpY*8(PHhR!RRE+(zVbb!``sZ8GuhXWwh<)R5g~ z|5<$D3<*h5MGn{yfq(kYBL?`~QLX<+DQNJ7|3?Bc>MoVIY(GRWE1-!aIsmTwEa(J0 zhv)w<3=w^Amr?hU8IMkAOz2e>r#5fa7_mLl)^YiBJsLOnj}>Io@`9s0o6cVu&_c>X ztU)Tc#6z!t7Gt`K;N$>TBaWqDNGJymys0pJ;{`|6fc^cQows0}h4{t&Z|-u+5xlT# zWV8b9zY??<4-jP&5=aXH3WxOW&bzW5i6Vvt;y(1^tSPTa!>&1D8`k7z z$k5FUA|;J`=`wWd?fnMwURZh63R`QokXsmaRt;r$=Rd46!O}S%uK&O^@?bZS`k2dw zO@k-B2cx2LpOlvMIgZw+h2zdmOWtQITB=2br58FD*&5c|JG6<*$@12cMXVXhFiL{@XSYSSC?Uzq`&Cv1>(BD;RW6=DOAG5Q!_YT;n@XpW# zHeh4uZ+{J73|2twLV;n25I@skl_7Hyx~^h~WrN&6CXA^MFeDHq9Ac}_B>@a&1PDL$ zMEqi*VV<|;VP=-L=*hT3PfuS6Fb^F$>@^)0=09me_)(2Q8)Wco$9Cv)@e>~=8FWuN$aTI zF^)TYOo!yG*lXbc)w*W!3}9=|F@BRh;3V3 zsq|KE%=LT3Q@zLD2e+jyN*1pAEQn;kD2C&y8OSnI@h+o3hXKE00TDzEqIG@a1w1@~ zXeVHC)2(Gjxj?HO1&3E`DM)nX0DQcTFxjnF;nS;Rb%L?UerKqg7r>B~)*s{gRFhN)K_s*FPj4f)<*hcqf!Ci)M>qvazp z0pUeOZo{89Egd<&U0{ndnkrjPbFO}UK1d9v%Wy~SVi5=}(W^_}&u^(UDH$slAVB9h zmI08POl=1U5{!nnClTfT%`8FaAgcZ^R!SwfaDwHJ+<|Z@KY=^&!1>;5IgG|-wiZTdpA3I1VA|{gDL#x{ShNy6!#K8m`WI(n)SQHJhFu~w7sYvjyfXCH zAB4YLK{@^_(R9{&1Wv$#f8L#rMyrg_rE+`M>VvV_v82kO`%(WKw^?o@umFtOKSn2i zu!uDEpCit0Toi$Ez)~Zok>HGy6Lk>SV>!Bu)&W5l{mVL&E40`KE8Zg9JCWyHeSfWc zgRRh|lR-64r5m!oFy%S1eXWDt=~G9C9%~EO+@ff%qnv2!qmt3-0djr;Y+w7OsbvRK zIALTHjz#FP+Pz^)kuFDZJL;L%*6l89-)_=P@Pjt~T(QaU=d06(6tR!zk@>B{B!;Zh`5 zgXi|&d)}*M_KUGrp_x>^c@}om+V5-)ml`%UpQI>L(yBfOj6sq(G_N0y*D>qrXpoLGDoRWk|^RJ-w`inb+p}-=p5c--Zd+yX!3Ah z{EOu)hU>PwVJ|5=;BVWxL;V;&iEjabvz=J}KoPLF+N)(C2LZi^Js0A6B;>pR#Gli% z+I~QFJkZSb`*=GQgfSJ?FzI){KMrU>EA&4N4GkKN5B5}4U&g*KlUvv1HSOcuo!?P` zsaILVjYSA$rynSlxqFvhVLvF|c&p9)ruX3%okS#1Z;Xe3pH(f^wPVw%N=;C4#g6Jt zEKJ={lrXa@h%|s^-Kli^KOwJ(#V)_S8IIWOxE}*kZt4xKLmGO96uhI3n*%e$nO#Ob zfvjl)=08#Erk&rX6{u5X3&^{Y?A8SxogoqK?meWVNNRi#tGSw!K+x*Y)2RE7tY7}= zk8JH*Y+y-+Si}Q^kJ0B~^$p-qx`?X^V6R~?ww8m<6|3XbBYi1LVGX7aj#^^~rcERS zbB*wX#O-ihx3zl%2F0JyRTkk7F5kNAY(0Dx7-r;xrGKgBehPM6-h2gQ&+D3biS6@r z+9f8Kz+>wS((|RybhzRC3ai+>1bi2RmIDLtLicCUOV!hd_LN2)*?$UsmbnJ=w_NO; z6xp-59m?db?f)u$KEqrL>B-2 z@o**CY(~~%@Z=@sOuwYAR0WT&)OWtD_bwzbYMj5}`}XX4O3Hkf=|RcRT*7y2lFI6y z8IO~hdbz6QLr3=y`fX}mW+aG+US0@)P?K0(crB*f;gRy z?3YfRE5TIl`+5(!?lLt zIoQJC(ZuqfTXe!yk&|*djlY-|A6thN(kQ4>8Ok{}d1ai)b6}NA-#q0DU*I8mf$3!S z58q1P{OAi`80R8cRlcx#0cgd`&I=nN^6+K7|8JG{+oz|IANul{EG^J#mdutr8&*iy`zgB3%}@PSFIR&D~e)O`fM)~Ed=|5ld*w>BVhIbf|E1*nk7Oq@eD^h^Q( zv55ffHn4UI5=1xwF^B>(90mRUcI~khAjWHXu!}XhF2K-oH{+AA~oW!J+P1K9IP@=5j*Zladvt!q*?ypJVvMHL-CO*(|O*-r}@cOJ`_uJMh&q zSX=+&ykC;DHN`}xbEp$}z(2qBr-{B~q|!OQ(Eu9q)aynlfPNSkMrHK>y0d&V%$1|k z@R2{ul%wBzg)tuB$02%;g_}sEDnTElyv#o@fz5UpTAD?Oarq16-EV@mY!O@Ht*>Mf z0|&e7IWWo|$F6Vo8#L|IKnVAEq7xV15y+Gu!!JrK9t(r_Y$1F^17IEb=XnbzuV(0l z?;fOpG$7fe^T2TroH^hzjT-6Qyas&PbV@dzwmE(0ayoI(0dL)^9|>;RrC#g37cTLx z;fk!c-8yu|&EywhX;h**7sHwe2&|P zO06wCwigP*sG~~o-+y$qb?k1GusZssP4D^4!K>-?;eJibUfO0^brGGg%*BICjVI8;?0-1y+1biZ2$64pi74>3Y~q*`f>0o_pP14OpoK z0gyD=$qhnyN$lv>{!w_SW~KWy#Sy!fo>OG;v%xM%yrG%&#_PLK7t8a+j{pU4mfAZT z_%QS+^$>1nR6eA<(fhtTgO4AK*Qpo?`OSNla0OyvD0%yGNCZT~3;+JfkY7e_@Z2LhLC3$EZAlrD)eZ0u9F#ZK1O2C!?EY&R+v z>Xvr|*>xmF-2kkUoACIcBfnI1#|8VsYq5i#?Az_FvFEtno_C)eS_W*+*;Nu8JLZZk zl}j8fcMDZGKC_%`sdl`Q`Q`a#ee==0v*18Cchyr<z9T>t94T zb=3N<^)-8Q9P@xOVM}2|0pTN&BlfMFC6KS<(*YQN4wzInlcRddrDJ?T>m^j2V@q&k z+`)nx?w&04t36E{5;IW5&oKbkg=c-e4E%=G2>>`wAq^on2ExgMFN z{(jZL_F(#Kp6K|%v%w;C`;Zg)hRy!=2bB`jZEMV<%od{O(uWW6P8+yy*zh6sb zlce+DDGq>Uvn8qp8rt)du2vL%0{T`In)%$kbuQD9TCZPt)_Y}L%HXUR63@1(@;`5M z5gfdRi^zc+=g~C@v{#2(tG+6YxL^)5%_<|dy!dosZ`I_d#@H{IAur-H)`L}lBJ{kb z{cEb9?Ui;f{HTB7zgOBGf2-RDU1G^FGr2f~oh@~bAJV)$3?N(6p!RLE+vNQYu#4`= z;3He>7x@cpmzNC;f#K*3pXV^hoF~Oh;N@?6Yo?szCsM&512|SpLAhSs_$S*AnC6O0 zJluwM@+Uxqzc)A4kGTh4dsJ9g{JD1Qg*2VT=G~3BPSkpip|wHa0pU&&F54m-L27m6 z*5Y7;EoqmJVz&ZC!C&G^qVw(IJ(5xe)EC$+sI@TWUAspQ;w z64d_jD1O9UO{7!$#&$5V=6aa$Rwq@?7o0C@M5i2K)E2zH`pa!?Kxk&F7{uDHUiGK& zNE%$LR`BjkcqYuz;N$2666i&Ea*=Mp9Kr60e`{G{Z1ec9YE5k)aQ97*05)Gbp2jHGFu|jOC4#fuNdd zb8=(Z%40kAX6tK#jWGy@5nf*=;g|~8E)N~{q~_Ey0!GyOo=<2n*(HHC$Sd47B0}Lu zY^LV{mE#9)II*bn?2r0Vp5Tl8Ex0j=XTHy2>B!*Z%OS6H`_bLq>OOsC8yn=LDb*rF z|20P7vD=Vd?fsb9cn@#Td+yu*W<>Lra%bhUwv19Sly+r@@%2NmmRWSUXG{*dQn;f@ zgF$)Fr@jDLS>(w2dT8Dhr00!>{OQ{aT8(<1z&W#+N$|)$+?fm(00ErsVXfr31t4C$ zB$&7m!hznZ<^K~FP;wcw`{tA{80t=}7y_2PIm|7-&)i%qQ@n9OHUjItyPMjSVAn`X zzkzXadvThMJGv)#zLcUV`2D(_({3a=3-`v*D|q$0CxuA5dd&m2T97AY9q!HN-K4(+ z83^y`F1bju6V_xdB^0iJTf}VfDnxEEcA}y zkNim9y`>CB{9gh~$s-1}dFArPWXQ~KsxsCoXXb$^v zet}j+Bm2=$>&nHFl~usr3hsQx7A=*!6hW2Lk|`n=j)ozti8UvKGVKB)0^uN!(Xvh8}l}O$o`0gkYn*mXPFcGX-4}EiRD>^23PN&eoGO@jdb}P1XT%$jtD>25|Li5OUXEl_`yGK( zS5^~TrG!)W7-Ov}x>AQb`p5_n`Vz2-8m{rGB;e2>Q%3u0>?7`OHxtx~oatgkojz9! z+LxJb9sXq$YI-WcW65n)sPx{vv$IC`@A1h?0yQmw+gib2UBZvi9GZBwez>fXzk=7d zxa6_eh_){Fd&Nb_;DeP49&k~s{i`8>YGC5t`eC_nso%Styvq9A@{;#9L=WHMzt1c5 zH+WF=#OCbEgKuo%*R{7U3uWKW8FKb&dN`_qj80TPE9>SAg?u*C(1C#um=U-o?vxT; z`BSLJ_ff)-*xhBsf4HTq@V{;a%+vSe{J-vVRyV6=yt{`no-a+`Qjjf$;4G7UmuH2W zXy@VmTY^J1Qca)iMdeyQ#HP}%h$fb^TdM_?oH> zs+D4>rdf7&05;SCrvcRsez-QNf+znp#AFeo$ws}lM@}>3Eqy!r+0&~?weQP^fe!e7 zr!ah*ntXEp!1JAJIN6=yJCsV`$>sf@ekfaY&IeLiyM&R$meGQagl-efA4*7fT^=X( zrSfdqtE25LVfVmo@DwmVk0S}sR!mAK@oYb6spy3a*-+@=@mc&+vidS)i={8#1P_f$ zH7%_CoIG#?OYIjOvWqZ`MuphlEphD3ixRZdZ>D_ zS`dTK)1PxYtA?3}WANk2g`Wg6U8YrQi(r<+Uh(@Mr1E>m2WyrOExC7G) z2^`^2PVxocG^hY>m~a1b2(}t5WEnINfhz!y5E|NRsDUC=rOKs>`DegF)C5vy<=z~| z3MgJ%kOzQ37kaf73t>xvlN@b5SQrV?@A z$T&;O`wfnCQyA$LLs>BNUU1Y5Hq8<+?s&v{LH?%?vU1>I%T9Y0wF7}5`&P@N)gS@1 z7~k&{Sr+w}&nZ3dFkV_5ip2|AOOVq7MuAPTlP&jsgqh_OrxG`;s+0QSv zi9Ap4k{cj$d3c2IsW70B@DA+n*67Yy$8TvcFuEBe2FVv&FX#91Iu%-W|NfW05f?Ub z>R|T0YgO>HVM6Rat#4>kLL0o;S8S6@N4F5dZHGrlSVulsUg?vjp{y&bnydi($8B8Z!M_lB`p5PeHKhHU3mMx zfZ6zFRBHByKosF55&7XUsYCdM$k9fMMsnu8W4gw$3k{FL+_?tFd_ zbKr9t4=3f`Xi*VtU~lc4kGDt4>{75)?CPw8ooAV0$C5X^ai_7~HLQz9DdZ+kR4QL1 zxpX@C1q{TPCd7QIWtzkF1v9l0?9y;4_oyM1*Yl+i2=s7e*Uuqq(nhs|8j0ER3&vBG z3Q21@#(*^mDiyCy5ycaQBBB;!GhSs?`ScQ-i!S9FHS}xpGgSHvRmu1=OiW^@*8g5; zC^j~D75FGOhrvLzVK~ANa6oS~tOm6Na3O&86SP{D8iyEbz(ZCp=GK=!-;ey3d)t5& z3XQJj|2uZi7+iy&b>of-#IaFvRm=1r9+f1Xay3@E^Tv zpj3AMQ2-24wgYa2%K?vvTvDe|aFm(@V6n1L1&sh2pt>A*Lp71xpj|GkaGD>2JD*}w z2)Ja05V7?Y)QomtcEM`+8FZ-|T%549*BziC9|iWMB>>SSzz=qHPO8}cXI*Eq!Al?v z0s&w(RBm)BSiyT1fPpHOK0CV(xP01M3+eq2s~~k$0LG@+-w0o=lSUnX3_?q+0IebF zCxRUZd8n#=R)+hqSL^TpL@_oPeK?^Uz<)C`oovF$|M^eAtAB}T{bxUX1~>MR7U8SA z2mH`CWetF)3ov60(wrW_TsfqzLw1Sclq>_d?mStpS`PyIW^<@0Eo_aPkn(+I0wWZnjp%k-0VZ=fb_xp422cy`Z|21gJbnCtbq3J3`UL&GQf2= ziq?0})X3A1m!S7OjdbB+uu+Q(3b}8X?HmEIVhARzHZE8%!Fq|hHr`hQ>jcu(Ky<9% z+g`RoOi^qwh#CU4XFqC@EDE5I@z75T%yx6L88c1y05w+|kVjq>_FB*?fnDIk5RX zTLi;QG~6MG=ahiwj_p2d#fPha)OErFZrbrX6MROJ@f{xR?LE|Gm~Q6zZT{(Q0!U_d z*t=7BKAR#4&v&PhLh)0g*OoTf5%5sjum3`tS1WAIh}M9_&KPdzscaA)Y2@fcN-+CU z#6iax1wl?ClYzGuax0e#YrOV#Aim1*8Iid20uA*XihK%OT-L@6;4O;%aM>C(-9OYa z@G2Gq3~TEfg%aEkRF~nHXfU9py{IVZ%Sy$(1EpoR4M_N{VBr@de>d#`(-j&?PrmzR z0q?I3QR~`1uU>7uM1K$t%nP^iBPfiALYZV1z(E~h`m8pcw*68y73a{gnk|917a^AY z*dP_4JBkC`kg@r=yqZ1s%PVRtO6%vmnios4)&P(e!QiCbGLc{#mzEL-J|J`Q95R{S z9pnUs8@|2fXm5UHl7RH`N^|M8aRM@WWs88+MBoxj)ES#;KfJ zsV89ANCcI@<%AGi$pwnzwdzr;Vgx(X&!#XM=&& zpiS?h7WE3!bLR-cmn0J? z`NS6EK66pYQ86{?YDnmL)ag%fAx8@gn{25c8!ukhI#lfutF+Cn~oYW|nLrFt3dDeIK5;c-N z1o?A`V$dTc^rq`1>)7@jl2>ZF@hmHwGpQIRe{b*`bB*LwK`9%)Eev`Qc$blBBK(5v zSB94b^ApUk#Sb>_WcdFjejrV~{Lyy)s|Lf%A?K7&js8O;Q&^s^J z*TP4l0~&L&OOGP_Q@T*ADpqt}Y&+@Jb)g&!^$l*2J-z^c?1uYu7#a~2-^2MPN9zls z&$SA@3fum>sV#&^I^QUyyxHh~CcievlWSt4Q-(Sh5B?$x{Z~J}$>1n+zRr5{35?+d zfI_zt1{mZ4A2HjJbj0BwIXHH<4x`AS|)tBV)U_am!#%+GNX z{n&5jm&bB-O|KF|$X*Q~j?PQvGdfTwinSjBKXk*&7ia>^B zX!#SmA6tedftyNA$`~BV@3zZ7#+s3)xcLV~KE6Tpn;Vi4i+}{b*9kH0JF!O^l0#Cc zbFwMwQVM!&j#oJAnML4@{?&^_FE7$c?4bfL=qAm7)x?X_y7r|;zvFr(~A}d$?M!54jveN;4r9a}NM5z6mm@>cS&Cv!_ zyj_Nhvr%A-(f_VUBD0yRYXu3FBp-~uQtyCA(CRQ$rF}M#)hFixANN-VTlx1A+VQ#F zX=K{QI9d2f#FKN|mi%WW9lNIlKQMC?Eq@B&T5FU9Y1{j4A$F#u3cVq&%QxGm7Y)S z!hM_DC>iS4Y?CicZ-kSDcJ;jD3)RG4iI_2&McWTNcSgc%_#-})jIwi&@Wz(uOecs7 z&7UYQdpMoxe`E`0=YUNOKLZ)?Q7@p*MlsRS*0>Pow`>Le=SN;Z>-XHJhxgKWE)cgx zwC_`j^RBoH|p1k4yX0hr|W7Gi?4! zgm?eGgp75#T8Is>4QedX0@j=8J=*A@GqZK~wCATiZ-r607VMo~LdXFmd{La#_D0D& zE;jFJ3z~wy5jLLc#Zjm3=oH^?1E=>t0i>aR*c?gU!h;?~-LtRw{?x2@h*Zm#UK`A- zT6;&mOgcw=N_Ibh*;c|-q+@x%hvuR?Z0Ap(SeTu32r#cApVQHkaDO0 zm5*vhRMKyui!OYm=Fe1`4D_Zzw|qES_u$@sxM2aVa;gaO+kvt=*(MeZ@;z*Nf#k9q-fv{0g!MYtrhpuR^P|<32Fiev zmyEKo#NNlLr8e;VDZV>JVRQd;ZeqWlOML%1ybu9A9Jls6eD{&6RbO11wi=7YJ+m=R<#s2xF6rGGhnF7pZvCPEVZJ8LGPJQfcopIoG!7)z_=(L|C5MuU zim8azNj5@d=nK%++3LLietzmn2$PA@lbc<~hRL27FE$_HJ%Mlm*WE>r2)v zL2)0WGj6%6B87X?IXnubV#`yKa{*^cGKD@ghQPpjp9{rFW2XNp?5agaXv7F{z3x+R zBBNJU|Kvx--uvAU#0BC?KmA^kUwoLpK^~$KibHttUcLW5$ij2da_ojW(j%>%I=i<< z!9#lIO`PTg+*zhj8gG#C1lhC`-;~~y-KURm9((#ClcUboY%c;&$ZUj8TKgHdFcX$e6Rz(a!N8%om!^=aW<% znujU2OE5a6y=s)KFITrk<4Ed_&2Qt6ym9i;<#+MxrybNKlpIP=N^J3CeQ`bMj6}PZ zdbkqx#-29yK-Rs>RsF>@JkVN_kz1Jh3432R5hw5WyJkojvZq=pq&yY(6f{2KeHn5g zr=G(YZmnt4W2n?-x&Og+5dz<17+@`19!tZz^LpP+|3QuQ8SG zoRU4}d-G!;IpCSoswSPdo-TAHSO%@@wu?GuVA~jdS$X?`${(Efcd`t#g?%e#;WXWC zxa9z2t+~m2y2Ut?E#H%kP$AB#uYOB@dYtG?R|QbPD50W!G}mIkG<$UF$7Iw)G91Y< zK4#NHLdB20&GOX4S(vouwVcw?IrO6AY7U8qV(ts8IwXKNuM8s)JohZatWhWl|lgDbI-mQ!3>;J52yasR?<6GJKZn(jqOI2{~7fh4w)P8!!j z6LcU}AY#M5lkxiwl3g1tBrId^EI<(A9gFLq8w}Cn^M>!|j{gYD^()}KSK50@O<{ZS zF6!YNfJ=6xwLp89KAuMWGT9;2ffYE~$-rB91LsQViXmukJy}7ApA1L&UMF}TkxBGH z`nk`8&Q=hovO?*8w4VRu>zAh4oq@dF*`F!NfPKCib)9^#41-Q7s6f^d4HFsaunNny zpqqYaT{iyuhqh__^zt8vPBB5qV-)wXnco5?G!X-mDO76(RiFe?E;N-%-1J+03L2Ya zWI}JCw9w?_-AK&n^hYeCbA13PJ~ZBYfpwYj>;XnDJem$r^C*+zVo1QGW60cbDOxb`Tz!y zAn;BE@Kcqc3p%XcgGTx_z3bR>Wf(3Wh>6&aNlFONghPhW#Vq~miq0oBFt#)D-~=MB z#JKoxX2}U?ARk`cGI9r$E%^<-`*(>QP#;l4w)7A4<3k|IQ)WS+2`ivv$xyGpSth&I zjP!OA!2MnN0XmPvn#W(jcxmDXg_7v`Ebt;Cz{R@Hb4nO^ZXAG1)*BG;TxJKTv(_4q zb|X2L=(cYUKZMZNKGsx*l<~e~$YR=I)&axOKlOVgTxE=W;AC3643XsC`{QpI_V&vm z7Vx@WhAgpSzS=|;>S6Hc8@p<1P#`WEN<-^^cYk|XX(Q2TXJuS`*$i9bWd`S`Hx~ZX z!!O>@izkEdYR5I@)f=cin_P1nQvq&%-55kS?~uB~D!U8o^vzQB_NX#+AzyL60x9X& z44WOE08T0?%az%~Ov^CFCyYFxxADmaBcEhwNSa(9EZc}}fp+ROD7mO>qU3|{q~UX1 zOCln)C6`BQaf(|NYg7|ajv(h^qzyP%fa=7#Z4#&eyd?!3;S!HT#;Vj*n3RiNc%X`^#=5)x~hJG*A z#f!K&ZbR?Vo}AU9e^q%urZJ@6HRtNoO zr;jURyZ}Mh{@$_$={Erb#u#Qb#`AA=0RCO+%?4zzxpWa|Hkf3vv}NA&gbrXUsn8wq zLYXYMBK6w!^C-9t&(^L(7cM%!ThFeukRz#6tabkhEI273b#2c;K--Yo){DhEsIJ#z zUfDS=E%`*OFYd?i!F=4qbvV8T;z7=h(o-AJTPg(6!&@~7bZ8o_p=L0?P>@3{yAP6M zTKBokbWf3lmeUZ^^R4JLzrl~o_tH+Oi%&V?IlP6g?v;zc!A7@|Hx?k&yKq6s5dWzj z@PYrKC^$0J6nK%Jy~B@Te#1z?St0YJP; z`5N5H!#huH289MsqXP&6sU_zKsn#bd7)cwcdiT_mlNfbCn(c? zfaT5C8BsmZBd)BQB^Lo1Pk&qC&i(fipg7(P@El1XZ8wG~cDYdw0sh$kDDO(+q1@Z> zgBfEVkqC!59Ub9>L?khzEay~+7!gfPj%4j1yJ1?TMZ4WtvPGhrJ+8m4UG+#AV5cC zP_~jzlN=L8VCPs53=;D8i(miYH_NYZUO3)$y4=!T03$|vkDn0{E zvlB=?B_m5Ca}4!DrUkf+%HHRinio$zoIMsZ-+^`#pPEMG0)hEP;}KWd_#*}EC|95U zd^{*?bg4+Ra8xo7JCjcAkeH-lY%c&0vL-$&1WH1bK$a&!xv&sviQw? z|A?0gZF{M@Z#EqagIvfxs7m2EY~rY|t8vOsnDE8t20%|6mY%b(Q`bgRnkv@-N`f+US%loW8nu~{h(;*aY1#5JOZHIY$ z*Mf1>c1YRL)$P$}XSM|h!Aon!XIEgy`ATn-MKtt)Q4=!O5N#K(zg|qkj!hmpCG=n;rsbOoH^5--S;L3 zNm_tAg_x7>!XhM*ig@>#eTAJ?e_Z|OxNk=+A%?L-Hm(xm9!M^69hfgWa>w{wObJDD zf*n+=%Oab;Asl8q07MDsp(B?jcWglRWqzh)m$3hg%Qh0gAG{Hol9 zM&z%@Bk9G60D2NdHwS?Q=z@w5CNq3coK^z*!SpUnLb{IG^3a{wYSS6fEwo{n4Pgnr zjKxuj{Kax{i|Z6ujw^o5qY-#Iwb;FgqbT}e`4`CIq6Q`kZO%WW(**p!b5s2rP20XX)`a7KTm26fwloCr7E@EXV zUn@*#L9;~DzPXs8SwutK=tbqsTb&6M3$oRzZ6*voBity=Tk#0(a6Si2d5?u)*`8L0 zn`~N26~PWW2Q%Ofk9;a)gp+8_=;jfNeX9Vp1%@m%{&e%g*o9VAF<4rK-wT4~ONlT$N054z93VHDo=*to-Hq9h@9~rtyLkrB= zXyf?X_$}%OBs};}Y>~X<;F2#OsefGI=z`#ye$%(ZLq6u=G7aiRV^0v=ifGWfrxf%P zw5DL4M>H)sms#VMrWpi*i(a8)?y?7>5_0gUIp( zoB%6UFDeib_(rp>KXOy|K(5NdLrcPUPbXm5)DVg2{|LaK*C(GSj>f^gF+DNn0^{t;-{6ne-dV7IGDg!f8DNUnOG5kEMvop%nzLD|51~^AqauUDNIO8Ud(o z^)*>1EY%NG#~d3Ny}0T!&$1&ADajpdgr>&n@5*FrN2)bNBJD*7JBbs( z3OvO=aNW*eS&awPdbPXH4QGU{yhrT~;l2TIrK>%&DukU~PQfgy*a5XwyX}T&S{zd9 zB`}x}JFf}o75wfSiywerA2GkD)G|rdC&o80n2hFvU>~*C=e}f=Lc{48o!hV~;&`L} z3gs6LUtcVLWKB!ic#id;f5-I=+A2Zd)`^_5qeAw4<9Vz-IoF?W5I#j=cPPpIqH*QZ zn5kW`@nGGUkmVFrSF8U zc&19>bY`la;76n0Xf9P=Kjz#ud&JfK19E0tYmt*C%ma^-VE=)X_odyMJb>kS0#zFY z=i?dW)Z79yu@_63wL}$h3AWy4(*QSi%MT_>xUF&?{HC()_3i}|aYD>V zN&TdQbo?95%y2qeD@ll>Bz8rHw3ihz%ZrfpJQRDR&nYEDBERUvCFW=CH2arTjHC7N zXsyvb<5z(R=@p5K@6T~_Uk`Z)33P20g5-Q+BzB_>hzulakL1OP9^-mDkq(9GWnNm% zCGvPz@#p7l1lJ3ku_Iv=tTa7C^yQ@H5-v{$&D@Eo;V{XT%Prd;{dVTC_f(~_2sTl~ z?N--W35~A@QY@-$yNcP&VIOz(xc2PGNK^=)_~UJN#MkW0W=e(cjzxg1zgN=w;<9rW z*{HG^7!4h03E({l(_%OWjF%kZ95!E+ z;AuV5*Co01`TyDbJJ=*EOw3C$x2XM9#x1LAIx2oDw&sWU1BMu)T(MiJI-?V-0 z6Q#}?;aV4Z8yAC5SyN606FM}>=heON#k_*kJGPvb$-8(7^4LfyCRnaxlfw01xlQeW z4wXW2Li=&3i8u*U6=Z*qO7N&_QxdaPfd-~Sq4OJ$IQ35g`E>PcOKBvnD%lgxBwZME z8rX4Lo!M{Y4QjKG(@igr@{4Xf&`nn$aF!U0FPI}fdot;9r0cB4*@k>j54a~Z<=c5V zRFkPiL)=w+*(Ys-3w@2VZdzPrakxF+Y+{H!%BjgG`M<;%kME}OqG8lp~&^2He zx>TU-_IBn4;3I(Wp0g8Sw?g+it3Ltf73*FA@XwzP?`&*!>a8VvM!fm69ot@;3yjJ7 zJ@8XW2^O)fX|O%Te-`kFZriT-$%H^pOFcVugSM~)Hkd{k76Tq8Q zD`2+#JvH!}YL^U!OmCxs$Z&@mJg_QM1!NXS#L+K;`JtGX17`e~oj9rDBq@D$Fpe4XqADdRWI|*PA zxTS z34`V4q1b099d#M8?VIBFXS{p}N6QdVToUj5-2;oMotgUaJ7lXgJ?~_I5ZxL!QNAx{ z2iF&A{a%9vuQeEI6x1y#iL|Qx$H$iYKUu8d?f>n&{{Q*srkZuU?1AjbyS|Qv=+6FX LWw-yaC58Skgq<$1 diff --git a/benchmarks/results/smoke_postpush_20260423_123452/charts/membench_comparison.png b/benchmarks/results/smoke_postpush_20260423_123452/charts/membench_comparison.png deleted file mode 100644 index 30a6b2575088134a38bfbd7efd972152367f5e81..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 33079 zcmeFZ2T;`Ow=LMBpaOzQXp~?&h-45YNK!cJQ52M%5fBlHP0pYalxQMI6eZ`JGc>KB zfMm%*L}HU;1G7K8|Noo0^KQNSs-|jcs^(OkYDM}NzOeV&YpwmgyeEJ22<1si3tx;IzGwwT+3T zg%F>h0G}w&>BqLVRyJ4o`OW|R8+?}5#{4x}cFk~=!&d)j*kCYBk?8+@x9pz3!(co= z-TM2QvSZZDz~S3+EU-(VOkP`Gc1nu8wO14zfd) zZnD!|8L|VVcCv&Z);e;m_R&Oz*cjHUwCGVGr|K z>DH6($kLO}H~VpMrl4DQB-&w+)4D$`t1_ai=t(D6M~V_hdc|t*NShr0k-SIWxZ0OC zCX-@is@%70uS}=YO?9S)_4Xzy#0r@8WM5=G)#uiu@9Vuf+m|%)Bl-TP?1ctqUw{5e zQ4(pXC4nuhtlMR-gkxLsaqEZHPd9^G^G(`2f|{eneND3}gh>m8Q~qXk2T!>km6zK1 zxtxAd#3CvvICeV5W$t}5bL@C6X+nDUurHhQ<4yVW{I=VZ8V$s6SUuNcjdjeH<;Q}g zHh*^{QXDxMPi%*`XQq*uub%(?DL9d+b?dx*6jQUr(pOu(h46m2Xvfi)BX1R?I9Hq( z-v5fRT;5(A^37|$!p+Jo{mC(QL1=3xuf>mL?dMwsA6AEc!TiVH&(0Q&_%J69T_eed zjbF*B@YrmRt&K3vd|&CGmRb6w$6?5=4qljXV?)PjJY-Vjc$Q9@YR0l;@8eH_wLgqw zSl>a-I@cbXeFTax{Dq{yl%KVdK*0)j+GU-V35J?Jwp z;w>gtZc#qzObyxL+v$Dy_AaIS#(3DX6s06aTjF3vy<4hEnyLVdzhc&9leYM0nL1^m zj%ITO8^0p4q_wJDo#CD^J$JFHmKVp)QC_kAeQ$KZZmNTmNy^pn%CFS4%-iN%CRMEV zJ#h~P%blny-{0BUV*4gycS*)I z?pw2JQ)2f0IKpw_Ilt)^$+dpP*lTlku-?NqqaD{re5q75O%jd=oA&0L)OhSH#Xe(S z*&{-+x?-$aJANA1S8f33RgZm+dZ+-nnyxNNFAUvuyj z{nuFOs$rNqz7@@zr`(p$W(hs|8g5(pCMZj{!nwwC-~QVY1lS@)pC{V0_3NHZ#ClZL zk5PHVjuAh`6K&=;+*dl)sN8-fMOinV?ORNe-ZptUHoD|r72?fWwAjdvEr&(TK3JE# z3FGf~1LD#yr?dFiVGUEd zZ%!uJM(=EIVJ}(qi<$?UUtoK}Ky$H6+veLd6;+Ld;UMw33+wRfN?NB}W^$f|@@oAm zB6wclJ7rb9kNne(z}x1V>tn%GL06`dNlp|rEX7yFsq~WEwpJ%~v&!a5pYd0ba)*6S zxnE!sHdC)@fDP4Z^Y=kE#|OQ-uD@a!iXA})`t-~&*s@z`fdv=q}DgJ{}lKjR| ztHyVnR6A5UaV~-ST}~6xVYSKY*9KiC)?T*6{bd|&r}`S>Fh5*FPRAjCXM#z@ zLc0b+jYc+H2-|PdBFXN~p;_=Gq>`@BEblI5_~U`w;p z17!|1N(z!H^RNKoTev7}Ja)H+sD$B53kUZxN|vbDgiToplU?SLSw`!?~2NIacAsd^`;aLEk(m;HRcL5(^^P+0*sJ zHdyWvWCQEeV|Jqx$GZ7vSQf;Vr#iU{Mmp8>c-NQP6(AVGI-_KB9De%juE@ZpQ|=+l zl3XM=3kci2{4v}+zeb7YV}y2m7#T2?>!beHYnLWsT=?9=+;=u6mRp|2$@(&~z-&#r zX!gT3b^aO-CNvNg%XQ7ch<4x$7k8K=+hVc)d}c#{yV8JM z7bmFQq!JNA{}28skA;?bK_}(;JK^^Xv0*Bk@?@J7B!qdgK3J@F1 zd8EJZ0pFbc)y!@m? z{lWINgRJic$JSkO+J$oiW&GI+N%GM$VSGAmi&~ZyDi8a=N6{?FljBL3k#@DEFM_ewxokCJ_8+ zQS8O`aJUmj9Ua--Cj9*kck#F*{{9oqZnThOKV6SWiZ`Ys%mnDd^Lo!Q6Ep&FL5(fqryY3OIBY63Gw3`F^*^VlFYB6n;7+D0R24ij0YEk z=^nm&m{HKx$WP2~mrscKBz|)pPnTr71in@KJLiZV zBtnA$(|1(Y#@y&*?>$_fADMIumR$XZ&iCD#!$*z0NBSWus<)aTZVVVii^yMDT<-nGaLTK5|>@!42~>uzC; ztCE@>4xBO4KApJ4*-`Vr7GWP>$l%)Q?`nnY!v4Yr@xK(Eo`a|y_2_W0&~d_v2py z9kbq|-*M$yEXx;=Th#GumfD(eU3aKIb~AqNYplnP6G~g{l?!(81c|dH_X>EPE{yuA zL?9?XWj~N>)YQQ)syc-~EuhK>$mOg#_f8q_^n{9|i%G27b+=Rd9$aUK$|IE9yDpe% zhcpy#rGtI+uc-qPb`Sfp$Jo+sf@_Wo=?l=xw~cE;(R~$?uc9H z`2!hWiHhkD%OC-?EutJ5YS_cbeSJ*SccF|?FTtBI&RaO^$DMg-$bHLvefT+TEyC*^ z-r}nK4MTP;4mb$Adc)zhYibd-ePO;WR2bC*WLGf{N)(HyM-T{Br9b6C!lm>Xkarlz zx`l>PDg1pbZ0rDQh*Or})h~C=76x62-s5*&Z<57L5SktBfXjhqHb0M==7 zqv?quZdu|p*|L?CIDmWlPRI8*ir$i2Zk44SC*0xj7yYH(49Qz_${qLU7ZVwy^dlz{ zpX6$f;l_CFP(tm1iQM!c^Yd0w8w(@9F~Jy*oelP&jrT6AfWlNxN@rh^^`X8+6l5BF zl9s`hUx9hWus2rIizm{&KSo-Va+Z@8o2Uk)RVByI(SmpI>(b@-JL5l2rM>tQq_6B)fTF}6KWPccZzy@qs2q_ zccph4KaKMHE5SG|Hz9MZqQ6J=+PJ40YjvJYkw{P;NsC)}&fDGcR>vLKdt&i>Q( z&K7ZLErKe3v=T?=-^j@E<;9r9pk24lMqvZdF_=xV&DwS>P=rzJynNxYG~-W9f9W|- zCQRD+uWk35W+f1RLEfqOe8Rq|?l4ExtJ3YAmR*v9BP6tT$@&&f)vXockaQH3ejD$d z$B5@R?5_vU28~V>57_s40}Ma29Xe_`MfBKpNmUJ0`LM8iVZCo?M|EV445M*BFP9mE zxu;DE-csLi7=C&%5p$+lJ0h|o|1X&E%w=Dux`;YjX$f)@c5hqUKn{UTYcxkN+yquO zq$IDNL7O&_QM%t{jnyMco=Sew`N+Y^#_iC2;0+m463nZG=!G( z6cIT4{!xRgOlf!5pnA-faQylDrs6znnjj?O8x4iOO3ks1vaL+V8(N839^0nQ42&KM z3DHI6cpQnZ^!G=_B0^&o(?u+8Ju0*4*L55Q>`ZDPPu$M4P$Dy3UELts);F-(u~JM)-2G2)WRo6;P#<5Q+`J~mJW)F_&bi4)7;K5L|5 zz2fD(GL`V&5DKB$4m&X)tCHL0M!Zrj>Tc`6FC_SVQOBL=#!t|5pQF9<`-AIsJ)Gdz zfKIC%Zd|`ekSw-JZh>vgCXmh$Dl<-fr<}>e$V7;#@&0Qk%J7-Fja(^Brs)CVD;-L5 z5}BwlX~}|OchZF+c)Bx9ZF6SSfkR~S&h?X*9^acvWR`w=M5JH9uKk-^m|@Ylw%ur` zmW9S>{F$6w{+-ORiT=d9k+!=FHbGU-nYd8(b-h9Q0ERI`^BT)hV6c)et@Ya|h)yMc zYRmPIJcwzGq`ZoO|NfW6l@OHx`n`wY|IOe37l+iDK0}fHcxAeq*5=pO!vNt@35YR# zbw-*rFPjYS87^k?$XiDiQ%v?^WT@O}iL2}cy3xBlqt_dUFlv~R#;PxGIn?Zx&`&Si z%+M;LVfe=6iT?J#&AOq7L0=3>`tafO*GLimiv4nlP`}092s}CI=JXdL={&j4`?yK( zl5ExitJhgpZJ#GSw?0}|1xvsiNE$B~Up_Vc4}1Jx>Z^^V317>~jq~0fS2rl8g^ZiO zR>AccQq}aNZVc^>!BvJuC_=~Usu}{BeSMBF`-A{nP?Do8q^K{UtU_o$we<+l8Z+ygR=_50YJle@Z?}>3b~a_TdX2-X2;Zy#=OWLGK0hWz==+U4qU1S`3`*V3WoRvpM~I z9hZ7^j~9J)Sdcn!M3?&J#_khFO*Gb??Di$tZMu%Up4#+ zhCN)u$yyZFKjI8!qp#S>j5P$xuFVb90gH9)*SE-`o))u?(XmOpL?IH3v1{HMn@sy3(6tPn2lH|JJ3u-*~lYCW2yT^-=|_ekxYXEb|yVR zQ{}x8q8D;r{@n`9<2`XIm(^WSft;_X~ICGg;CF>pFcABs_u2HNIqIXv+sLlORHUcT@E zVEj0UCi;-M6xL4eFZm5@AuSZUu9Otr8phGtU(QE*0mKv|$EoQ$8KFR33n1x)s;<*G z{>e08C9J&k_A=r+vu6wY5=|@NUL%0?{%JWQ)jhhiT2$rjzP;&bij?!X^LAh*crkE8m=OW~5s0Z%mV-u1Wh;i06xm7u~hp!|;L zzd|?Q{o)*mgClS~sxVz=6Ut1+@iJm16@o96yB5VGK32cp8~-Xg0(srrUTzE(!*E{@ zl>Zl}DQMZE$OwoOH7gph=d|US>(a73Qq*rPCkmtowuh19yW;{L+n$qiQGg?e=+JhH zBOzyp4`>cs(&+D@Dv#9D0WFyj4@?FC1u_-(82HcE(<#JreL;0?l?Sy3%1p{DB0z`% zMRkx~Y^WGFw%F+=cqj1MBWg^fh-FOHqrb5a;Qc(wxyB5zZGcfEy-iEz`8936T~_MU zSO&gQ@+%sgKXiYM4*{P$ivwu zAr-tF(d#I+o9-yA9nn*U;=~GAJ8!s7;zWM20sF{!m|hI+?nfs12ELm6)T~rxKi|l; zxln8R!?m2JGq7EdP4N~tRX&j+ZQ{eL{&l;9thVOn#Fm%kb9 zE8DulIzKi?mbu>=aFtsEIN}ABtVjTppg~;s4!jd1H-^VNbXe5smU8jOi#|c$0UL*& zf1v;6JvkFx_jH}LiVMCI@Z!|WFP=589}y^6g~V~+0rhQGeP*YC;J z=S=q}BFwLD*Qv5(c&1@C$Hn4$lM0nQa1r0CzrOjC>*(9^j9W-|O4;u&3flf|?bt5T zuk$fc^uPYYzR$dUYd@{Tz0pSlBv?B9N;LftnSfX~sgczgaXV-K4#i*A2o z75jH6>sm`}CImcoNLJlC4k44_qQd(09MU7Z|8b{WsF;(c@*p`0N7;30T(6B+H+KLb zC21T;#>*gn7{6}elYj92!?ouEPdbvxAN>rZv+YMk{|d?Q^j^)Qv4$ju!&gVmDsrOz zKVA1{G|{dyX;0)m%2x@b`Unufv`x1y7>B9&%LN7$a1fEHP@%R%1lpF}+cQUI0jdhk zN3+_4?+>74qFy~WL8}`V)2O7zs%|miC$!su$xw)u8Y%3vkQ;bV1RF^J+tGI7$j#+7#|B7c(>ML7`(h; z4y07v&0w}T;IvG%T<26m_JEM50c=P{C{ZisfpO;~H^^z^^RWZ6@K)R~1o&v8g(f4~ zWT~tEK%g|oxGsL7ztAvM#njIm$b^wv{CqB@mxzk+btpvZAXKr&I8UX}kMh1`x-GR( zPp6-jQAjyP2v(SgfSg0Q|KO36^q~nLx2ctZLgL?_(sI|B1%Bxu|mrF@ye?sw}M=@i*p zsn+DDdD4b{#c=gNqonj6ZF<2D@k4%+rn1E)g&AeK16?o8!mR*z-T;j z@0%Z~tFtJZEmV%xN`DfNrS6U0UkwkLDI-((l8z&B;>7-T03K!aZU*z%wEa~;r@PKG z#V)|mF#xe&V%4dl<}E%~{HecBRb_8cdXh`nbv{+uOpySrssA-x5Gl=aBibX4#mVx~ zS5gczib2=O`ez-B2yM=4|4vT?=A=me5JZph6%RkN4P=^Pum>3Gb3*7*{F53;#PlsV z$l6{YarrXT0B+WJvW~@DR3n=Eb!y#u0Dr}D8$G6h>sDx#;2!ezdpV%;sRv)=xLzc{ z?)?m4qnwVfT;Cu}^zzPZ(Ga6XEc0{XD_I)O14yLHf;cZKdwc~hxr ziGuWDKTAM2NUk@o8$GrMN8h~#DV)D^mEk_ikfr`;K&Ag1{6wzAf-PFTw^RPv0-=a& z_D4Qw>JpX6$l$jG@<5)5lc@Dpk;^3=|ATAW0n;w;01uXM!kQax%*a6`FdQ!; z{P1ZK8jb|zN9OtGw%S&h+(hl(YxqA8K%WO|)r(K3H2$U7DrhMJz^k0?GyWFAfHWLR zHwy&&PKsM0_z(U8syj!m0Yip0*=ZJ8L=Q>>tAD)6vhc?v6P`-6!+@E{`dkRenWt4a zRGRE6yoKlJ(_1T3`mj^pYoI)f zts^)NaKyS^48&v8_HeFs$qnj@S*25{wp9@9s6az7cD@UkqpIbEanvy=M(IoKXHPZ+ zu`(kf5fzHd?#thAhRKO1i*^D&qvV<=y))#ru-LSOaZo>wDsY6(HRoF6Q0*Oh@Nfu8 zT#KfahI;2Ri>91-jWhe0E@d%qU^*1>kF zxnO#X>M^pb@KaCf-#_w<1-iM}1u0I$-~S)B!rUXzH|^$APE+N*6E1LuhaV+}-sMK_ zEU)oSz~ygyQ61yMpb~NzLx3>416d)7))YVkHRw%uP7UsoT6TFlo%CE6@YyKPL2>B| z8Qcfz z{|KxD8ecFbC`Mb(^yG|yfrADuz!r(6JBR|TgLHHpX|Szj4n$=fuz(z)T3cvy=ANhU z$PqC@DZh;Ky{ZvewPCQNFlpA(g9@_QUA^?F&ak;Iw0iE{yLS}n=h;im;uGBj{uxlP z-#4F1!$?C#Wc^qYE|$%MA9P<}!0%QPzkj;<7&conEaBgS6)s3tPa&WpgalrRJ7axF zw?i(LIJnD5{9AcSz^+`mM1OP2^qd_r{;;AKi4^0mRP5 zU|j&tAIT;t$hk@5{2qq^dh@O9fXEhYg~XEyUC=?7#cb#L#iDGQ#YTW!Zq_?z5jjOWi(1fG$WU>fk5xY18 z5siIE#DlsPVGAp8gCrc~qHuv=aFa6AqH22;(dQj$0vr?%k+EhE&e2Mfva8GYv-4ti zTp(!a&A6O|nw@gWz>|?^Sh3P2C&>)_X7?oz8586ZAu9tj!$lpDLF4kQe;4Z~X?Q8@ z1pf-yH};NGsqLN#Fr8FT)$7a@R>IoShovZAGlpOg#K1_~X70@5vn;l)*Vx(UsH%~_ zprfJ~fS5iMt#`~J|I6=H=1u%nQLFybtFVFdS?U^@m>AuSZeL@tgQ26Z?*pBx3A@z; zaQieXvmR2dUT#GJb84~$M0Zt(xj8UH&aecl+rXJ2O$oeKm@e*p+xSjd6@VRbpu--$ zHV@ObzSUc}+H1;hr46pI_ClbmGlr2ZCU+KG8h84x*w3gVTNh~)aU2;Qz)mktKl&D- zg3#16q?3ODxI_+>zsqZz5ZkXG^Fat;Rf4+h8C<36b0HFGGO|GbY?P$zgA!}0WeliL z&V_!oAQ3>Kg3Yb=a2q6WDjHxLZD+o4#Fkg%N^H;zvi}lZup9BlGCJwzxB)$>(03z< zMdN2N3cwjzFv2y%HP5ldgJ8vZSGg;O^qF31smQi|08l2eJ&Atn(Y*fsH_3;Ozeq^B z54Ti&Z+5Fo-7xU%79!-hV<5Wlm_aFZy(=1ma~ve65hw_F+Y&^Y#pWSRdLvo@A>5SQ ztZXF5s2p<#2sr{^>04Hd%0k6_E!BpdW%K+qpm@Uy|9JSe7{bdvIFEvcN z(%*9(!m5?8!77mWR&&oW#<#zp2G4!Y5dVmt`v9%zuqSzQ5d=1Gz{w)YmsK7y4lmL- zIoXWIC785+xaPbIMR}ub^SUhOT^{p5A?jVA;r$0x;jKTv7qQf(bTkM*K9ncita1It z*v`bWqNsJ!Xb&#Og>S{-fZjCyH*=PD)2MBYu zL1&#Pt!7V$N%i91)K>d(7sGYc0=*7-SyLp{YQps z>t#}E_YHsi>8u{QN-`9UppzwBpvcXUdAqBCDL!C5;0^_y#*|dm&Zg;W{+}P;t&VYN zvoaZ{gIi%`to-BiTl?<`lD4jJKNCG%8!WvO9%>l&b=_dZU|jQ-b?4ih(zJD(fVZmk zCycR=K=0$R)}EnY*z&E=j+{gMX{|?^u14 zST_rubY1RpNr$YQB14mIallMU>sTUpq2Qng_2wS6N3i&w<7>Duq0(#fRiz$3CRh3> zGnH>HtWVdmDpx8j!bmhR-jw%_t2>NzC2qaAU+BW!>uS5#-6!QvhdB~1Huta=$KxQFZs6t(vEc;XTVL=DZUcSi4&W>u_e zQihDC7t5V(R=cNoRZPFu>pG2+>Kpab3}u%Ir@1Q$t!eA4J(3O@Dy9t|aNq+<#ZU)B+&N&B@6vhJTcjlc{a_HyFRy*FY^x#W4ZaR9GUO`sEuuXA=c9d<5R%%r>5;tx zxP@k}wDdRDG3s&Q<6qRsJ*kd)C@&D~fpy`}&H(U`BJIG^v4pf*cekh!sOy4meQ_Xt z$L{2y`1pg-=9#D%0qmoZ&POdEca2ouzp-r_i>c@8n_H~hkLY>v1tdX)ON>f z@uFjvnYmUbWYrKvC0W1?mm=xQV*O9*`F`+UDUJ8T4rcxj7Kr5>anzq*zDZE_<-~Mf zk>yljd8evYLJoCbp*eG?xB3e^JcC;NL_TN|dm#?xOOzm~0VTLQLtfU}*fvy~0;c`k zxF~L>p=S;utBTzC#(;fH2U{cnH7$8~K!TavhSmR1yDe6}#-`nY@bv!JjChV$esNC$ z5*}!gdobe7mDMsm;0^1LzP%YiIJE(2u=36vqF@^PcGhMKWh6V!_-^m+7G~YS#dyMt zOHs2)h}yZiL77<##8Ox(p`rW?g@vG%rfEh2qo}Ztvzm^bihD`6x)*)fVcHWH|BB}J zagd|KG(4C?sdXU4aIGI4D_5eR3g%4LI&K_=yYU#3Fy&FhkO7p?{61QgjE7#j_D_Fd zy0%pf`J~1XCV|s|Ve634lUmq=zB3U8Ex36e$qH%*#+j#k1-u>qdly{HvZ~75>WtCZ zVf&E*r3Z@h#!E*ZS&RlD)xzXg1k)UB>f;9|0TSVO@aztmvQ3XpjOS!7bT{JXD=zBpG4e>n;M%M%EW(l?JIPyajL zXV0F||HKd5fboe2NS`?}^Wc#ozOtF)0Mkw&tef8gON`ZrhV`T_IOLYi{0;`Zi~dt@ zhl3`S)XL@KA(&hSOnDs&R6|mLA?SZIIKZI#!mbiwasC~*VFv?apXx3SG=h>NB7T!7 zt1nQvApfxAOb$VDrLX$w{v`rnK8F#AU|-Z)q5%kRnAiEz(gHKpr|4#plPXvVp_Lkg z?8uade}H8_JjR&%4l)ZIUwc`jmrg^1aI{!s=-L2vk(wiTbZS-9bQtD}$AYXawN%dR zEoDZ5pV>)i%SySdXRNnB2Azvv5Lr+(hb|c5%a$(+5|28Fgv}KW3r7o(UdZGu@c(dHS25GWBl? z2`^iMz=QQ75d;~n*UBe4}Uu;aF?tIzega5NlFK__$qL(_${ZX_M zz|RRcEW3Y;k<;iTdeV1_uU7%iEBEG^plT!*bm^CUa$0(WPr5Us2ge+}sM10oZXl}> zP?gYuA~m@k=gW3m=%=cWdC`Dyg*4RO>=wa!N5YJ&MW71trg)v;9fzv!lKkiY(d={K z5!fsbAugZJW(Fuu{i%+W>xGEolmJf{!)AYw*st(Q_QTLWQ498b#_-cWfi2|^o6ffw zxKiN(ObollkXhchHg#8|p1>N?duptiz`2}LT*gy<@X6*C{>W!czDvEUv+l?M3`~wK&FSfkk=;NeUC6R&LNc{ z;-b-Q_gGZQP@lYX!FIOyasaEiumljkaUd^EZUY6)7iNDCx7T2$JiNh4Q0(tt9>Qdc zu2xx!&{97*!DQXj9wXFLA!hT-t3`TOg4Y4Wi4JJ}$c;P@m%7`m$^RTmF7)94gH}p^ zWF&%q;F+)n3&IOpfJ(n^5x$6Dv77q&U2PxPRrGJv>mUZS{mp7*2lNmnChgF^)9~Y~ z6w5O|guHI$3dLEKhL{N2!h;rI%bs7YI>P$J1HB?bHpyy0Rffa?yW}^8B z#5i-rXYOSgNfyw<_u7+kBth1(7X_zLyO7C`T%%;LxI)fv#~<52}l8NAfSAK$l*ZZ9o5V(V5mI?E*)YnQsgF2 z*iU!yy7?@KJq-C1edqy5xg(#G;=O!Gu}Q7J0qN36IxinW-X784AD%1rQH&P-Y z?(D5ua!U}531oX&i!vE}tRC^hW??$^hmqNYa*_r_U@JthZfbzQD5Q^alTzZHTN6as z!GF@?=hx@zz$oyhwg!j^@4Y50pw7Kb1^+^=WO9<0)V({4VO3tJ_(zRvoUa0A18RCu z3s%qWGb+AAF#K+%&F3S{-UPjTI)Eg^-nJ@_UHqmL>i(K^0^MH{I-(>IAC+<>SYq*? z9>^56fKXLOaUNTJLyCu-Kp{#r2K$6dVv7;Ol`rjb2s82z^J>+g;)cd zeYXz{RQ7nNmJh;(oPUcW^Tos_pd!W3SZ<<0azTsK*3TmRlLGZ?4iNd0{HkDOx;`ze zZ{_vT`L`^@ieLpji#x2uk&TI11qRRWU;$Gs#%9@lyWAb`!pW!W03!S(n+OQJBplL( z)&Wh$q_P6L8=JaFska2lOfJq?Uyx@86bQ<{*C0*NUk$xZJw2KipyKZ>i9}RH<#oOh zyb##=Qfzm#yGoJ4Abr#SscE`8k6C{)ZiBM(78fu+j(`YzcOu}a;U!QaY+YUL z5!fyU23EE!`A<5*8-?kFBBO;2o-e-SswK!08<1Fa)D^Bw#^+18uFs#HC>{0Vt_2mQ z4iV1uhj%|Z27#MUh?ig!`j^iW$j!fNsrc^@1>^m@zj=GyPrnh+>C9e`kgdKQMzw2A zz)zJj4dXqt9m#dmy?EGmt_!Y{+GBU#rT#_=@;%j@XW5SHVOpp5S2MLTWc@-;kHwaJ z$8{{ge&>`(%Vy9y-!;?qVPlo}z*yoi#gyKas={qY9yCyzcVjPtUgb0x*T4bKN*cPC z@aUWJ5%u0x(LwEI(#BF9D$7C*RdjC|s11u+3_jk*l)f&wco zKYzpD3|Q_R193w;eHNE&%c08ag@kE5--tla(bHG9fX*7<{{?@n58HxG;{=T; z=2S2d5MN^~zk^3k1T57-xif=;1Qw;lC281HRBo$+GXX7Jsx%EQ>abd6L;vW>x4(lq zi@Xc}iqB8QO7H*=AC%wtqghrDR)NNfAHNLL+v^tbIlXc!^IPU$3OMHU+E%r-494|u zA#0~D`%600zkyKU%ayWc57O~6>>g?RIak1g(xmd2qU?n=%2s>@ODlcqt}KC2CT{kHHhUyxup5rP&7LogvKC8l{kF8wOr(D}lIL-qBf_ zDtP!1Z1bz+{tPzA*Ydj6)ajVNsg6bgOUGG(jamPT!`_KcO8@gm zo*7qU+5~A7%cwpCCY4Kqa_LiL$b)Aa4Vr>eiR)^U-(L8AaN>nfcUPv4IAY)Rq3XF_ zN#{*nkI^Iyu0r>>m$QB$*3f68E>;A9^jFdNqprZ&STIAro+pCnNCS*(r-6>R zLuHShs85g>*jw#Gf{5&2zLBvQU*qGn4S4J13NdJ-3gKYoaVCz-B>-RiwYfP8ks+H_ zW6#ZrQKtZnEP^&1)lSMQg&_QSBN^|T6?0u77-_BEtES-R@*yw$W9~0>5~`jW6S4s0 zv=0nfgJ{P1K5#wxQ_$_%ARz|AEtypiwJ&ArY9sTE4+LkooJ@~~6UOlmEWjqD4Z(w`EM^q^tD9{>R;eX-KmcmhOdWJk=V;&SVy3FT3*N1@aPGSgC&3xZc7Tw zA3e@-9j zUbCi6{*#P%(~&|A*WOu%A9EACL{Tz~a`}~MUeQD<%(&%eeFc|vo;8U4bjU9N8U=Fc zhLmPu1o}n~oDy{P53w)#BBj1sm1h*P^_z?uWvzer*b5=y@v4~m_fk2<%il5vM86U@ zyYFQ<5d?@^!7?i+J7rYAMlQM=iRgP@B>GzJSdIiUEdJuQzwjHs!d0wFp6}3h8=G(Z{2jSop6+di{|qZHoVO1P%=#VU8+&NL zL0aOHR?c} z_DSSM71llWGPhZY$plD1(fG}D&CE6l7WeA8I6N13QkBC@iOQzyJ~~B1=kv9V#^*;(EfbHfWH?A)C@Q?C{LR1v9pB&;60}*AbX2C8_&ur4 zRAG>=u;<|?@)$ypWD&K?>JL7%))d*-D?{#UvOU|aO0})}b#;acRYzC*+z6z?K`l$R zhVr%hlpDg+x%%#IB$t)YG_N0d6JB9{XYrD}WnP8h3Eu`u7VQ|@O;E0#q*#-~a%WO`LQ z|JbrG;59hdbPd+Ac72Y|$!n_~!wr4XcC2q`x-U;+{miB-Rv0HGRV&6_H&%Ymch+0h zuJ)N7E1dq$>pH(+%$d%ud#w1ceZz||NEVy1?to2Fq4&JFe2CLzqd;WnFz`bu^Y*IO z|Kyn4IYf&-{^cL_3mY*WcFL(!Tf~mowjuJk(_hzDit@MH`78NyJ=QFkK_{OAnrlM= z>DjUs&UQ--?t+-8jXvVY_axnarZ?yEOizl;5A3?KL|n395}qyJq1;Y?(v^N<)2yhI zY+5U`SK^X++Xa%uY?#2x#N?!9W7vOS^;yj<-Ko5nXLN_?TBzu%-YbHN@pZl6^ALB2 zvbgP#*_iUa_02xRI(f;W+YIi&@~g8l?XP~kZfco;nI9Lqk#tzrplARDZ(>jsm5VAk zM1^Ug(&EBteWF-V!ibu#&QLzl=`3{hfD=V97_R#!Z!PMlD%^slguFd&vK1RMU7p#Q zAHrpM|9Cri*rv*jKfq<%!?>GQ;_yH6ncPJGtjhO=RT2iaJyrujCLe^3SbX!y(JCLg|?s>Xli%%P?~wvPvb6{}rkCQ+4{%$dwPPGbos64gC{mxD~O zZmTNaGppDG_pi52Tl(ajay_27Bf-1OML&?*u~Cw_UUrM-RYWQJyF@pcWtC&bw-?1;E`NTl%L-Hr9U`PNqBrLo=d zpQ+n(&W|6j&osT%5A4)pd@pKfLRJaZ(vaec&L?S!Yy2GYG#;d!;lEkxbY0z2G8M-E zxOsRIrAkDR3cJjKPHeettVUXi?uG|+Y>L% z%U>HC{Zbm9MAt1oL8Tr@rV350S8NXz>{ycUjbHz7Ud}&fxd15^c`9j#rMZISb5T0; z6YU_!a!(q(2CN zUM%?U-7u_{HmaUe@1GXj+mA7#Z%t`4;0`%iMns2~trnc(xJO5T(amb|PCHQtJ3i zUY~J8uH$ei%mly7NBCVT-1R%m2pcOx{;_+@!DhC zepeQ5rMSCNK2jBda`(<1uR8Tb^FNoVxT_cT)P zkPp)-NW88sX6iz^+FnlwkgDqI;B%@4yDQU;0H$N8@$X_RF}I0NZe$3UkK>hqIBhFy zo*(puqJqQQL+~LB_+}h|B;a1IL|1;GpLz%52z*-N8frm-BNTphLspeQMphGWuUpap z7r@`XVKDxHKJ6ex4b_KzNIL+|q%2ZCa$TcbXU4<0mo10L_g?o-yYmY z%kEGEctPcq^0|=dY$fXpSOZ7ryHbU83{N@nV|`{7 zV$4o%WdXMm=dw1paqH@+sOJI^aRzX3N^#U2BE2z;dLf4}#$Ur<4^DSy=}k%DtY7^G zV=iaA*jUNv*BY$0)wN<6JEf!Hd0?0Jg*+_<&dt=X=4K!RrSv7L-xb1ZcBrEJyD^TB}Mf8Xqc zj9D+QXF|u5RkPT{&u#~x?9fpToCjQips54FJFK?)`g;kd#j|Pq{ab{~&u)!$KWnrg zyBc!ec#t+^;D9Gr+!vw|v-}~?1T}uy6Sr00FdFzVkt_p|QUc1*5 zk)k*Q_mX_Sn*N`sG)@+wne${&fT1_U_V!#BaRw3=+am>KlJdS{glw-J#VQ%0G2Y&;NPXo2Cg}YliB~p>w7dexn-emRrBNYqy zYI^6jIh~){AgFvf=!Yz*;UH_go<+?~OcIV(nscDXkq@H4iQ&->2|w@`v=3L*nj}Rt zLc8)&)T%^3)d{C!p^oP}k;1&+e!D<36;|vKhh>LExfi#F!Vbn?G2_FB z4-a0u+@CaLx&hQo^H&a_*mwJ`pY-%oGp^W@c;spHpf%n>!WcOBL2{*wOX*s`|0jV8 zzn67XUjI+4tN)5o|6deu|3wn}@6!AK|JPxU{D1dwy#MzfJ`JGGIvDF}!R&VmY21i; z^Z^5|FYrks=BU-Z4*bZq;OV3T9a8DXUDQVH$snIV;R#ZiE7I!MfkKxB41^B(dknIu z`_}rxSRLwzU<0umbt@`AQbL;518-#$9Z%_<^|5SF?0qNOlYBvG|2JrPQirTNK=tI+PXsXdAw%tB(2OW~0lIW% zgK@_joRsPPy?e?8w=#x_gM@ftY{?K@xDL#nP2&;B;Br!KK2T)vP^5XG#&5{};Sj?` zq4`M$`Xp;X!>ClB*aEj|+s3FUYLzuWveh3`C9-3KXTb}Yw0ClQT{{oFr^x`@jG@!I zc&w|CD_5n1Hv=651P*X%I1BAG>a>l6BNR-wk&~{r3iJ^#f4L*z2DWK`E_6 zkg!5j)dsg;!jn@l>6nL^R8}n9mPkF(dE5;`Vk@{<@}+`WupAu-ebr`F?VLo`(`?kB z)L8(2kjOzmTa3L$mU^R7NHhz-WTMzB;`4?|2uKg*eN5(n8#$|yE?tM#Cer>&Z6}W- z>qo}`YUjkHKEyCd!t;ZDgk}VKdvCve{5P~}YdGV7*ut8du>w~-|B78DOu+IWXab6B zFBv?&*K|6Hy#%8I24S#}?R8=oAV!pp;squGb`Q33oxpS)QR*RlIAxYdz0zxPSM3Kah=;?CA+2 zWq4aQ3d+dNb8#<*NP6m=0|&yE3qct4(^8Cltco;!e(y+j!%=>1{U#_5me5=@0t5;F zRr|2SHEWmeDL&6i0ss&D#2egMbLV>nlePZKaF;m4FMrGWarP-@p~vmT=g+|NPq}p7 zna2$2v$Q7fBj$Vr=7n=f}#IDjM}^U|i^ zv3bVES90fp8Lh1QNQxf>q)lC@{}6nVLvg=*LV3YKHDyA`1L}< zBnfqBW>oJSD2Wo?Pi2>D*t_dXGpsV)Irce*Z@@b$73_a}E2!3y!a#V6&JHHMV1nX} zsUEFQE^TXk>36%QcqgQ`^l1WZz_)MZ{ee0NcXTa4c3-VwT)!To@8N*MXRr% zYThl<_=iGGkbFIr_Y!Fmquec%-BD)Ud-R0}P&Q~yqF){90;6YXi*XsMATcu68$Lqc z0&S46(3MJusS|BPj%E88DcxP2Qx2JUOrkRhUJ-V@sX(PrKltf=w8%yAOZ3xydWBR!G^60tIXamK6 z&%$gsAJgn-1IJM7>HUs=LjmJCRgC^Qu3`nKNtF$u!o*|&3ZYATA(TB~UXAass8^j$ z-#X9Eh->330GcKi7@Ha_ho*Fqq6DLVT3vTBHE+-P%L6s9(aR?80OJa5NGOe4opvUPt^wJNR_~^xOH6YG%0+GkMF`tNfbp#@t z&|S99D0MAd-Dv$Wk_L+#!)Y&%ZVWMSa^|;6$F8KK- z`W1r+-5rb*l$PF**87sf4r#23!2iVx0%JZJLci8SKb!Yffsik8uBQKJhhWU@M<-?l*)ep zlql?E3|XX0`&<$1E@YO2E*AW`tv0KFu@(G$5SZ0Ui!*Jg{)R|_b*`Wef&LSF7Yi?6E`UKlwdV@^7i#8|Q>d7ga|@lfEx1L^nk3`g%*~^4PC< z$I$GEEU&RCAendbOL#LO@->3N!H+7+$q5X{Z6Z#z$A9=8ld$gVKt|PG05>=ofkB2T z$ov`}cfk_11g1oYXW|V#x~)9FTUaFEjtNbDa|nLZjbwO2@YES~<)#NSX?8vD4vVJg zS>n|u;8hb5wRmLuzHg@@OavHy%K79yBN^dC4=SFnzTEQNqYnnD9 zQ#QEMEk3Ww$mhQw?LAZkYq96@$w!7{nGn$2BAeLbo0q?un_Etsh-wz%%If_2^=_v~ z*PzfL%SSd8$$dk8f1BVDp;!Ch^cVq_l_?&`t-qDw+XjFUe0mFOjX zpHZq5{v8PWSO1Fl_#fK(#~QM^s{$wr(eSn+-jgke4fxl-Smld8Wf1ieTPyQ`B~z(g;|5(_p?Z#1c^tes7;FbS^R72tAv+z7Brk2l=lw5!JtQIzH=J z4|*yw5OQ;>TYp~p>wZ*8WAhOxgC+RcDB=7I&{blfQ(ltl|M14q>jRPY$PJL7{b@yq zY$$?)n-|=??HPAi!?0pi0Cb43|FZ)*<6hDZnX1OD1`55@I#0O8w$44_m(?$|egf&;-hyzSZjF+hjlYL3wF1!V_9OsU$BVpvF zvM(qseU!hpZ80S$*dUqorQk9ewWo=1i}Ic#wS%x=@!f^7u`uYRuZ;+spl zjBLi3>*ORmNP12)N2^Hl(X%7ben0x7D0L-`P+liUq}sBPf%YgJ>b!WC@JDKr zEt~;5YE~SeUo&vgHJj2rs`amO1DEz~0B1 zE}1ktR?x`vI=O6xVKBiSLEtZywlQKCmSzDQCbl`&!ja=RDyDmngmN;}&vkorQ7gHcbr5JhGnkqgr zyzJviXH3qOrdaCZx<$L-(KON?12kYa1tO zUXX4>CG4?9_$XZ*aKt)BB&&Hn`4xV6b63XpD#*gAk8(8UI7kMPd?tbIL4qR@Yf1L* za)@=-e!!QPKO`N4F42wCfElutmvlaTUeY4nliNN2aVWVx_Mj2;MHcfLoN`!EM$sKS z@hU4K%-?1;(kz=rukHcx=vgxp_rU0+Va(@%);}8t`FE<+m4GImt3z{` zY0!EEDxG{-^$X!WbI8(&{3C)@{b>z{yXl-n2O9AKv-KTEei$u^?CYpS%UtYM2T`sB zi7QMyYcj-#6&t-L7vHI+YbK@AyEGyN=jyU21g89BI{abf^7V=;CJ)7OAH`}>Ps-R;K>Z`RYkrGe3{J#wm zqk4xSpfW}4_T`l;oX&LGbBHv}<4NCDC-eZy0-j(;=i=FVQ(N*M##jGSXzQY2%C%36 zAGyaq`EC}wRrou#=nx zT7KGd=XKHQT!o&g0wDqFfMeZAtv+V6!hR3b&tc{T-*cCt-q#oa^oa61CUfJHFx%-PjP=AwZZqH8 zYX-skh7gD6PdgYtX>a+J>FCh(q7K6^JhiH~P@;eg1f*!ls`-lLLm8~RH1W?4HDrF< z!p9hI?bu!Ka|wL4BW}4hwwi~P=<(d#h#w^P;lCSjU4&l!9lX6-o5ENwk%V^XY1?Eo znOMYV&}^);o?=j`-O?-75$1n5Ck*LCCROxsm?_yvP24W$-qP`X5OwQeJo}fV7sDCf zCj+lbUp?MPP5jctb!x?K%tamyk2Q_GMY#8>J@d7fB%06(kCqg(OHg>a7~_;4SWYpF zu=HLfuxw*14(6^#Vw3w?hdys9!)USy)w|-z8lnPnjchALI+~ zgSm+u;kUU*fN`|BVYg{Upe*P#xbMh<+x+@ObeXBcMT{O15ZOX?=lQyjBRW*%TTG3} z=Ni^yKefO3I&yn%9cF!vq&PAEhGY6L?imk%2QkP}#QWZ0Cp~lWpaqFTVfK*LBX-df zd*Lxmcn3l4TXSO9@vvD|)~UOJ73YzUBRFi49vK0;UT`ARKGU7Jh-~&~!$n%>$o2UP zACnUxZJ7`xEAD@k13qwG_|CEU1uZa(V8?D`384R4d@I?sW{p?$cqS=;a_`OViY}Nw zjw9a=GBWFV4@hA7Xiwz!6J*Rv-!cN@*GBlK@Uw~*6bY}vWmvXlI%6_5Mz3My2oDn; z7BE9ui4?}<pwypTg?4n$hYM57MwIPG3gs@Dn);XTSvGNJJVtT;k(FAPvmfdsDfX5IL@y z4AZ<9(lxCvq7qyLzwn5=z(`XZ;>H*Id= zLMiwhZw#Xc@OC6lg9#Px=ig|k16HZsiuwzn=V5D{4%rDWde-`l#Gvgg+C?OfRS-qe z%$hwOk?d(R3^|PFS$b3kGk-FCEla>XG3fco`p;lbu&$rw5NMK>yJt}-tj2@iCSu@Z zse4bM-)k{+&_I3J#u1C5&(lOAMDi{LS5g+z%ErzSiXu&vc@|UWBul>^H z4XEe!rTe)UmB*t!L6@x|+XDjWSmA9hrs@tQbij)(D0gFP*klKy=fUj-6tlzh&A@ zpCQYcn+Baej|Zq{6Hp#YTsS<6*_sj>89$Z@=R2Y6WILq}6Q&!T>5zuH?ts*iC^D8< zNSb0y%R|8@_FRE4i%)%W#EF(ZXe*<-lK~?(yXpUNCy&xaate#H3bE9v3^p5%!LBbQhcS!!=r zfm~<`nz~xr2ZUaWlpt^_q;jo^nurQPqt=rg3290?rA_*t0%Wks-3iMzODEfe z4a^ziwbL?LjZft5rrbH}1IB^Dv4ISJwM{sx_YJ<>#d!kJZ|S%#lt@9cU^)M~DXhXA z(~YvL%JL2-rg%tdL^y5VC3csbwKS{X%UOJ!9^#&N*pxKK&@i%8hC*<7P=R-Yz;hy^ z5$QTrI)^;wELv&|BJa-C{Z%er5!{7m>%`!`!HW#w!8dXZ#~Bsz1)yXLsWTtoBu?7L zor9rq<-1wQzY%Q})$uN>Ew?F}Z@qYqe}u}TZY=P~_l4mfPs|D2c)uRy)>)XLGU0Yr z@i*Q>z_|D2Q^GlSCnXaH16As4^i7?tygCd!x;$__8>Vm-~6{qa^qHk)Kod(N!0#>2^o9O z94`9s6yy6!gQMigM6LQXb6GuJfM;?<&!9@8*ijXUjTPI3c!{JI^5w!wS@N6Rxq}bF zrNij1|fL9l5;yUZYfgMxV%49~{$n>*H16|XD7jiZbgL+P9+==b(bp>3a; zu#rGFpMH7p(iQ8=$Qy(9G-ZRO;v3=x0%jSC9g_Jry?hAtuqCHB2E+IKpYbSYN!wx9 z`R2l9RPE@!>ICGLMY>!sV6lI4zKAn;H_t@8k~4q8ce_l9FhoV*Jp|{1WOh9}!dRfA zGIw|IPv4r;tWqA&Z(FU^dFJwo>w8KEa@W4*2sY6a_rL!?_$bG&Bj##w4ZCX9~fY4c7;x$=#r9DOKXe`&RBpNre z3s4`to2bXyeC;C^yDzqx@~xd`yAua&arA}!WShCeqnMl1M$?TiEOQa1!BsZ-QPBKf z)5w|S*pV3)#lx^)E7dI?*5ucEI)bS#G)@UJ0gJ0)bQBXp;L1%%SEWfq*g;YsThH+i z^7+3M8s=hjq_`0q<_TrBF2_kQ_Wf97q`hb)YkW{FZ3Y#6U2O$&Zi3)M80!~_Ji?y| zcc$$}5%%%bB@U8xWT*j}gdmo0uq>Pc-E0PkAPNJVx%Jt1=i0hkUk*nL#)W{Woe81E zvmxT(f5v5Oz3mBSHvm4!?x%owI*9UkjPe;(17B{@7Ga*OPHds ziRn##-9aA1b4Y*FHcaG#SiVIy+45tKWE{v%SITMngpsBwjlzho@g)qSER4-ARgf_mY&ECC4TeR}jJdrs583-~} z&j)~_tIslCr5|Q9SDG{7tKIV>)Kr9SoLe@Ccr_H2@F-bOE)a7!mFK&sn802K#2Zl` zVyd$4>4;LLbi#zV&@ggNuhB7o|6uPz z)%!&5(-nPaZf+w|G%nya-zY}KmaOj+@tG&e)_ZoFl(G09;Zt-|9RboVK;)Tdn zmZPvC;k|6$$ic2atC`+j3;A}OU(1;*^j6~Li#uUc9`hy#DY}==ox3iuqTn-ko(4D8 zGPiKo0YRIXIXQPEZ9?UenvAm#iydD!mT6daL~axlKXCJPnuF0xcYWgzOxU&~B{jTG zSE2{TGI{SuW7ny^-cR1I`QB;X$A{q0xcLkr`YN(qtRd4;UQhbhpElt;GmNJRZWDh-Pe$3w kFf%Xy@e%(" - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "example_count": 0, - "measured": false, - "metadata": {}, - "metrics": {}, - "mode": "raw", - "notes": [ - "limit_per_category=1", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": null, - "reference_kind": "historical", - "runtime_seconds": null, - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "blocked", - "system_name": "mempalace" -} \ No newline at end of file diff --git a/benchmarks/results/smoke_postpush_20260423_123452/runs/convomem_new_brainctl_cmd.json b/benchmarks/results/smoke_postpush_20260423_123452/runs/convomem_new_brainctl_cmd.json deleted file mode 100644 index c2c7c04..0000000 --- a/benchmarks/results/smoke_postpush_20260423_123452/runs/convomem_new_brainctl_cmd.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "artifacts": { - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\convomem_new_brainctl_cmd.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" - }, - "benchmark": "convomem", - "caveats": [ - "Blocked while loading ConvoMem evidence data: HTTP Error 404: Not Found" - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "example_count": 0, - "measured": true, - "metadata": {}, - "metrics": {}, - "mode": "cmd", - "notes": [ - "limit_per_category=1", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": null, - "reference_kind": "measured", - "rows": [], - "runtime_seconds": null, - "series_name": "new_brainctl", - "source_path": null, - "status": "blocked", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/smoke_postpush_20260423_123452/runs/convomem_old_brainctl_cmd.json b/benchmarks/results/smoke_postpush_20260423_123452/runs/convomem_old_brainctl_cmd.json deleted file mode 100644 index 72c554f..0000000 --- a/benchmarks/results/smoke_postpush_20260423_123452/runs/convomem_old_brainctl_cmd.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\convomem_old_brainctl_cmd.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" - }, - "benchmark": "convomem", - "caveats": [ - "Blocked while loading ConvoMem evidence data: " - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "example_count": 0, - "measured": false, - "metadata": {}, - "metrics": {}, - "mode": "cmd", - "notes": [ - "limit_per_category=1", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": null, - "reference_kind": "historical", - "runtime_seconds": null, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "blocked", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/smoke_postpush_20260423_123452/runs/locomo_mempalace_raw_session.json b/benchmarks/results/smoke_postpush_20260423_123452/runs/locomo_mempalace_raw_session.json deleted file mode 100644 index 3280184..0000000 --- a/benchmarks/results/smoke_postpush_20260423_123452/runs/locomo_mempalace_raw_session.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\locomo_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\locomo_mempalace_raw_session.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" - }, - "benchmark": "locomo", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "measured": false, - "metadata": {}, - "metrics": { - "avg_recall": 0.6028, - "cat_1_recall": 0.5899, - "cat_2_recall": 0.6921, - "cat_3_recall": 0.4604, - "cat_4_recall": 0.5809, - "cat_5_recall": 0.6188, - "perfect_rate": 0.5534, - "top_k": 10, - "zero_rate": 0.3499 - }, - "mode": "raw_session", - "notes": [ - "granularity=session", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": 0.6028, - "reference_kind": "historical", - "runtime_seconds": 2106.411, - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "mempalace" -} \ No newline at end of file diff --git a/benchmarks/results/smoke_postpush_20260423_123452/runs/locomo_new_brainctl_cmd_session.json b/benchmarks/results/smoke_postpush_20260423_123452/runs/locomo_new_brainctl_cmd_session.json deleted file mode 100644 index 658b086..0000000 --- a/benchmarks/results/smoke_postpush_20260423_123452/runs/locomo_new_brainctl_cmd_session.json +++ /dev/null @@ -1,4455 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\locomo_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\locomo_new_brainctl_cmd_session.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" - }, - "benchmark": "locomo", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 199, - "measured": true, - "metadata": {}, - "metrics": { - "avg_recall": 0.9413, - "cat_1_recall": 0.8641, - "cat_2_recall": 1.0, - "cat_3_recall": 0.7436, - "cat_4_recall": 0.9571, - "cat_5_recall": 0.9787, - "perfect_rate": 0.9196, - "top_k": 10, - "zero_rate": 0.0402 - }, - "mode": "cmd_session", - "notes": [ - "granularity=session", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": 0.9413, - "reference_kind": "measured", - "rows": [ - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_1" - ], - "question": "When did Caroline go to the LGBTQ support group?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_10", - "session_13", - "session_12", - "session_4", - "session_14", - "session_5", - "session_7", - "session_11", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_1" - ], - "question": "When did Melanie paint a sunrise?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_14", - "session_13", - "session_8", - "session_11", - "session_16", - "session_17", - "session_9", - "session_18", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_1" - ], - "question": "What fields would Caroline be likely to pursue in her educaton?", - "recall": 0.0, - "retrieved_ids": [ - "session_7", - "session_13", - "session_18", - "session_10", - "session_4", - "session_14", - "session_9", - "session_17", - "session_5", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_2" - ], - "question": "What did Caroline research?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_1", - "session_2", - "session_8", - "session_14", - "session_13", - "session_18", - "session_15", - "session_5", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1" - ], - "question": "What is Caroline's identity?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_11", - "session_16", - "session_17", - "session_3", - "session_1", - "session_8", - "session_9", - "session_12", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_2" - ], - "question": "When did Melanie run a charity race?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_7", - "session_8", - "session_14", - "session_13", - "session_18", - "session_15", - "session_1", - "session_5", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_2" - ], - "question": "When is Melanie planning on going camping?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_2", - "session_16", - "session_12", - "session_7", - "session_17", - "session_14", - "session_18", - "session_9", - "session_6" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_2", - "session_3" - ], - "question": "What is Caroline's relationship status?", - "recall": 0.0, - "retrieved_ids": [ - "session_16", - "session_1", - "session_8", - "session_9", - "session_12", - "session_7", - "session_5", - "session_14", - "session_18", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_3" - ], - "question": "When did Caroline give a speech at a school?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_13", - "session_18", - "session_5", - "session_14", - "session_10", - "session_7", - "session_8", - "session_19", - "session_2" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_3" - ], - "question": "When did Caroline meet up with her friends, family, and mentors?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_7", - "session_9", - "session_2", - "session_16", - "session_5", - "session_17", - "session_3", - "session_8", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_3" - ], - "question": "How long has Caroline had her current group of friends for?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_7", - "session_1", - "session_4", - "session_12", - "session_13", - "session_16", - "session_3", - "session_6", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_3", - "session_4" - ], - "question": "Where did Caroline move from 4 years ago?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_8", - "session_19", - "session_4", - "session_16", - "session_15", - "session_7", - "session_17", - "session_9", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_4" - ], - "question": "How long ago was Caroline's 18th birthday?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_11", - "session_16", - "session_15", - "session_3", - "session_17", - "session_8", - "session_9", - "session_7", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_4" - ], - "question": "What career path has Caroline decided to persue?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_7", - "session_14", - "session_1", - "session_3", - "session_16", - "session_15", - "session_8", - "session_9", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_3", - "session_4" - ], - "question": "Would Caroline still want to pursue counseling as a career if she hadn't received support growing up?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_7", - "session_17", - "session_1", - "session_18", - "session_6", - "session_2", - "session_10", - "session_5", - "session_3" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_5", - "session_9" - ], - "question": "What activities does Melanie partake in?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_15", - "session_8", - "session_18", - "session_1", - "session_4", - "session_13", - "session_9", - "session_19", - "session_17" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_5" - ], - "question": "When did Melanie sign up for a pottery class?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_16", - "session_14", - "session_10", - "session_12", - "session_17", - "session_8", - "session_18", - "session_11", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_5" - ], - "question": "When is Caroline going to the transgender conference?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_7", - "session_17", - "session_1", - "session_14", - "session_9", - "session_3", - "session_4", - "session_13", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_4", - "session_6", - "session_8" - ], - "question": "Where has Melanie camped?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_18", - "session_9", - "session_6", - "session_2", - "session_4", - "session_16", - "session_8", - "session_1", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_4", - "session_6" - ], - "question": "What do Melanie's kids like?", - "recall": 0.5, - "retrieved_ids": [ - "session_9", - "session_10", - "session_15", - "session_18", - "session_8", - "session_17", - "session_1", - "session_16", - "session_4", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_6" - ], - "question": "When did Melanie go to the museum?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_13", - "session_1", - "session_14", - "session_5", - "session_11", - "session_18", - "session_8", - "session_7", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_6" - ], - "question": "When did Caroline have a picnic?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_8", - "session_14", - "session_13", - "session_18", - "session_15", - "session_1", - "session_5", - "session_11", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_6" - ], - "question": "Would Caroline likely have Dr. Seuss books on her bookshelf?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_18", - "session_10", - "session_6", - "session_17", - "session_4", - "session_14", - "session_9", - "session_5", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_6", - "session_7" - ], - "question": "What books has Melanie read?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_7", - "session_17", - "session_2", - "session_8", - "session_18", - "session_1", - "session_4", - "session_13", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_5", - "session_7" - ], - "question": "What does Melanie do to destress?", - "recall": 0.5, - "retrieved_ids": [ - "session_7", - "session_8", - "session_18", - "session_1", - "session_4", - "session_13", - "session_9", - "session_19", - "session_17", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_7" - ], - "question": "When did Caroline go to the LGBTQ conference?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_7", - "session_14", - "session_1", - "session_13", - "session_11", - "session_10", - "session_9", - "session_4", - "session_3" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_7" - ], - "question": "When did Melanie read the book \"nothing is impossible\"?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_7", - "session_17", - "session_2", - "session_8", - "session_14", - "session_13", - "session_18", - "session_15", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_7" - ], - "question": "Would Caroline pursue writing as a career option?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_1", - "session_4", - "session_18", - "session_8", - "session_9", - "session_12", - "session_5", - "session_14", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_8" - ], - "question": "When did Caroline go to the adoption meeting?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_2", - "session_17", - "session_19", - "session_13", - "session_10", - "session_5", - "session_15", - "session_7", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_8" - ], - "question": "When did Melanie go to the pottery workshop?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_4", - "session_5", - "session_16", - "session_12", - "session_17", - "session_14", - "session_13", - "session_1", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [], - "question": "Would Melanie be considered a member of the LGBTQ community?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_5", - "session_14", - "session_11", - "session_9", - "session_3", - "session_7", - "session_10", - "session_15", - "session_16" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_4" - ], - "question": "When did Melanie go camping in June?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_18", - "session_9", - "session_6", - "session_2", - "session_4", - "session_16", - "session_8", - "session_13", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_3", - "session_5", - "session_8" - ], - "question": "What LGBTQ+ events has Caroline participated in?", - "recall": 0.75, - "retrieved_ids": [ - "session_9", - "session_10", - "session_3", - "session_2", - "session_11", - "session_7", - "session_17", - "session_5", - "session_14", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_5" - ], - "question": "When did Caroline go to a pride parade during the summer?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_10", - "session_11", - "session_5", - "session_18", - "session_16", - "session_12", - "session_2", - "session_15", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_3", - "session_9" - ], - "question": "What events has Caroline participated in to help children?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_9", - "session_10", - "session_3", - "session_2", - "session_11", - "session_7", - "session_17", - "session_19", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_9" - ], - "question": "When did Melanie go camping in July?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_18", - "session_9", - "session_6", - "session_2", - "session_4", - "session_16", - "session_8", - "session_13", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_9" - ], - "question": "When did Caroline join a mentorship program?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_10", - "session_3", - "session_8", - "session_14", - "session_13", - "session_18", - "session_15", - "session_1", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_8" - ], - "question": "What did Melanie paint recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_17", - "session_11", - "session_12", - "session_18", - "session_10", - "session_14", - "session_1", - "session_8", - "session_16" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_3", - "session_6", - "session_8", - "session_9" - ], - "question": "What activities has Melanie done with her family?", - "recall": 0.4, - "retrieved_ids": [ - "session_10", - "session_5", - "session_15", - "session_7", - "session_1", - "session_6", - "session_13", - "session_16", - "session_17", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_5", - "session_9" - ], - "question": "In what ways is Caroline participating in the LGBTQ community?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_11", - "session_9", - "session_15", - "session_14", - "session_3", - "session_7", - "session_10", - "session_16", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_6" - ], - "question": "How many times has Melanie gone to the beach in 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_3", - "session_14", - "session_6", - "session_4", - "session_7", - "session_17", - "session_8", - "session_18", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_10" - ], - "question": "When did Caroline join a new activist group?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_1", - "session_9", - "session_13", - "session_12", - "session_4", - "session_3", - "session_8", - "session_14", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_10" - ], - "question": "Would Melanie be more interested in going to a national park or a theme park?", - "recall": 0.0, - "retrieved_ids": [ - "session_11", - "session_18", - "session_15", - "session_16", - "session_7", - "session_14", - "session_5", - "session_4", - "session_3", - "session_6" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_11", - "session_9" - ], - "question": "What kind of art does Caroline make?", - "recall": 0.5, - "retrieved_ids": [ - "session_2", - "session_4", - "session_3", - "session_5", - "session_6", - "session_11", - "session_10", - "session_15", - "session_14", - "session_16" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_11" - ], - "question": "When is Melanie's daughter's birthday?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_4", - "session_8", - "session_18", - "session_1", - "session_13", - "session_9", - "session_19", - "session_17", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_11" - ], - "question": "When did Caroline attend a pride parade in August?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_5", - "session_11", - "session_10", - "session_8", - "session_9", - "session_14", - "session_12", - "session_17", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [], - "question": "Would Melanie be considered an ally to the transgender community?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_18", - "session_17", - "session_1", - "session_14", - "session_9", - "session_5", - "session_11", - "session_15", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_12", - "session_3" - ], - "question": "Who supports Caroline when she has a negative experience?", - "recall": 0.5, - "retrieved_ids": [ - "session_4", - "session_10", - "session_12", - "session_19", - "session_9", - "session_5", - "session_7", - "session_15", - "session_11", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_12", - "session_5", - "session_8" - ], - "question": "What types of pottery have Melanie and her kids made?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_10", - "session_15", - "session_7", - "session_5", - "session_16", - "session_12", - "session_17", - "session_8", - "session_2" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_12" - ], - "question": "When did Caroline and Melanie go to a pride fesetival together?", - "recall": 1.0, - "retrieved_ids": [ - "session_9", - "session_14", - "session_5", - "session_11", - "session_12", - "session_10", - "session_17", - "session_8", - "session_18", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_12" - ], - "question": "What would Caroline's political leaning likely be?", - "recall": 0.0, - "retrieved_ids": [ - "session_18", - "session_4", - "session_14", - "session_9", - "session_17", - "session_5", - "session_10", - "session_13", - "session_7", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_1", - "session_13", - "session_8" - ], - "question": "What has Melanie painted?", - "recall": 1.0, - "retrieved_ids": [ - "session_1", - "session_13", - "session_9", - "session_14", - "session_17", - "session_16", - "session_11", - "session_8", - "session_12", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_13", - "session_7" - ], - "question": "What are Melanie's pets' names?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_7", - "session_8", - "session_18", - "session_1", - "session_4", - "session_9", - "session_19", - "session_17", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_13" - ], - "question": "When did Caroline apply to adoption agencies?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_2", - "session_19", - "session_17", - "session_8", - "session_14", - "session_18", - "session_15", - "session_1", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_13" - ], - "question": "When did Caroline draw a self-portrait?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_14", - "session_16", - "session_11", - "session_17", - "session_7", - "session_8", - "session_19", - "session_2", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_14", - "session_8" - ], - "question": "What subject have Caroline and Melanie both painted?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_9", - "session_19", - "session_13", - "session_12", - "session_16", - "session_15", - "session_3", - "session_1", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_14", - "session_4" - ], - "question": "What symbols are important to Caroline?", - "recall": 0.5, - "retrieved_ids": [ - "session_14", - "session_7", - "session_2", - "session_11", - "session_6", - "session_18", - "session_15", - "session_3", - "session_8", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_14" - ], - "question": "When did Caroline encounter people on a hike and have a negative experience?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_4", - "session_16", - "session_8", - "session_14", - "session_11", - "session_5", - "session_15", - "session_18", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_14" - ], - "question": "When did Melanie make a plate in pottery class?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_5", - "session_16", - "session_12", - "session_17", - "session_8", - "session_15", - "session_7", - "session_10", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_12", - "session_14" - ], - "question": "Would Caroline be considered religious?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_18", - "session_1", - "session_8", - "session_9", - "session_7", - "session_5", - "session_14", - "session_4", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_15", - "session_2" - ], - "question": "What instruments does Melanie play?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_6", - "session_2", - "session_3", - "session_8", - "session_18", - "session_1", - "session_4", - "session_13", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_11", - "session_15" - ], - "question": "What musical artists/bands has Melanie seen?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_11", - "session_17", - "session_14", - "session_8", - "session_18", - "session_1", - "session_4", - "session_13", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_15" - ], - "question": "When did Melanie go to the park?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_16", - "session_13", - "session_1", - "session_14", - "session_5", - "session_11", - "session_18", - "session_8", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_15" - ], - "question": "When is Caroline's youth center putting on a talent show?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_9", - "session_11", - "session_14", - "session_5", - "session_19", - "session_12", - "session_10", - "session_16", - "session_17" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_15" - ], - "question": "Would Melanie likely enjoy the song \"The Four Seasons\" by Vivaldi?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_15", - "session_11", - "session_8", - "session_10", - "session_4", - "session_14", - "session_9", - "session_17", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_11", - "session_16" - ], - "question": "What are some changes Caroline has faced during her transition journey?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_16", - "session_3", - "session_8", - "session_7", - "session_18", - "session_11", - "session_14", - "session_9", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_10", - "session_16" - ], - "question": "What does Melanie do with her family on hikes?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_7", - "session_12", - "session_4", - "session_16", - "session_8", - "session_14", - "session_19", - "session_2", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_16" - ], - "question": "When did Caroline go biking with friends?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_6", - "session_3", - "session_8", - "session_19", - "session_2", - "session_12", - "session_4", - "session_17", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_16" - ], - "question": "How long has Melanie been practicing art?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_3", - "session_15", - "session_17", - "session_1", - "session_6", - "session_19", - "session_4", - "session_8", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_13", - "session_16", - "session_7" - ], - "question": "What personality traits might Melanie say Caroline has?", - "recall": 0.6667, - "retrieved_ids": [ - "session_10", - "session_16", - "session_15", - "session_8", - "session_14", - "session_1", - "session_18", - "session_9", - "session_4", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_15", - "session_17" - ], - "question": "What transgender-specific events has Caroline attended?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_15", - "session_9", - "session_17", - "session_3", - "session_1", - "session_14", - "session_5", - "session_10", - "session_2" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_17", - "session_7" - ], - "question": "What book did Melanie read from Caroline's suggestion?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_7", - "session_17", - "session_2", - "session_8", - "session_14", - "session_13", - "session_18", - "session_15", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_17" - ], - "question": "When did Melanie's friend adopt a child?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_2", - "session_17", - "session_19", - "session_8", - "session_6", - "session_16", - "session_3", - "session_12", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_17" - ], - "question": "When did Melanie get hurt?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_8", - "session_5", - "session_13", - "session_1", - "session_16", - "session_18", - "session_14", - "session_15", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_18" - ], - "question": "When did Melanie's family go on a roadtrip?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_10", - "session_7", - "session_17", - "session_4", - "session_3", - "session_13", - "session_2", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_18" - ], - "question": "How many children does Melanie have?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_10", - "session_3", - "session_18", - "session_1", - "session_4", - "session_13", - "session_9", - "session_19", - "session_17" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_18" - ], - "question": "When did Melanie go on a hike after the roadtrip?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_4", - "session_14", - "session_12", - "session_16", - "session_8", - "session_2", - "session_9", - "session_1", - "session_3" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_18" - ], - "question": "Would Melanie go on another roadtrip soon?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_1", - "session_9", - "session_13", - "session_11", - "session_12", - "session_16", - "session_8", - "session_4", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 1, - "category_name": "Single-hop", - "evidence_ids": [ - "session_19", - "session_7" - ], - "question": "What items has Melanie bought?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_4", - "session_8", - "session_18", - "session_1", - "session_13", - "session_9", - "session_17", - "session_15", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_19" - ], - "question": "When did Caroline pass the adoption interview?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_3", - "session_2", - "session_17", - "session_13", - "session_8", - "session_14", - "session_18", - "session_15", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 2, - "category_name": "Temporal", - "evidence_ids": [ - "session_19" - ], - "question": "When did Melanie buy the figurines?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_8", - "session_14", - "session_13", - "session_18", - "session_15", - "session_1", - "session_5", - "session_11", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 3, - "category_name": "Temporal-inference", - "evidence_ids": [ - "session_19" - ], - "question": "Would Caroline want to move back to her home country soon?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_19", - "session_18", - "session_8", - "session_1", - "session_4", - "session_7", - "session_10", - "session_2", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What did the charity race raise awareness for?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_7", - "session_3", - "session_8", - "session_14", - "session_13", - "session_18", - "session_15", - "session_1", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What did Melanie realize after the charity race?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_7", - "session_8", - "session_18", - "session_9", - "session_1", - "session_4", - "session_3", - "session_14", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "How does Melanie prioritize self-care?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_9", - "session_13", - "session_4", - "session_1", - "session_7", - "session_11", - "session_17", - "session_19", - "session_16" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What are Caroline's plans for the summer?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_2", - "session_10", - "session_11", - "session_15", - "session_16", - "session_7", - "session_17", - "session_14", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What type of individuals does the adoption agency Caroline is considering support?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_11", - "session_19", - "session_13", - "session_17", - "session_15", - "session_14", - "session_8", - "session_1", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "Why did Caroline choose the adoption agency?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_2", - "session_13", - "session_17", - "session_16", - "session_8", - "session_14", - "session_18", - "session_15", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What is Caroline excited about in the adoption process?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_2", - "session_19", - "session_13", - "session_8", - "session_5", - "session_16", - "session_1", - "session_9", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_2" - ], - "question": "What does Melanie think about Caroline's decision to adopt?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_2", - "session_17", - "session_19", - "session_8", - "session_12", - "session_4", - "session_5", - "session_16", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_3" - ], - "question": "How long have Mel and her husband been married?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_10", - "session_7", - "session_8", - "session_16", - "session_15", - "session_17", - "session_1", - "session_6", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What does Caroline's necklace symbolize?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_14", - "session_1", - "session_8", - "session_9", - "session_12", - "session_7", - "session_5", - "session_18", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What country is Caroline's grandma from?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_3", - "session_1", - "session_8", - "session_9", - "session_12", - "session_7", - "session_5", - "session_14", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What was grandma's gift to Caroline?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_19", - "session_1", - "session_8", - "session_9", - "session_12", - "session_7", - "session_5", - "session_14", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What is Melanie's hand-painted bowl a reminder of?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_8", - "session_5", - "session_12", - "session_16", - "session_11", - "session_14", - "session_1", - "session_13", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What did Melanie and her family do while camping?", - "recall": 0.0, - "retrieved_ids": [ - "session_10", - "session_7", - "session_16", - "session_8", - "session_17", - "session_13", - "session_18", - "session_9", - "session_6", - "session_2" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What kind of counseling and mental health services is Caroline interested in pursuing?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_7", - "session_11", - "session_5", - "session_6", - "session_1", - "session_2", - "session_8", - "session_3", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What workshop did Caroline attend recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_4", - "session_8", - "session_17", - "session_11", - "session_12", - "session_18", - "session_10", - "session_14", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What was discussed in the LGBTQ+ counseling workshop?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_8", - "session_7", - "session_1", - "session_5", - "session_6", - "session_9", - "session_11", - "session_14", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What motivated Caroline to pursue counseling?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_7", - "session_5", - "session_1", - "session_6", - "session_3", - "session_18", - "session_14", - "session_8", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_4" - ], - "question": "What kind of place does Caroline want to create for people?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_14", - "session_17", - "session_2", - "session_3", - "session_5", - "session_6", - "session_11", - "session_10", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_5" - ], - "question": "Did Melanie make the black and white bowl in the photo?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_16", - "session_12", - "session_4", - "session_6", - "session_13", - "session_3", - "session_8", - "session_14", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_6" - ], - "question": "What kind of books does Caroline have in her library?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_7", - "session_10", - "session_17", - "session_2", - "session_4", - "session_3", - "session_5", - "session_11", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_6" - ], - "question": "What was Melanie's favorite book from her childhood?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_7", - "session_10", - "session_17", - "session_8", - "session_14", - "session_18", - "session_1", - "session_4", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What book did Caroline recommend to Melanie?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_17", - "session_6", - "session_8", - "session_14", - "session_13", - "session_18", - "session_15", - "session_1", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What did Caroline take away from the book \"Becoming Nicole\"?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_13", - "session_6", - "session_17", - "session_2", - "session_1", - "session_14", - "session_5", - "session_4", - "session_3" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What are the new shoes that Melanie got used for?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_17", - "session_6", - "session_15", - "session_9", - "session_4", - "session_1", - "session_16", - "session_19", - "session_2" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What is Melanie's reason for getting into running?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_2", - "session_16", - "session_18", - "session_4", - "session_10", - "session_15", - "session_14", - "session_6", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_7" - ], - "question": "What does Melanie say running has been great for?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_10", - "session_2", - "session_6", - "session_17", - "session_15", - "session_9", - "session_8", - "session_3", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What did Mel and her kids make during the pottery workshop?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_16", - "session_10", - "session_4", - "session_7", - "session_18", - "session_5", - "session_12", - "session_17", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What kind of pot did Mel and her kids make with clay?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_10", - "session_5", - "session_7", - "session_2", - "session_4", - "session_3", - "session_6", - "session_11", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What creative project do Mel and her kids do together besides pottery?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_12", - "session_14", - "session_10", - "session_7", - "session_5", - "session_16", - "session_17", - "session_15", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What did Mel and her kids paint in their latest project in July 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_10", - "session_7", - "session_12", - "session_15", - "session_14", - "session_17", - "session_1", - "session_13", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What did Caroline see at the council meeting for adoption?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_2", - "session_17", - "session_19", - "session_13", - "session_10", - "session_5", - "session_15", - "session_7", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "What do sunflowers represent according to Caroline?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_11", - "session_14", - "session_1", - "session_9", - "session_12", - "session_7", - "session_5", - "session_18", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_8" - ], - "question": "Why are flowers important to Melanie?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_14", - "session_7", - "session_2", - "session_11", - "session_6", - "session_18", - "session_15", - "session_3", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_9" - ], - "question": "What inspired Caroline's painting for the art show?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_16", - "session_9", - "session_17", - "session_11", - "session_8", - "session_12", - "session_13", - "session_4", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "How often does Melanie go to the beach with her kids?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_7", - "session_14", - "session_6", - "session_1", - "session_9", - "session_11", - "session_2", - "session_18", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "What did Melanie and her family see during their camping trip last year?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_18", - "session_8", - "session_7", - "session_16", - "session_12", - "session_17", - "session_4", - "session_9", - "session_6" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_10" - ], - "question": "How did Melanie feel while watching the meteor shower?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_8", - "session_17", - "session_13", - "session_16", - "session_14", - "session_1", - "session_5", - "session_15", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_11" - ], - "question": "Whose birthday did Melanie celebrate recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_4", - "session_17", - "session_10", - "session_14", - "session_8", - "session_13", - "session_12", - "session_18", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_11" - ], - "question": "Who performed at the concert at Melanie's daughter's birthday?", - "recall": 1.0, - "retrieved_ids": [ - "session_11", - "session_4", - "session_8", - "session_18", - "session_1", - "session_13", - "session_9", - "session_19", - "session_17", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_12" - ], - "question": "Why did Melanie choose to use colors and patterns in her pottery project?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_16", - "session_19", - "session_10", - "session_4", - "session_14", - "session_2", - "session_8", - "session_7", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "What pet does Caroline have?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_13", - "session_1", - "session_8", - "session_9", - "session_12", - "session_5", - "session_14", - "session_18", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "What pets does Melanie have?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_13", - "session_8", - "session_18", - "session_1", - "session_4", - "session_9", - "session_19", - "session_17", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "Where did Oliver hide his bone once?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_10", - "session_6", - "session_7", - "session_11", - "session_8", - "session_14", - "session_18", - "session_15", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_13" - ], - "question": "What activity did Caroline used to do with her dad?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_10", - "session_5", - "session_15", - "session_7", - "session_8", - "session_14", - "session_18", - "session_1", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_14" - ], - "question": "What did Caroline make for a local church?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_8", - "session_15", - "session_5", - "session_7", - "session_10", - "session_16", - "session_18", - "session_17", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_14" - ], - "question": "What did Caroline find in her neighborhood during her walk?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_7", - "session_10", - "session_16", - "session_18", - "session_8", - "session_19", - "session_9", - "session_11", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "Which song motivates Caroline to be courageous?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_3", - "session_11", - "session_14", - "session_1", - "session_8", - "session_5", - "session_18", - "session_4", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "Which classical musicians does Melanie enjoy listening to?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_15", - "session_8", - "session_18", - "session_10", - "session_1", - "session_4", - "session_13", - "session_9", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_15" - ], - "question": "Who is Melanie a fan of in terms of modern music?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_5", - "session_11", - "session_8", - "session_18", - "session_1", - "session_4", - "session_13", - "session_9", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "How long has Melanie been creating art?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_3", - "session_15", - "session_17", - "session_1", - "session_6", - "session_19", - "session_4", - "session_8", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_16" - ], - "question": "What precautionary sign did Melanie see at the caf\u00e9?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_5", - "session_10", - "session_15", - "session_14", - "session_1", - "session_11", - "session_18", - "session_8", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What advice does Caroline give for getting started with adoption?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_13", - "session_2", - "session_19", - "session_8", - "session_15", - "session_12", - "session_18", - "session_4", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What setback did Melanie face in October 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_9", - "session_10", - "session_15", - "session_3", - "session_8", - "session_14", - "session_13", - "session_18", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What does Melanie do to keep herself busy during her pottery break?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_8", - "session_16", - "session_10", - "session_14", - "session_2", - "session_7", - "session_18", - "session_5", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What painting did Melanie show to Caroline on October 13, 2023?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_13", - "session_8", - "session_17", - "session_16", - "session_11", - "session_9", - "session_1", - "session_15", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What kind of painting did Caroline share with Melanie on October 13, 2023?", - "recall": 0.0, - "retrieved_ids": [ - "session_2", - "session_4", - "session_3", - "session_5", - "session_6", - "session_11", - "session_10", - "session_15", - "session_14", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What was the poetry reading that Caroline attended about?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_13", - "session_6", - "session_7", - "session_2", - "session_4", - "session_15", - "session_3", - "session_12", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What did the posters at the poetry reading say?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_10", - "session_6", - "session_7", - "session_2", - "session_8", - "session_14", - "session_13", - "session_18", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "What does Caroline's drawing symbolize for her?", - "recall": 0.0, - "retrieved_ids": [ - "session_14", - "session_10", - "session_16", - "session_7", - "session_1", - "session_8", - "session_9", - "session_12", - "session_5", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_17" - ], - "question": "How do Melanie and Caroline describe their journey through life together?", - "recall": 1.0, - "retrieved_ids": [ - "session_3", - "session_8", - "session_4", - "session_17", - "session_14", - "session_19", - "session_7", - "session_15", - "session_10", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What happened to Melanie's son on their road trip?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_10", - "session_12", - "session_17", - "session_7", - "session_6", - "session_14", - "session_2", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "How did Melanie's son handle the accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_16", - "session_8", - "session_14", - "session_13", - "session_15", - "session_1", - "session_5", - "session_11", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "How did Melanie feel about her family after the accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_10", - "session_7", - "session_2", - "session_9", - "session_1", - "session_4", - "session_3", - "session_14", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "How did Melanie's children handle the accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_16", - "session_14", - "session_13", - "session_15", - "session_1", - "session_5", - "session_11", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "How did Melanie feel after the accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_2", - "session_9", - "session_1", - "session_4", - "session_3", - "session_14", - "session_13", - "session_8", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What was Melanie's reaction to her children enjoying the Grand Canyon?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_10", - "session_3", - "session_7", - "session_1", - "session_4", - "session_13", - "session_9", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What do Melanie's family give her?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_7", - "session_3", - "session_19", - "session_2", - "session_18", - "session_6", - "session_4", - "session_8", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "How did Melanie feel about her family supporting her?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_7", - "session_8", - "session_17", - "session_16", - "session_3", - "session_2", - "session_15", - "session_18", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 4, - "category_name": "Open-domain", - "evidence_ids": [ - "session_18" - ], - "question": "What did Melanie do after the road trip to relax?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_17", - "session_1", - "session_10", - "session_14", - "session_12", - "session_15", - "session_2", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_2" - ], - "question": "What did Caroline realize after her charity race?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_7", - "session_10", - "session_8", - "session_18", - "session_9", - "session_1", - "session_4", - "session_3", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_2" - ], - "question": "What are Melanie's plans for the summer with respect to adoption?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_2", - "session_12", - "session_17", - "session_10", - "session_13", - "session_8", - "session_11", - "session_15", - "session_16" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_2" - ], - "question": "What type of individuals does the adoption agency Melanie is considering support?", - "recall": 1.0, - "retrieved_ids": [ - "session_2", - "session_11", - "session_19", - "session_13", - "session_17", - "session_15", - "session_14", - "session_8", - "session_1", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_2" - ], - "question": "Why did Melanie choose the adoption agency?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_2", - "session_13", - "session_17", - "session_16", - "session_8", - "session_14", - "session_18", - "session_15", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_2" - ], - "question": "What is Melanie excited about in her adoption process?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_2", - "session_19", - "session_10", - "session_7", - "session_13", - "session_8", - "session_5", - "session_16", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What does Melanie's necklace symbolize?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_14", - "session_8", - "session_18", - "session_1", - "session_13", - "session_9", - "session_19", - "session_17", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What country is Melanie's grandma from?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_3", - "session_8", - "session_18", - "session_1", - "session_13", - "session_9", - "session_19", - "session_17", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What was grandma's gift to Melanie?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_19", - "session_8", - "session_18", - "session_1", - "session_13", - "session_9", - "session_17", - "session_15", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What was grandpa's gift to Caroline?", - "recall": 1.0, - "retrieved_ids": [ - "session_19", - "session_4", - "session_1", - "session_8", - "session_9", - "session_12", - "session_7", - "session_5", - "session_14", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What is Caroline's hand-painted bowl a reminder of?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_8", - "session_5", - "session_12", - "session_16", - "session_11", - "session_14", - "session_1", - "session_9", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What did Caroline and her family do while camping?", - "recall": 0.0, - "retrieved_ids": [ - "session_10", - "session_7", - "session_16", - "session_8", - "session_17", - "session_13", - "session_18", - "session_9", - "session_6", - "session_2" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What kind of counseling and mental health services is Melanie interested in pursuing?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_7", - "session_11", - "session_5", - "session_6", - "session_1", - "session_2", - "session_8", - "session_3", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What kind of counseling workshop did Melanie attend recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_13", - "session_8", - "session_5", - "session_6", - "session_7", - "session_1", - "session_11", - "session_10", - "session_17" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What motivated Melanie to pursue counseling?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_7", - "session_5", - "session_1", - "session_6", - "session_3", - "session_18", - "session_14", - "session_8", - "session_13" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_4" - ], - "question": "What kind of place does Melanie want to create for people?", - "recall": 1.0, - "retrieved_ids": [ - "session_4", - "session_14", - "session_17", - "session_2", - "session_3", - "session_5", - "session_6", - "session_11", - "session_10", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_5" - ], - "question": "Did Caroline make the black and white bowl in the photo?", - "recall": 1.0, - "retrieved_ids": [ - "session_5", - "session_16", - "session_12", - "session_4", - "session_6", - "session_13", - "session_3", - "session_8", - "session_14", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_7" - ], - "question": "What are the new shoes that Caroline got used for?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_17", - "session_6", - "session_15", - "session_9", - "session_4", - "session_1", - "session_16", - "session_19", - "session_2" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_7" - ], - "question": "What is Caroline's reason for getting into running?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_2", - "session_16", - "session_18", - "session_4", - "session_10", - "session_15", - "session_14", - "session_5", - "session_6" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_7" - ], - "question": "What does Caroline say running has been great for?", - "recall": 1.0, - "retrieved_ids": [ - "session_7", - "session_10", - "session_2", - "session_6", - "session_17", - "session_15", - "session_9", - "session_8", - "session_3", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_8" - ], - "question": "What did Melanie see at the council meeting for adoption?", - "recall": 1.0, - "retrieved_ids": [ - "session_8", - "session_2", - "session_17", - "session_19", - "session_13", - "session_10", - "session_5", - "session_15", - "session_7", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_9" - ], - "question": "What inspired Melanie's painting for the art show?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_16", - "session_9", - "session_17", - "session_11", - "session_8", - "session_12", - "session_13", - "session_4", - "session_1" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_9" - ], - "question": "What inspired Caroline's sculpture for the art show?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_16", - "session_9", - "session_17", - "session_11", - "session_12", - "session_8", - "session_13", - "session_5", - "session_19" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_10" - ], - "question": "How often does Caroline go to the beach with her kids?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_7", - "session_14", - "session_6", - "session_1", - "session_9", - "session_11", - "session_2", - "session_18", - "session_4" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_10" - ], - "question": "What did Caroline and her family see during their camping trip last year?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_18", - "session_8", - "session_7", - "session_16", - "session_12", - "session_17", - "session_4", - "session_9", - "session_6" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_10" - ], - "question": "How did Caroline feel while watching the meteor shower?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_8", - "session_17", - "session_13", - "session_16", - "session_14", - "session_1", - "session_5", - "session_15", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_12" - ], - "question": "Why did Caroline choose to use colors and patterns in her pottery project?", - "recall": 1.0, - "retrieved_ids": [ - "session_12", - "session_16", - "session_19", - "session_10", - "session_4", - "session_14", - "session_2", - "session_8", - "session_7", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_13" - ], - "question": "Is Oscar Melanie's pet?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_7", - "session_8", - "session_18", - "session_1", - "session_4", - "session_9", - "session_19", - "session_17", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_13" - ], - "question": "Where did Oscar hide his bone once?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_10", - "session_6", - "session_11", - "session_8", - "session_14", - "session_18", - "session_15", - "session_1", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_13" - ], - "question": "What activity did Melanie used to do with her dad?", - "recall": 1.0, - "retrieved_ids": [ - "session_13", - "session_10", - "session_5", - "session_15", - "session_7", - "session_8", - "session_14", - "session_18", - "session_1", - "session_11" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_14" - ], - "question": "What did Melanie make for a local church?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_8", - "session_15", - "session_5", - "session_7", - "session_10", - "session_16", - "session_18", - "session_17", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_14" - ], - "question": "What did Melanie find in her neighborhood during her walk?", - "recall": 1.0, - "retrieved_ids": [ - "session_14", - "session_7", - "session_10", - "session_16", - "session_18", - "session_8", - "session_19", - "session_9", - "session_11", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_15" - ], - "question": "Which song motivates Melanie to be courageous?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_3", - "session_11", - "session_14", - "session_1", - "session_8", - "session_5", - "session_18", - "session_4", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_15" - ], - "question": "What type of instrument does Caroline play?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_6", - "session_14", - "session_2", - "session_3", - "session_1", - "session_8", - "session_9", - "session_12", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_15" - ], - "question": "Which classical musicians does Caroline enjoy listening to?", - "recall": 1.0, - "retrieved_ids": [ - "session_6", - "session_15", - "session_8", - "session_18", - "session_10", - "session_1", - "session_9", - "session_12", - "session_7", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_15" - ], - "question": "Who is Caroline a fan of in terms of modern music?", - "recall": 1.0, - "retrieved_ids": [ - "session_15", - "session_5", - "session_11", - "session_1", - "session_8", - "session_9", - "session_12", - "session_7", - "session_14", - "session_18" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_16" - ], - "question": "What precautionary sign did Caroline see at the caf\u00e9?", - "recall": 1.0, - "retrieved_ids": [ - "session_16", - "session_5", - "session_10", - "session_15", - "session_14", - "session_1", - "session_11", - "session_18", - "session_8", - "session_7" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_17" - ], - "question": "What setback did Caroline face recently?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_10", - "session_9", - "session_15", - "session_3", - "session_13", - "session_11", - "session_12", - "session_18", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_17" - ], - "question": "What does Caroline do to keep herself busy during her pottery break?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_8", - "session_16", - "session_10", - "session_14", - "session_2", - "session_7", - "session_18", - "session_5", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_17" - ], - "question": "What was the poetry reading that Melanie attended about?", - "recall": 1.0, - "retrieved_ids": [ - "session_17", - "session_13", - "session_6", - "session_7", - "session_2", - "session_4", - "session_15", - "session_3", - "session_11", - "session_12" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "What happened to Caroline's son on their road trip?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_10", - "session_12", - "session_17", - "session_7", - "session_6", - "session_14", - "session_2", - "session_15" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "How did Caroline's son handle the accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_16", - "session_8", - "session_14", - "session_13", - "session_15", - "session_1", - "session_5", - "session_11", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "How did Caroline feel about her family after the accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_10", - "session_7", - "session_2", - "session_9", - "session_1", - "session_4", - "session_3", - "session_14", - "session_8" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "How did Caroline's children handle the accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_16", - "session_14", - "session_13", - "session_15", - "session_1", - "session_5", - "session_11", - "session_10" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "How did Caroline feel after the accident?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_2", - "session_9", - "session_1", - "session_4", - "session_3", - "session_14", - "session_13", - "session_8", - "session_5" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "What was Caroline's reaction to her children enjoying the Grand Canyon?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_10", - "session_3", - "session_7", - "session_1", - "session_9", - "session_12", - "session_5", - "session_14" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "What did Caroline do after the road trip to relax?", - "recall": 1.0, - "retrieved_ids": [ - "session_18", - "session_8", - "session_17", - "session_1", - "session_10", - "session_14", - "session_12", - "session_15", - "session_2", - "session_9" - ], - "sample_id": "conv-26" - }, - { - "category": 5, - "category_name": "Adversarial", - "evidence_ids": [ - "session_18" - ], - "question": "What does Caroline love most about camping with her family?", - "recall": 1.0, - "retrieved_ids": [ - "session_10", - "session_18", - "session_7", - "session_14", - "session_9", - "session_6", - "session_2", - "session_4", - "session_16", - "session_8" - ], - "sample_id": "conv-26" - } - ], - "runtime_seconds": 5.736, - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/smoke_postpush_20260423_123452/runs/locomo_old_brainctl_cmd_session.json b/benchmarks/results/smoke_postpush_20260423_123452/runs/locomo_old_brainctl_cmd_session.json deleted file mode 100644 index bbe862c..0000000 --- a/benchmarks/results/smoke_postpush_20260423_123452/runs/locomo_old_brainctl_cmd_session.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\locomo_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\locomo_old_brainctl_cmd_session.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" - }, - "benchmark": "locomo", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "measured": false, - "metadata": {}, - "metrics": { - "avg_recall": 0.9217, - "cat_1_recall": 0.7614, - "cat_2_recall": 0.92, - "cat_3_recall": 0.6616, - "cat_4_recall": 0.9738, - "cat_5_recall": 0.9821, - "perfect_rate": 0.8817, - "top_k": 10, - "zero_rate": 0.0438 - }, - "mode": "cmd_session", - "notes": [ - "granularity=session", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": 0.9217, - "reference_kind": "historical", - "runtime_seconds": 445.74, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_mempalace_raw_session.json b/benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_mempalace_raw_session.json deleted file mode 100644 index 6191fa1..0000000 --- a/benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_mempalace_raw_session.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\longmemeval_mempalace_raw_session.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "measured": false, - "metadata": {}, - "metrics": { - "ndcg_at_10": 0.8948, - "ndcg_at_5": 0.893, - "r_at_10": 0.983, - "r_at_5": 0.966 - }, - "mode": "raw_session", - "notes": [ - "top_k=10", - "Runs MemPalace benchmark module raw session retrieval logic directly." - ], - "primary_metric": "r_at_5", - "primary_metric_value": 0.966, - "reference_kind": "historical", - "runtime_seconds": 695.36, - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "mempalace" -} \ No newline at end of file diff --git a/benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_new_brainctl_brain.json b/benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_new_brainctl_brain.json deleted file mode 100644 index 7f89cbf..0000000 --- a/benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_new_brainctl_brain.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\longmemeval_new_brainctl_brain.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 1, - "measured": true, - "metadata": {}, - "metrics": { - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0 - }, - "mode": "brain", - "notes": [ - "top_k=10", - "Legacy 470-question session-level slice." - ], - "primary_metric": "r_at_5", - "primary_metric_value": 1.0, - "reference_kind": "measured", - "rows": [ - { - "answer_session_ids": [ - "answer_280352e9" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e47becba", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_280352e9", - "02bd2b90_3", - "sharegpt_Cr2tc1f_0", - "f6859b48_2", - "ultrachat_214101", - "5dac7cc2_1" - ] - } - ], - "runtime_seconds": 2.644, - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_new_brainctl_cmd.json b/benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_new_brainctl_cmd.json deleted file mode 100644 index 86be500..0000000 --- a/benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_new_brainctl_cmd.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\longmemeval_new_brainctl_cmd.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 1, - "measured": true, - "metadata": {}, - "metrics": { - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0 - }, - "mode": "cmd", - "notes": [ - "top_k=10", - "Legacy 470-question session-level slice." - ], - "primary_metric": "r_at_5", - "primary_metric_value": 1.0, - "reference_kind": "measured", - "rows": [ - { - "answer_session_ids": [ - "answer_280352e9" - ], - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "question_id": "e47becba", - "question_type": "single-session-user", - "r_all_at_10": 1.0, - "r_all_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0, - "top_session_ids": [ - "answer_280352e9", - "02bd2b90_3", - "sharegpt_Cr2tc1f_0", - "f6859b48_2", - "ultrachat_214101", - "5dac7cc2_1" - ] - } - ], - "runtime_seconds": 2.577, - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_old_brainctl_brain.json b/benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_old_brainctl_brain.json deleted file mode 100644 index ef92691..0000000 --- a/benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_old_brainctl_brain.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\longmemeval_old_brainctl_brain.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "measured": false, - "metadata": {}, - "metrics": { - "ndcg_at_10": 0.9253, - "ndcg_at_5": 0.9204, - "r_at_10": 0.9894, - "r_at_5": 0.9681 - }, - "mode": "brain", - "notes": [ - "top_k=10" - ], - "primary_metric": "r_at_5", - "primary_metric_value": 0.9681, - "reference_kind": "historical", - "runtime_seconds": 85.439, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_old_brainctl_cmd.json b/benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_old_brainctl_cmd.json deleted file mode 100644 index 594d9a6..0000000 --- a/benchmarks/results/smoke_postpush_20260423_123452/runs/longmemeval_old_brainctl_cmd.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\longmemeval_old_brainctl_cmd.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "measured": false, - "metadata": {}, - "metrics": { - "ndcg_at_10": 0.9247, - "ndcg_at_5": 0.9206, - "r_at_10": 0.9894, - "r_at_5": 0.9702 - }, - "mode": "cmd", - "notes": [ - "top_k=10" - ], - "primary_metric": "r_at_5", - "primary_metric_value": 0.9702, - "reference_kind": "historical", - "runtime_seconds": 130.863, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/smoke_postpush_20260423_123452/runs/membench_mempalace_raw_turn.json b/benchmarks/results/smoke_postpush_20260423_123452/runs/membench_mempalace_raw_turn.json deleted file mode 100644 index edb0934..0000000 --- a/benchmarks/results/smoke_postpush_20260423_123452/runs/membench_mempalace_raw_turn.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\membench_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\membench_mempalace_raw_turn.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" - }, - "benchmark": "membench", - "caveats": [ - "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 200, - "measured": false, - "metadata": {}, - "metrics": { - "hit_at_5": 0.885, - "simple_hit_at_5": 0.885, - "top_k": 5 - }, - "mode": "raw_turn", - "notes": [ - "FirstAgent slice only", - "turn-level retrieval", - "topic=all" - ], - "primary_metric": "hit_at_5", - "primary_metric_value": 0.885, - "reference_kind": "historical", - "runtime_seconds": 804.35, - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "partial", - "system_name": "mempalace" -} \ No newline at end of file diff --git a/benchmarks/results/smoke_postpush_20260423_123452/runs/membench_new_brainctl_cmd_turn.json b/benchmarks/results/smoke_postpush_20260423_123452/runs/membench_new_brainctl_cmd_turn.json deleted file mode 100644 index 4c6d1ae..0000000 --- a/benchmarks/results/smoke_postpush_20260423_123452/runs/membench_new_brainctl_cmd_turn.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\membench_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\membench_new_brainctl_cmd_turn.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" - }, - "benchmark": "membench", - "caveats": [ - "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 1, - "measured": true, - "metadata": {}, - "metrics": { - "hit_at_5": 1.0, - "simple_hit_at_5": 1.0, - "top_k": 5 - }, - "mode": "cmd_turn", - "notes": [ - "FirstAgent slice only", - "turn-level retrieval", - "topic=all" - ], - "primary_metric": "hit_at_5", - "primary_metric_value": 1.0, - "reference_kind": "measured", - "rows": [ - { - "category": "simple", - "hit_at_k": true, - "question": "What is the name of my niece's company?", - "retrieved_ids": [ - "simple_roles_0|sid=119|g=119|s=5|t=18", - "simple_roles_0|sid=101|g=101|s=5|t=0", - "simple_roles_0|sid=135|g=135|s=6|t=12", - "simple_roles_0|sid=11|g=11|s=0|t=11", - "simple_roles_0|sid=25|g=25|s=1|t=5" - ], - "target_ids": [ - "119" - ], - "tid": 0, - "topic": "roles" - } - ], - "runtime_seconds": 0.567, - "series_name": "new_brainctl", - "source_path": null, - "status": "partial", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/smoke_postpush_20260423_123452/runs/membench_old_brainctl_cmd_turn.json b/benchmarks/results/smoke_postpush_20260423_123452/runs/membench_old_brainctl_cmd_turn.json deleted file mode 100644 index 070c2db..0000000 --- a/benchmarks/results/smoke_postpush_20260423_123452/runs/membench_old_brainctl_cmd_turn.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\membench_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\membench_old_brainctl_cmd_turn.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" - }, - "benchmark": "membench", - "caveats": [ - "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 200, - "measured": false, - "metadata": {}, - "metrics": { - "hit_at_5": 0.93, - "simple_hit_at_5": 0.93, - "top_k": 5 - }, - "mode": "cmd_turn", - "notes": [ - "FirstAgent slice only", - "turn-level retrieval", - "topic=all" - ], - "primary_metric": "hit_at_5", - "primary_metric_value": 0.93, - "reference_kind": "historical", - "runtime_seconds": 140.592, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "partial", - "system_name": "brainctl" -} \ No newline at end of file diff --git a/benchmarks/results/smoke_postpush_20260423_123452/summary.csv b/benchmarks/results/smoke_postpush_20260423_123452/summary.csv deleted file mode 100644 index 766000a..0000000 --- a/benchmarks/results/smoke_postpush_20260423_123452/summary.csv +++ /dev/null @@ -1,15 +0,0 @@ -benchmark,series_name,system_name,mode,reference_kind,status,example_count,primary_metric,primary_metric_value,runtime_seconds,dataset_path,source_path,notes,caveats,avg_recall,cat_1_recall,cat_2_recall,cat_3_recall,cat_4_recall,cat_5_recall,hit_at_5,ndcg_at_10,ndcg_at_5,perfect_rate,r_at_10,r_at_5,simple_hit_at_5,top_k,zero_rate -longmemeval,old_brainctl,brainctl,brain,historical,full_same_machine,470,r_at_5,0.9681,85.439,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,top_k=10,,,,,,,,,0.9253,0.9204,,0.9894,0.9681,,, -longmemeval,old_brainctl,brainctl,cmd,historical,full_same_machine,470,r_at_5,0.9702,130.863,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,top_k=10,,,,,,,,,0.9247,0.9206,,0.9894,0.9702,,, -longmemeval,mempalace,mempalace,raw_session,historical,full_same_machine,470,r_at_5,0.966,695.36,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,top_k=10 | Runs MemPalace benchmark module raw session retrieval logic directly.,,,,,,,,,0.8948,0.893,,0.983,0.966,,, -locomo,old_brainctl,brainctl,cmd_session,historical,full_same_machine,1986,avg_recall,0.9217,445.74,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,granularity=session | top_k=10,,0.9217,0.7614,0.92,0.6616,0.9738,0.9821,,,,0.8817,,,,10,0.0438 -locomo,mempalace,mempalace,raw_session,historical,full_same_machine,1986,avg_recall,0.6028,2106.411,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,granularity=session | top_k=10,,0.6028,0.5899,0.6921,0.4604,0.5809,0.6188,,,,0.5534,,,,10,0.3499 -membench,old_brainctl,brainctl,cmd_turn,historical,partial,200,hit_at_5,0.93,140.592,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,FirstAgent slice only | turn-level retrieval | topic=all,MemBench comparison is partial because ThirdAgent and noise-extended slices are not included.,,,,,,,0.93,,,,,,0.93,5, -membench,mempalace,mempalace,raw_turn,historical,partial,200,hit_at_5,0.885,804.35,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,FirstAgent slice only | turn-level retrieval | topic=all,MemBench comparison is partial because ThirdAgent and noise-extended slices are not included.,,,,,,,0.885,,,,,,0.885,5, -convomem,old_brainctl,brainctl,cmd,historical,blocked,0,avg_recall,,,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,limit_per_category=1 | top_k=10,Blocked while loading ConvoMem evidence data: ,,,,,,,,,,,,,,, -convomem,mempalace,mempalace,raw,historical,blocked,0,avg_recall,,,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache,C:\Users\mario\Documents\GitHub\brainctl\benchmarks\results\full_compare_20260418_033425\summary.json,limit_per_category=1 | top_k=10,Blocked while loading ConvoMem evidence data: ,,,,,,,,,,,,,,, -longmemeval,new_brainctl,brainctl,brain,measured,full_same_machine,1,r_at_5,1.0,2.644,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,,top_k=10 | Legacy 470-question session-level slice.,,,,,,,,,1.0,1.0,,1.0,1.0,,, -longmemeval,new_brainctl,brainctl,cmd,measured,full_same_machine,1,r_at_5,1.0,2.577,C:\Users\mario\Documents\GitHub\LongMemEval\data\longmemeval_s_cleaned.json,,top_k=10 | Legacy 470-question session-level slice.,,,,,,,,,1.0,1.0,,1.0,1.0,,, -locomo,new_brainctl,brainctl,cmd_session,measured,full_same_machine,199,avg_recall,0.9413,5.736,C:\Users\mario\Documents\GitHub\locomo\data\locomo10.json,,granularity=session | top_k=10,,0.9413,0.8641,1.0,0.7436,0.9571,0.9787,,,,0.9196,,,,10,0.0402 -membench,new_brainctl,brainctl,cmd_turn,measured,partial,1,hit_at_5,1.0,0.567,C:\Users\mario\Documents\GitHub\Membench\MemData\FirstAgent,,FirstAgent slice only | turn-level retrieval | topic=all,MemBench comparison is partial because ThirdAgent and noise-extended slices are not included.,,,,,,,1.0,,,,,,1.0,5, -convomem,new_brainctl,brainctl,cmd,measured,blocked,0,avg_recall,,,C:\Users\mario\Documents\GitHub\mempalace\benchmarks\convomem_cache,,limit_per_category=1 | top_k=10,Blocked while loading ConvoMem evidence data: HTTP Error 404: Not Found,,,,,,,,,,,,,,, diff --git a/benchmarks/results/smoke_postpush_20260423_123452/summary.json b/benchmarks/results/smoke_postpush_20260423_123452/summary.json deleted file mode 100644 index e7f5f05..0000000 --- a/benchmarks/results/smoke_postpush_20260423_123452/summary.json +++ /dev/null @@ -1,509 +0,0 @@ -{ - "generated_at_utc": "2026-04-23T12:35:06.799013+00:00", - "metadata": { - "argv": [ - "benchmarks\\compare_memory_engines.py", - "--label", - "smoke_postpush", - "--longmemeval-limit", - "1", - "--locomo-limit", - "1", - "--membench-limit", - "1", - "--convomem-limit-per-category", - "1" - ], - "cwd": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl", - "datasets": { - "convomem_cache": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "locomo_data": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "longmemeval_data": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "membench_data": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent" - }, - "generated_at_utc": "2026-04-23T12:35:06.703127+00:00", - "git_commit": "b9a75f6b975c5330e3d35a2627a5e132b2e06e93", - "historical_summary_mode": "recovered", - "historical_summary_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "machine": "AMD64", - "platform": "Windows-11-10.0.26200-SP0", - "processor": "Intel64 Family 6 Model 189 Stepping 1, GenuineIntel", - "python_version": "3.14.4 (tags/v3.14.4:23116f9, Apr 7 2026, 14:10:54) [MSC v.1944 64 bit (AMD64)]" - }, - "notes": [ - "Historical old-BrainCTL and MemPalace series come from the recovered 2026-04-18 comparison bundle.", - "New BrainCTL series are rerun in the current checked-out repo using the legacy benchmark definitions.", - "MemBench remains intentionally partial because the legacy comparison only covered the FirstAgent slice.", - "ConvoMem remains a coverage/status benchmark here; it has no dedicated comparison chart in the legacy chart pack." - ], - "runs": [ - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\longmemeval_old_brainctl_brain.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "measured": false, - "metadata": {}, - "metrics": { - "ndcg_at_10": 0.9253, - "ndcg_at_5": 0.9204, - "r_at_10": 0.9894, - "r_at_5": 0.9681 - }, - "mode": "brain", - "notes": [ - "top_k=10" - ], - "primary_metric": "r_at_5", - "primary_metric_value": 0.9681, - "reference_kind": "historical", - "runtime_seconds": 85.439, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\longmemeval_old_brainctl_cmd.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "measured": false, - "metadata": {}, - "metrics": { - "ndcg_at_10": 0.9247, - "ndcg_at_5": 0.9206, - "r_at_10": 0.9894, - "r_at_5": 0.9702 - }, - "mode": "cmd", - "notes": [ - "top_k=10" - ], - "primary_metric": "r_at_5", - "primary_metric_value": 0.9702, - "reference_kind": "historical", - "runtime_seconds": 130.863, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\longmemeval_mempalace_raw_session.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 470, - "measured": false, - "metadata": {}, - "metrics": { - "ndcg_at_10": 0.8948, - "ndcg_at_5": 0.893, - "r_at_10": 0.983, - "r_at_5": 0.966 - }, - "mode": "raw_session", - "notes": [ - "top_k=10", - "Runs MemPalace benchmark module raw session retrieval logic directly." - ], - "primary_metric": "r_at_5", - "primary_metric_value": 0.966, - "reference_kind": "historical", - "runtime_seconds": 695.36, - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "mempalace" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\locomo_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\locomo_old_brainctl_cmd_session.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" - }, - "benchmark": "locomo", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "measured": false, - "metadata": {}, - "metrics": { - "avg_recall": 0.9217, - "cat_1_recall": 0.7614, - "cat_2_recall": 0.92, - "cat_3_recall": 0.6616, - "cat_4_recall": 0.9738, - "cat_5_recall": 0.9821, - "perfect_rate": 0.8817, - "top_k": 10, - "zero_rate": 0.0438 - }, - "mode": "cmd_session", - "notes": [ - "granularity=session", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": 0.9217, - "reference_kind": "historical", - "runtime_seconds": 445.74, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "brainctl" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\locomo_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\locomo_mempalace_raw_session.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" - }, - "benchmark": "locomo", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 1986, - "measured": false, - "metadata": {}, - "metrics": { - "avg_recall": 0.6028, - "cat_1_recall": 0.5899, - "cat_2_recall": 0.6921, - "cat_3_recall": 0.4604, - "cat_4_recall": 0.5809, - "cat_5_recall": 0.6188, - "perfect_rate": 0.5534, - "top_k": 10, - "zero_rate": 0.3499 - }, - "mode": "raw_session", - "notes": [ - "granularity=session", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": 0.6028, - "reference_kind": "historical", - "runtime_seconds": 2106.411, - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "full_same_machine", - "system_name": "mempalace" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\membench_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\membench_old_brainctl_cmd_turn.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" - }, - "benchmark": "membench", - "caveats": [ - "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 200, - "measured": false, - "metadata": {}, - "metrics": { - "hit_at_5": 0.93, - "simple_hit_at_5": 0.93, - "top_k": 5 - }, - "mode": "cmd_turn", - "notes": [ - "FirstAgent slice only", - "turn-level retrieval", - "topic=all" - ], - "primary_metric": "hit_at_5", - "primary_metric_value": 0.93, - "reference_kind": "historical", - "runtime_seconds": 140.592, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "partial", - "system_name": "brainctl" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\membench_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\membench_mempalace_raw_turn.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" - }, - "benchmark": "membench", - "caveats": [ - "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 200, - "measured": false, - "metadata": {}, - "metrics": { - "hit_at_5": 0.885, - "simple_hit_at_5": 0.885, - "top_k": 5 - }, - "mode": "raw_turn", - "notes": [ - "FirstAgent slice only", - "turn-level retrieval", - "topic=all" - ], - "primary_metric": "hit_at_5", - "primary_metric_value": 0.885, - "reference_kind": "historical", - "runtime_seconds": 804.35, - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "partial", - "system_name": "mempalace" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\convomem_old_brainctl_cmd.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" - }, - "benchmark": "convomem", - "caveats": [ - "Blocked while loading ConvoMem evidence data: " - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "example_count": 0, - "measured": false, - "metadata": {}, - "metrics": {}, - "mode": "cmd", - "notes": [ - "limit_per_category=1", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": null, - "reference_kind": "historical", - "runtime_seconds": null, - "series_name": "old_brainctl", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "blocked", - "system_name": "brainctl" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\charts\\aggregate_primary_metrics.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\convomem_mempalace_raw.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" - }, - "benchmark": "convomem", - "caveats": [ - "Blocked while loading ConvoMem evidence data: " - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "example_count": 0, - "measured": false, - "metadata": {}, - "metrics": {}, - "mode": "raw", - "notes": [ - "limit_per_category=1", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": null, - "reference_kind": "historical", - "runtime_seconds": null, - "series_name": "mempalace", - "source_path": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\benchmarks\\results\\full_compare_20260418_033425\\summary.json", - "status": "blocked", - "system_name": "mempalace" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\longmemeval_new_brainctl_brain.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 1, - "measured": true, - "metadata": {}, - "metrics": { - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0 - }, - "mode": "brain", - "notes": [ - "top_k=10", - "Legacy 470-question session-level slice." - ], - "primary_metric": "r_at_5", - "primary_metric_value": 1.0, - "reference_kind": "measured", - "runtime_seconds": 2.644, - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\longmemeval_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\longmemeval_new_brainctl_cmd.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" - }, - "benchmark": "longmemeval", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\LongMemEval\\data\\longmemeval_s_cleaned.json", - "example_count": 1, - "measured": true, - "metadata": {}, - "metrics": { - "ndcg_at_10": 1.0, - "ndcg_at_5": 1.0, - "r_at_10": 1.0, - "r_at_5": 1.0 - }, - "mode": "cmd", - "notes": [ - "top_k=10", - "Legacy 470-question session-level slice." - ], - "primary_metric": "r_at_5", - "primary_metric_value": 1.0, - "reference_kind": "measured", - "runtime_seconds": 2.577, - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\locomo_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\locomo_new_brainctl_cmd_session.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" - }, - "benchmark": "locomo", - "caveats": [], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\locomo\\data\\locomo10.json", - "example_count": 199, - "measured": true, - "metadata": {}, - "metrics": { - "avg_recall": 0.9413, - "cat_1_recall": 0.8641, - "cat_2_recall": 1.0, - "cat_3_recall": 0.7436, - "cat_4_recall": 0.9571, - "cat_5_recall": 0.9787, - "perfect_rate": 0.9196, - "top_k": 10, - "zero_rate": 0.0402 - }, - "mode": "cmd_session", - "notes": [ - "granularity=session", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": 0.9413, - "reference_kind": "measured", - "runtime_seconds": 5.736, - "series_name": "new_brainctl", - "source_path": null, - "status": "full_same_machine", - "system_name": "brainctl" - }, - { - "artifacts": { - "aggregate_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\aggregate_primary_metrics.png", - "benchmark_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\membench_comparison.png", - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\membench_new_brainctl_cmd_turn.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" - }, - "benchmark": "membench", - "caveats": [ - "MemBench comparison is partial because ThirdAgent and noise-extended slices are not included." - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\Membench\\MemData\\FirstAgent", - "example_count": 1, - "measured": true, - "metadata": {}, - "metrics": { - "hit_at_5": 1.0, - "simple_hit_at_5": 1.0, - "top_k": 5 - }, - "mode": "cmd_turn", - "notes": [ - "FirstAgent slice only", - "turn-level retrieval", - "topic=all" - ], - "primary_metric": "hit_at_5", - "primary_metric_value": 1.0, - "reference_kind": "measured", - "runtime_seconds": 0.567, - "series_name": "new_brainctl", - "source_path": null, - "status": "partial", - "system_name": "brainctl" - }, - { - "artifacts": { - "run_json": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\runs\\convomem_new_brainctl_cmd.json", - "status_chart": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\results\\smoke_postpush_20260423_123452\\charts\\coverage_status.png" - }, - "benchmark": "convomem", - "caveats": [ - "Blocked while loading ConvoMem evidence data: HTTP Error 404: Not Found" - ], - "dataset_path": "C:\\Users\\mario\\Documents\\GitHub\\mempalace\\benchmarks\\convomem_cache", - "example_count": 0, - "measured": true, - "metadata": {}, - "metrics": {}, - "mode": "cmd", - "notes": [ - "limit_per_category=1", - "top_k=10" - ], - "primary_metric": "avg_recall", - "primary_metric_value": null, - "reference_kind": "measured", - "runtime_seconds": null, - "series_name": "new_brainctl", - "source_path": null, - "status": "blocked", - "system_name": "brainctl" - } - ] -} \ No newline at end of file diff --git a/docs/PROCEDURAL_MEMORY_MIGRATION.md b/docs/PROCEDURAL_MEMORY_MIGRATION.md new file mode 100644 index 0000000..878df23 --- /dev/null +++ b/docs/PROCEDURAL_MEMORY_MIGRATION.md @@ -0,0 +1,72 @@ +# Procedural Memory Migration Notes + +This note documents the safety boundary for +`db/migrations/052_procedural_memory_layer.sql`. + +## What Changes + +Migration 052 adds procedural memory as a first-class layer: + +- widens `memories.memory_type` from `episodic|semantic` to + `episodic|semantic|procedural`; +- adds canonical procedure tables: + `procedures`, `procedure_steps`, `procedure_sources`, `procedure_runs`, and + `procedure_candidates`; +- adds `procedures_fts` plus triggers so procedural records are searchable + with plain SQLite FTS5; +- keeps a one-to-one bridge row in `memories` through + `procedures.memory_id` so older generic memory search surfaces still have a + human-readable synopsis. + +## Transaction Safety + +The migration runs inside SQLite transaction semantics used by the existing +migration runner. The `memories` table is rebuilt to widen the CHECK +constraint because SQLite cannot alter CHECK constraints in place. The rebuild +copies existing rows forward and preserves existing memory IDs before swapping +the replacement table into place. + +The procedural companion tables are additive. They do not delete or compress +episodic evidence, semantic facts, events, decisions, entities, or graph edges. + +## Backwards Compatibility + +Newer brainctl versions can read older databases and apply migration 052. + +Older brainctl versions are expected to keep reading migrated databases for +ordinary episodic and semantic rows because the existing `memories` columns are +preserved. Older versions will not understand canonical procedure tables or +`memory_type='procedural'` rows. Operators that need strict older-version +compatibility should not write procedural rows before rolling all clients +forward. + +## Failure and Rollback + +If migration application fails before commit, SQLite rolls the transaction back +and the original schema remains in place. + +If an operator needs to roll back after a successful migration, use the normal +local-first backup path: + +1. stop writers using the target `brain.db`; +2. restore the pre-migration `brain.db` backup if one was taken; +3. otherwise run a forward-only corrective migration rather than editing + migration 052 in place. + +Migration files remain append-only. Do not modify 052 after release; add a new +numbered migration for corrections. + +## Versioning Notes + +This schema should ship with a version bump because it introduces a new +user-visible memory type and new public procedure APIs. The compatibility +matrix should state that procedural-memory writes require a version at or above +the release containing migration 052, while older clients may still read +non-procedural rows from the migrated database. + +## Fresh Install Parity + +`db/init_schema.sql` and `src/agentmemory/db/init_schema.sql` must include the +same procedural schema as migration 052 so fresh installs and upgraded +databases converge. Keep `tests/test_schema_parity.py` and +`tests/test_migrate.py` passing when changing either schema path. diff --git a/docs/RERANKER.md b/docs/RERANKER.md index 38c2b69..de8c603 100644 --- a/docs/RERANKER.md +++ b/docs/RERANKER.md @@ -103,6 +103,27 @@ Models load from the Hugging Face Hub on first use (cached at `~/.cache/huggingface/`). After the first call the model is held in the per-process module cache. +## Second-stage tiny MLP artifact policy + +The local second-stage reranker can optionally load a tiny JSON MLP artifact +from `src/agentmemory/retrieval/models/tiny_mlp_v1.json`, or from an explicit +path passed through the internal reranker configuration. That artifact is not +checked into git. If the file is absent, the second-stage path falls back to +the deterministic heuristic slate scorer and search remains fully functional. + +This keeps the default package local-first and reviewable: + +- no mandatory network fetch, +- no opaque weights bundled in source, +- no hard dependency on numpy at import time, +- no failure when the model artifact is unavailable. + +Training and calibration scripts live under `benchmarks/` and emit JSON +artifacts into ignored benchmark/training output directories. If a trained +artifact is published later, it should be attached as a release asset or LFS +object with a short provenance record containing the source commit, training +bundle, feature version, and held-out metrics. + ## Latency / quality tradeoff Measured on Apple Silicon M-series, CPU only (no MPS), Python 3.14, diff --git a/pyproject.toml b/pyproject.toml index 142fadf..a4a3dbd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -151,4 +151,4 @@ Documentation = "https://github.com/TSchonleber/brainctl/blob/main/docs/AGENT_ON where = ["src"] [tool.setuptools.package-data] -agentmemory = ["db/*.sql", "retrieval/models/*.json"] +agentmemory = ["db/*.sql"] diff --git a/src/agentmemory/retrieval/models/tiny_mlp_v1.json b/src/agentmemory/retrieval/models/tiny_mlp_v1.json deleted file mode 100644 index 75e172a..0000000 --- a/src/agentmemory/retrieval/models/tiny_mlp_v1.json +++ /dev/null @@ -1,1964 +0,0 @@ -{ - "feature_version": "v1", - "feature_order": [ - "base_score", - "retrieval_score", - "rrf_score", - "confidence", - "query_overlap", - "informative_overlap", - "tfidf_cosine", - "exact_phrase", - "entity_overlap", - "alias_overlap", - "query_temporal", - "candidate_temporal", - "temporal_anchor_overlap", - "query_session_hint", - "candidate_session_hint", - "session_gap_score", - "intent_bucket_fit", - "source_keyword", - "source_semantic", - "source_both", - "source_graph", - "bucket_memories", - "bucket_events", - "bucket_entities", - "bucket_procedures", - "bucket_decisions", - "candidate_age_score", - "support_evidence_score", - "status_active", - "status_stale", - "status_needs_review", - "position_score", - "neighbor_margin", - "query_length_score", - "candidate_length_score", - "procedural_candidate" - ], - "norm_mean": [ - 0.41749247, - 0.99348148, - 0.0, - 1.0, - 0.4938599, - 0.49430492, - 0.04392525, - 0.0, - 0.61979308, - 0.0, - 0.30617284, - 0.99259259, - 0.0054321, - 0.0, - 0.0, - 0.0, - 1.0, - 1.0, - 0.0, - 0.0, - 0.0, - 1.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.99999986, - 0.0, - 1.0, - 0.0, - 0.0, - 0.29330117, - 0.41749247, - 0.70833333, - 1.0, - 0.0 - ], - "norm_std": [ - 0.06543539, - 0.06219823, - 1.0, - 1.0, - 0.21382714, - 0.21424068, - 0.02311982, - 1.0, - 0.40672457, - 1.0, - 0.46090241, - 0.08574694, - 0.07350232, - 1.0, - 1.0, - 1.0, - 1.0, - 1.0, - 1.0, - 1.0, - 1.0, - 1.0, - 1.0, - 1.0, - 1.0, - 1.0, - 3.5e-07, - 1.0, - 1.0, - 1.0, - 1.0, - 0.26323239, - 0.06543539, - 0.21390693, - 1.0, - 1.0 - ], - "w1": [ - [ - 0.03560616, - -0.0627443, - 0.08493239, - -0.15795253, - -0.07903827, - -0.11793049, - -0.04711614, - 0.010806, - 0.05168679, - -0.03489014, - 0.09624207, - -0.06908551, - -0.19136466, - 0.13007085, - 0.13310104, - 0.02663946, - 0.01187928, - -0.03025753, - 0.15956304, - -0.20128861, - -0.17845711, - -0.08524863, - 0.12528894, - 0.09287848, - -0.15283449, - -0.10091259, - -0.13670618, - -0.08281627, - -0.08072781, - -0.02773686, - 0.19308194, - 0.2129777, - -0.25250718, - 0.02021468, - 0.0780084, - 0.05377544 - ], - [ - -0.12247778, - -0.09727918, - 0.09473897, - -0.11908801, - -0.10917026, - -0.08511965, - 0.17624795, - 0.02725429, - -0.04348743, - -0.01235824, - -0.02271716, - 0.09714493, - -0.09480573, - 0.158548, - 0.04578166, - 0.17114986, - -0.01028186, - -0.01900935, - 0.14902617, - 0.06425034, - 0.08349582, - -0.15725163, - -0.11654586, - 0.09637033, - 0.20083713, - -0.12941183, - -0.04491017, - 0.09923915, - -0.10297524, - 0.00904144, - 0.01564684, - 0.15967851, - -0.04180626, - 0.04847615, - -0.03304263, - 0.00037121 - ], - [ - 0.08655218, - 0.07345125, - -0.04164363, - 0.04773975, - -0.04876091, - 0.25205024, - -0.1326336, - 0.30062865, - -0.01592228, - -0.03009033, - 0.13975931, - 0.12756869, - 0.04987138, - -0.00825623, - -0.01566002, - 0.01092905, - 0.09443552, - 0.02428797, - -0.03015481, - -0.12351374, - 0.24512503, - 0.09980949, - -0.10873792, - -0.1934399, - 0.20647106, - 0.04115161, - -0.09005481, - -0.16027824, - 0.05718578, - 0.00071849, - -0.11969669, - -0.0148447, - -0.26341796, - 0.01473061, - -0.07118302, - -0.0895403 - ], - [ - 0.11357714, - 0.13461218, - -0.05521256, - -0.10812939, - -0.16226841, - -0.09597153, - -0.1059543, - 0.22412674, - -0.16334151, - 0.01821852, - 0.08926422, - 0.02763969, - 0.06413619, - 0.16164087, - 0.04164969, - 0.06935453, - 0.04115642, - -0.12043613, - 0.04340255, - 0.02809592, - 0.02053495, - 0.04171833, - 0.06670012, - -0.26836158, - 0.16231375, - 0.0452924, - -0.10625103, - -0.04859591, - -0.18338143, - 0.05353742, - -0.0131033, - 0.04171462, - 0.00556409, - 0.0512178, - 0.00101571, - -0.02816198 - ], - [ - -0.24038783, - -0.01415554, - 0.10245672, - -0.04515896, - 0.0698126, - 0.09413812, - 0.0713224, - -0.10189157, - -0.03088761, - 0.17572084, - 0.21737282, - 0.02761021, - 0.03334866, - 0.01100147, - 0.23298364, - -0.00678087, - 0.07980934, - 0.08440971, - -0.28778513, - -0.17001778, - -0.04028229, - 0.2845227, - -0.02645513, - 0.11962521, - 0.03047676, - 0.15372894, - -0.04070144, - -0.06984379, - 0.04653573, - 0.13915728, - -0.00425252, - 0.07983171, - -0.00795474, - -0.01693339, - 0.09492825, - -0.02199367 - ], - [ - -0.15911493, - -0.10015459, - -0.02284495, - 0.15514966, - -0.0268189, - -0.10809812, - -0.04782187, - -0.0343184, - -0.03249869, - -0.30650209, - 0.08479067, - 0.03216638, - -0.16860774, - -0.0999994, - 0.24802633, - -0.02034952, - -0.0822032, - 0.07913344, - -0.13808732, - 0.05329835, - -0.01695757, - 0.05017757, - 0.07732054, - 0.14183431, - 0.16128731, - 0.13135603, - 0.16968437, - -0.0055633, - 0.01224657, - 0.19672649, - -0.16297306, - 0.00926604, - 0.12124126, - 0.17227825, - 0.02153848, - -0.03226106 - ], - [ - 0.01741444, - -0.09854849, - -0.15233836, - -0.0425439, - -0.17371613, - 0.11316355, - 0.00083508, - -0.17475953, - 0.02797641, - -0.0282839, - -0.18757592, - -0.10346714, - -0.27590027, - -0.07098138, - 0.00828527, - -0.09308328, - 0.10721423, - 0.04597999, - -0.03508213, - -0.09632145, - 0.07347198, - 0.04629826, - -0.00162966, - -0.12187952, - 0.00143935, - -0.01579032, - 0.21951773, - 0.03330108, - -0.01760711, - 0.03697383, - -0.03055064, - 0.01704189, - 0.14397163, - -0.05500865, - -0.07834397, - 0.21152787 - ], - [ - -0.04175293, - 0.07761583, - -0.13533351, - 0.08807174, - -0.12340773, - 0.04383488, - -0.02108443, - -0.07054033, - -0.06657603, - 0.02107855, - -0.00845367, - -0.01786699, - 0.00560739, - -0.17680091, - 0.01912945, - 0.0513852, - 0.19452241, - 0.06645943, - -0.12803067, - -0.1531681, - -0.20667123, - -0.019934, - 0.08379041, - -0.22209556, - 0.0242174, - -0.14857906, - -0.04013221, - -0.12035933, - 0.18966682, - 0.07040179, - -0.08863019, - 0.09370273, - 0.02022649, - 0.13533322, - 0.14643996, - -0.0117561 - ], - [ - 0.00234343, - 0.08897077, - -0.10979803, - -0.11148962, - 0.04197479, - -0.01417014, - 0.04459943, - 0.03768853, - 0.05802625, - 0.03534665, - -0.14127705, - -0.01832664, - -0.05648871, - -0.10605809, - 0.12852114, - -0.10168783, - -0.11585213, - 0.03539728, - 0.08531093, - 0.0852421, - 0.01963101, - 0.09753675, - -0.1236057, - 0.01182642, - -0.13057883, - -0.03810668, - 0.22831653, - 0.08649151, - -0.07430201, - -0.13743254, - 0.11038383, - 2.404e-05, - -0.05540174, - -0.07749389, - 0.18858125, - -0.02912529 - ], - [ - -0.1082256, - 0.06450271, - 0.05936934, - -0.02453269, - 0.09267135, - -0.01228072, - 0.16125621, - 0.14399916, - 0.11825327, - -0.04441284, - -0.05999637, - 0.04543412, - 0.05292828, - -0.0427532, - -0.10098612, - 0.07948205, - -0.10600576, - 0.2430219, - 0.23851072, - 0.02885641, - -0.04662795, - 0.07464566, - -0.00144311, - 0.11115771, - 0.04740742, - 0.02594105, - 0.00307946, - 0.00752354, - 0.24603418, - -0.01049396, - 0.00413324, - -0.09577953, - 0.05157854, - -0.00523842, - 0.05905836, - -0.25047084 - ], - [ - 0.10172078, - -0.079691, - 0.01700802, - -0.11344858, - 0.23717787, - -0.07948596, - 0.00938344, - -0.08706485, - 0.01831632, - -0.20978206, - 0.17938602, - 0.1193088, - 0.08401807, - 0.09596154, - 0.03977419, - 0.12960148, - 0.15951507, - -0.01040051, - -0.14050754, - -0.07331913, - 0.22066132, - 0.14947697, - -0.02515992, - 0.21466295, - 0.00721109, - -0.02413213, - 0.20904697, - -0.22593013, - -0.0269203, - 0.11228634, - -0.03377056, - -0.05420135, - -0.06633632, - -0.12881651, - 0.11627176, - -0.10678336 - ], - [ - 0.08705564, - 0.0274381, - 0.08245554, - -0.04048624, - 0.34456371, - 0.04986273, - 0.07359808, - -0.07811606, - 0.04132152, - 0.03916817, - 0.07560008, - -0.12705897, - 0.01576878, - 0.2054888, - -0.00308846, - 0.04377, - -0.02284968, - -0.03667089, - -0.10000718, - 0.17329511, - -0.02079915, - -0.06225471, - -0.14519778, - 0.06165459, - -0.15555852, - -0.06901025, - -0.01160653, - -0.23388869, - -0.15249719, - 0.10341119, - -0.0126799, - 0.06806008, - -0.00599208, - 0.05327136, - 0.14831087, - -0.03141538 - ], - [ - 0.00248131, - 0.01382137, - -0.05102114, - 0.10034689, - 0.04707867, - -0.05673636, - -0.24695362, - -0.25642235, - 0.0053212, - 0.2062746, - -0.08461769, - -0.01505946, - 0.09012134, - -0.16505567, - 0.03748591, - -0.0341829, - 0.16763983, - -0.08998384, - 0.02811649, - -0.05262129, - 0.19917377, - -0.05199496, - -0.1867055, - -0.04438921, - -0.00611379, - 0.03019869, - 0.01613229, - -0.00147113, - 0.00834957, - 0.02526975, - 0.02664451, - -0.00557047, - 0.11235736, - 0.17141486, - 0.13495282, - -0.08190232 - ], - [ - 0.13210261, - 0.02608092, - 0.0189323, - -0.20627105, - -0.11916235, - -0.14734558, - -0.00693601, - -0.01942504, - -0.00034062, - -0.18316879, - -0.12195589, - 0.17684984, - 0.02642982, - 0.04691019, - -0.09951822, - 0.05421111, - -0.05284619, - -0.12326949, - 0.19239431, - 0.0038342, - -0.13180516, - -0.05721296, - 0.08188994, - -0.10665478, - -0.00952105, - -0.06018101, - 0.04388688, - -0.02638721, - -0.12852174, - 0.10585031, - 0.07365797, - 0.03685765, - -0.06911617, - -0.01004123, - 0.07333375, - 0.16499907 - ], - [ - 0.05966595, - 0.10423752, - 0.07470599, - 0.05187747, - -0.25258115, - -0.15058583, - -0.09942808, - -0.12687011, - -0.08721881, - 0.10315556, - 0.00440108, - -0.08803297, - 0.06397317, - -0.12425841, - -0.18982103, - -0.03686074, - 0.17375686, - -0.14861082, - -0.14597201, - 0.03211278, - 0.0701286, - 0.094435, - -0.0419096, - 0.00136748, - 0.21465897, - -0.07500163, - 0.38023162, - -0.0123501, - -0.08977226, - 0.05860578, - -0.11938242, - 0.08597149, - 0.10004904, - -0.02306738, - 0.07145268, - -0.01969634 - ], - [ - -0.1019137, - 0.02661659, - -0.03694117, - 0.02838962, - 0.03308644, - 0.02175547, - -0.14487732, - -0.06322395, - 0.03806621, - -0.03923142, - -0.14517346, - -0.09825995, - -0.0841438, - 0.0566868, - -0.24754901, - 0.11172003, - 0.01570161, - -0.10613731, - 0.02973909, - -0.07399851, - 0.03814175, - 0.17893105, - -0.12207742, - -0.03573723, - 0.10678408, - 0.0371927, - 0.10153805, - -0.00334157, - 0.04741244, - 0.14333671, - -0.12438304, - -0.04357777, - 0.08159333, - -0.13365333, - 0.06205619, - 0.15386187 - ], - [ - 0.04373752, - 0.08113852, - 0.05454663, - -0.07095148, - -0.09738484, - 0.18825577, - -0.10541704, - -0.10471189, - -0.01199727, - -0.00732316, - -0.08017028, - 0.02421221, - -0.02141335, - -0.01565395, - -0.13343441, - -0.21870062, - 0.03083685, - -0.00844042, - 0.21749365, - 0.05626158, - -0.10377883, - -0.05479325, - -0.01148538, - -0.12121614, - 0.00136678, - -0.04800466, - -0.08509618, - 0.02693563, - 0.06634582, - 0.03455434, - 0.13191724, - 0.08910484, - -0.11431568, - -0.02803116, - -0.13104053, - 0.00736857 - ], - [ - -0.11423616, - 0.00805504, - -0.07904507, - -0.17268357, - -0.04954119, - 0.01919434, - -0.03978332, - -0.01125653, - 0.24979031, - -0.12573375, - 0.0377463, - 0.10080928, - 0.02395664, - -0.21864088, - -0.05477353, - -0.04007715, - 0.18685359, - 0.03992045, - -0.19724776, - -0.06370316, - 0.02118407, - -0.05072511, - 0.13470438, - 0.24465581, - 0.02970938, - 0.02915016, - 0.11566215, - 0.11316527, - -0.07429726, - -0.04247635, - -0.04923994, - 0.04538945, - 0.11656982, - -0.19026933, - 0.0836781, - 0.00444563 - ], - [ - 0.09593605, - 0.03421868, - -0.04335472, - 0.00861347, - -0.07956931, - -0.02059617, - 0.10370228, - -0.20990227, - 0.18629368, - -0.03993968, - 0.13700316, - 0.00107125, - 0.09620356, - 0.11085424, - -0.035012, - -0.23773636, - -0.04320147, - 0.00610722, - -0.15298116, - -0.04915652, - 0.1447951, - 0.03750609, - -0.27235837, - 0.21317907, - 0.0052797, - 0.03262515, - -0.04304481, - -0.13268457, - 0.11791274, - 0.04010508, - -0.16919472, - -0.15722663, - 0.07445489, - -0.10935331, - -0.16190694, - -0.01060379 - ], - [ - 0.00690927, - 0.07617659, - -0.04558591, - -0.06323032, - -0.00826627, - 0.04259575, - -0.1516996, - -0.17518982, - 0.04793752, - 0.15524714, - 0.0723479, - 0.15935153, - -0.04720119, - -0.07224724, - 0.23133791, - -0.17853535, - -0.11238577, - -0.09141776, - -0.05058574, - 0.1627224, - -0.03866616, - -0.02934805, - -0.17872378, - 0.13566328, - -0.02423134, - -0.13606694, - -0.05594612, - -0.13995308, - 0.13822585, - -0.34996215, - 0.05299869, - 0.19209274, - 0.02504293, - 0.02661301, - -0.09488851, - 0.00045334 - ], - [ - -0.02318739, - -0.17331286, - -0.14280332, - 0.02778544, - 0.12629738, - 0.15493911, - 0.00297189, - 0.25426784, - -0.09138193, - 0.06957881, - -0.27307865, - 0.10223514, - 0.06122647, - -0.06375665, - 0.13207417, - 0.16286804, - -0.05356609, - 0.10749714, - -0.06216699, - -0.12426344, - -0.20204873, - 0.11369118, - -0.11020813, - -0.10996499, - -0.12925997, - -0.05746827, - 0.1083113, - -0.13056122, - 0.17151853, - 0.04572298, - 0.05533413, - -0.0605972, - -0.09214594, - 0.15880309, - 0.15566754, - 0.20525297 - ], - [ - -0.08216716, - -0.03864507, - 0.05815269, - 0.00260951, - 0.01757535, - 0.02501611, - -0.05914076, - -0.15373986, - -0.13361046, - 0.20686708, - 0.03662696, - 0.1005868, - -0.0316816, - -0.12774639, - -0.11488982, - 0.10690011, - 0.05401626, - 0.08829828, - 0.09703823, - -0.28812646, - -0.0020973, - -0.2689, - 0.17448928, - 0.10210366, - -0.01803813, - 0.17169469, - 0.00339603, - 0.0345015, - 0.06322085, - -0.43568172, - -0.18279309, - -0.19842252, - -0.10345636, - 0.33524853, - 0.10033639, - -0.27700045 - ], - [ - 0.14614664, - -0.05609601, - -0.05605452, - 0.19127928, - -0.01877323, - -0.04893666, - -0.03919627, - -0.1309746, - 0.14196424, - 0.14060274, - 0.00883373, - 0.06609614, - -0.01395025, - -0.0781324, - 0.04152222, - -0.08591801, - -0.18697791, - -0.01906469, - 0.02885823, - 0.19237291, - -0.10776446, - -0.09872246, - 0.03374563, - 0.07638213, - -0.08909665, - -0.1387718, - 0.03279936, - 0.14866761, - 0.16281627, - -0.20581045, - 0.02740397, - -0.12431817, - -0.06405125, - -0.06969794, - 0.17762032, - -0.23901815 - ], - [ - -0.02287583, - -0.07715234, - 0.00149201, - -0.02858308, - -0.12667721, - 0.12910833, - 0.11697609, - 0.2193583, - 0.03071399, - 0.05243432, - 0.04926346, - 0.27790869, - 0.09853811, - 0.0510973, - -0.04861197, - -0.17942413, - 0.07612472, - -0.07796914, - -0.21196026, - 0.30443254, - -0.04088125, - -0.09343364, - 0.09163056, - 0.05284737, - -0.14930866, - -0.25276406, - 0.00112552, - -0.05150917, - -0.2245989, - 0.05394875, - 0.08784097, - -0.1254563, - 0.06037247, - 0.17172835, - -0.03244248, - -0.06486371 - ], - [ - -0.04811214, - -0.03238361, - 0.05740922, - -0.12222278, - -0.19749302, - 0.05366402, - 0.0662674, - 0.34691377, - 0.05846913, - 0.20825505, - 0.19372345, - -0.02424424, - -0.23789188, - -0.02259895, - -0.03395781, - -0.35401449, - -0.06433834, - 0.0654915, - 0.06154865, - -0.04839595, - -0.00974295, - -0.27708945, - -0.13615435, - 0.149231, - 0.06104846, - -0.22233556, - 0.04058664, - -0.29874494, - -0.03796338, - 0.05702502, - 0.04470803, - 0.008909, - -0.11108029, - 0.02877535, - -0.13759785, - 0.00173487 - ], - [ - -0.04199364, - 0.17852177, - 0.05332332, - 0.02140852, - -0.05718739, - 0.1842773, - 0.16038615, - -0.13990472, - -0.20770115, - 0.05242315, - -0.2462415, - -0.23918485, - -0.15482104, - 0.03924776, - 0.02213099, - -0.06490248, - 0.13706809, - 0.0224472, - -0.06896784, - -0.23129097, - -0.2036835, - -0.1150746, - -0.13369137, - 0.07587398, - 0.04672357, - 0.00347622, - 0.05070789, - -0.20348534, - -0.10353498, - -0.1388135, - 0.07546919, - -0.03381477, - 0.05230332, - -0.01846291, - -0.02871224, - 0.08241005 - ], - [ - 0.06619578, - -0.10296794, - 0.07945815, - 0.0262713, - -0.00427586, - 0.02397648, - -0.0171129, - -0.04397511, - 0.11081175, - 0.09887568, - -0.07023213, - 0.19146539, - -0.17690413, - 0.04321954, - 0.07393942, - 0.28903817, - -0.28591486, - -0.1729307, - 0.15219038, - -0.037077, - -0.19293672, - -0.109285, - 0.05347647, - 0.08837012, - -0.21336286, - 0.00369204, - -0.23189702, - -0.09947856, - 0.01423757, - -0.08503693, - -0.1676933, - -0.03832558, - -0.14565732, - 0.05144183, - 0.01218425, - 0.05649455 - ], - [ - 0.04579647, - 0.11568024, - -0.01176082, - 0.16230981, - 0.21280394, - -0.14453532, - -0.08168277, - 0.04078746, - 0.05448564, - -0.03541555, - 0.07112035, - -0.05467179, - -0.27870929, - 0.15770916, - -0.04051316, - 0.05193248, - -0.09392915, - -0.00811798, - -0.07494448, - -0.03417982, - 0.05756687, - -0.02401527, - 0.00695893, - 0.0760573, - -0.0146506, - -0.01404471, - 0.07949468, - -0.06660219, - -0.06824063, - 0.16366559, - 0.03953175, - 0.07638046, - -0.02221289, - 0.0075497, - 0.00943258, - -0.04588541 - ], - [ - 0.04797041, - -0.20121693, - -0.05054892, - 0.0997263, - 0.01445835, - -0.16447287, - -0.02780565, - 0.20643551, - -0.13330268, - 0.00794669, - -0.18869184, - 0.01289104, - -0.08100612, - -0.04093442, - 0.12704173, - -0.06682232, - -0.2013926, - 0.03129147, - -0.07602255, - -0.02268009, - -0.06242138, - 0.1329068, - 0.06552862, - 0.04069621, - 0.11890345, - 0.14499464, - -0.117626, - -0.04876846, - -0.01984203, - -0.05780132, - -0.03613788, - 0.03890619, - 0.01177699, - 0.01304849, - 0.13538047, - 0.12169092 - ], - [ - 0.06006328, - -0.03944994, - -0.00951969, - 0.04261639, - 0.1206877, - 0.20048033, - 0.03257966, - -0.1178473, - -0.05662421, - -0.08328411, - 0.17606165, - 0.15682822, - 0.08864125, - -0.17636161, - -0.13636659, - 0.05553838, - -0.09866566, - -0.10743866, - 0.0646202, - -0.13295722, - -0.30627351, - -0.02926435, - -0.02241107, - -0.2129931, - 0.12648909, - -0.31918129, - -0.16955234, - 0.00460723, - 0.22476327, - 0.26784233, - -0.05765325, - 0.05324364, - 0.3006056, - -0.04828446, - -0.04314175, - 0.12302262 - ], - [ - 0.2527439, - 0.01931682, - -0.20149605, - 0.17474292, - -0.06182885, - 0.20365184, - 0.01960956, - -0.02929029, - 0.03178645, - 0.11817293, - 0.04411025, - -0.19158546, - -0.03393629, - 0.12744427, - 0.00075699, - -0.18640462, - 0.02957546, - 0.02267049, - 0.09110623, - 0.06920933, - 0.09372355, - -0.12309646, - 0.03321503, - 0.00998576, - 0.12250213, - 0.04728115, - -0.0067059, - -0.03722774, - -0.02026739, - -0.00022926, - 0.10395895, - 0.04508873, - -0.16747937, - -0.16690733, - 0.04203882, - 0.02197959 - ], - [ - -0.04611023, - 0.07005009, - -0.17280951, - -0.14195826, - -0.14033296, - -0.02027851, - -0.12879549, - 0.09282715, - 0.00655107, - -0.14070923, - 0.10181729, - -0.14956261, - 0.02448953, - -0.03958525, - 0.31020584, - -0.0355054, - -0.02140267, - -0.17373022, - 0.05351058, - 0.06263498, - 0.03252552, - -0.00680133, - 0.01888207, - -0.06641865, - 0.00464689, - 0.18646192, - 0.00931591, - 0.12530876, - 0.04941375, - 0.04872637, - 0.1766502, - 0.07536327, - 0.17820578, - -0.18311787, - -0.11805867, - 0.11495747 - ] - ], - "b1": [ - 0.00196937, - -0.00037225, - -0.00028717, - -0.00197042, - -0.00443127, - -0.0013593, - 0.00047665, - -0.00072591, - 0.00204353, - -0.00343355, - -0.00100886, - -0.0027349, - -0.00112659, - -0.00070595, - 0.00166679, - -8.147e-05, - 0.00061991, - -0.00028349, - -0.00293703, - 0.00755559, - -0.00093398, - 0.00068876, - -0.00086624, - -0.00060589, - -0.00153215, - 7.15e-06, - -0.00127016, - 0.00049314, - 0.00013518, - 0.00517144, - -0.00113493, - 0.00043336 - ], - "w2": [ - [ - 0.03429766, - -0.04560188, - -0.00127876, - -0.15457293, - -0.21399308, - -0.13972045, - -0.01411688, - -0.0025633, - 0.14509003, - -0.18593507, - -0.07606387, - 0.00235123, - -0.12129521, - -0.04394316, - 0.07089626, - -0.06539088, - 0.00175297, - 0.02476123, - -0.07798005, - 0.28347975, - 0.00119744, - 0.03920252, - -0.02615003, - -0.1499084, - 0.08961679, - -0.06334643, - 0.08680762, - -0.00816018, - -0.00499551, - 0.37000231, - 0.02652476, - -0.01769801 - ], - [ - -0.06653104, - -0.04301215, - -0.16434716, - -0.05719357, - -0.26866194, - 0.01084099, - -0.18599356, - -0.0134559, - -0.25903442, - -0.07898428, - 0.04823889, - 0.0886109, - 0.2762187, - 0.11015847, - 0.02252327, - 0.05559074, - 0.06144833, - -0.10880886, - 0.04306304, - 0.01089718, - -0.26983964, - 0.02188271, - 0.05958972, - -0.32837577, - -0.20135523, - -0.04342517, - -0.18949315, - -0.19355945, - 0.13598805, - -0.2122015, - -0.00203461, - -0.08253614 - ], - [ - 0.08338055, - -0.13719762, - -0.03774681, - 0.05050198, - -0.1427142, - 0.03909324, - 0.11569735, - -0.09967201, - -0.24051152, - -0.13822121, - 0.06753143, - -0.2241865, - 0.03916588, - -0.20681907, - 0.03523033, - -0.22345099, - -0.04994399, - 0.09847235, - -0.16605588, - 0.21353753, - -0.02281422, - -0.07368999, - -0.2516424, - -0.02795873, - -0.11109833, - -0.16419424, - 0.00292467, - -0.27983376, - 0.15111101, - 0.03920507, - 0.03391535, - -0.06018601 - ], - [ - 0.01328226, - -0.22786798, - 0.04367201, - -0.01218587, - 0.15822336, - -0.09457487, - 0.18105676, - 0.07905877, - 0.09503726, - -0.11528522, - 0.21919927, - -0.1286572, - -0.08744177, - 0.10297459, - 0.17600544, - 0.22247413, - 0.26622602, - 0.10996652, - 0.17181348, - 0.04249817, - -0.01946072, - -0.00256137, - -0.00509794, - -0.0224812, - 0.18116639, - -0.07508817, - -0.02621115, - 0.11246318, - -0.16975699, - -0.09120722, - 0.12935966, - 0.15198137 - ], - [ - -0.0001619, - -0.01379371, - 0.07320605, - -0.07753854, - -0.00499501, - 0.00258234, - -0.09450649, - 0.07808871, - 0.00707622, - 0.00667178, - -0.02361775, - 0.10802386, - 0.10948048, - -0.04573059, - 0.14622733, - 0.07123783, - -0.18684069, - 0.07823346, - -0.01700459, - 0.11673401, - 0.25037271, - -0.07563561, - 0.23799873, - -0.05047589, - -0.03172117, - -0.08627651, - 0.04038998, - -0.1164684, - -0.03765861, - 0.07345288, - -0.20935098, - 0.06446052 - ], - [ - 0.17734167, - -0.11915261, - -0.01618277, - -0.07962959, - 0.15389639, - 0.06910934, - 0.03434151, - 0.07086111, - 0.02893482, - -0.24976256, - 0.04343829, - 0.02524846, - -0.01306084, - 0.04513885, - -0.34223498, - -0.02099936, - 0.14967928, - 0.08794806, - 0.03458945, - -0.08905981, - -0.1749678, - -0.1260701, - 0.01647745, - 0.1292002, - 0.10167254, - 0.09019891, - -0.00041903, - -0.05618957, - 0.16316041, - -0.00282543, - 0.11296007, - -0.04622841 - ], - [ - -0.29267501, - -0.01015285, - 0.18290184, - -0.08512424, - 0.04903818, - 0.07383068, - -0.08676388, - 0.18350033, - -0.15846489, - 0.07324827, - -0.17748137, - 0.0486868, - -0.03026208, - -0.13459166, - -0.03794671, - 0.06873218, - 0.16330207, - -0.03241835, - 0.27765925, - -0.10209869, - -0.04676954, - -0.04220477, - 0.00811791, - 0.14306605, - 0.02832193, - 0.09643381, - -0.17361211, - -0.16568168, - 0.06065134, - 0.09034955, - 0.00074388, - -0.02829556 - ], - [ - -0.16954701, - -0.19203422, - 0.1201108, - -0.10700465, - 0.09189501, - 0.18561998, - -0.07910199, - 0.17166136, - -0.00419339, - 0.08773094, - 0.13791409, - 0.20500771, - 0.06591668, - 0.15897407, - -0.02101547, - -0.16760041, - -0.10341213, - -0.11307912, - 0.17918511, - 0.04866367, - 0.20195524, - -0.12934254, - 0.06614909, - -0.0216321, - -0.0119192, - -0.02172883, - -0.15073647, - -0.17304099, - 0.02350505, - 0.11896853, - -0.03216596, - -0.18928925 - ], - [ - -0.1420778, - -0.09104979, - -0.03080536, - 0.27221826, - -0.10756145, - 0.15214398, - 0.09274366, - -0.04478585, - 0.23921175, - -0.03000053, - -0.09068356, - 0.05695906, - -0.15435204, - 0.14855358, - -0.20944908, - 0.14095938, - 0.26635589, - -0.10363976, - -0.03668884, - 0.11856426, - 0.05369262, - 0.03789264, - 0.10674311, - -0.25500801, - 0.162527, - -0.04659194, - 0.00330514, - -0.02039369, - -0.02615833, - 0.02428332, - -0.17517032, - 0.04648334 - ], - [ - -0.04295201, - 0.01913865, - 0.08941513, - 0.0364854, - 0.06334408, - -0.11887691, - -0.00195216, - 0.04580442, - 0.09969148, - -0.29499885, - -0.0650717, - 0.0854579, - 0.04290411, - 0.11516077, - 0.01323483, - -0.0385593, - 0.11976477, - -0.1939243, - -0.0699621, - 0.00794991, - 0.06876896, - -0.13767376, - -0.23859084, - 0.07459248, - -0.02824027, - 0.01942016, - 0.12243855, - -0.08488264, - -0.04546411, - 0.1439702, - 0.06366091, - -0.20880981 - ], - [ - -0.02974994, - 0.04376367, - 0.23131969, - 0.10653928, - 0.08726453, - 0.20180255, - 0.20747097, - -0.074475, - 0.02025284, - -0.11908976, - -0.00271304, - 0.32292598, - 0.2284816, - -0.18069845, - 0.14941911, - -0.02405956, - 0.11582927, - 0.02808227, - -0.03902498, - -0.15443809, - 0.01376139, - 0.08679999, - -0.20078811, - 0.06643476, - -0.21179974, - -0.09739104, - -0.2639807, - 0.04464895, - 0.03488319, - -0.28111167, - -0.28843781, - 0.08980877 - ], - [ - -0.1801343, - 0.0491963, - 0.23360958, - -0.06008171, - -0.07070128, - -0.06345462, - -0.06165247, - 0.0017824, - -0.05481267, - 0.14388679, - -0.19126183, - -0.0399862, - 0.05438321, - 0.09884513, - -0.13035389, - -0.05911431, - 0.00198846, - -0.012547, - 0.05883228, - -0.16637059, - 0.03623264, - -0.14119068, - 0.02516784, - 0.03522996, - -0.09595736, - -0.14212536, - 0.04100391, - 0.05085428, - -0.03056572, - 0.17199256, - 0.06998061, - 0.01387065 - ], - [ - -0.11941817, - -0.15861942, - -0.14690619, - -0.02241434, - 0.0662634, - 0.12299136, - 0.11977162, - -0.07154237, - -0.047431, - -0.33170437, - 0.02717803, - -0.09490496, - -0.04468394, - 0.04803596, - 0.03967852, - -0.06984498, - -0.02577174, - -0.05287281, - -0.01252779, - -0.16964182, - 0.09229982, - -0.06957105, - -0.08581756, - 0.11699777, - 0.06424206, - -0.1120628, - 0.14138985, - 0.03872028, - -0.00561418, - 0.04636024, - -0.17776693, - 0.28520324 - ], - [ - -0.10281586, - 0.10212244, - -0.11056783, - -0.0850553, - 0.06786135, - 0.00811472, - 0.01207705, - 0.19360553, - -0.25412053, - -0.04129009, - 0.11605744, - 0.05294472, - -0.06042106, - -0.19168196, - -0.10926887, - 0.08296959, - -0.02487325, - -0.35280197, - -0.04695221, - -0.21045875, - -0.09645094, - -0.07460011, - 0.15667572, - -0.13894349, - 0.04720454, - -0.07530807, - -0.01818459, - 0.02895305, - 0.00940061, - 0.13127451, - 0.0072958, - -0.13114248 - ], - [ - 0.29545482, - -0.09467129, - 0.17797674, - -0.31534029, - -0.0649652, - -0.0473803, - 0.09512377, - 0.04399461, - 0.03088335, - -0.14096128, - 0.03473734, - 0.05137572, - -0.04868198, - -0.00225438, - -0.079406, - 0.03098636, - 0.04657169, - -0.14919995, - 0.17505804, - 0.01278448, - 0.10518397, - 0.16139606, - -0.03720567, - 0.24036851, - -0.18538461, - -0.13889716, - 0.06723214, - 0.18803595, - -0.07961564, - -0.14658821, - -0.01432602, - -0.0119934 - ], - [ - 0.21551848, - 0.07536598, - -0.12615861, - -0.16416756, - -0.13411485, - -0.2155955, - -0.05349927, - -0.02613664, - 0.10120401, - 0.09623937, - -0.18968477, - -0.06343746, - -0.03026075, - 0.04375666, - 0.17671492, - 0.07846314, - 0.16710939, - 0.13399871, - -0.01334278, - 0.13363101, - -0.1639659, - 0.04008857, - -0.23062853, - -0.11453942, - -0.08612538, - 0.01949231, - -0.08362308, - -0.12786494, - 0.06369067, - -0.10011735, - 0.13949364, - 0.21482437 - ] - ], - "b2": [ - 0.01885969, - 0.00170239, - 0.00050057, - 3.708e-05, - 0.00050642, - -0.00307421, - -0.0004431, - -0.01001486, - -0.00038455, - 0.00768781, - 0.00076421, - -0.01321739, - -0.00194406, - 0.00162896, - 0.00318053, - 0.00098175 - ], - "w3": [ - [ - 0.19036689, - -0.08403602, - 0.02800143, - -0.00779111, - 0.03950601, - -0.0811864, - -0.0030103, - -0.15454482, - 0.01133027, - 0.07498601, - -0.0376662, - -0.18525029, - 0.11770436, - 0.04727888, - -0.09068078, - -0.0334385 - ] - ], - "b3": [ - -0.0 - ], - "metadata": { - "generated_at_utc": "2026-04-23T19:40:50.652698+00:00", - "source_data": "C:\\Users\\mario\\Documents\\GitHub\\brainctl\\BRAINCTL WITH PRO\\brainctl\\benchmarks\\training_data\\hard_negatives_v1.jsonl", - "objective": "lambda_weighted_pairwise_ndcg_at_5", - "train_records": 2025, - "heldout_records": 560, - "longmemeval_extra_epoch_applied": false - } -} \ No newline at end of file